Diff of the two buildlogs: -- --- b1/build.log 2021-08-13 15:26:13.244123851 +0000 +++ b2/build.log 2021-08-13 15:41:48.873082566 +0000 @@ -1,6 +1,6 @@ I: pbuilder: network access will be disabled during build -I: Current time: Thu Sep 15 09:32:54 -12 2022 -I: pbuilder-time-stamp: 1663277574 +I: Current time: Sat Aug 14 05:26:18 +14 2021 +I: pbuilder-time-stamp: 1628868378 I: Building the build Environment I: extracting base tarball [/var/cache/pbuilder/bullseye-reproducible-base.tgz] I: copying local configuration @@ -16,8 +16,8 @@ I: copying [./consul_1.8.7+dfsg1-2.debian.tar.xz] I: Extracting source gpgv: unknown type of key resource 'trustedkeys.kbx' -gpgv: keyblock resource '/tmp/dpkg-verify-sig.EupHXmaI/trustedkeys.kbx': General error -gpgv: Signature made Fri Apr 23 23:25:37 2021 -12 +gpgv: keyblock resource '/tmp/dpkg-verify-sig.xi7Q0ckC/trustedkeys.kbx': General error +gpgv: Signature made Sun Apr 25 01:25:37 2021 +14 gpgv: using RSA key C5A5B9DDC33D93FBB63D67C83287D89A97CDA87B gpgv: issuer "vvidic@debian.org" gpgv: Can't check signature: No public key @@ -31,135 +31,169 @@ dpkg-source: info: applying CVE-2020-25864.patch I: using fakeroot in build. I: Installing the build-deps -I: user script /srv/workspace/pbuilder/17411/tmp/hooks/D02_print_environment starting +I: user script /srv/workspace/pbuilder/20322/tmp/hooks/D01_modify_environment starting +debug: Running on codethink14-arm64. +I: Changing host+domainname to test build reproducibility +I: Adding a custom variable just for the fun of it... +I: Changing /bin/sh to bash +Removing 'diversion of /bin/sh to /bin/sh.distrib by dash' +Adding 'diversion of /bin/sh to /bin/sh.distrib by bash' +Removing 'diversion of /usr/share/man/man1/sh.1.gz to /usr/share/man/man1/sh.distrib.1.gz by dash' +Adding 'diversion of /usr/share/man/man1/sh.1.gz to /usr/share/man/man1/sh.distrib.1.gz by bash' +I: Setting pbuilder2's login shell to /bin/bash +I: Setting pbuilder2's GECOS to second user,second room,second work-phone,second home-phone,second other +I: user script /srv/workspace/pbuilder/20322/tmp/hooks/D01_modify_environment finished +I: user script /srv/workspace/pbuilder/20322/tmp/hooks/D02_print_environment starting I: set - BUILDDIR='/build' - BUILDUSERGECOS='first user,first room,first work-phone,first home-phone,first other' - BUILDUSERNAME='pbuilder1' - BUILD_ARCH='arm64' - DEBIAN_FRONTEND='noninteractive' + BASH=/bin/sh + BASHOPTS=checkwinsize:cmdhist:complete_fullquote:extquote:force_fignore:globasciiranges:hostcomplete:interactive_comments:progcomp:promptvars:sourcepath + BASH_ALIASES=() + BASH_ARGC=() + BASH_ARGV=() + BASH_CMDS=() + BASH_LINENO=([0]="12" [1]="0") + BASH_SOURCE=([0]="/tmp/hooks/D02_print_environment" [1]="/tmp/hooks/D02_print_environment") + BASH_VERSINFO=([0]="5" [1]="1" [2]="4" [3]="1" [4]="release" [5]="aarch64-unknown-linux-gnu") + BASH_VERSION='5.1.4(1)-release' + BUILDDIR=/build + BUILDUSERGECOS='second user,second room,second work-phone,second home-phone,second other' + BUILDUSERNAME=pbuilder2 + BUILD_ARCH=arm64 + DEBIAN_FRONTEND=noninteractive DEB_BUILD_OPTIONS='buildinfo=+all reproducible=+all,-fixfilepath parallel=8' - DISTRIBUTION='' - HOME='/var/lib/jenkins' - HOST_ARCH='arm64' + DIRSTACK=() + DISTRIBUTION= + EUID=0 + FUNCNAME=([0]="Echo" [1]="main") + GROUPS=() + HOME=/var/lib/jenkins + HOSTNAME=i-capture-the-hostname + HOSTTYPE=aarch64 + HOST_ARCH=arm64 IFS=' ' - LANG='C' - LANGUAGE='en_US:en' - LC_ALL='C' - MAIL='/var/mail/root' - OPTIND='1' - PATH='/usr/sbin:/usr/bin:/sbin:/bin:/usr/games' - PBCURRENTCOMMANDLINEOPERATION='build' - PBUILDER_OPERATION='build' - PBUILDER_PKGDATADIR='/usr/share/pbuilder' - PBUILDER_PKGLIBDIR='/usr/lib/pbuilder' - PBUILDER_SYSCONFDIR='/etc' - PPID='17411' - PS1='# ' - PS2='> ' + LANG=C + LANGUAGE=nl_BE:nl + LC_ALL=C + MACHTYPE=aarch64-unknown-linux-gnu + MAIL=/var/mail/root + OPTERR=1 + OPTIND=1 + OSTYPE=linux-gnu + PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path + PBCURRENTCOMMANDLINEOPERATION=build + PBUILDER_OPERATION=build + PBUILDER_PKGDATADIR=/usr/share/pbuilder + PBUILDER_PKGLIBDIR=/usr/lib/pbuilder + PBUILDER_SYSCONFDIR=/etc + PIPESTATUS=([0]="0") + POSIXLY_CORRECT=y + PPID=20322 PS4='+ ' - PWD='/' - SHELL='/bin/bash' - SHLVL='2' - SUDO_COMMAND='/usr/bin/timeout -k 18.1h 18h /usr/bin/ionice -c 3 /usr/bin/nice /usr/sbin/pbuilder --build --configfile /srv/reproducible-results/rbuild-debian/tmp.XA4NF3aLng/pbuilderrc_6MUi --hookdir /etc/pbuilder/first-build-hooks --debbuildopts -b --basetgz /var/cache/pbuilder/bullseye-reproducible-base.tgz --buildresult /srv/reproducible-results/rbuild-debian/tmp.XA4NF3aLng/b1 --logfile b1/build.log consul_1.8.7+dfsg1-2.dsc' - SUDO_GID='117' - SUDO_UID='110' - SUDO_USER='jenkins' - TERM='unknown' - TZ='/usr/share/zoneinfo/Etc/GMT+12' - USER='root' - USERNAME='root' - _='/usr/bin/systemd-run' - http_proxy='http://192.168.101.16:3128' + PWD=/ + SHELL=/bin/bash + SHELLOPTS=braceexpand:errexit:hashall:interactive-comments:posix + SHLVL=3 + SUDO_COMMAND='/usr/bin/timeout -k 24.1h 24h /usr/bin/ionice -c 3 /usr/bin/nice -n 11 /usr/bin/unshare --uts -- /usr/sbin/pbuilder --build --configfile /srv/reproducible-results/rbuild-debian/tmp.XA4NF3aLng/pbuilderrc_bJAF --hookdir /etc/pbuilder/rebuild-hooks --debbuildopts -b --basetgz /var/cache/pbuilder/bullseye-reproducible-base.tgz --buildresult /srv/reproducible-results/rbuild-debian/tmp.XA4NF3aLng/b2 --logfile b2/build.log --extrapackages usrmerge consul_1.8.7+dfsg1-2.dsc' + SUDO_GID=117 + SUDO_UID=110 + SUDO_USER=jenkins + TERM=unknown + TZ=/usr/share/zoneinfo/Etc/GMT-14 + UID=0 + USER=root + USERNAME=root + _='I: set' + http_proxy=http://192.168.101.16:3128 I: uname -a - Linux codethink9-arm64 4.15.0-153-generic #160-Ubuntu SMP Thu Jul 29 07:06:07 UTC 2021 aarch64 GNU/Linux + Linux i-capture-the-hostname 4.15.0-153-generic #160-Ubuntu SMP Thu Jul 29 07:06:07 UTC 2021 aarch64 GNU/Linux I: ls -l /bin total 5252 - -rwxr-xr-x 1 root root 1282512 Aug 4 2021 bash - -rwxr-xr-x 3 root root 34808 Jul 20 2020 bunzip2 - -rwxr-xr-x 3 root root 34808 Jul 20 2020 bzcat - lrwxrwxrwx 1 root root 6 Jul 20 2020 bzcmp -> bzdiff - -rwxr-xr-x 1 root root 2225 Jul 20 2020 bzdiff - lrwxrwxrwx 1 root root 6 Jul 20 2020 bzegrep -> bzgrep - -rwxr-xr-x 1 root root 4877 Sep 4 2019 bzexe - lrwxrwxrwx 1 root root 6 Jul 20 2020 bzfgrep -> bzgrep - -rwxr-xr-x 1 root root 3775 Jul 20 2020 bzgrep - -rwxr-xr-x 3 root root 34808 Jul 20 2020 bzip2 - -rwxr-xr-x 1 root root 14264 Jul 20 2020 bzip2recover - lrwxrwxrwx 1 root root 6 Jul 20 2020 bzless -> bzmore - -rwxr-xr-x 1 root root 1297 Jul 20 2020 bzmore - -rwxr-xr-x 1 root root 39832 Sep 22 2020 cat - -rwxr-xr-x 1 root root 64512 Sep 22 2020 chgrp - -rwxr-xr-x 1 root root 60368 Sep 22 2020 chmod - -rwxr-xr-x 1 root root 64528 Sep 22 2020 chown - -rwxr-xr-x 1 root root 138896 Sep 22 2020 cp - -rwxr-xr-x 1 root root 129544 Dec 10 2020 dash - -rwxr-xr-x 1 root root 101384 Sep 22 2020 date - -rwxr-xr-x 1 root root 80984 Sep 22 2020 dd - -rwxr-xr-x 1 root root 89824 Sep 22 2020 df - -rwxr-xr-x 1 root root 143088 Sep 22 2020 dir - -rwxr-xr-x 1 root root 76152 Jul 28 2021 dmesg - lrwxrwxrwx 1 root root 8 Nov 6 2019 dnsdomainname -> hostname - lrwxrwxrwx 1 root root 8 Nov 6 2019 domainname -> hostname - -rwxr-xr-x 1 root root 35632 Sep 22 2020 echo - -rwxr-xr-x 1 root root 28 Nov 9 2020 egrep - -rwxr-xr-x 1 root root 31512 Sep 22 2020 false - -rwxr-xr-x 1 root root 28 Nov 9 2020 fgrep - -rwxr-xr-x 1 root root 64856 Jul 28 2021 findmnt - -rwsr-xr-x 1 root root 34824 Feb 26 2021 fusermount - -rwxr-xr-x 1 root root 178400 Nov 9 2020 grep - -rwxr-xr-x 2 root root 2346 Mar 2 2021 gunzip - -rwxr-xr-x 1 root root 6376 Mar 2 2021 gzexe - -rwxr-xr-x 1 root root 93744 Mar 2 2021 gzip - -rwxr-xr-x 1 root root 18440 Nov 6 2019 hostname - -rwxr-xr-x 1 root root 68720 Sep 22 2020 ln - -rwxr-xr-x 1 root root 52720 Feb 7 2020 login - -rwxr-xr-x 1 root root 143088 Sep 22 2020 ls - -rwxr-xr-x 1 root root 161960 Jul 28 2021 lsblk - -rwxr-xr-x 1 root root 85200 Sep 22 2020 mkdir - -rwxr-xr-x 1 root root 68744 Sep 22 2020 mknod - -rwxr-xr-x 1 root root 43976 Sep 22 2020 mktemp - -rwxr-xr-x 1 root root 51368 Jul 28 2021 more - -rwsr-xr-x 1 root root 51360 Jul 28 2021 mount - -rwxr-xr-x 1 root root 14496 Jul 28 2021 mountpoint - -rwxr-xr-x 1 root root 134808 Sep 22 2020 mv - lrwxrwxrwx 1 root root 8 Nov 6 2019 nisdomainname -> hostname - lrwxrwxrwx 1 root root 14 Apr 18 2021 pidof -> /sbin/killall5 - -rwxr-xr-x 1 root root 35720 Sep 22 2020 pwd - lrwxrwxrwx 1 root root 4 Aug 4 2021 rbash -> bash - -rwxr-xr-x 1 root root 43872 Sep 22 2020 readlink - -rwxr-xr-x 1 root root 68592 Sep 22 2020 rm - -rwxr-xr-x 1 root root 43880 Sep 22 2020 rmdir - -rwxr-xr-x 1 root root 19208 Sep 27 2020 run-parts - -rwxr-xr-x 1 root root 114016 Dec 22 2018 sed - lrwxrwxrwx 1 root root 4 Sep 14 03:48 sh -> dash - -rwxr-xr-x 1 root root 35656 Sep 22 2020 sleep - -rwxr-xr-x 1 root root 72640 Sep 22 2020 stty - -rwsr-xr-x 1 root root 67776 Jul 28 2021 su - -rwxr-xr-x 1 root root 35672 Sep 22 2020 sync - -rwxr-xr-x 1 root root 535768 Feb 16 2021 tar - -rwxr-xr-x 1 root root 10568 Sep 27 2020 tempfile - -rwxr-xr-x 1 root root 89120 Sep 22 2020 touch - -rwxr-xr-x 1 root root 31512 Sep 22 2020 true - -rwxr-xr-x 1 root root 14264 Feb 26 2021 ulockmgr_server - -rwsr-xr-x 1 root root 30880 Jul 28 2021 umount - -rwxr-xr-x 1 root root 35640 Sep 22 2020 uname - -rwxr-xr-x 2 root root 2346 Mar 2 2021 uncompress - -rwxr-xr-x 1 root root 143088 Sep 22 2020 vdir - -rwxr-xr-x 1 root root 59584 Jul 28 2021 wdctl - lrwxrwxrwx 1 root root 8 Nov 6 2019 ypdomainname -> hostname - -rwxr-xr-x 1 root root 1984 Mar 2 2021 zcat - -rwxr-xr-x 1 root root 1678 Mar 2 2021 zcmp - -rwxr-xr-x 1 root root 5880 Mar 2 2021 zdiff - -rwxr-xr-x 1 root root 29 Mar 2 2021 zegrep - -rwxr-xr-x 1 root root 29 Mar 2 2021 zfgrep - -rwxr-xr-x 1 root root 2081 Mar 2 2021 zforce - -rwxr-xr-x 1 root root 7585 Mar 2 2021 zgrep - -rwxr-xr-x 1 root root 2206 Mar 2 2021 zless - -rwxr-xr-x 1 root root 1842 Mar 2 2021 zmore - -rwxr-xr-x 1 root root 4553 Mar 2 2021 znew -I: user script /srv/workspace/pbuilder/17411/tmp/hooks/D02_print_environment finished + -rwxr-xr-x 1 root root 1282512 Aug 5 10:25 bash + -rwxr-xr-x 3 root root 34808 Jul 21 2020 bunzip2 + -rwxr-xr-x 3 root root 34808 Jul 21 2020 bzcat + lrwxrwxrwx 1 root root 6 Jul 21 2020 bzcmp -> bzdiff + -rwxr-xr-x 1 root root 2225 Jul 21 2020 bzdiff + lrwxrwxrwx 1 root root 6 Jul 21 2020 bzegrep -> bzgrep + -rwxr-xr-x 1 root root 4877 Sep 5 2019 bzexe + lrwxrwxrwx 1 root root 6 Jul 21 2020 bzfgrep -> bzgrep + -rwxr-xr-x 1 root root 3775 Jul 21 2020 bzgrep + -rwxr-xr-x 3 root root 34808 Jul 21 2020 bzip2 + -rwxr-xr-x 1 root root 14264 Jul 21 2020 bzip2recover + lrwxrwxrwx 1 root root 6 Jul 21 2020 bzless -> bzmore + -rwxr-xr-x 1 root root 1297 Jul 21 2020 bzmore + -rwxr-xr-x 1 root root 39832 Sep 23 2020 cat + -rwxr-xr-x 1 root root 64512 Sep 23 2020 chgrp + -rwxr-xr-x 1 root root 60368 Sep 23 2020 chmod + -rwxr-xr-x 1 root root 64528 Sep 23 2020 chown + -rwxr-xr-x 1 root root 138896 Sep 23 2020 cp + -rwxr-xr-x 1 root root 129544 Dec 11 2020 dash + -rwxr-xr-x 1 root root 101384 Sep 23 2020 date + -rwxr-xr-x 1 root root 80984 Sep 23 2020 dd + -rwxr-xr-x 1 root root 89824 Sep 23 2020 df + -rwxr-xr-x 1 root root 143088 Sep 23 2020 dir + -rwxr-xr-x 1 root root 76152 Jul 29 09:09 dmesg + lrwxrwxrwx 1 root root 8 Nov 8 2019 dnsdomainname -> hostname + lrwxrwxrwx 1 root root 8 Nov 8 2019 domainname -> hostname + -rwxr-xr-x 1 root root 35632 Sep 23 2020 echo + -rwxr-xr-x 1 root root 28 Nov 10 2020 egrep + -rwxr-xr-x 1 root root 31512 Sep 23 2020 false + -rwxr-xr-x 1 root root 28 Nov 10 2020 fgrep + -rwxr-xr-x 1 root root 64856 Jul 29 09:09 findmnt + -rwsr-xr-x 1 root root 34824 Feb 27 06:12 fusermount + -rwxr-xr-x 1 root root 178400 Nov 10 2020 grep + -rwxr-xr-x 2 root root 2346 Mar 3 13:30 gunzip + -rwxr-xr-x 1 root root 6376 Mar 3 13:30 gzexe + -rwxr-xr-x 1 root root 93744 Mar 3 13:30 gzip + -rwxr-xr-x 1 root root 18440 Nov 8 2019 hostname + -rwxr-xr-x 1 root root 68720 Sep 23 2020 ln + -rwxr-xr-x 1 root root 52720 Feb 8 2020 login + -rwxr-xr-x 1 root root 143088 Sep 23 2020 ls + -rwxr-xr-x 1 root root 161960 Jul 29 09:09 lsblk + -rwxr-xr-x 1 root root 85200 Sep 23 2020 mkdir + -rwxr-xr-x 1 root root 68744 Sep 23 2020 mknod + -rwxr-xr-x 1 root root 43976 Sep 23 2020 mktemp + -rwxr-xr-x 1 root root 51368 Jul 29 09:09 more + -rwsr-xr-x 1 root root 51360 Jul 29 09:09 mount + -rwxr-xr-x 1 root root 14496 Jul 29 09:09 mountpoint + -rwxr-xr-x 1 root root 134808 Sep 23 2020 mv + lrwxrwxrwx 1 root root 8 Nov 8 2019 nisdomainname -> hostname + lrwxrwxrwx 1 root root 14 Apr 19 05:38 pidof -> /sbin/killall5 + -rwxr-xr-x 1 root root 35720 Sep 23 2020 pwd + lrwxrwxrwx 1 root root 4 Aug 5 10:25 rbash -> bash + -rwxr-xr-x 1 root root 43872 Sep 23 2020 readlink + -rwxr-xr-x 1 root root 68592 Sep 23 2020 rm + -rwxr-xr-x 1 root root 43880 Sep 23 2020 rmdir + -rwxr-xr-x 1 root root 19208 Sep 28 2020 run-parts + -rwxr-xr-x 1 root root 114016 Dec 23 2018 sed + lrwxrwxrwx 1 root root 4 Aug 14 05:26 sh -> bash + lrwxrwxrwx 1 root root 4 Aug 12 23:25 sh.distrib -> dash + -rwxr-xr-x 1 root root 35656 Sep 23 2020 sleep + -rwxr-xr-x 1 root root 72640 Sep 23 2020 stty + -rwsr-xr-x 1 root root 67776 Jul 29 09:09 su + -rwxr-xr-x 1 root root 35672 Sep 23 2020 sync + -rwxr-xr-x 1 root root 535768 Feb 17 23:55 tar + -rwxr-xr-x 1 root root 10568 Sep 28 2020 tempfile + -rwxr-xr-x 1 root root 89120 Sep 23 2020 touch + -rwxr-xr-x 1 root root 31512 Sep 23 2020 true + -rwxr-xr-x 1 root root 14264 Feb 27 06:12 ulockmgr_server + -rwsr-xr-x 1 root root 30880 Jul 29 09:09 umount + -rwxr-xr-x 1 root root 35640 Sep 23 2020 uname + -rwxr-xr-x 2 root root 2346 Mar 3 13:30 uncompress + -rwxr-xr-x 1 root root 143088 Sep 23 2020 vdir + -rwxr-xr-x 1 root root 59584 Jul 29 09:09 wdctl + lrwxrwxrwx 1 root root 8 Nov 8 2019 ypdomainname -> hostname + -rwxr-xr-x 1 root root 1984 Mar 3 13:30 zcat + -rwxr-xr-x 1 root root 1678 Mar 3 13:30 zcmp + -rwxr-xr-x 1 root root 5880 Mar 3 13:30 zdiff + -rwxr-xr-x 1 root root 29 Mar 3 13:30 zegrep + -rwxr-xr-x 1 root root 29 Mar 3 13:30 zfgrep + -rwxr-xr-x 1 root root 2081 Mar 3 13:30 zforce + -rwxr-xr-x 1 root root 7585 Mar 3 13:30 zgrep + -rwxr-xr-x 1 root root 2206 Mar 3 13:30 zless + -rwxr-xr-x 1 root root 1842 Mar 3 13:30 zmore + -rwxr-xr-x 1 root root 4553 Mar 3 13:30 znew +I: user script /srv/workspace/pbuilder/20322/tmp/hooks/D02_print_environment finished -> Attempting to satisfy build-dependencies -> Creating pbuilder-satisfydepends-dummy package Package: pbuilder-satisfydepends-dummy @@ -559,7 +593,7 @@ Get: 210 http://deb.debian.org/debian bullseye/main arm64 golang-k8s-sigs-yaml-dev all 1.2.0-3 [15.0 kB] Get: 211 http://deb.debian.org/debian bullseye/main arm64 mockery arm64 0.0~git20181123.e78b021-3+b6 [1362 kB] Get: 212 http://deb.debian.org/debian bullseye/main arm64 golang-github-hashicorp-go-rootcerts-dev all 1.0.2-2 [8356 B] -Fetched 121 MB in 3s (44.6 MB/s) +Fetched 121 MB in 2s (53.4 MB/s) debconf: delaying package configuration, since apt-utils is not installed Selecting previously unselected package bsdextrautils. (Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 19646 files and directories currently installed.) @@ -1434,9 +1468,42 @@ Building dependency tree... Reading state information... fakeroot is already the newest version (1.25.3-1.1). -0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded. +The following additional packages will be installed: + libfile-find-rule-perl libnumber-compare-perl libtext-glob-perl +The following NEW packages will be installed: + libfile-find-rule-perl libnumber-compare-perl libtext-glob-perl usrmerge +0 upgraded, 4 newly installed, 0 to remove and 0 not upgraded. +Need to get 59.5 kB of archives. +After this operation, 157 kB of additional disk space will be used. +Get:1 http://deb.debian.org/debian bullseye/main arm64 libnumber-compare-perl all 0.03-1.1 [6956 B] +Get:2 http://deb.debian.org/debian bullseye/main arm64 libtext-glob-perl all 0.11-1 [8888 B] +Get:3 http://deb.debian.org/debian bullseye/main arm64 libfile-find-rule-perl all 0.34-1 [30.6 kB] +Get:4 http://deb.debian.org/debian bullseye/main arm64 usrmerge all 25 [13.0 kB] +debconf: delaying package configuration, since apt-utils is not installed +Fetched 59.5 kB in 0s (2814 kB/s) +Selecting previously unselected package libnumber-compare-perl. +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 53971 files and directories currently installed.) +Preparing to unpack .../libnumber-compare-perl_0.03-1.1_all.deb ... +Unpacking libnumber-compare-perl (0.03-1.1) ... +Selecting previously unselected package libtext-glob-perl. +Preparing to unpack .../libtext-glob-perl_0.11-1_all.deb ... +Unpacking libtext-glob-perl (0.11-1) ... +Selecting previously unselected package libfile-find-rule-perl. +Preparing to unpack .../libfile-find-rule-perl_0.34-1_all.deb ... +Unpacking libfile-find-rule-perl (0.34-1) ... +Selecting previously unselected package usrmerge. +Preparing to unpack .../archives/usrmerge_25_all.deb ... +Unpacking usrmerge (25) ... +Setting up libtext-glob-perl (0.11-1) ... +Setting up libnumber-compare-perl (0.03-1.1) ... +Setting up libfile-find-rule-perl (0.34-1) ... +Setting up usrmerge (25) ... +The system has been successfully converted. +Processing triggers for man-db (2.9.4-2) ... +Not building database; man-db/auto-update is not 'true'. I: Building the package -I: Running cd /build/consul-1.8.7+dfsg1/ && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games" HOME="/nonexistent/first-build" dpkg-buildpackage -us -uc -b && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games" HOME="/nonexistent/first-build" dpkg-genchanges -S > ../consul_1.8.7+dfsg1-2_source.changes +hostname: Temporary failure in name resolution +I: Running cd /build/consul-1.8.7+dfsg1/ && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path" HOME="/nonexistent/second-build" dpkg-buildpackage -us -uc -b && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path" HOME="/nonexistent/second-build" dpkg-genchanges -S > ../consul_1.8.7+dfsg1-2_source.changes dpkg-buildpackage: info: source package consul dpkg-buildpackage: info: source version 1.8.7+dfsg1-2 dpkg-buildpackage: info: source distribution unstable @@ -1567,7 +1634,7 @@ CGO_FFLAGS="-g -O2 -fdebug-prefix-map=/build/consul-1.8.7+dfsg1=. -fstack-protector-strong" CGO_LDFLAGS="-Wl,-z,relro" PKG_CONFIG="pkg-config" -GOGCCFLAGS="-fPIC -pthread -fmessage-length=0 -fdebug-prefix-map=/tmp/go-build383947929=/tmp/go-build -gno-record-gcc-switches" +GOGCCFLAGS="-fPIC -pthread -fmessage-length=0 -fdebug-prefix-map=/tmp/go-build342129244=/tmp/go-build -gno-record-gcc-switches" dh_auto_build: warning: github.com/hashicorp/consul/agent/routine-leak-checker contains no non-test Go files, removing it from build cd _build && go generate -v github.com/hashicorp/consul github.com/hashicorp/consul/acl github.com/hashicorp/consul/agent github.com/hashicorp/consul/agent/ae github.com/hashicorp/consul/agent/auto-config github.com/hashicorp/consul/agent/cache github.com/hashicorp/consul/agent/cache-types github.com/hashicorp/consul/agent/checks github.com/hashicorp/consul/agent/config github.com/hashicorp/consul/agent/connect github.com/hashicorp/consul/agent/connect/ca github.com/hashicorp/consul/agent/consul github.com/hashicorp/consul/agent/consul/authmethod github.com/hashicorp/consul/agent/consul/authmethod/kubeauth github.com/hashicorp/consul/agent/consul/authmethod/ssoauth github.com/hashicorp/consul/agent/consul/authmethod/testauth github.com/hashicorp/consul/agent/consul/autopilot github.com/hashicorp/consul/agent/consul/discoverychain github.com/hashicorp/consul/agent/consul/fsm github.com/hashicorp/consul/agent/consul/prepared_query github.com/hashicorp/consul/agent/consul/state github.com/hashicorp/consul/agent/consul/wanfed github.com/hashicorp/consul/agent/debug github.com/hashicorp/consul/agent/dns github.com/hashicorp/consul/agent/exec github.com/hashicorp/consul/agent/local github.com/hashicorp/consul/agent/metadata github.com/hashicorp/consul/agent/mock github.com/hashicorp/consul/agent/pool github.com/hashicorp/consul/agent/proxycfg github.com/hashicorp/consul/agent/router github.com/hashicorp/consul/agent/structs github.com/hashicorp/consul/agent/systemd github.com/hashicorp/consul/agent/token github.com/hashicorp/consul/agent/xds github.com/hashicorp/consul/agent/xds/proxysupport github.com/hashicorp/consul/api github.com/hashicorp/consul/api/watch github.com/hashicorp/consul/command github.com/hashicorp/consul/command/acl github.com/hashicorp/consul/command/acl/agenttokens github.com/hashicorp/consul/command/acl/authmethod github.com/hashicorp/consul/command/acl/authmethod/create github.com/hashicorp/consul/command/acl/authmethod/delete github.com/hashicorp/consul/command/acl/authmethod/list github.com/hashicorp/consul/command/acl/authmethod/read github.com/hashicorp/consul/command/acl/authmethod/update github.com/hashicorp/consul/command/acl/bindingrule github.com/hashicorp/consul/command/acl/bindingrule/create github.com/hashicorp/consul/command/acl/bindingrule/delete github.com/hashicorp/consul/command/acl/bindingrule/list github.com/hashicorp/consul/command/acl/bindingrule/read github.com/hashicorp/consul/command/acl/bindingrule/update github.com/hashicorp/consul/command/acl/bootstrap github.com/hashicorp/consul/command/acl/policy github.com/hashicorp/consul/command/acl/policy/create github.com/hashicorp/consul/command/acl/policy/delete github.com/hashicorp/consul/command/acl/policy/list github.com/hashicorp/consul/command/acl/policy/read github.com/hashicorp/consul/command/acl/policy/update github.com/hashicorp/consul/command/acl/role github.com/hashicorp/consul/command/acl/role/create github.com/hashicorp/consul/command/acl/role/delete github.com/hashicorp/consul/command/acl/role/list github.com/hashicorp/consul/command/acl/role/read github.com/hashicorp/consul/command/acl/role/update github.com/hashicorp/consul/command/acl/rules github.com/hashicorp/consul/command/acl/token github.com/hashicorp/consul/command/acl/token/clone github.com/hashicorp/consul/command/acl/token/create github.com/hashicorp/consul/command/acl/token/delete github.com/hashicorp/consul/command/acl/token/list github.com/hashicorp/consul/command/acl/token/read github.com/hashicorp/consul/command/acl/token/update github.com/hashicorp/consul/command/agent github.com/hashicorp/consul/command/catalog github.com/hashicorp/consul/command/catalog/list/dc github.com/hashicorp/consul/command/catalog/list/nodes github.com/hashicorp/consul/command/catalog/list/services github.com/hashicorp/consul/command/config github.com/hashicorp/consul/command/config/delete github.com/hashicorp/consul/command/config/list github.com/hashicorp/consul/command/config/read github.com/hashicorp/consul/command/config/write github.com/hashicorp/consul/command/connect github.com/hashicorp/consul/command/connect/ca github.com/hashicorp/consul/command/connect/ca/get github.com/hashicorp/consul/command/connect/ca/set github.com/hashicorp/consul/command/connect/envoy github.com/hashicorp/consul/command/connect/envoy/pipe-bootstrap github.com/hashicorp/consul/command/connect/expose github.com/hashicorp/consul/command/connect/proxy github.com/hashicorp/consul/command/debug github.com/hashicorp/consul/command/event github.com/hashicorp/consul/command/exec github.com/hashicorp/consul/command/flags github.com/hashicorp/consul/command/forceleave github.com/hashicorp/consul/command/helpers github.com/hashicorp/consul/command/info github.com/hashicorp/consul/command/intention github.com/hashicorp/consul/command/intention/check github.com/hashicorp/consul/command/intention/create github.com/hashicorp/consul/command/intention/delete github.com/hashicorp/consul/command/intention/finder github.com/hashicorp/consul/command/intention/get github.com/hashicorp/consul/command/intention/match github.com/hashicorp/consul/command/join github.com/hashicorp/consul/command/keygen github.com/hashicorp/consul/command/keyring github.com/hashicorp/consul/command/kv github.com/hashicorp/consul/command/kv/del github.com/hashicorp/consul/command/kv/exp github.com/hashicorp/consul/command/kv/get github.com/hashicorp/consul/command/kv/imp github.com/hashicorp/consul/command/kv/impexp github.com/hashicorp/consul/command/kv/put github.com/hashicorp/consul/command/leave github.com/hashicorp/consul/command/lock github.com/hashicorp/consul/command/login github.com/hashicorp/consul/command/logout github.com/hashicorp/consul/command/maint github.com/hashicorp/consul/command/members github.com/hashicorp/consul/command/monitor github.com/hashicorp/consul/command/operator github.com/hashicorp/consul/command/operator/autopilot github.com/hashicorp/consul/command/operator/autopilot/get github.com/hashicorp/consul/command/operator/autopilot/set github.com/hashicorp/consul/command/operator/raft github.com/hashicorp/consul/command/operator/raft/listpeers github.com/hashicorp/consul/command/operator/raft/removepeer github.com/hashicorp/consul/command/reload github.com/hashicorp/consul/command/rtt github.com/hashicorp/consul/command/services github.com/hashicorp/consul/command/services/deregister github.com/hashicorp/consul/command/services/register github.com/hashicorp/consul/command/snapshot github.com/hashicorp/consul/command/snapshot/inspect github.com/hashicorp/consul/command/snapshot/restore github.com/hashicorp/consul/command/snapshot/save github.com/hashicorp/consul/command/tls github.com/hashicorp/consul/command/tls/ca github.com/hashicorp/consul/command/tls/ca/create github.com/hashicorp/consul/command/tls/cert github.com/hashicorp/consul/command/tls/cert/create github.com/hashicorp/consul/command/validate github.com/hashicorp/consul/command/version github.com/hashicorp/consul/command/watch github.com/hashicorp/consul/connect github.com/hashicorp/consul/connect/certgen github.com/hashicorp/consul/connect/proxy github.com/hashicorp/consul/internal/go-sso/oidcauth github.com/hashicorp/consul/internal/go-sso/oidcauth/internal/strutil github.com/hashicorp/consul/internal/go-sso/oidcauth/oidcauthtest github.com/hashicorp/consul/ipaddr github.com/hashicorp/consul/lib github.com/hashicorp/consul/lib/decode github.com/hashicorp/consul/lib/file github.com/hashicorp/consul/lib/semaphore github.com/hashicorp/consul/lib/stringslice github.com/hashicorp/consul/lib/template github.com/hashicorp/consul/logging github.com/hashicorp/consul/logging/monitor github.com/hashicorp/consul/proto github.com/hashicorp/consul/proto/pbacl github.com/hashicorp/consul/proto/pbautoconf github.com/hashicorp/consul/proto/pbcommon github.com/hashicorp/consul/proto/pbconfig github.com/hashicorp/consul/proto/pbconnect github.com/hashicorp/consul/sdk/freeport github.com/hashicorp/consul/sdk/testutil github.com/hashicorp/consul/sdk/testutil/retry github.com/hashicorp/consul/sentinel github.com/hashicorp/consul/service_os github.com/hashicorp/consul/snapshot github.com/hashicorp/consul/testrpc github.com/hashicorp/consul/tlsutil github.com/hashicorp/consul/types github.com/hashicorp/consul/version src/github.com/hashicorp/consul/main.go @@ -2569,26 +2636,26 @@ src/github.com/hashicorp/consul/types/node_id.go src/github.com/hashicorp/consul/version/version.go cd _build && go install -trimpath -v -p 8 github.com/hashicorp/consul github.com/hashicorp/consul/acl github.com/hashicorp/consul/agent github.com/hashicorp/consul/agent/ae github.com/hashicorp/consul/agent/auto-config github.com/hashicorp/consul/agent/cache github.com/hashicorp/consul/agent/cache-types github.com/hashicorp/consul/agent/checks github.com/hashicorp/consul/agent/config github.com/hashicorp/consul/agent/connect github.com/hashicorp/consul/agent/connect/ca github.com/hashicorp/consul/agent/consul github.com/hashicorp/consul/agent/consul/authmethod github.com/hashicorp/consul/agent/consul/authmethod/kubeauth github.com/hashicorp/consul/agent/consul/authmethod/ssoauth github.com/hashicorp/consul/agent/consul/authmethod/testauth github.com/hashicorp/consul/agent/consul/autopilot github.com/hashicorp/consul/agent/consul/discoverychain github.com/hashicorp/consul/agent/consul/fsm github.com/hashicorp/consul/agent/consul/prepared_query github.com/hashicorp/consul/agent/consul/state github.com/hashicorp/consul/agent/consul/wanfed github.com/hashicorp/consul/agent/debug github.com/hashicorp/consul/agent/dns github.com/hashicorp/consul/agent/exec github.com/hashicorp/consul/agent/local github.com/hashicorp/consul/agent/metadata github.com/hashicorp/consul/agent/mock github.com/hashicorp/consul/agent/pool github.com/hashicorp/consul/agent/proxycfg github.com/hashicorp/consul/agent/router github.com/hashicorp/consul/agent/structs github.com/hashicorp/consul/agent/systemd github.com/hashicorp/consul/agent/token github.com/hashicorp/consul/agent/xds github.com/hashicorp/consul/agent/xds/proxysupport github.com/hashicorp/consul/api github.com/hashicorp/consul/api/watch github.com/hashicorp/consul/command github.com/hashicorp/consul/command/acl github.com/hashicorp/consul/command/acl/agenttokens github.com/hashicorp/consul/command/acl/authmethod github.com/hashicorp/consul/command/acl/authmethod/create github.com/hashicorp/consul/command/acl/authmethod/delete github.com/hashicorp/consul/command/acl/authmethod/list github.com/hashicorp/consul/command/acl/authmethod/read github.com/hashicorp/consul/command/acl/authmethod/update github.com/hashicorp/consul/command/acl/bindingrule github.com/hashicorp/consul/command/acl/bindingrule/create github.com/hashicorp/consul/command/acl/bindingrule/delete github.com/hashicorp/consul/command/acl/bindingrule/list github.com/hashicorp/consul/command/acl/bindingrule/read github.com/hashicorp/consul/command/acl/bindingrule/update github.com/hashicorp/consul/command/acl/bootstrap github.com/hashicorp/consul/command/acl/policy github.com/hashicorp/consul/command/acl/policy/create github.com/hashicorp/consul/command/acl/policy/delete github.com/hashicorp/consul/command/acl/policy/list github.com/hashicorp/consul/command/acl/policy/read github.com/hashicorp/consul/command/acl/policy/update github.com/hashicorp/consul/command/acl/role github.com/hashicorp/consul/command/acl/role/create github.com/hashicorp/consul/command/acl/role/delete github.com/hashicorp/consul/command/acl/role/list github.com/hashicorp/consul/command/acl/role/read github.com/hashicorp/consul/command/acl/role/update github.com/hashicorp/consul/command/acl/rules github.com/hashicorp/consul/command/acl/token github.com/hashicorp/consul/command/acl/token/clone github.com/hashicorp/consul/command/acl/token/create github.com/hashicorp/consul/command/acl/token/delete github.com/hashicorp/consul/command/acl/token/list github.com/hashicorp/consul/command/acl/token/read github.com/hashicorp/consul/command/acl/token/update github.com/hashicorp/consul/command/agent github.com/hashicorp/consul/command/catalog github.com/hashicorp/consul/command/catalog/list/dc github.com/hashicorp/consul/command/catalog/list/nodes github.com/hashicorp/consul/command/catalog/list/services github.com/hashicorp/consul/command/config github.com/hashicorp/consul/command/config/delete github.com/hashicorp/consul/command/config/list github.com/hashicorp/consul/command/config/read github.com/hashicorp/consul/command/config/write github.com/hashicorp/consul/command/connect github.com/hashicorp/consul/command/connect/ca github.com/hashicorp/consul/command/connect/ca/get github.com/hashicorp/consul/command/connect/ca/set github.com/hashicorp/consul/command/connect/envoy github.com/hashicorp/consul/command/connect/envoy/pipe-bootstrap github.com/hashicorp/consul/command/connect/expose github.com/hashicorp/consul/command/connect/proxy github.com/hashicorp/consul/command/debug github.com/hashicorp/consul/command/event github.com/hashicorp/consul/command/exec github.com/hashicorp/consul/command/flags github.com/hashicorp/consul/command/forceleave github.com/hashicorp/consul/command/helpers github.com/hashicorp/consul/command/info github.com/hashicorp/consul/command/intention github.com/hashicorp/consul/command/intention/check github.com/hashicorp/consul/command/intention/create github.com/hashicorp/consul/command/intention/delete github.com/hashicorp/consul/command/intention/finder github.com/hashicorp/consul/command/intention/get github.com/hashicorp/consul/command/intention/match github.com/hashicorp/consul/command/join github.com/hashicorp/consul/command/keygen github.com/hashicorp/consul/command/keyring github.com/hashicorp/consul/command/kv github.com/hashicorp/consul/command/kv/del github.com/hashicorp/consul/command/kv/exp github.com/hashicorp/consul/command/kv/get github.com/hashicorp/consul/command/kv/imp github.com/hashicorp/consul/command/kv/impexp github.com/hashicorp/consul/command/kv/put github.com/hashicorp/consul/command/leave github.com/hashicorp/consul/command/lock github.com/hashicorp/consul/command/login github.com/hashicorp/consul/command/logout github.com/hashicorp/consul/command/maint github.com/hashicorp/consul/command/members github.com/hashicorp/consul/command/monitor github.com/hashicorp/consul/command/operator github.com/hashicorp/consul/command/operator/autopilot github.com/hashicorp/consul/command/operator/autopilot/get github.com/hashicorp/consul/command/operator/autopilot/set github.com/hashicorp/consul/command/operator/raft github.com/hashicorp/consul/command/operator/raft/listpeers github.com/hashicorp/consul/command/operator/raft/removepeer github.com/hashicorp/consul/command/reload github.com/hashicorp/consul/command/rtt github.com/hashicorp/consul/command/services github.com/hashicorp/consul/command/services/deregister github.com/hashicorp/consul/command/services/register github.com/hashicorp/consul/command/snapshot github.com/hashicorp/consul/command/snapshot/inspect github.com/hashicorp/consul/command/snapshot/restore github.com/hashicorp/consul/command/snapshot/save github.com/hashicorp/consul/command/tls github.com/hashicorp/consul/command/tls/ca github.com/hashicorp/consul/command/tls/ca/create github.com/hashicorp/consul/command/tls/cert github.com/hashicorp/consul/command/tls/cert/create github.com/hashicorp/consul/command/validate github.com/hashicorp/consul/command/version github.com/hashicorp/consul/command/watch github.com/hashicorp/consul/connect github.com/hashicorp/consul/connect/certgen github.com/hashicorp/consul/connect/proxy github.com/hashicorp/consul/internal/go-sso/oidcauth github.com/hashicorp/consul/internal/go-sso/oidcauth/internal/strutil github.com/hashicorp/consul/internal/go-sso/oidcauth/oidcauthtest github.com/hashicorp/consul/ipaddr github.com/hashicorp/consul/lib github.com/hashicorp/consul/lib/decode github.com/hashicorp/consul/lib/file github.com/hashicorp/consul/lib/semaphore github.com/hashicorp/consul/lib/stringslice github.com/hashicorp/consul/lib/template github.com/hashicorp/consul/logging github.com/hashicorp/consul/logging/monitor github.com/hashicorp/consul/proto github.com/hashicorp/consul/proto/pbacl github.com/hashicorp/consul/proto/pbautoconf github.com/hashicorp/consul/proto/pbcommon github.com/hashicorp/consul/proto/pbconfig github.com/hashicorp/consul/proto/pbconnect github.com/hashicorp/consul/sdk/freeport github.com/hashicorp/consul/sdk/testutil github.com/hashicorp/consul/sdk/testutil/retry github.com/hashicorp/consul/sentinel github.com/hashicorp/consul/service_os github.com/hashicorp/consul/snapshot github.com/hashicorp/consul/testrpc github.com/hashicorp/consul/tlsutil github.com/hashicorp/consul/types github.com/hashicorp/consul/version +internal/cpu internal/unsafeheader runtime/internal/sys unicode/utf8 -math/bits -internal/cpu -internal/race -unicode runtime/internal/atomic +internal/race +math/bits sync/atomic -runtime/internal/math +unicode encoding +runtime/internal/math unicode/utf16 internal/bytealg container/list -internal/testlog internal/nettrace math crypto/internal/subtle -runtime/cgo +internal/testlog crypto/subtle +runtime/cgo vendor/golang.org/x/crypto/cryptobyte/asn1 vendor/golang.org/x/crypto/internal/subtle github.com/circonus-labs/circonus-gometrics/api/config @@ -2596,8 +2663,8 @@ golang.org/x/net/internal/iana github.com/hashicorp/consul/lib/stringslice github.com/hashicorp/consul/types -google.golang.org/grpc/serviceconfig runtime +google.golang.org/grpc/serviceconfig github.com/aws/aws-sdk-go/aws/client/metadata github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/selection github.com/hashicorp/consul/vendor/k8s.io/utils/integer @@ -2612,10 +2679,10 @@ github.com/aws/aws-sdk-go/internal/sdkmath internal/reflectlite sync -internal/singleflight google.golang.org/grpc/internal/buffer -google.golang.org/grpc/internal/grpcsync +internal/singleflight github.com/aws/aws-sdk-go/internal/sync/singleflight +google.golang.org/grpc/internal/grpcsync github.com/hashicorp/consul/agent/token math/rand errors @@ -2630,15 +2697,15 @@ container/heap github.com/beorn7/perks/quantile github.com/gogo/protobuf/sortkeys -bytes hash +text/tabwriter strings +bytes crypto/internal/randutil -text/tabwriter hash/crc32 crypto/hmac -vendor/golang.org/x/crypto/hkdf hash/crc64 +vendor/golang.org/x/crypto/hkdf hash/fnv reflect crypto @@ -2648,6 +2715,7 @@ github.com/kr/text golang.org/x/text/transform github.com/aws/aws-sdk-go/internal/sdkio +golang.org/x/crypto/pbkdf2 github.com/armon/go-radix regexp/syntax bufio @@ -2656,13 +2724,12 @@ github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg html google.golang.org/grpc/encoding -github.com/aws/aws-sdk-go/internal/strings -github.com/aws/aws-sdk-go/internal/sdkuri internal/syscall/unix -internal/syscall/execenv time +internal/syscall/execenv +github.com/aws/aws-sdk-go/internal/strings +github.com/aws/aws-sdk-go/internal/sdkuri github.com/ryanuber/go-glob -golang.org/x/crypto/pbkdf2 regexp internal/poll context @@ -2675,87 +2742,84 @@ google.golang.org/grpc/tap github.com/hashicorp/consul/lib/semaphore golang.org/x/net/context -github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/version github.com/hashicorp/consul/agent/dns google.golang.org/api/internal/third_party/uritemplates +github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/version +os +github.com/hashicorp/errwrap internal/fmtsort encoding/binary -github.com/hashicorp/errwrap github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/util/sets -os +encoding/base64 crypto/md5 golang.org/x/crypto/blake2b crypto/sha512 crypto/cipher -encoding/base64 crypto/ed25519/internal/edwards25519 crypto/sha1 +fmt +net crypto/sha256 encoding/pem -vendor/golang.org/x/crypto/poly1305 +path/filepath crypto/aes -vendor/golang.org/x/crypto/chacha20 crypto/des -github.com/cespare/xxhash -fmt -net -path/filepath +vendor/golang.org/x/crypto/chacha20 +vendor/golang.org/x/crypto/poly1305 os/signal -vendor/golang.org/x/crypto/chacha20poly1305 +github.com/cespare/xxhash +io/ioutil golang.org/x/sys/unix runtime/debug +os/exec google.golang.org/grpc/internal/envconfig +github.com/prometheus/procfs/internal/util +vendor/golang.org/x/crypto/chacha20poly1305 +github.com/aws/aws-sdk-go/internal/shareddefaults github.com/golang/snappy github.com/pierrec/lz4/internal/xxh32 -io/ioutil -os/exec -github.com/aws/aws-sdk-go/internal/shareddefaults -github.com/google/gofuzz/bytesource -internal/lazyregexp -github.com/prometheus/procfs/internal/util -github.com/hashicorp/consul/vendor/github.com/rboyer/safeio +encoding/json github.com/hashicorp/hcl/hcl/token log -encoding/json compress/flate -math/big -encoding/hex github.com/hashicorp/hcl/hcl/ast github.com/hashicorp/hcl/hcl/scanner github.com/hashicorp/hcl/json/token -net/url +math/big +encoding/hex github.com/hashicorp/hcl/json/scanner github.com/hashicorp/hcl/hcl/parser +net/url github.com/hashicorp/hcl/json/parser -compress/gzip github.com/hashicorp/hcl/hcl/printer -vendor/golang.org/x/crypto/curve25519 +compress/gzip github.com/hashicorp/hcl +vendor/golang.org/x/crypto/curve25519 vendor/golang.org/x/text/unicode/bidi -vendor/golang.org/x/text/unicode/norm vendor/golang.org/x/net/http2/hpack mime -mime/quotedprintable +vendor/golang.org/x/text/unicode/norm github.com/hashicorp/consul/acl +mime/quotedprintable +vendor/golang.org/x/text/secure/bidirule net/http/internal github.com/pkg/errors -vendor/golang.org/x/text/secure/bidirule github.com/circonus-labs/circonusllhist github.com/golang/protobuf/proto github.com/prometheus/common/model crypto/rand crypto/elliptic encoding/asn1 -crypto/ed25519 crypto/rsa +crypto/ed25519 crypto/dsa vendor/golang.org/x/net/idna github.com/prometheus/procfs/internal/fs github.com/mitchellh/go-testing-interface github.com/hashicorp/consul/version github.com/hashicorp/go-uuid -go/token encoding/gob +go/token text/template/parse compress/lzw vendor/golang.org/x/crypto/cryptobyte @@ -2764,65 +2828,65 @@ github.com/hashicorp/go-multierror golang.org/x/crypto/ed25519 golang.org/x/net/bpf +crypto/ecdsa github.com/hashicorp/memberlist/vendor/github.com/sean-/seed github.com/mitchellh/reflectwalk github.com/davecgh/go-spew/spew -crypto/ecdsa github.com/mitchellh/copystructure -text/template github.com/pmezard/go-difflib/difflib github.com/stretchr/objx +text/template gopkg.in/yaml.v3 flag golang.org/x/time/rate github.com/mattn/go-isatty -github.com/hashicorp/go-version github.com/mattn/go-colorable +github.com/hashicorp/go-version runtime/trace github.com/fatih/color -github.com/hashicorp/consul/lib/decode testing -github.com/hashicorp/golang-lru +github.com/hashicorp/consul/lib/decode github.com/golang/protobuf/ptypes/any github.com/golang/protobuf/ptypes/duration github.com/golang/protobuf/ptypes/timestamp github.com/matttproud/golang_protobuf_extensions/pbutil github.com/hashicorp/go-hclog -github.com/mitchellh/hashstructure -github.com/bgentry/speakeasy +github.com/hashicorp/golang-lru html/template +github.com/mitchellh/hashstructure github.com/golang/protobuf/ptypes github.com/prometheus/client_model/go +github.com/bgentry/speakeasy os/user github.com/hashicorp/consul/command/helpers crypto/x509 net/textproto vendor/golang.org/x/net/http/httpproxy -github.com/DataDog/datadog-go/statsd github.com/prometheus/client_golang/prometheus/internal +github.com/DataDog/datadog-go/statsd github.com/prometheus/procfs -vendor/golang.org/x/net/http/httpguts -mime/multipart github.com/hashicorp/consul/ipaddr github.com/hashicorp/consul/sdk/freeport github.com/hashicorp/go-sockaddr +vendor/golang.org/x/net/http/httpguts +mime/multipart golang.org/x/net/internal/socket github.com/hashicorp/yamux github.com/mitchellh/mapstructure log/syslog github.com/hashicorp/go-syslog -github.com/armon/circbuf golang.org/x/net/ipv4 +crypto/tls golang.org/x/net/ipv6 github.com/hashicorp/consul/logging -crypto/tls +github.com/armon/circbuf github.com/golang/protobuf/ptypes/struct golang.org/x/net/internal/socks github.com/golang/protobuf/jsonpb -golang.org/x/net/proxy github.com/hashicorp/consul/agent/exec golang.org/x/net/internal/timeseries google.golang.org/grpc/grpclog +golang.org/x/net/proxy google.golang.org/grpc/credentials/internal google.golang.org/grpc/metadata google.golang.org/grpc/connectivity @@ -2834,46 +2898,46 @@ google.golang.org/grpc/binarylog/grpc_binarylog_v1 google.golang.org/genproto/googleapis/rpc/status golang.org/x/text/unicode/bidi +golang.org/x/net/http2/hpack golang.org/x/text/unicode/norm google.golang.org/grpc/status -golang.org/x/net/http2/hpack google.golang.org/grpc/internal/syscall -google.golang.org/grpc/stats google.golang.org/grpc/internal/binarylog +google.golang.org/grpc/stats google.golang.org/grpc/naming github.com/aws/aws-sdk-go/aws/awserr github.com/jmespath/go-jmespath -github.com/aws/aws-sdk-go/internal/ini -github.com/aws/aws-sdk-go/aws/endpoints golang.org/x/text/secure/bidirule encoding/xml github.com/hashicorp/hil/ast +github.com/aws/aws-sdk-go/internal/ini +github.com/aws/aws-sdk-go/aws/endpoints golang.org/x/net/idna github.com/hashicorp/hil github.com/aws/aws-sdk-go/aws/credentials github.com/posener/complete/cmd/install github.com/aws/aws-sdk-go/aws/awsutil -github.com/posener/complete/cmd -github.com/aws/aws-sdk-go/aws/credentials/processcreds +golang.org/x/net/http/httpguts net/http/httptrace -github.com/hashicorp/go-rootcerts github.com/miekg/dns -github.com/posener/complete +github.com/hashicorp/go-rootcerts +github.com/posener/complete/cmd google.golang.org/grpc/credentials -golang.org/x/net/http/httpguts -net/http -github.com/gogo/protobuf/proto -github.com/mitchellh/cli +github.com/aws/aws-sdk-go/aws/credentials/processcreds github.com/hashicorp/go-memdb -github.com/hashicorp/consul/sdk/testutil/retry +github.com/gogo/protobuf/proto +net/http +github.com/posener/complete google.golang.org/grpc/resolver google.golang.org/grpc/internal/channelz google.golang.org/grpc/balancer +github.com/mitchellh/cli google.golang.org/grpc/balancer/base -google.golang.org/grpc/internal/resolver/dns google.golang.org/grpc/balancer/roundrobin +google.golang.org/grpc/internal/resolver/dns google.golang.org/grpc/internal/resolver/passthrough google.golang.org/grpc/peer +github.com/hashicorp/consul/sdk/testutil/retry github.com/hashicorp/consul/vendor/github.com/hashicorp/vault/sdk/helper/consts github.com/hashicorp/consul/vendor/github.com/hashicorp/vault/sdk/helper/hclutil github.com/pierrec/lz4 @@ -2881,57 +2945,58 @@ gopkg.in/square/go-jose.v2/cipher gopkg.in/square/go-jose.v2/json github.com/hashicorp/consul/vendor/github.com/hashicorp/vault/sdk/helper/parseutil -github.com/hashicorp/consul/vendor/github.com/hashicorp/vault/sdk/helper/compressutil github.com/hashicorp/go-bexpr +github.com/hashicorp/consul/vendor/github.com/hashicorp/vault/sdk/helper/compressutil +github.com/google/gofuzz/bytesource github.com/google/gofuzz github.com/hashicorp/consul/vendor/github.com/hashicorp/vault/sdk/helper/jsonutil gopkg.in/inf.v0 github.com/hashicorp/consul/vendor/k8s.io/apimachinery/third_party/forked/golang/reflect -gopkg.in/square/go-jose.v2 github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/fields +gopkg.in/square/go-jose.v2 github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/conversion github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/util/errors github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/util/validation/field -github.com/hashicorp/consul/vendor/k8s.io/klog github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/util/validation +github.com/hashicorp/consul/vendor/k8s.io/klog go/scanner +internal/lazyregexp github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/conversion/queryparams -github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/labels -github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/util/json go/ast +github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/util/json +github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/labels github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/util/naming -gopkg.in/square/go-jose.v2/jwt github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/types github.com/googleapis/gnostic/extensions -github.com/modern-go/concurrent +gopkg.in/square/go-jose.v2/jwt gopkg.in/yaml.v2 -github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/util/framer -github.com/hashicorp/consul/vendor/k8s.io/client-go/pkg/version -golang.org/x/term +github.com/modern-go/concurrent github.com/modern-go/reflect2 +github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/util/framer go/doc +go/parser github.com/gogo/protobuf/protoc-gen-gogo/descriptor +github.com/hashicorp/consul/vendor/github.com/coredns/coredns/plugin/pkg/dnsutil github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/api/resource -go/parser github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/runtime/schema +github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/util/intstr github.com/armon/go-metrics github.com/hashicorp/go-cleanhttp -github.com/hashicorp/go-retryablehttp github.com/tv42/httpunix expvar github.com/prometheus/common/expfmt +github.com/hashicorp/go-retryablehttp net/rpc net/http/httptest +github.com/NYTimes/gziphandler github.com/armon/go-metrics/datadog -github.com/circonus-labs/circonus-gometrics/api github.com/hashicorp/serf/coordinate -github.com/NYTimes/gziphandler -github.com/hashicorp/consul/vendor/github.com/coredns/coredns/plugin/pkg/dnsutil -github.com/hashicorp/go-msgpack/codec +github.com/circonus-labs/circonus-gometrics/api github.com/elazarl/go-bindata-assetfs +github.com/hashicorp/go-msgpack/codec +github.com/docker/go-connections/sockets github.com/stretchr/testify/assert github.com/hashicorp/consul/api -github.com/docker/go-connections/sockets github.com/prometheus/client_golang/prometheus golang.org/x/net/trace golang.org/x/net/http2 @@ -2939,22 +3004,23 @@ github.com/circonus-labs/circonus-gometrics/checkmgr github.com/stretchr/testify/mock github.com/hashicorp/memberlist -github.com/prometheus/client_golang/prometheus/push github.com/circonus-labs/circonus-gometrics github.com/stretchr/testify/require +github.com/prometheus/client_golang/prometheus/push github.com/hashicorp/raft github.com/armon/go-metrics/prometheus -github.com/gogo/protobuf/gogoproto -github.com/hashicorp/consul/proto/pbacl github.com/armon/go-metrics/circonus +github.com/gogo/protobuf/gogoproto github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/util/runtime github.com/hashicorp/consul/command/flags -github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/runtime -github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/util/intstr github.com/googleapis/gnostic/compiler -github.com/json-iterator/go -github.com/hashicorp/serf/serf +github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/runtime +github.com/hashicorp/consul/proto/pbacl +github.com/googleapis/gnostic/OpenAPIv2 +google.golang.org/grpc/internal/transport github.com/hashicorp/consul/command/acl/agenttokens +github.com/hashicorp/consul/vendor/github.com/hashicorp/vault/api +github.com/hashicorp/serf/serf github.com/hashicorp/consul/command/acl/authmethod github.com/hashicorp/consul/command/acl/authmethod/create github.com/hashicorp/consul/command/acl/authmethod/delete @@ -2962,91 +3028,90 @@ github.com/hashicorp/consul/command/acl/authmethod/read github.com/hashicorp/consul/command/acl/authmethod/update github.com/hashicorp/consul/command/acl/bindingrule -github.com/hashicorp/consul/command/acl/token github.com/hashicorp/consul/command/acl/bindingrule/create github.com/hashicorp/consul/command/acl/bindingrule/list -github.com/hashicorp/consul/command/acl/bootstrap +github.com/hashicorp/consul/command/acl/token github.com/hashicorp/consul/command/acl/policy -github.com/hashicorp/consul/command/acl/policy/list -github.com/hashicorp/consul/command/acl/role -github.com/hashicorp/consul/command/acl/token/list -google.golang.org/grpc/internal/transport -github.com/hashicorp/consul/vendor/github.com/hashicorp/vault/api -github.com/hashicorp/consul/command/acl/role/list github.com/hashicorp/consul/lib +github.com/hashicorp/consul/command/acl/role github.com/hashicorp/consul/agent/consul/autopilot +github.com/aws/aws-sdk-go/aws +google.golang.org/grpc +github.com/hashicorp/consul/command/acl/bootstrap +github.com/hashicorp/consul/command/acl/policy/list +github.com/hashicorp/consul/command/acl/role/list +github.com/hashicorp/consul/command/acl/token/list github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/util/net -github.com/googleapis/gnostic/OpenAPIv2 +github.com/json-iterator/go +github.com/hashicorp/consul/agent/cache +github.com/hashicorp/consul/agent/ae +github.com/aws/aws-sdk-go/aws/request github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/runtime/serializer/recognizer sigs.k8s.io/yaml +github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming +github.com/hashicorp/consul/vendor/k8s.io/client-go/pkg/version github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/watch -github.com/hashicorp/consul/agent/cache -github.com/hashicorp/consul/agent/ae +golang.org/x/term +github.com/hashicorp/consul/agent/structs github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/util/yaml +github.com/hashicorp/consul/vendor/k8s.io/client-go/tools/clientcmd/api github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/apis/meta/v1 -github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming -github.com/hashicorp/consul/agent/structs golang.org/x/crypto/ssh/terminal -github.com/hashicorp/consul/vendor/k8s.io/client-go/tools/clientcmd/api +github.com/aws/aws-sdk-go/aws/corehandlers +github.com/aws/aws-sdk-go/aws/client +github.com/aws/aws-sdk-go/private/protocol +github.com/aws/aws-sdk-go/aws/csm +github.com/aws/aws-sdk-go/aws/ec2metadata golang.org/x/net/context/ctxhttp -github.com/aws/aws-sdk-go/aws +github.com/aws/aws-sdk-go/private/protocol/rest +github.com/aws/aws-sdk-go/private/protocol/query/queryutil +github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil +google.golang.org/grpc/health/grpc_health_v1 +github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds +github.com/aws/aws-sdk-go/aws/signer/v4 +github.com/aws/aws-sdk-go/private/protocol/json/jsonutil golang.org/x/oauth2/internal +github.com/aws/aws-sdk-go/private/protocol/query github.com/hashicorp/consul/vendor/k8s.io/client-go/util/connrotation github.com/hashicorp/consul/vendor/k8s.io/client-go/tools/metrics -github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/runtime/serializer/json github.com/hashicorp/consul/vendor/k8s.io/client-go/util/keyutil -github.com/aws/aws-sdk-go/aws/request golang.org/x/oauth2 -github.com/hashicorp/consul/vendor/k8s.io/client-go/util/cert +github.com/aws/aws-sdk-go/service/sts github.com/hashicorp/consul/vendor/k8s.io/client-go/util/flowcontrol +github.com/hashicorp/consul/vendor/k8s.io/client-go/util/cert +github.com/aws/aws-sdk-go/aws/credentials/endpointcreds +github.com/aws/aws-sdk-go/private/protocol/jsonrpc github.com/hashicorp/consul/vendor/k8s.io/client-go/transport +github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/runtime/serializer/json +github.com/aws/aws-sdk-go/aws/defaults github.com/pquerna/cachecontrol/cacheobject +github.com/aws/aws-sdk-go/service/acmpca github.com/hashicorp/consul/vendor/github.com/mitchellh/pointerstructure -google.golang.org/grpc github.com/patrickmn/go-cache github.com/hashicorp/go-raftchunking/types github.com/pquerna/cachecontrol +github.com/aws/aws-sdk-go/service/sts/stsiface github.com/coreos/go-oidc github.com/hashicorp/go-raftchunking -github.com/aws/aws-sdk-go/aws/client -github.com/aws/aws-sdk-go/aws/corehandlers -github.com/aws/aws-sdk-go/private/protocol -github.com/aws/aws-sdk-go/aws/csm -github.com/aws/aws-sdk-go/aws/ec2metadata -github.com/hashicorp/consul/internal/go-sso/oidcauth github.com/hashicorp/net-rpc-msgpackrpc -github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds -github.com/aws/aws-sdk-go/private/protocol/rest -github.com/aws/aws-sdk-go/private/protocol/query/queryutil -github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil -github.com/aws/aws-sdk-go/private/protocol/json/jsonutil +github.com/aws/aws-sdk-go/aws/credentials/stscreds github.com/hashicorp/consul/lib/template github.com/gogo/protobuf/types -github.com/aws/aws-sdk-go/aws/signer/v4 -github.com/aws/aws-sdk-go/aws/credentials/endpointcreds -github.com/aws/aws-sdk-go/private/protocol/query -github.com/aws/aws-sdk-go/private/protocol/jsonrpc -github.com/aws/aws-sdk-go/service/sts -github.com/aws/aws-sdk-go/aws/defaults -github.com/aws/aws-sdk-go/service/acmpca github.com/hashicorp/consul/proto/pbconfig +archive/tar +github.com/hashicorp/consul/internal/go-sso/oidcauth +github.com/aws/aws-sdk-go/aws/session github.com/hashicorp/consul/vendor/k8s.io/api/authentication/v1 -google.golang.org/grpc/health/grpc_health_v1 github.com/hashicorp/consul/vendor/k8s.io/api/core/v1 github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/api/errors -github.com/aws/aws-sdk-go/service/sts/stsiface github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/runtime/serializer/protobuf -github.com/aws/aws-sdk-go/aws/credentials/stscreds github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured github.com/hashicorp/consul/vendor/k8s.io/api/admissionregistration/v1 -github.com/aws/aws-sdk-go/aws/session github.com/hashicorp/consul/vendor/k8s.io/api/admissionregistration/v1beta1 -github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/runtime/serializer/versioning github.com/hashicorp/consul/vendor/k8s.io/api/auditregistration/v1alpha1 +github.com/hashicorp/consul/vendor/k8s.io/api/authentication/v1beta1 github.com/hashicorp/consul/command/acl github.com/hashicorp/consul/agent/connect -github.com/hashicorp/consul/agent/checks -github.com/hashicorp/consul/agent/consul/prepared_query github.com/hashicorp/consul/command/acl/bindingrule/delete github.com/hashicorp/consul/command/acl/bindingrule/read github.com/hashicorp/consul/command/acl/bindingrule/update @@ -3064,125 +3129,126 @@ github.com/hashicorp/consul/command/acl/token/delete github.com/hashicorp/consul/command/acl/token/read github.com/hashicorp/consul/command/acl/token/update +github.com/hashicorp/consul/agent/checks github.com/hashicorp/consul/agent/consul/discoverychain +github.com/hashicorp/consul/agent/consul/prepared_query github.com/hashicorp/consul/agent/consul/authmethod -github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/runtime/serializer -github.com/hashicorp/consul/vendor/k8s.io/api/authentication/v1beta1 +github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/runtime/serializer/versioning github.com/hashicorp/consul/vendor/k8s.io/api/authorization/v1 github.com/hashicorp/consul/vendor/k8s.io/api/authorization/v1beta1 github.com/hashicorp/consul/vendor/k8s.io/api/certificates/v1beta1 github.com/hashicorp/consul/vendor/k8s.io/api/coordination/v1 -github.com/hashicorp/consul/vendor/k8s.io/api/coordination/v1beta1 +github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/runtime/serializer github.com/hashicorp/consul/agent/consul/state +github.com/hashicorp/consul/vendor/k8s.io/api/coordination/v1beta1 github.com/hashicorp/consul/vendor/k8s.io/api/rbac/v1 github.com/hashicorp/consul/vendor/k8s.io/api/rbac/v1alpha1 github.com/hashicorp/consul/vendor/k8s.io/api/rbac/v1beta1 github.com/hashicorp/consul/vendor/k8s.io/client-go/pkg/apis/clientauthentication +github.com/hashicorp/consul/vendor/k8s.io/client-go/rest/watch github.com/hashicorp/consul/vendor/k8s.io/client-go/pkg/apis/clientauthentication/v1alpha1 github.com/hashicorp/consul/vendor/k8s.io/client-go/pkg/apis/clientauthentication/v1beta1 -github.com/hashicorp/consul/vendor/k8s.io/client-go/rest/watch github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/api/meta github.com/hashicorp/consul/agent/consul/authmethod/ssoauth github.com/hashicorp/consul/vendor/k8s.io/client-go/plugin/pkg/client/auth/exec github.com/hashicorp/consul/tlsutil github.com/hashicorp/consul/agent/metadata -github.com/hashicorp/consul/agent/router github.com/hashicorp/consul/proto +github.com/hashicorp/consul/agent/router github.com/hashicorp/consul/proto/pbcommon -archive/tar github.com/hashicorp/consul/vendor/k8s.io/client-go/rest +github.com/hashicorp/consul/snapshot github.com/hashicorp/consul/agent/pool github.com/hashicorp/consul/vendor/github.com/hashicorp/go-connlimit -golang.org/x/sync/singleflight github.com/boltdb/bolt -github.com/hashicorp/consul/agent/consul/wanfed github.com/hashicorp/consul/proto/pbconnect +golang.org/x/sync/singleflight github.com/hashicorp/consul/agent/local +github.com/hashicorp/consul/agent/consul/wanfed github.com/hashicorp/go-sockaddr/template github.com/denverdino/aliyungo/util -github.com/hashicorp/consul/snapshot github.com/aws/aws-sdk-go/private/protocol/ec2query cloud.google.com/go/compute/metadata -golang.org/x/oauth2/jws +github.com/hashicorp/consul/agent/connect/ca +github.com/hashicorp/consul/agent/consul/fsm github.com/hashicorp/consul/proto/pbautoconf +golang.org/x/oauth2/jws github.com/denverdino/aliyungo/common github.com/aws/aws-sdk-go/service/ec2 -google.golang.org/api/googleapi -golang.org/x/oauth2/jwt -github.com/hashicorp/consul/agent/connect/ca -github.com/hashicorp/consul/agent/consul/fsm github.com/hashicorp/raft-boltdb +golang.org/x/oauth2/jwt +google.golang.org/api/googleapi golang.org/x/oauth2/google github.com/googleapis/gax-go/v2 -github.com/denverdino/aliyungo/ecs -google.golang.org/api/internal/gensupport go.opencensus.io/internal go.opencensus.io/trace/tracestate -google.golang.org/api/internal -go.opencensus.io/trace +github.com/denverdino/aliyungo/ecs +google.golang.org/api/internal/gensupport go.opencensus.io/resource -google.golang.org/api/option +go.opencensus.io/trace +google.golang.org/api/internal go.opencensus.io/metric/metricdata runtime/pprof +google.golang.org/api/option go.opencensus.io/metric/metricproducer google.golang.org/api/googleapi/transport -google.golang.org/api/option/internaloption google.golang.org/api/transport/cert -github.com/hashicorp/mdns go.opencensus.io/trace/propagation -github.com/gophercloud/gophercloud -github.com/packethost/packngo +github.com/hashicorp/mdns +google.golang.org/api/option/internaloption go.opencensus.io/plugin/ochttp/propagation/b3 google.golang.org/api/transport/http/internal/propagation +github.com/gophercloud/gophercloud +github.com/packethost/packngo github.com/shirou/gopsutil/internal/common github.com/hashicorp/go-discover/provider/mdns +go.opencensus.io/tag github.com/hashicorp/go-discover/provider/aliyun github.com/hashicorp/consul/agent/systemd -go.opencensus.io/tag github.com/golang/protobuf/protoc-gen-go/descriptor -net/mail go.opencensus.io/stats/internal -github.com/shirou/gopsutil/cpu +net/mail go.opencensus.io/stats -github.com/shirou/gopsutil/disk go.opencensus.io/stats/view +github.com/shirou/gopsutil/cpu +github.com/shirou/gopsutil/disk github.com/shirou/gopsutil/host github.com/shirou/gopsutil/mem -github.com/gophercloud/gophercloud/openstack/identity/v3/tokens +go.opencensus.io/plugin/ochttp github.com/gophercloud/gophercloud/pagination +github.com/gophercloud/gophercloud/openstack/identity/v3/tokens github.com/gophercloud/gophercloud/openstack/utils github.com/hashicorp/go-discover/provider/packet -go.opencensus.io/plugin/ochttp -github.com/gophercloud/gophercloud/openstack/identity/v2/tenants -github.com/gophercloud/gophercloud/openstack/compute/v2/servers github.com/hashicorp/consul/agent/debug -github.com/gophercloud/gophercloud/openstack/identity/v3/extensions/ec2tokens -github.com/gophercloud/gophercloud/openstack/identity/v3/extensions/oauth1 github.com/hashicorp/consul/vendor/github.com/cncf/udpa/go/udpa/annotations -github.com/gophercloud/gophercloud/openstack/identity/v2/tokens github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/annotations github.com/hashicorp/consul/vendor/github.com/envoyproxy/protoc-gen-validate/validate +github.com/gophercloud/gophercloud/openstack/identity/v2/tenants +github.com/gophercloud/gophercloud/openstack/identity/v3/extensions/ec2tokens +github.com/gophercloud/gophercloud/openstack/identity/v3/extensions/oauth1 +github.com/gophercloud/gophercloud/openstack/compute/v2/servers github.com/golang/protobuf/ptypes/wrappers +github.com/gophercloud/gophercloud/openstack/identity/v2/tokens +google.golang.org/api/transport/http github.com/golang/protobuf/ptypes/empty github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/config/listener/v2 google.golang.org/genproto/googleapis/api/annotations github.com/census-instrumentation/opencensus-proto/gen-go/resource/v1 +google.golang.org/api/compute/v1 github.com/census-instrumentation/opencensus-proto/gen-go/trace/v1 -github.com/gophercloud/gophercloud/openstack github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/pkg/conversion -github.com/hashicorp/consul/api/watch -github.com/hashicorp/consul/lib/file -github.com/hashicorp/consul/logging/monitor +github.com/gophercloud/gophercloud/openstack github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/type github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/type/metadata/v2 -google.golang.org/api/transport/http +github.com/hashicorp/consul/api/watch github.com/hashicorp/go-discover/provider/os +github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/type/tracing/v2 +github.com/hashicorp/consul/lib/file +github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/type/matcher +github.com/hashicorp/consul/logging/monitor github.com/hashicorp/consul/sdk/testutil github.com/imdario/mergo github.com/prometheus/client_golang/prometheus/promhttp -github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/type/tracing/v2 -github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/type/matcher -google.golang.org/api/compute/v1 internal/profile github.com/hashicorp/go-checkpoint github.com/hashicorp/consul/command/catalog @@ -3190,15 +3256,15 @@ github.com/ryanuber/columnize github.com/hashicorp/consul/command/catalog/list/services github.com/hashicorp/consul/command/catalog/list/nodes +github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/api/v2/core github.com/hashicorp/consul/command/config +net/http/pprof github.com/hashicorp/consul/command/config/delete github.com/hashicorp/consul/command/config/list github.com/hashicorp/consul/command/config/read github.com/hashicorp/consul/command/config/write github.com/hashicorp/consul/command/connect github.com/hashicorp/consul/command/connect/ca -github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/api/v2/core -net/http/pprof github.com/hashicorp/consul/command/connect/ca/get github.com/hashicorp/consul/command/connect/ca/set github.com/hashicorp/consul/connect @@ -3226,18 +3292,6 @@ github.com/hashicorp/consul/command/kv/exp github.com/hashicorp/consul/command/kv/imp github.com/hashicorp/consul/command/kv/put -github.com/hashicorp/consul/command/leave -github.com/hashicorp/consul/command/login -github.com/hashicorp/consul/command/logout -github.com/hashicorp/consul/command/maint -github.com/hashicorp/consul/command/members -github.com/hashicorp/consul/command/monitor -github.com/hashicorp/consul/command/operator -github.com/hashicorp/consul/command/operator/autopilot -github.com/hashicorp/consul/command/operator/autopilot/get -github.com/hashicorp/consul/command/operator/autopilot/set -github.com/hashicorp/consul/command/operator/raft -github.com/hashicorp/consul/command/operator/raft/listpeers github.com/hashicorp/consul/vendor/k8s.io/api/apps/v1 github.com/hashicorp/consul/vendor/k8s.io/api/apps/v1beta1 github.com/hashicorp/consul/vendor/k8s.io/api/apps/v1beta2 @@ -3269,24 +3323,18 @@ github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/api/v2/route github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/config/filter/network/ext_authz/v2 github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/config/trace/v2 -github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/api/v2/listener github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/service/auth/v2 -github.com/hashicorp/consul/command/operator/raft/removepeer -github.com/hashicorp/consul/command/reload -github.com/hashicorp/consul/command/rtt -github.com/hashicorp/consul/command/snapshot -github.com/hashicorp/consul/command/snapshot/inspect -github.com/hashicorp/consul/command/snapshot/restore +github.com/hashicorp/consul/command/leave +github.com/hashicorp/consul/command/login +github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/api/v2/listener +github.com/hashicorp/consul/command/logout +github.com/hashicorp/consul/command/maint github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/service/auth/v2alpha -github.com/hashicorp/consul/command/snapshot/save -github.com/hashicorp/consul/command/tls -github.com/hashicorp/consul/command/tls/ca -github.com/hashicorp/consul/command/tls/ca/create +github.com/hashicorp/consul/command/members github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/scheme -github.com/hashicorp/consul/command/tls/cert -github.com/hashicorp/consul/command/tls/cert/create -github.com/hashicorp/consul/agent/consul/authmethod/testauth -github.com/hashicorp/consul/agent/mock +github.com/hashicorp/consul/command/monitor +github.com/hashicorp/consul/command/operator +github.com/hashicorp/consul/command/operator/autopilot github.com/hashicorp/consul/vendor/k8s.io/client-go/discovery github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/admissionregistration/v1 github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/admissionregistration/v1beta1 @@ -3327,13 +3375,32 @@ github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/storage/v1alpha1 github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/storage/v1beta1 github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/config/filter/accesslog/v2 +github.com/hashicorp/consul/command/operator/autopilot/get +github.com/hashicorp/consul/command/operator/autopilot/set +github.com/hashicorp/consul/command/operator/raft +github.com/hashicorp/consul/command/operator/raft/listpeers +github.com/hashicorp/consul/command/operator/raft/removepeer +github.com/hashicorp/consul/command/reload +github.com/hashicorp/consul/command/rtt +github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes +github.com/hashicorp/consul/command/snapshot +github.com/hashicorp/consul/command/snapshot/inspect +github.com/hashicorp/consul/command/snapshot/restore +github.com/hashicorp/consul/vendor/github.com/rboyer/safeio +github.com/hashicorp/consul/command/tls +github.com/hashicorp/consul/command/tls/ca +github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/api/v2 +github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/config/filter/network/tcp_proxy/v2 +github.com/hashicorp/consul/command/snapshot/save +github.com/hashicorp/consul/command/tls/ca/create +github.com/hashicorp/consul/command/tls/cert +github.com/hashicorp/consul/command/tls/cert/create +github.com/hashicorp/consul/agent/consul/authmethod/testauth +github.com/hashicorp/consul/agent/mock github.com/hashicorp/consul/connect/certgen github.com/hashicorp/consul/internal/go-sso/oidcauth/oidcauthtest github.com/hashicorp/consul/sentinel github.com/hashicorp/consul/testrpc -github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes -github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/config/filter/network/tcp_proxy/v2 -github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/api/v2 github.com/hashicorp/consul/agent/consul/authmethod/kubeauth github.com/hashicorp/consul/agent/consul github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/config/filter/network/http_connection_manager/v2 @@ -3345,19 +3412,19 @@ github.com/hashicorp/consul/agent/cache-types github.com/hashicorp/consul/agent/config github.com/hashicorp/consul/agent/proxycfg +github.com/hashicorp/consul/agent/xds github.com/hashicorp/consul/command/validate github.com/hashicorp/consul/command/services github.com/hashicorp/consul/agent/auto-config -github.com/hashicorp/consul/agent/xds github.com/hashicorp/consul/command/services/deregister github.com/hashicorp/consul/command/services/register github.com/hashicorp/consul/command/connect/envoy github.com/hashicorp/consul/agent -github.com/hashicorp/consul/command/agent -github.com/hashicorp/consul/command/connect/expose -github.com/hashicorp/consul/command/lock github.com/hashicorp/consul/command/keyring github.com/hashicorp/consul/command/watch +github.com/hashicorp/consul/command/connect/expose +github.com/hashicorp/consul/command/agent +github.com/hashicorp/consul/command/lock github.com/hashicorp/consul/command github.com/hashicorp/consul make[1]: Leaving directory '/build/consul-1.8.7+dfsg1' @@ -3369,7 +3436,7 @@ cd _build && go test -vet=off -v -p 4 -short -failfast -timeout 8m github.com/hashicorp/consul github.com/hashicorp/consul/acl github.com/hashicorp/consul/agent github.com/hashicorp/consul/agent/ae github.com/hashicorp/consul/agent/auto-config github.com/hashicorp/consul/agent/config github.com/hashicorp/consul/agent/debug github.com/hashicorp/consul/agent/dns github.com/hashicorp/consul/agent/exec github.com/hashicorp/consul/agent/local github.com/hashicorp/consul/agent/metadata github.com/hashicorp/consul/agent/mock github.com/hashicorp/consul/agent/pool github.com/hashicorp/consul/agent/router github.com/hashicorp/consul/agent/structs github.com/hashicorp/consul/agent/systemd github.com/hashicorp/consul/agent/token github.com/hashicorp/consul/agent/xds github.com/hashicorp/consul/agent/xds/proxysupport github.com/hashicorp/consul/command github.com/hashicorp/consul/command/acl github.com/hashicorp/consul/command/acl/agenttokens github.com/hashicorp/consul/command/acl/authmethod github.com/hashicorp/consul/command/acl/authmethod/create github.com/hashicorp/consul/command/acl/authmethod/delete github.com/hashicorp/consul/command/acl/authmethod/list github.com/hashicorp/consul/command/acl/authmethod/read github.com/hashicorp/consul/command/acl/authmethod/update github.com/hashicorp/consul/command/acl/bindingrule github.com/hashicorp/consul/command/acl/bindingrule/create github.com/hashicorp/consul/command/acl/bindingrule/delete github.com/hashicorp/consul/command/acl/bindingrule/list github.com/hashicorp/consul/command/acl/bindingrule/read github.com/hashicorp/consul/command/acl/bindingrule/update github.com/hashicorp/consul/command/acl/bootstrap github.com/hashicorp/consul/command/acl/policy github.com/hashicorp/consul/command/acl/policy/create github.com/hashicorp/consul/command/acl/policy/delete github.com/hashicorp/consul/command/acl/policy/list github.com/hashicorp/consul/command/acl/policy/read github.com/hashicorp/consul/command/acl/policy/update github.com/hashicorp/consul/command/acl/role github.com/hashicorp/consul/command/acl/role/create github.com/hashicorp/consul/command/acl/role/delete github.com/hashicorp/consul/command/acl/role/list github.com/hashicorp/consul/command/acl/role/read github.com/hashicorp/consul/command/acl/role/update github.com/hashicorp/consul/command/acl/rules github.com/hashicorp/consul/command/acl/token github.com/hashicorp/consul/command/acl/token/clone github.com/hashicorp/consul/command/acl/token/create github.com/hashicorp/consul/command/acl/token/delete github.com/hashicorp/consul/command/acl/token/list github.com/hashicorp/consul/command/acl/token/read github.com/hashicorp/consul/command/acl/token/update github.com/hashicorp/consul/command/agent github.com/hashicorp/consul/command/catalog github.com/hashicorp/consul/command/catalog/list/dc github.com/hashicorp/consul/command/catalog/list/nodes github.com/hashicorp/consul/command/catalog/list/services github.com/hashicorp/consul/command/config github.com/hashicorp/consul/command/config/delete github.com/hashicorp/consul/command/config/list github.com/hashicorp/consul/command/config/read github.com/hashicorp/consul/command/config/write github.com/hashicorp/consul/command/connect github.com/hashicorp/consul/command/connect/ca github.com/hashicorp/consul/command/connect/ca/get github.com/hashicorp/consul/command/connect/ca/set github.com/hashicorp/consul/command/connect/envoy github.com/hashicorp/consul/command/connect/envoy/pipe-bootstrap github.com/hashicorp/consul/command/connect/expose github.com/hashicorp/consul/command/connect/proxy github.com/hashicorp/consul/command/event github.com/hashicorp/consul/command/exec github.com/hashicorp/consul/command/forceleave github.com/hashicorp/consul/command/helpers github.com/hashicorp/consul/command/info github.com/hashicorp/consul/command/intention github.com/hashicorp/consul/command/intention/check github.com/hashicorp/consul/command/intention/create github.com/hashicorp/consul/command/intention/delete github.com/hashicorp/consul/command/intention/finder github.com/hashicorp/consul/command/intention/get github.com/hashicorp/consul/command/intention/match github.com/hashicorp/consul/command/join github.com/hashicorp/consul/command/keygen github.com/hashicorp/consul/command/keyring github.com/hashicorp/consul/command/kv github.com/hashicorp/consul/command/kv/del github.com/hashicorp/consul/command/kv/exp github.com/hashicorp/consul/command/kv/get github.com/hashicorp/consul/command/kv/imp github.com/hashicorp/consul/command/kv/impexp github.com/hashicorp/consul/command/kv/put github.com/hashicorp/consul/command/leave github.com/hashicorp/consul/command/lock github.com/hashicorp/consul/command/login github.com/hashicorp/consul/command/logout github.com/hashicorp/consul/command/maint github.com/hashicorp/consul/command/members github.com/hashicorp/consul/command/operator github.com/hashicorp/consul/command/operator/autopilot github.com/hashicorp/consul/command/operator/autopilot/get github.com/hashicorp/consul/command/operator/autopilot/set github.com/hashicorp/consul/command/operator/raft github.com/hashicorp/consul/command/operator/raft/listpeers github.com/hashicorp/consul/command/operator/raft/removepeer github.com/hashicorp/consul/command/reload github.com/hashicorp/consul/command/rtt github.com/hashicorp/consul/command/services github.com/hashicorp/consul/command/services/deregister github.com/hashicorp/consul/command/services/register github.com/hashicorp/consul/command/snapshot github.com/hashicorp/consul/command/snapshot/inspect github.com/hashicorp/consul/command/snapshot/restore github.com/hashicorp/consul/command/snapshot/save github.com/hashicorp/consul/command/validate github.com/hashicorp/consul/command/version github.com/hashicorp/consul/command/watch github.com/hashicorp/consul/connect github.com/hashicorp/consul/connect/certgen github.com/hashicorp/consul/connect/proxy github.com/hashicorp/consul/internal/go-sso/oidcauth github.com/hashicorp/consul/internal/go-sso/oidcauth/internal/strutil github.com/hashicorp/consul/internal/go-sso/oidcauth/oidcauthtest github.com/hashicorp/consul/ipaddr github.com/hashicorp/consul/lib github.com/hashicorp/consul/lib/decode github.com/hashicorp/consul/lib/file github.com/hashicorp/consul/lib/semaphore github.com/hashicorp/consul/lib/stringslice github.com/hashicorp/consul/lib/template github.com/hashicorp/consul/logging github.com/hashicorp/consul/logging/monitor github.com/hashicorp/consul/proto github.com/hashicorp/consul/proto/pbacl github.com/hashicorp/consul/proto/pbautoconf github.com/hashicorp/consul/proto/pbcommon github.com/hashicorp/consul/proto/pbconfig github.com/hashicorp/consul/proto/pbconnect github.com/hashicorp/consul/sdk/freeport github.com/hashicorp/consul/sdk/testutil github.com/hashicorp/consul/sdk/testutil/retry github.com/hashicorp/consul/sentinel github.com/hashicorp/consul/service_os github.com/hashicorp/consul/snapshot github.com/hashicorp/consul/testrpc github.com/hashicorp/consul/tlsutil github.com/hashicorp/consul/types github.com/hashicorp/consul/version testing: warning: no tests to run PASS -ok github.com/hashicorp/consul 0.096s [no tests to run] +ok github.com/hashicorp/consul 0.080s [no tests to run] === RUN TestACL === RUN TestACL/DenyAll === RUN TestACL/DenyAll/DenyACLRead @@ -3975,7 +4042,7 @@ === RUN TestACL/KeyWritePrefixDefaultAllow === RUN TestACL/KeyWritePrefixDefaultAllow/KeyWritePrefixDenied.Prefix(foo) === RUN TestACL/KeyWritePrefixDefaultAllow/KeyWritePrefixAllowed.Prefix(bar) ---- PASS: TestACL (0.05s) +--- PASS: TestACL (0.04s) --- PASS: TestACL/DenyAll (0.00s) --- PASS: TestACL/DenyAll/DenyACLRead (0.00s) --- PASS: TestACL/DenyAll/DenyACLWrite (0.00s) @@ -4685,7 +4752,7 @@ === RUN TestMergePolicies/Write_Precedence === RUN TestMergePolicies/Deny_Precedence === RUN TestMergePolicies/Read_Precedence ---- PASS: TestMergePolicies (0.02s) +--- PASS: TestMergePolicies (0.00s) --- PASS: TestMergePolicies/Agents (0.00s) --- PASS: TestMergePolicies/Events (0.00s) --- PASS: TestMergePolicies/Node (0.00s) @@ -4746,146 +4813,163 @@ === PAUSE TestStaticAuthorizer === CONT TestACL_Enforce === RUN TestACL_Enforce/acl/read/Deny + authorizer_test.go:618: PASS: ACLRead(*acl.AuthorizerContext) +=== RUN TestACL_Enforce/acl/read/Allow === CONT TestAllAllowed -=== RUN TestAllAllowed/prefix-write-allowed -=== RUN TestAllAllowed/prefix-read-allowed -=== RUN TestAllAllowed/prefix-allow-other-write-exact -=== RUN TestAllAllowed/prefix-allow-other-list-prefix -=== RUN TestAllAllowed/prefix-allow-other-read-prefix -=== RUN TestAllAllowed/prefix-allow-other-deny-exact -=== RUN TestAllAllowed/no-rules-default -=== RUN TestAllAllowed/prefix-deny -=== RUN TestAllAllowed/prefix-allow-other-write-prefix -=== RUN TestAllAllowed/prefix-allow-other-list-exact -=== RUN TestAllAllowed/prefix-allow-other-read-exact -=== RUN TestAllAllowed/prefix-allow-other-deny-prefix -=== RUN TestAllAllowed/prefix-list-allowed ---- PASS: TestAllAllowed (0.00s) - --- PASS: TestAllAllowed/prefix-write-allowed (0.00s) - --- PASS: TestAllAllowed/prefix-read-allowed (0.00s) - --- PASS: TestAllAllowed/prefix-allow-other-write-exact (0.00s) - --- PASS: TestAllAllowed/prefix-allow-other-list-prefix (0.00s) - --- PASS: TestAllAllowed/prefix-allow-other-read-prefix (0.00s) - --- PASS: TestAllAllowed/prefix-allow-other-deny-exact (0.00s) - --- PASS: TestAllAllowed/no-rules-default (0.00s) - --- PASS: TestAllAllowed/prefix-deny (0.00s) - --- PASS: TestAllAllowed/prefix-allow-other-write-prefix (0.00s) - --- PASS: TestAllAllowed/prefix-allow-other-list-exact (0.00s) - --- PASS: TestAllAllowed/prefix-allow-other-read-exact (0.00s) - --- PASS: TestAllAllowed/prefix-allow-other-deny-prefix (0.00s) - --- PASS: TestAllAllowed/prefix-list-allowed (0.00s) -=== CONT TestStaticAuthorizer -=== RUN TestStaticAuthorizer/AllowAll -=== PAUSE TestStaticAuthorizer/AllowAll -=== RUN TestStaticAuthorizer/DenyAll -=== PAUSE TestStaticAuthorizer/DenyAll -=== RUN TestStaticAuthorizer/ManageAll -=== PAUSE TestStaticAuthorizer/ManageAll -=== CONT TestStaticAuthorizer/AllowAll -=== CONT TestStaticAuthorizer/ManageAll -=== CONT TestStaticAuthorizer/DenyAll ---- PASS: TestStaticAuthorizer (0.00s) - --- PASS: TestStaticAuthorizer/AllowAll (0.00s) - --- PASS: TestStaticAuthorizer/ManageAll (0.00s) - --- PASS: TestStaticAuthorizer/DenyAll (0.00s) === CONT TestPolicyAuthorizer +=== RUN TestAllAllowed/prefix-allow-other-write-prefix +=== RUN TestPolicyAuthorizer/Intention_Wildcards_-_all_default +=== CONT TestAnyAllowed +=== RUN TestAnyAllowed/prefix-list-allowed +=== CONT TestChainedAuthorizer +=== RUN TestChainedAuthorizer/No_Authorizers +=== RUN TestAnyAllowed/prefix-deny-other-write-exact +=== RUN TestAnyAllowed/prefix-deny-other-list-exact +=== RUN TestAnyAllowed/prefix-deny-other-read-prefix +=== RUN TestAnyAllowed/prefix-deny-other-read-exact +=== RUN TestAnyAllowed/prefix-deny-other-deny-prefix +=== RUN TestAnyAllowed/no-rules-default +=== PAUSE TestChainedAuthorizer/No_Authorizers +=== CONT TestACL_Enforce/acl/read/Allow + authorizer_test.go:618: PASS: ACLRead(*acl.AuthorizerContext) +=== RUN TestACL_Enforce/acl/write/Deny + authorizer_test.go:618: PASS: ACLWrite(*acl.AuthorizerContext) +=== RUN TestACL_Enforce/acl/write/Allow + authorizer_test.go:618: PASS: ACLWrite(*acl.AuthorizerContext) +=== RUN TestACL_Enforce/acl/list/Deny +=== RUN TestACL_Enforce/operator/read/Deny + authorizer_test.go:618: PASS: OperatorRead(*acl.AuthorizerContext) +=== RUN TestACL_Enforce/operator/read/Allow + authorizer_test.go:618: PASS: OperatorRead(*acl.AuthorizerContext) +=== RUN TestACL_Enforce/operator/write/Deny + authorizer_test.go:618: PASS: OperatorWrite(*acl.AuthorizerContext) +=== RUN TestACL_Enforce/operator/write/Allow + authorizer_test.go:618: PASS: OperatorWrite(*acl.AuthorizerContext) +=== RUN TestACL_Enforce/operator/list/Deny +=== RUN TestACL_Enforce/keyring/read/Deny + authorizer_test.go:618: PASS: KeyringRead(*acl.AuthorizerContext) +=== RUN TestACL_Enforce/keyring/read/Allow + authorizer_test.go:618: PASS: KeyringRead(*acl.AuthorizerContext) +=== RUN TestACL_Enforce/keyring/write/Deny + authorizer_test.go:618: PASS: KeyringWrite(*acl.AuthorizerContext) +=== RUN TestACL_Enforce/keyring/write/Allow + authorizer_test.go:618: PASS: KeyringWrite(*acl.AuthorizerContext) +=== RUN TestACL_Enforce/keyring/list/Deny +=== RUN TestACL_Enforce/agent/foo/read/Deny + authorizer_test.go:618: PASS: AgentRead(string,*acl.AuthorizerContext) +=== RUN TestACL_Enforce/agent/foo/read/Allow + authorizer_test.go:618: PASS: AgentRead(string,*acl.AuthorizerContext) +=== RUN TestACL_Enforce/agent/foo/write/Deny + authorizer_test.go:618: PASS: AgentWrite(string,*acl.AuthorizerContext) +=== RUN TestACL_Enforce/agent/foo/write/Allow +=== RUN TestAllAllowed/prefix-allow-other-write-exact +=== PAUSE TestPolicyAuthorizer/Intention_Wildcards_-_all_default === RUN TestPolicyAuthorizer/Intention_Wildcards_-_any_default +=== RUN TestAllAllowed/prefix-allow-other-list-prefix === PAUSE TestPolicyAuthorizer/Intention_Wildcards_-_any_default === RUN TestPolicyAuthorizer/Defaults === PAUSE TestPolicyAuthorizer/Defaults === RUN TestPolicyAuthorizer/Prefer_Exact_Matches === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches +=== RUN TestAllAllowed/no-rules-default +=== CONT TestACL_Enforce/agent/foo/write/Allow + authorizer_test.go:618: PASS: AgentWrite(string,*acl.AuthorizerContext) +=== RUN TestACL_Enforce/agent/foo/list/Deny === RUN TestPolicyAuthorizer/Intention_Wildcards_-_prefix_denied === PAUSE TestPolicyAuthorizer/Intention_Wildcards_-_prefix_denied === RUN TestPolicyAuthorizer/Intention_Wildcards_-_prefix_allowed +=== RUN TestACL_Enforce/event/foo/read/Deny === PAUSE TestPolicyAuthorizer/Intention_Wildcards_-_prefix_allowed === RUN TestPolicyAuthorizer/Intention_Wildcards_-_all_allowed === PAUSE TestPolicyAuthorizer/Intention_Wildcards_-_all_allowed -=== RUN TestPolicyAuthorizer/Intention_Wildcards_-_all_default -=== PAUSE TestPolicyAuthorizer/Intention_Wildcards_-_all_default -=== CONT TestPolicyAuthorizer/Intention_Wildcards_-_any_default -=== RUN TestPolicyAuthorizer/Intention_Wildcards_-_any_default/AnyDefault.Prefix(*) -=== PAUSE TestPolicyAuthorizer/Intention_Wildcards_-_any_default/AnyDefault.Prefix(*) -=== RUN TestPolicyAuthorizer/Intention_Wildcards_-_any_default/AllDenied.Prefix(*) -=== PAUSE TestPolicyAuthorizer/Intention_Wildcards_-_any_default/AllDenied.Prefix(*) -=== CONT TestPolicyAuthorizer/Intention_Wildcards_-_any_default/AnyDefault.Prefix(*) -=== CONT TestAnyAllowed -=== RUN TestAnyAllowed/prefix-read-allowed -=== RUN TestAnyAllowed/prefix-deny-other-write-prefix -=== RUN TestAnyAllowed/prefix-deny-other-write-exact -=== RUN TestAnyAllowed/prefix-deny-other-list-exact -=== RUN TestAnyAllowed/prefix-deny-other-deny-prefix -=== RUN TestAnyAllowed/prefix-deny-other-deny-exact -=== RUN TestAnyAllowed/prefix-write-allowed -=== RUN TestAnyAllowed/prefix-list-allowed -=== RUN TestAnyAllowed/prefix-deny-other-list-prefix -=== RUN TestAnyAllowed/prefix-deny-other-read-prefix -=== RUN TestAnyAllowed/prefix-deny-other-read-exact -=== RUN TestAnyAllowed/no-rules-default -=== RUN TestAnyAllowed/prefix-deny ---- PASS: TestAnyAllowed (0.00s) - --- PASS: TestAnyAllowed/prefix-read-allowed (0.00s) - --- PASS: TestAnyAllowed/prefix-deny-other-write-prefix (0.00s) - --- PASS: TestAnyAllowed/prefix-deny-other-write-exact (0.00s) - --- PASS: TestAnyAllowed/prefix-deny-other-list-exact (0.00s) - --- PASS: TestAnyAllowed/prefix-deny-other-deny-prefix (0.00s) - --- PASS: TestAnyAllowed/prefix-deny-other-deny-exact (0.00s) - --- PASS: TestAnyAllowed/prefix-write-allowed (0.00s) - --- PASS: TestAnyAllowed/prefix-list-allowed (0.00s) - --- PASS: TestAnyAllowed/prefix-deny-other-list-prefix (0.00s) - --- PASS: TestAnyAllowed/prefix-deny-other-read-prefix (0.00s) - --- PASS: TestAnyAllowed/prefix-deny-other-read-exact (0.00s) - --- PASS: TestAnyAllowed/no-rules-default (0.00s) - --- PASS: TestAnyAllowed/prefix-deny (0.00s) === CONT TestPolicyAuthorizer/Intention_Wildcards_-_all_default === RUN TestPolicyAuthorizer/Intention_Wildcards_-_all_default/AnyAllowed.Prefix(*) === PAUSE TestPolicyAuthorizer/Intention_Wildcards_-_all_default/AnyAllowed.Prefix(*) -=== RUN TestPolicyAuthorizer/Intention_Wildcards_-_all_default/AllDefault.Prefix(*) -=== PAUSE TestPolicyAuthorizer/Intention_Wildcards_-_all_default/AllDefault.Prefix(*) -=== CONT TestPolicyAuthorizer/Intention_Wildcards_-_all_default/AnyAllowed.Prefix(*) -=== CONT TestPolicyAuthorizer/Intention_Wildcards_-_all_allowed -=== RUN TestPolicyAuthorizer/Intention_Wildcards_-_all_allowed/AnyAllowed.Prefix(*) -=== PAUSE TestPolicyAuthorizer/Intention_Wildcards_-_all_allowed/AnyAllowed.Prefix(*) -=== RUN TestPolicyAuthorizer/Intention_Wildcards_-_all_allowed/AllAllowed.Prefix(*) -=== PAUSE TestPolicyAuthorizer/Intention_Wildcards_-_all_allowed/AllAllowed.Prefix(*) -=== CONT TestPolicyAuthorizer/Intention_Wildcards_-_all_allowed/AnyAllowed.Prefix(*) +=== CONT TestACL_Enforce/event/foo/read/Deny + authorizer_test.go:618: PASS: EventRead(string,*acl.AuthorizerContext) +=== RUN TestAllAllowed/prefix-write-allowed +=== RUN TestACL_Enforce/event/foo/read/Allow +=== RUN TestAllAllowed/prefix-list-allowed +=== RUN TestAllAllowed/prefix-read-allowed +=== CONT TestACL_Enforce/event/foo/read/Allow + authorizer_test.go:618: PASS: EventRead(string,*acl.AuthorizerContext) +=== CONT TestStaticAuthorizer +=== RUN TestStaticAuthorizer/AllowAll +=== RUN TestAllAllowed/prefix-deny === CONT TestPolicyAuthorizer/Intention_Wildcards_-_prefix_allowed -=== RUN TestPolicyAuthorizer/Intention_Wildcards_-_prefix_allowed/AnyAllowed.Prefix(*) -=== PAUSE TestPolicyAuthorizer/Intention_Wildcards_-_prefix_allowed/AnyAllowed.Prefix(*) -=== RUN TestPolicyAuthorizer/Intention_Wildcards_-_prefix_allowed/AllDenied.Prefix(*) -=== PAUSE TestPolicyAuthorizer/Intention_Wildcards_-_prefix_allowed/AllDenied.Prefix(*) -=== CONT TestPolicyAuthorizer/Intention_Wildcards_-_prefix_allowed/AnyAllowed.Prefix(*) -=== CONT TestPolicyAuthorizer/Intention_Wildcards_-_prefix_denied -=== RUN TestPolicyAuthorizer/Intention_Wildcards_-_prefix_denied/AnyAllowed.Prefix(*) -=== PAUSE TestPolicyAuthorizer/Intention_Wildcards_-_prefix_denied/AnyAllowed.Prefix(*) -=== RUN TestPolicyAuthorizer/Intention_Wildcards_-_prefix_denied/AllDenied.Prefix(*) -=== PAUSE TestPolicyAuthorizer/Intention_Wildcards_-_prefix_denied/AllDenied.Prefix(*) -=== CONT TestPolicyAuthorizer/Intention_Wildcards_-_prefix_denied/AnyAllowed.Prefix(*) +=== PAUSE TestStaticAuthorizer/AllowAll +=== RUN TestAnyAllowed/prefix-read-allowed +=== RUN TestStaticAuthorizer/DenyAll +=== PAUSE TestStaticAuthorizer/DenyAll +=== RUN TestStaticAuthorizer/ManageAll +=== RUN TestAnyAllowed/prefix-deny +=== PAUSE TestStaticAuthorizer/ManageAll +=== RUN TestChainedAuthorizer/Authorizer_Defaults +=== PAUSE TestChainedAuthorizer/Authorizer_Defaults === CONT TestPolicyAuthorizer/Prefer_Exact_Matches +=== RUN TestChainedAuthorizer/Authorizer_No_Defaults +=== PAUSE TestChainedAuthorizer/Authorizer_No_Defaults +=== RUN TestChainedAuthorizer/First_Found +=== PAUSE TestChainedAuthorizer/First_Found +=== RUN TestAllAllowed/prefix-allow-other-read-prefix === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadPrefixAllowed.Prefix(fo) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadPrefixAllowed.Prefix(fo) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWritePrefixDenied.Prefix(fo) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWritePrefixDenied.Prefix(fo) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadPrefixAllowed.Prefix(for) +=== RUN TestAnyAllowed/prefix-deny-other-write-prefix === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadPrefixAllowed.Prefix(for) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWritePrefixDenied.Prefix(for) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWritePrefixDenied.Prefix(for) +=== RUN TestAllAllowed/prefix-allow-other-deny-prefix === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadAllowed.Prefix(foo) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadAllowed.Prefix(foo) -=== CONT TestChainedAuthorizer -=== RUN TestChainedAuthorizer/No_Authorizers -=== PAUSE TestChainedAuthorizer/No_Authorizers -=== RUN TestChainedAuthorizer/Authorizer_Defaults -=== PAUSE TestChainedAuthorizer/Authorizer_Defaults -=== RUN TestChainedAuthorizer/Authorizer_No_Defaults -=== PAUSE TestChainedAuthorizer/Authorizer_No_Defaults -=== RUN TestChainedAuthorizer/First_Found -=== PAUSE TestChainedAuthorizer/First_Found -=== CONT TestPolicyAuthorizer/Intention_Wildcards_-_all_default/AllDefault.Prefix(*) -=== CONT TestPolicyAuthorizer/Intention_Wildcards_-_prefix_allowed/AllDenied.Prefix(*) -=== CONT TestChainedAuthorizer/First_Found -=== CONT TestChainedAuthorizer/Authorizer_No_Defaults -=== CONT TestChainedAuthorizer/Authorizer_Defaults +=== RUN TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWriteAllowed.Prefix(foo) +=== PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWriteAllowed.Prefix(foo) +=== RUN TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadPrefixAllowed.Prefix(foot) +=== PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadPrefixAllowed.Prefix(foot) +=== RUN TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWritePrefixDenied.Prefix(foot) +=== PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWritePrefixDenied.Prefix(foot) +=== RUN TestAnyAllowed/prefix-deny-other-list-prefix +=== RUN TestAnyAllowed/prefix-deny-other-deny-exact +=== RUN TestAnyAllowed/prefix-write-allowed +--- PASS: TestAnyAllowed (0.01s) + --- PASS: TestAnyAllowed/prefix-list-allowed (0.00s) + --- PASS: TestAnyAllowed/prefix-deny-other-write-exact (0.00s) + --- PASS: TestAnyAllowed/prefix-deny-other-list-exact (0.00s) + --- PASS: TestAnyAllowed/prefix-deny-other-read-prefix (0.00s) + --- PASS: TestAnyAllowed/prefix-deny-other-read-exact (0.00s) + --- PASS: TestAnyAllowed/prefix-deny-other-deny-prefix (0.00s) + --- PASS: TestAnyAllowed/no-rules-default (0.00s) + --- PASS: TestAnyAllowed/prefix-read-allowed (0.00s) + --- PASS: TestAnyAllowed/prefix-deny (0.00s) + --- PASS: TestAnyAllowed/prefix-deny-other-write-prefix (0.00s) + --- PASS: TestAnyAllowed/prefix-deny-other-list-prefix (0.00s) + --- PASS: TestAnyAllowed/prefix-deny-other-deny-exact (0.00s) + --- PASS: TestAnyAllowed/prefix-write-allowed (0.00s) +=== CONT TestPolicyAuthorizer/Intention_Wildcards_-_any_default +=== RUN TestPolicyAuthorizer/Intention_Wildcards_-_any_default/AnyDefault.Prefix(*) +=== PAUSE TestPolicyAuthorizer/Intention_Wildcards_-_any_default/AnyDefault.Prefix(*) +=== RUN TestPolicyAuthorizer/Intention_Wildcards_-_any_default/AllDenied.Prefix(*) +=== PAUSE TestPolicyAuthorizer/Intention_Wildcards_-_any_default/AllDenied.Prefix(*) +=== RUN TestAllAllowed/prefix-allow-other-deny-exact +=== RUN TestAllAllowed/prefix-allow-other-list-exact +=== RUN TestAllAllowed/prefix-allow-other-read-exact +--- PASS: TestAllAllowed (0.01s) + --- PASS: TestAllAllowed/prefix-allow-other-write-prefix (0.00s) + --- PASS: TestAllAllowed/prefix-allow-other-write-exact (0.00s) + --- PASS: TestAllAllowed/prefix-allow-other-list-prefix (0.00s) + --- PASS: TestAllAllowed/no-rules-default (0.00s) + --- PASS: TestAllAllowed/prefix-write-allowed (0.00s) + --- PASS: TestAllAllowed/prefix-list-allowed (0.00s) + --- PASS: TestAllAllowed/prefix-read-allowed (0.00s) + --- PASS: TestAllAllowed/prefix-deny (0.00s) + --- PASS: TestAllAllowed/prefix-allow-other-read-prefix (0.00s) + --- PASS: TestAllAllowed/prefix-allow-other-deny-prefix (0.00s) + --- PASS: TestAllAllowed/prefix-allow-other-deny-exact (0.00s) + --- PASS: TestAllAllowed/prefix-allow-other-list-exact (0.00s) + --- PASS: TestAllAllowed/prefix-allow-other-read-exact (0.00s) +=== CONT TestStaticAuthorizer/AllowAll === CONT TestPolicyAuthorizer/Defaults === RUN TestPolicyAuthorizer/Defaults/DefaultACLRead.Prefix(foo) === PAUSE TestPolicyAuthorizer/Defaults/DefaultACLRead.Prefix(foo) @@ -4939,79 +5023,113 @@ === PAUSE TestPolicyAuthorizer/Defaults/DefaultSessionWrite.Prefix(foo) === RUN TestPolicyAuthorizer/Defaults/DefaultSnapshot.Prefix(foo) === PAUSE TestPolicyAuthorizer/Defaults/DefaultSnapshot.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultACLRead.Prefix(foo) +=== RUN TestPolicyAuthorizer/Intention_Wildcards_-_all_default/AllDefault.Prefix(*) +=== PAUSE TestPolicyAuthorizer/Intention_Wildcards_-_all_default/AllDefault.Prefix(*) +=== CONT TestStaticAuthorizer/ManageAll +=== CONT TestChainedAuthorizer/First_Found === CONT TestChainedAuthorizer/No_Authorizers ---- PASS: TestChainedAuthorizer (0.00s) - --- PASS: TestChainedAuthorizer/First_Found (0.00s) - --- PASS: TestChainedAuthorizer/Authorizer_No_Defaults (0.00s) - --- PASS: TestChainedAuthorizer/Authorizer_Defaults (0.00s) - --- PASS: TestChainedAuthorizer/No_Authorizers (0.00s) -=== CONT TestPolicyAuthorizer/Intention_Wildcards_-_any_default/AllDenied.Prefix(*) -=== CONT TestPolicyAuthorizer/Defaults/DefaultSnapshot.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultSessionWrite.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultSessionRead.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultServiceWrite.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultServiceRead.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultPreparedQueryWrite.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultPreparedQueryRead.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultOperatorWrite.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultOperatorRead.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultNodeWrite.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultNodeRead.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultKeyWritePrefix.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultKeyWrite.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultKeyringWrite.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultKeyringRead.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultEventRead.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultKeyList.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultKeyRead.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultIntentionWrite.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultIntentionRead.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultIntentionDefaultAllow.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultEventWrite.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultACLWrite.Prefix(foo) -=== CONT TestPolicyAuthorizer/Intention_Wildcards_-_prefix_denied/AllDenied.Prefix(*) -=== CONT TestPolicyAuthorizer/Defaults/DefaultAgentWrite.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultAgentRead.Prefix(foo) -=== CONT TestPolicyAuthorizer/Intention_Wildcards_-_all_allowed/AllAllowed.Prefix(*) -=== CONT TestACL_Enforce/acl/read/Deny - authorizer_test.go:618: PASS: ACLRead(*acl.AuthorizerContext) -=== RUN TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWriteAllowed.Prefix(foo) -=== RUN TestACL_Enforce/acl/read/Allow -=== PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWriteAllowed.Prefix(foo) -=== RUN TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadPrefixAllowed.Prefix(foot) -=== PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadPrefixAllowed.Prefix(foot) -=== RUN TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWritePrefixDenied.Prefix(foot) -=== PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWritePrefixDenied.Prefix(foot) +=== CONT TestStaticAuthorizer/DenyAll +=== RUN TestPolicyAuthorizer/Intention_Wildcards_-_prefix_allowed/AnyAllowed.Prefix(*) +=== CONT TestPolicyAuthorizer/Intention_Wildcards_-_prefix_denied +=== RUN TestACL_Enforce/event/foo/write/Deny +=== CONT TestPolicyAuthorizer/Intention_Wildcards_-_all_allowed === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadPrefixAllowed.Prefix(foot2) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadPrefixAllowed.Prefix(foot2) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWritePrefixDenied.Prefix(foot2) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWritePrefixDenied.Prefix(foot2) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadPrefixAllowed.Prefix(food) +=== CONT TestChainedAuthorizer/Authorizer_No_Defaults +=== CONT TestChainedAuthorizer/Authorizer_Defaults +--- PASS: TestStaticAuthorizer (0.00s) + --- PASS: TestStaticAuthorizer/AllowAll (0.00s) + --- PASS: TestStaticAuthorizer/ManageAll (0.00s) + --- PASS: TestStaticAuthorizer/DenyAll (0.00s) +=== PAUSE TestPolicyAuthorizer/Intention_Wildcards_-_prefix_allowed/AnyAllowed.Prefix(*) +=== RUN TestPolicyAuthorizer/Intention_Wildcards_-_prefix_allowed/AllDenied.Prefix(*) +=== PAUSE TestPolicyAuthorizer/Intention_Wildcards_-_prefix_allowed/AllDenied.Prefix(*) +=== RUN TestPolicyAuthorizer/Intention_Wildcards_-_prefix_denied/AnyAllowed.Prefix(*) +=== CONT TestPolicyAuthorizer/Intention_Wildcards_-_any_default/AllDenied.Prefix(*) +=== PAUSE TestPolicyAuthorizer/Intention_Wildcards_-_prefix_denied/AnyAllowed.Prefix(*) +=== CONT TestPolicyAuthorizer/Intention_Wildcards_-_all_default/AnyAllowed.Prefix(*) +=== RUN TestPolicyAuthorizer/Intention_Wildcards_-_prefix_denied/AllDenied.Prefix(*) +=== PAUSE TestPolicyAuthorizer/Intention_Wildcards_-_prefix_denied/AllDenied.Prefix(*) +=== CONT TestPolicyAuthorizer/Defaults/DefaultACLRead.Prefix(foo) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadPrefixAllowed.Prefix(food) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWritePrefixDenied.Prefix(food) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWritePrefixDenied.Prefix(food) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadDenied.Prefix(football) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadDenied.Prefix(football) -=== CONT TestACL_Enforce/acl/read/Allow - authorizer_test.go:618: PASS: ACLRead(*acl.AuthorizerContext) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWriteDenied.Prefix(football) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWriteDenied.Prefix(football) -=== RUN TestACL_Enforce/acl/write/Deny === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadPrefixAllowed.Prefix(fo) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadPrefixAllowed.Prefix(fo) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWritePrefixDenied.Prefix(fo) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWritePrefixDenied.Prefix(fo) +=== CONT TestACL_Enforce/event/foo/write/Deny + authorizer_test.go:618: PASS: EventWrite(string,*acl.AuthorizerContext) +=== CONT TestPolicyAuthorizer/Defaults/DefaultSnapshot.Prefix(foo) +=== RUN TestACL_Enforce/event/foo/write/Allow +=== CONT TestPolicyAuthorizer/Defaults/DefaultSessionWrite.Prefix(foo) +=== CONT TestPolicyAuthorizer/Defaults/DefaultSessionRead.Prefix(foo) +=== CONT TestPolicyAuthorizer/Defaults/DefaultServiceWrite.Prefix(foo) +=== CONT TestPolicyAuthorizer/Defaults/DefaultServiceRead.Prefix(foo) +=== CONT TestPolicyAuthorizer/Defaults/DefaultPreparedQueryWrite.Prefix(foo) +=== CONT TestPolicyAuthorizer/Defaults/DefaultKeyWritePrefix.Prefix(foo) +=== CONT TestPolicyAuthorizer/Defaults/DefaultNodeRead.Prefix(foo) +=== CONT TestPolicyAuthorizer/Defaults/DefaultOperatorWrite.Prefix(foo) +=== RUN TestPolicyAuthorizer/Intention_Wildcards_-_all_allowed/AnyAllowed.Prefix(*) +=== PAUSE TestPolicyAuthorizer/Intention_Wildcards_-_all_allowed/AnyAllowed.Prefix(*) +=== CONT TestPolicyAuthorizer/Defaults/DefaultKeyWrite.Prefix(foo) +=== RUN TestPolicyAuthorizer/Intention_Wildcards_-_all_allowed/AllAllowed.Prefix(*) +=== PAUSE TestPolicyAuthorizer/Intention_Wildcards_-_all_allowed/AllAllowed.Prefix(*) +=== CONT TestPolicyAuthorizer/Defaults/DefaultKeyringWrite.Prefix(foo) +=== CONT TestPolicyAuthorizer/Defaults/DefaultNodeWrite.Prefix(foo) +=== CONT TestPolicyAuthorizer/Defaults/DefaultACLWrite.Prefix(foo) +=== CONT TestPolicyAuthorizer/Defaults/DefaultEventWrite.Prefix(foo) +=== CONT TestPolicyAuthorizer/Defaults/DefaultOperatorRead.Prefix(foo) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadPrefixAllowed.Prefix(for) +=== CONT TestPolicyAuthorizer/Defaults/DefaultAgentWrite.Prefix(foo) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadPrefixAllowed.Prefix(for) +=== CONT TestPolicyAuthorizer/Defaults/DefaultKeyList.Prefix(foo) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWritePrefixDenied.Prefix(for) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWritePrefixDenied.Prefix(for) +=== CONT TestPolicyAuthorizer/Defaults/DefaultKeyringRead.Prefix(foo) +=== CONT TestPolicyAuthorizer/Defaults/DefaultKeyRead.Prefix(foo) +=== CONT TestACL_Enforce/event/foo/write/Allow + authorizer_test.go:618: PASS: EventWrite(string,*acl.AuthorizerContext) +=== CONT TestPolicyAuthorizer/Defaults/DefaultIntentionWrite.Prefix(foo) +=== RUN TestACL_Enforce/event/foo/list/Deny +=== CONT TestPolicyAuthorizer/Defaults/DefaultAgentRead.Prefix(foo) +=== RUN TestACL_Enforce/intention/foo/read/Deny +=== CONT TestPolicyAuthorizer/Defaults/DefaultIntentionRead.Prefix(foo) +=== CONT TestPolicyAuthorizer/Intention_Wildcards_-_all_default/AllDefault.Prefix(*) +=== CONT TestPolicyAuthorizer/Defaults/DefaultIntentionDefaultAllow.Prefix(foo) +=== CONT TestPolicyAuthorizer/Intention_Wildcards_-_prefix_denied/AnyAllowed.Prefix(*) +=== CONT TestPolicyAuthorizer/Intention_Wildcards_-_prefix_allowed/AllDenied.Prefix(*) +--- PASS: TestChainedAuthorizer (0.01s) + --- PASS: TestChainedAuthorizer/No_Authorizers (0.00s) + --- PASS: TestChainedAuthorizer/Authorizer_No_Defaults (0.00s) + --- PASS: TestChainedAuthorizer/Authorizer_Defaults (0.00s) + --- PASS: TestChainedAuthorizer/First_Found (0.00s) +=== CONT TestPolicyAuthorizer/Intention_Wildcards_-_any_default/AnyDefault.Prefix(*) +=== CONT TestPolicyAuthorizer/Intention_Wildcards_-_all_allowed/AnyAllowed.Prefix(*) +=== CONT TestACL_Enforce/intention/foo/read/Deny + authorizer_test.go:618: PASS: IntentionRead(string,*acl.AuthorizerContext) +=== CONT TestPolicyAuthorizer/Defaults/DefaultEventRead.Prefix(foo) +=== CONT TestPolicyAuthorizer/Defaults/DefaultPreparedQueryRead.Prefix(foo) +=== RUN TestACL_Enforce/intention/foo/read/Allow === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadAllowed.Prefix(foo) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadAllowed.Prefix(foo) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWriteAllowed.Prefix(foo) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWriteAllowed.Prefix(foo) +=== CONT TestPolicyAuthorizer/Intention_Wildcards_-_all_allowed/AllAllowed.Prefix(*) +=== CONT TestPolicyAuthorizer/Intention_Wildcards_-_prefix_denied/AllDenied.Prefix(*) +=== CONT TestPolicyAuthorizer/Intention_Wildcards_-_prefix_allowed/AnyAllowed.Prefix(*) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadPrefixAllowed.Prefix(foot) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadPrefixAllowed.Prefix(foot) +=== CONT TestACL_Enforce/intention/foo/read/Allow + authorizer_test.go:618: PASS: IntentionRead(string,*acl.AuthorizerContext) +=== RUN TestACL_Enforce/intention/foo/write/Deny === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWritePrefixDenied.Prefix(foot) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWritePrefixDenied.Prefix(foot) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadPrefixAllowed.Prefix(foot2) @@ -5021,13 +5139,29 @@ === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadPrefixAllowed.Prefix(food) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadPrefixAllowed.Prefix(food) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWritePrefixDenied.Prefix(food) +=== CONT TestACL_Enforce/intention/foo/write/Deny + authorizer_test.go:618: PASS: IntentionWrite(string,*acl.AuthorizerContext) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWritePrefixDenied.Prefix(food) +=== RUN TestACL_Enforce/intention/foo/write/Allow === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadDenied.Prefix(football) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadDenied.Prefix(football) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWriteDenied.Prefix(football) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWriteDenied.Prefix(football) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(fo) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(fo) +=== CONT TestACL_Enforce/intention/foo/write/Allow + authorizer_test.go:618: PASS: IntentionWrite(string,*acl.AuthorizerContext) +=== RUN TestACL_Enforce/intention/foo/list/Deny +=== RUN TestACL_Enforce/node/foo/read/Deny + authorizer_test.go:618: PASS: NodeRead(string,*acl.AuthorizerContext) +=== RUN TestACL_Enforce/node/foo/read/Allow + authorizer_test.go:618: PASS: NodeRead(string,*acl.AuthorizerContext) +=== RUN TestACL_Enforce/node/foo/write/Deny + authorizer_test.go:618: PASS: NodeWrite(string,*acl.AuthorizerContext) +=== RUN TestACL_Enforce/node/foo/write/Allow + authorizer_test.go:618: PASS: NodeWrite(string,*acl.AuthorizerContext) +=== RUN TestACL_Enforce/node/foo/list/Deny +=== RUN TestACL_Enforce/query/foo/read/Deny === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(fo) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(fo) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(for) @@ -5036,14 +5170,14 @@ === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(for) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadAllowed.Prefix(foo) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadAllowed.Prefix(foo) +=== CONT TestACL_Enforce/query/foo/read/Deny + authorizer_test.go:618: PASS: PreparedQueryRead(string,*acl.AuthorizerContext) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWriteAllowed.Prefix(foo) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWriteAllowed.Prefix(foo) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(foot) +=== RUN TestACL_Enforce/query/foo/read/Allow === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(foot) -=== CONT TestACL_Enforce/acl/write/Deny - authorizer_test.go:618: PASS: ACLWrite(*acl.AuthorizerContext) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(foot) -=== RUN TestACL_Enforce/acl/write/Allow === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(foot) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(foot2) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(foot2) @@ -5053,19 +5187,19 @@ === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(food) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(food) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(food) +=== CONT TestACL_Enforce/query/foo/read/Allow + authorizer_test.go:618: PASS: PreparedQueryRead(string,*acl.AuthorizerContext) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadDenied.Prefix(football) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadDenied.Prefix(football) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWriteDenied.Prefix(football) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWriteDenied.Prefix(football) -=== CONT TestACL_Enforce/acl/write/Allow - authorizer_test.go:618: PASS: ACLWrite(*acl.AuthorizerContext) -=== RUN TestACL_Enforce/acl/list/Deny === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(fo) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(fo) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(fo) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(fo) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(for) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(for) +=== RUN TestACL_Enforce/query/foo/write/Deny === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(for) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(for) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadAllowed.Prefix(foo) @@ -5075,13 +5209,29 @@ === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(foot) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(foot) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(foot) +=== CONT TestACL_Enforce/query/foo/write/Deny + authorizer_test.go:618: PASS: PreparedQueryWrite(string,*acl.AuthorizerContext) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(foot) +=== RUN TestACL_Enforce/query/foo/write/Allow === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(foot2) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(foot2) +=== CONT TestACL_Enforce/query/foo/write/Allow + authorizer_test.go:618: PASS: PreparedQueryWrite(string,*acl.AuthorizerContext) +=== RUN TestACL_Enforce/query/foo/list/Deny +=== RUN TestACL_Enforce/service/foo/read/Deny + authorizer_test.go:618: PASS: ServiceRead(string,*acl.AuthorizerContext) +=== RUN TestACL_Enforce/service/foo/read/Allow + authorizer_test.go:618: PASS: ServiceRead(string,*acl.AuthorizerContext) +=== RUN TestACL_Enforce/service/foo/write/Deny + authorizer_test.go:618: PASS: ServiceWrite(string,*acl.AuthorizerContext) +=== RUN TestACL_Enforce/service/foo/write/Allow + authorizer_test.go:618: PASS: ServiceWrite(string,*acl.AuthorizerContext) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(foot2) +=== RUN TestACL_Enforce/session/foo/list/Deny === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(foot2) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(food) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(food) +=== RUN TestACL_Enforce/session/foo/read/Deny === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(food) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(food) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadDenied.Prefix(football) @@ -5089,6 +5239,8 @@ === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWriteDenied.Prefix(football) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWriteDenied.Prefix(football) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(fo)#01 +=== CONT TestACL_Enforce/session/foo/read/Deny + authorizer_test.go:618: PASS: SessionRead(string,*acl.AuthorizerContext) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(fo)#01 === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(fo)#01 === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(fo)#01 @@ -5136,6 +5288,7 @@ === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionReadPrefixAllowed.Prefix(foot2) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionWritePrefixDenied.Prefix(foot2) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionWritePrefixDenied.Prefix(foot2) +=== RUN TestACL_Enforce/session/foo/read/Allow === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionReadPrefixAllowed.Prefix(food) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionReadPrefixAllowed.Prefix(food) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionWritePrefixDenied.Prefix(food) @@ -5144,6 +5297,9 @@ === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionReadDenied.Prefix(football) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionWriteDenied.Prefix(football) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionWriteDenied.Prefix(football) +=== CONT TestACL_Enforce/session/foo/read/Allow + authorizer_test.go:618: PASS: SessionRead(string,*acl.AuthorizerContext) +=== RUN TestACL_Enforce/session/foo/write/Deny === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadPrefixAllowed.Prefix(fo) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadPrefixAllowed.Prefix(fo) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWritePrefixDenied.Prefix(fo) @@ -5152,8 +5308,11 @@ === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadPrefixAllowed.Prefix(for) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWritePrefixDenied.Prefix(for) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWritePrefixDenied.Prefix(for) +=== CONT TestACL_Enforce/session/foo/write/Deny + authorizer_test.go:618: PASS: SessionWrite(string,*acl.AuthorizerContext) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadAllowed.Prefix(foo) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadAllowed.Prefix(foo) +=== RUN TestACL_Enforce/session/foo/write/Allow === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWriteAllowed.Prefix(foo) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWriteAllowed.Prefix(foo) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadPrefixAllowed.Prefix(foot) @@ -5161,16 +5320,26 @@ === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWritePrefixDenied.Prefix(foot) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWritePrefixDenied.Prefix(foot) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadPrefixAllowed.Prefix(foot2) +=== CONT TestACL_Enforce/session/foo/write/Allow + authorizer_test.go:618: PASS: SessionWrite(string,*acl.AuthorizerContext) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadPrefixAllowed.Prefix(foot2) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWritePrefixDenied.Prefix(foot2) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWritePrefixDenied.Prefix(foot2) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadPrefixAllowed.Prefix(food) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadPrefixAllowed.Prefix(food) +=== RUN TestACL_Enforce/session/foo/list/Deny#01 +=== RUN TestACL_Enforce/key/foo/read/Deny + authorizer_test.go:618: PASS: KeyRead(string,*acl.AuthorizerContext) +=== RUN TestACL_Enforce/key/foo/read/Allow + authorizer_test.go:618: PASS: KeyRead(string,*acl.AuthorizerContext) +=== RUN TestACL_Enforce/key/foo/write/Deny === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWritePrefixDenied.Prefix(food) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWritePrefixDenied.Prefix(food) +=== CONT TestACL_Enforce/key/foo/write/Deny + authorizer_test.go:618: PASS: KeyWrite(string,*acl.AuthorizerContext) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadDenied.Prefix(football) -=== RUN TestACL_Enforce/operator/read/Deny === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadDenied.Prefix(football) +=== RUN TestACL_Enforce/key/foo/write/Allow === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWriteDenied.Prefix(football) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWriteDenied.Prefix(football) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(fo) @@ -5181,10 +5350,13 @@ === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(for) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(for) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(for) +=== CONT TestACL_Enforce/key/foo/write/Allow + authorizer_test.go:618: PASS: KeyWrite(string,*acl.AuthorizerContext) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadAllowed.Prefix(foo) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadAllowed.Prefix(foo) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/EventWriteAllowed.Prefix(foo) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/EventWriteAllowed.Prefix(foo) +=== RUN TestACL_Enforce/key/foo/list/Deny === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(foot) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(foot) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(foot) @@ -5193,18 +5365,84 @@ === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(foot2) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(foot2) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(foot2) +=== CONT TestACL_Enforce/key/foo/list/Deny + authorizer_test.go:618: PASS: KeyList(string,*acl.AuthorizerContext) +=== RUN TestACL_Enforce/key/foo/list/Allow === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(food) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(food) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(food) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(food) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadDenied.Prefix(football) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadDenied.Prefix(football) +=== CONT TestACL_Enforce/key/foo/list/Allow + authorizer_test.go:618: PASS: KeyList(string,*acl.AuthorizerContext) +=== RUN TestACL_Enforce/key/foo/deny/Deny === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/EventWriteDenied.Prefix(football) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/EventWriteDenied.Prefix(football) +=== RUN TestACL_Enforce/not-a-real-resource/read/Deny === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryReadPrefixAllowed.Prefix(fo) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryReadPrefixAllowed.Prefix(fo) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryWritePrefixDenied.Prefix(fo) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryWritePrefixDenied.Prefix(fo) +--- PASS: TestACL_Enforce (0.02s) + --- PASS: TestACL_Enforce/acl/read/Deny (0.00s) + --- PASS: TestACL_Enforce/acl/read/Allow (0.00s) + --- PASS: TestACL_Enforce/acl/write/Deny (0.00s) + --- PASS: TestACL_Enforce/acl/write/Allow (0.00s) + --- PASS: TestACL_Enforce/acl/list/Deny (0.00s) + --- PASS: TestACL_Enforce/operator/read/Deny (0.00s) + --- PASS: TestACL_Enforce/operator/read/Allow (0.00s) + --- PASS: TestACL_Enforce/operator/write/Deny (0.00s) + --- PASS: TestACL_Enforce/operator/write/Allow (0.00s) + --- PASS: TestACL_Enforce/operator/list/Deny (0.00s) + --- PASS: TestACL_Enforce/keyring/read/Deny (0.00s) + --- PASS: TestACL_Enforce/keyring/read/Allow (0.00s) + --- PASS: TestACL_Enforce/keyring/write/Deny (0.00s) + --- PASS: TestACL_Enforce/keyring/write/Allow (0.00s) + --- PASS: TestACL_Enforce/keyring/list/Deny (0.00s) + --- PASS: TestACL_Enforce/agent/foo/read/Deny (0.00s) + --- PASS: TestACL_Enforce/agent/foo/read/Allow (0.00s) + --- PASS: TestACL_Enforce/agent/foo/write/Deny (0.00s) + --- PASS: TestACL_Enforce/agent/foo/write/Allow (0.00s) + --- PASS: TestACL_Enforce/agent/foo/list/Deny (0.00s) + --- PASS: TestACL_Enforce/event/foo/read/Deny (0.00s) + --- PASS: TestACL_Enforce/event/foo/read/Allow (0.00s) + --- PASS: TestACL_Enforce/event/foo/write/Deny (0.00s) + --- PASS: TestACL_Enforce/event/foo/write/Allow (0.00s) + --- PASS: TestACL_Enforce/event/foo/list/Deny (0.00s) + --- PASS: TestACL_Enforce/intention/foo/read/Deny (0.00s) + --- PASS: TestACL_Enforce/intention/foo/read/Allow (0.00s) + --- PASS: TestACL_Enforce/intention/foo/write/Deny (0.00s) + --- PASS: TestACL_Enforce/intention/foo/write/Allow (0.00s) + --- PASS: TestACL_Enforce/intention/foo/list/Deny (0.00s) + --- PASS: TestACL_Enforce/node/foo/read/Deny (0.00s) + --- PASS: TestACL_Enforce/node/foo/read/Allow (0.00s) + --- PASS: TestACL_Enforce/node/foo/write/Deny (0.00s) + --- PASS: TestACL_Enforce/node/foo/write/Allow (0.00s) + --- PASS: TestACL_Enforce/node/foo/list/Deny (0.00s) + --- PASS: TestACL_Enforce/query/foo/read/Deny (0.00s) + --- PASS: TestACL_Enforce/query/foo/read/Allow (0.00s) + --- PASS: TestACL_Enforce/query/foo/write/Deny (0.00s) + --- PASS: TestACL_Enforce/query/foo/write/Allow (0.00s) + --- PASS: TestACL_Enforce/query/foo/list/Deny (0.00s) + --- PASS: TestACL_Enforce/service/foo/read/Deny (0.00s) + --- PASS: TestACL_Enforce/service/foo/read/Allow (0.00s) + --- PASS: TestACL_Enforce/service/foo/write/Deny (0.00s) + --- PASS: TestACL_Enforce/service/foo/write/Allow (0.00s) + --- PASS: TestACL_Enforce/session/foo/list/Deny (0.00s) + --- PASS: TestACL_Enforce/session/foo/read/Deny (0.00s) + --- PASS: TestACL_Enforce/session/foo/read/Allow (0.00s) + --- PASS: TestACL_Enforce/session/foo/write/Deny (0.00s) + --- PASS: TestACL_Enforce/session/foo/write/Allow (0.00s) + --- PASS: TestACL_Enforce/session/foo/list/Deny#01 (0.00s) + --- PASS: TestACL_Enforce/key/foo/read/Deny (0.00s) + --- PASS: TestACL_Enforce/key/foo/read/Allow (0.00s) + --- PASS: TestACL_Enforce/key/foo/write/Deny (0.00s) + --- PASS: TestACL_Enforce/key/foo/write/Allow (0.00s) + --- PASS: TestACL_Enforce/key/foo/list/Deny (0.00s) + --- PASS: TestACL_Enforce/key/foo/list/Allow (0.00s) + --- PASS: TestACL_Enforce/key/foo/deny/Deny (0.00s) + --- PASS: TestACL_Enforce/not-a-real-resource/read/Deny (0.00s) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryReadPrefixAllowed.Prefix(for) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryReadPrefixAllowed.Prefix(for) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryWritePrefixDenied.Prefix(for) @@ -5246,162 +5484,122 @@ === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryReadPrefixAllowed.Prefix(fo) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventWriteDenied.Prefix(football) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadDenied.Prefix(football) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(food) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(food) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(foot2) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(foot2) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(foot) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(foot) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventWriteAllowed.Prefix(foo) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadAllowed.Prefix(foo) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(for) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(for) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(fo) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(fo) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionWriteDenied.Prefix(football) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWriteDenied.Prefix(football) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionWritePrefixDenied.Prefix(food) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadPrefixAllowed.Prefix(foot2) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionReadDenied.Prefix(football) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadDenied.Prefix(football) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWritePrefixDenied.Prefix(food) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadPrefixAllowed.Prefix(food) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWritePrefixDenied.Prefix(foot2) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadPrefixAllowed.Prefix(foot2) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWritePrefixDenied.Prefix(foot) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadPrefixAllowed.Prefix(foot) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWriteAllowed.Prefix(foo) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadAllowed.Prefix(foo) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWritePrefixDenied.Prefix(for) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadPrefixAllowed.Prefix(for) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWritePrefixDenied.Prefix(fo) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadPrefixAllowed.Prefix(fo) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionWriteDenied.Prefix(football) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionReadDenied.Prefix(football) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionWritePrefixDenied.Prefix(food) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWritePrefixDenied.Prefix(foot2) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(food) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(foot2) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(foot2) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadPrefixAllowed.Prefix(foot) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(for) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventWriteAllowed.Prefix(foo) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(fo)#01 +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadAllowed.Prefix(foo) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(for) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadDenied.Prefix(football)#01 === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionReadPrefixAllowed.Prefix(food) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionWritePrefixDenied.Prefix(foot2) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWriteAllowed.Prefix(foo) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionReadPrefixAllowed.Prefix(foot2) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionWritePrefixDenied.Prefix(foot) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionReadPrefixAllowed.Prefix(foot) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(foot) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionWriteAllowed.Prefix(foo) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionReadAllowed.Prefix(foo) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(fo) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionWritePrefixDenied.Prefix(for) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(fo) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionReadPrefixAllowed.Prefix(for) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionWritePrefixDenied.Prefix(fo) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionReadPrefixAllowed.Prefix(fo) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWriteDenied.Prefix(football)#01 -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadDenied.Prefix(football)#01 === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(food)#01 === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(food)#01 +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionWritePrefixDenied.Prefix(foot) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(foot)#01 === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(foot2)#01 === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(foot2)#01 === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(foot)#01 -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(foot)#01 +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(for)#01 === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWriteAllowed.Prefix(foo)#01 +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(fo)#01 === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadAllowed.Prefix(foo)#01 -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(for)#01 === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(for)#01 -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(fo)#01 -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(fo)#01 +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(fo) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWriteDenied.Prefix(football) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadPrefixAllowed.Prefix(fo) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadDenied.Prefix(football) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(food) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadDenied.Prefix(football) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(food) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWritePrefixDenied.Prefix(fo) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(food) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(foot2) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(foot2) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(foot) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(foot) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadPrefixAllowed.Prefix(food) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionWritePrefixDenied.Prefix(foot2) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWriteDenied.Prefix(football) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWritePrefixDenied.Prefix(foot2) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(foot) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadPrefixAllowed.Prefix(foot2) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWritePrefixDenied.Prefix(foot) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWriteAllowed.Prefix(foo) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadAllowed.Prefix(foo) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadPrefixAllowed.Prefix(foot) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(for) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWriteAllowed.Prefix(foo) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(for) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadAllowed.Prefix(foo) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(fo) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWritePrefixDenied.Prefix(for) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(fo) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadPrefixAllowed.Prefix(for) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWriteDenied.Prefix(football) -=== CONT TestACL_Enforce/operator/read/Deny - authorizer_test.go:618: PASS: OperatorRead(*acl.AuthorizerContext) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(food) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWritePrefixDenied.Prefix(fo) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadDenied.Prefix(football) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(food) -=== RUN TestACL_Enforce/operator/read/Allow +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWriteAllowed.Prefix(foo) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadAllowed.Prefix(foo) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(foot2) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(for) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(foot2) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(for) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(foot) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(foot) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(foot) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWriteAllowed.Prefix(foo) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadAllowed.Prefix(foo) -=== CONT TestACL_Enforce/operator/read/Allow - authorizer_test.go:618: PASS: OperatorRead(*acl.AuthorizerContext) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(fo) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(fo) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadDenied.Prefix(football) -=== RUN TestACL_Enforce/operator/write/Deny -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(for) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadDenied.Prefix(football) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWriteDenied.Prefix(football) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWritePrefixDenied.Prefix(food) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(for) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadAllowed.Prefix(foo) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWritePrefixDenied.Prefix(foot) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(foot2) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadDenied.Prefix(football) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWritePrefixDenied.Prefix(foot2) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadPrefixAllowed.Prefix(foot2) -=== CONT TestACL_Enforce/operator/write/Deny - authorizer_test.go:618: PASS: OperatorWrite(*acl.AuthorizerContext) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWritePrefixDenied.Prefix(foot) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadPrefixAllowed.Prefix(foot) -=== RUN TestACL_Enforce/operator/write/Allow -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWriteAllowed.Prefix(foo) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadAllowed.Prefix(foo) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWritePrefixDenied.Prefix(for) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadPrefixAllowed.Prefix(for) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWritePrefixDenied.Prefix(fo) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadPrefixAllowed.Prefix(fo) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWriteDenied.Prefix(football) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadPrefixAllowed.Prefix(foot) -=== CONT TestACL_Enforce/operator/write/Allow - authorizer_test.go:618: PASS: OperatorWrite(*acl.AuthorizerContext) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWriteAllowed.Prefix(foo) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWritePrefixDenied.Prefix(food) -=== RUN TestACL_Enforce/operator/list/Deny -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadPrefixAllowed.Prefix(food) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadPrefixAllowed.Prefix(for) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(food) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWritePrefixDenied.Prefix(for) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWritePrefixDenied.Prefix(foot2) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(fo) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWritePrefixDenied.Prefix(fo) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWriteDenied.Prefix(football) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadPrefixAllowed.Prefix(foot2) -=== RUN TestACL_Enforce/keyring/read/Deny === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWritePrefixDenied.Prefix(foot) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWritePrefixDenied.Prefix(for) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWriteAllowed.Prefix(foo) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadAllowed.Prefix(foo) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadPrefixAllowed.Prefix(for) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWritePrefixDenied.Prefix(fo) -=== CONT TestACL_Enforce/keyring/read/Deny - authorizer_test.go:618: PASS: KeyringRead(*acl.AuthorizerContext) -=== RUN TestACL_Enforce/keyring/read/Allow - authorizer_test.go:618: PASS: KeyringRead(*acl.AuthorizerContext) -=== RUN TestACL_Enforce/keyring/write/Deny - authorizer_test.go:618: PASS: KeyringWrite(*acl.AuthorizerContext) -=== RUN TestACL_Enforce/keyring/write/Allow - authorizer_test.go:618: PASS: KeyringWrite(*acl.AuthorizerContext) -=== RUN TestACL_Enforce/keyring/list/Deny -=== RUN TestACL_Enforce/agent/foo/read/Deny - authorizer_test.go:618: PASS: AgentRead(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/agent/foo/read/Allow - authorizer_test.go:618: PASS: AgentRead(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/agent/foo/write/Deny - authorizer_test.go:618: PASS: AgentWrite(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/agent/foo/write/Allow - authorizer_test.go:618: PASS: AgentWrite(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/agent/foo/list/Deny -=== RUN TestACL_Enforce/event/foo/read/Deny - authorizer_test.go:618: PASS: EventRead(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/event/foo/read/Allow ---- PASS: TestPolicyAuthorizer (0.00s) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(foot) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadPrefixAllowed.Prefix(foot) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadPrefixAllowed.Prefix(food) +--- PASS: TestPolicyAuthorizer (0.01s) --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_all_default (0.00s) --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_all_default/AnyAllowed.Prefix(*) (0.00s) --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_all_default/AllDefault.Prefix(*) (0.00s) - --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_prefix_allowed (0.00s) - --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_prefix_allowed/AnyAllowed.Prefix(*) (0.00s) - --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_prefix_allowed/AllDenied.Prefix(*) (0.00s) --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_any_default (0.00s) - --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_any_default/AnyDefault.Prefix(*) (0.00s) --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_any_default/AllDenied.Prefix(*) (0.00s) - --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_prefix_denied (0.00s) - --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_prefix_denied/AnyAllowed.Prefix(*) (0.00s) - --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_prefix_denied/AllDenied.Prefix(*) (0.00s) + --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_any_default/AnyDefault.Prefix(*) (0.00s) --- PASS: TestPolicyAuthorizer/Defaults (0.00s) --- PASS: TestPolicyAuthorizer/Defaults/DefaultACLRead.Prefix(foo) (0.00s) --- PASS: TestPolicyAuthorizer/Defaults/DefaultSnapshot.Prefix(foo) (0.00s) @@ -5410,28 +5608,34 @@ --- PASS: TestPolicyAuthorizer/Defaults/DefaultServiceWrite.Prefix(foo) (0.00s) --- PASS: TestPolicyAuthorizer/Defaults/DefaultServiceRead.Prefix(foo) (0.00s) --- PASS: TestPolicyAuthorizer/Defaults/DefaultPreparedQueryWrite.Prefix(foo) (0.00s) - --- PASS: TestPolicyAuthorizer/Defaults/DefaultPreparedQueryRead.Prefix(foo) (0.00s) - --- PASS: TestPolicyAuthorizer/Defaults/DefaultOperatorWrite.Prefix(foo) (0.00s) - --- PASS: TestPolicyAuthorizer/Defaults/DefaultOperatorRead.Prefix(foo) (0.00s) - --- PASS: TestPolicyAuthorizer/Defaults/DefaultNodeWrite.Prefix(foo) (0.00s) - --- PASS: TestPolicyAuthorizer/Defaults/DefaultNodeRead.Prefix(foo) (0.00s) --- PASS: TestPolicyAuthorizer/Defaults/DefaultKeyWritePrefix.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Defaults/DefaultNodeRead.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Defaults/DefaultOperatorWrite.Prefix(foo) (0.00s) --- PASS: TestPolicyAuthorizer/Defaults/DefaultKeyWrite.Prefix(foo) (0.00s) --- PASS: TestPolicyAuthorizer/Defaults/DefaultKeyringWrite.Prefix(foo) (0.00s) - --- PASS: TestPolicyAuthorizer/Defaults/DefaultEventRead.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Defaults/DefaultNodeWrite.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Defaults/DefaultACLWrite.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Defaults/DefaultEventWrite.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Defaults/DefaultOperatorRead.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Defaults/DefaultAgentWrite.Prefix(foo) (0.00s) --- PASS: TestPolicyAuthorizer/Defaults/DefaultKeyList.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Defaults/DefaultKeyringRead.Prefix(foo) (0.00s) --- PASS: TestPolicyAuthorizer/Defaults/DefaultKeyRead.Prefix(foo) (0.00s) --- PASS: TestPolicyAuthorizer/Defaults/DefaultIntentionWrite.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Defaults/DefaultAgentRead.Prefix(foo) (0.00s) --- PASS: TestPolicyAuthorizer/Defaults/DefaultIntentionRead.Prefix(foo) (0.00s) --- PASS: TestPolicyAuthorizer/Defaults/DefaultIntentionDefaultAllow.Prefix(foo) (0.00s) - --- PASS: TestPolicyAuthorizer/Defaults/DefaultEventWrite.Prefix(foo) (0.00s) - --- PASS: TestPolicyAuthorizer/Defaults/DefaultKeyringRead.Prefix(foo) (0.00s) - --- PASS: TestPolicyAuthorizer/Defaults/DefaultACLWrite.Prefix(foo) (0.00s) - --- PASS: TestPolicyAuthorizer/Defaults/DefaultAgentWrite.Prefix(foo) (0.00s) - --- PASS: TestPolicyAuthorizer/Defaults/DefaultAgentRead.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Defaults/DefaultEventRead.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Defaults/DefaultPreparedQueryRead.Prefix(foo) (0.00s) --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_all_allowed (0.00s) --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_all_allowed/AnyAllowed.Prefix(*) (0.00s) --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_all_allowed/AllAllowed.Prefix(*) (0.00s) + --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_prefix_allowed (0.00s) + --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_prefix_allowed/AllDenied.Prefix(*) (0.00s) + --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_prefix_allowed/AnyAllowed.Prefix(*) (0.00s) + --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_prefix_denied (0.00s) + --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_prefix_denied/AnyAllowed.Prefix(*) (0.00s) + --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_prefix_denied/AllDenied.Prefix(*) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches (0.02s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadPrefixAllowed.Prefix(fo) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryWriteDenied.Prefix(football) (0.00s) @@ -5448,244 +5652,119 @@ --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryReadPrefixAllowed.Prefix(for) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryWritePrefixDenied.Prefix(fo) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryReadPrefixAllowed.Prefix(fo) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventWriteDenied.Prefix(football) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadDenied.Prefix(football) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(food) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(food) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(foot2) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(foot2) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(foot) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(foot) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventWriteAllowed.Prefix(foo) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadAllowed.Prefix(foo) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(for) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(for) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(fo) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(fo) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionWriteDenied.Prefix(football) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWriteDenied.Prefix(football) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionWritePrefixDenied.Prefix(food) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadPrefixAllowed.Prefix(foot2) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionReadDenied.Prefix(football) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadDenied.Prefix(football) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWritePrefixDenied.Prefix(food) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadPrefixAllowed.Prefix(food) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWritePrefixDenied.Prefix(foot2) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadPrefixAllowed.Prefix(foot2) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWritePrefixDenied.Prefix(foot) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadPrefixAllowed.Prefix(foot) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWriteAllowed.Prefix(foo) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadAllowed.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventWriteDenied.Prefix(football) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWritePrefixDenied.Prefix(for) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadPrefixAllowed.Prefix(for) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWritePrefixDenied.Prefix(fo) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadPrefixAllowed.Prefix(fo) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionWriteDenied.Prefix(football) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionReadDenied.Prefix(football) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionWritePrefixDenied.Prefix(food) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWritePrefixDenied.Prefix(foot2) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(food) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(foot2) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(foot2) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadPrefixAllowed.Prefix(foot) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(for) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventWriteAllowed.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadAllowed.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(fo)#01 (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(for) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadDenied.Prefix(football)#01 (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionReadPrefixAllowed.Prefix(food) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionWritePrefixDenied.Prefix(foot2) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionReadPrefixAllowed.Prefix(foot2) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionWritePrefixDenied.Prefix(foot) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWriteAllowed.Prefix(foo) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionReadPrefixAllowed.Prefix(foot) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(foot) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionWriteAllowed.Prefix(foo) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionReadAllowed.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(fo) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionWritePrefixDenied.Prefix(for) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(fo) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionReadPrefixAllowed.Prefix(for) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionWritePrefixDenied.Prefix(fo) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionReadPrefixAllowed.Prefix(fo) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(food)#01 (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWriteDenied.Prefix(football)#01 (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadDenied.Prefix(football)#01 (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(food)#01 (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(food)#01 (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(foot2)#01 (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionWritePrefixDenied.Prefix(foot) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(foot)#01 (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(foot2)#01 (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(foot)#01 (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(foot)#01 (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(for)#01 (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(foot2)#01 (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWriteAllowed.Prefix(foo)#01 (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(fo)#01 (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadAllowed.Prefix(foo)#01 (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(for)#01 (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(for)#01 (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(fo)#01 (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(fo)#01 (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWriteDenied.Prefix(football) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(fo) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadPrefixAllowed.Prefix(fo) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadDenied.Prefix(football) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(food) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadDenied.Prefix(football) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(food) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWritePrefixDenied.Prefix(fo) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(food) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(foot2) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(foot2) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(foot) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadPrefixAllowed.Prefix(food) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionWritePrefixDenied.Prefix(foot2) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWriteDenied.Prefix(football) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWritePrefixDenied.Prefix(foot2) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(foot) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadPrefixAllowed.Prefix(foot2) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWritePrefixDenied.Prefix(foot) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWriteAllowed.Prefix(foo) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadAllowed.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadPrefixAllowed.Prefix(foot) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWriteAllowed.Prefix(foo) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(for) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(for) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadAllowed.Prefix(foo) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(fo) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWritePrefixDenied.Prefix(for) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(fo) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadPrefixAllowed.Prefix(for) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWriteDenied.Prefix(football) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(food) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(food) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWriteAllowed.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadAllowed.Prefix(foo) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(foot2) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(for) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(foot2) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(for) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(foot) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(foot) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(foot) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWriteAllowed.Prefix(foo) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadAllowed.Prefix(foo) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(fo) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(fo) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadDenied.Prefix(football) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(for) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadDenied.Prefix(football) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWriteDenied.Prefix(football) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWritePrefixDenied.Prefix(food) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(for) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadAllowed.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWritePrefixDenied.Prefix(foot) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(foot2) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadDenied.Prefix(football) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWritePrefixDenied.Prefix(foot2) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadPrefixAllowed.Prefix(foot2) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWritePrefixDenied.Prefix(foot) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadPrefixAllowed.Prefix(foot) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWriteAllowed.Prefix(foo) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadAllowed.Prefix(foo) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWritePrefixDenied.Prefix(for) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadPrefixAllowed.Prefix(for) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWritePrefixDenied.Prefix(fo) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadPrefixAllowed.Prefix(fo) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWriteDenied.Prefix(football) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadPrefixAllowed.Prefix(foot) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWriteAllowed.Prefix(foo) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWritePrefixDenied.Prefix(food) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadPrefixAllowed.Prefix(food) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadPrefixAllowed.Prefix(for) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(food) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWritePrefixDenied.Prefix(for) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWritePrefixDenied.Prefix(foot2) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(fo) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWritePrefixDenied.Prefix(fo) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWriteDenied.Prefix(football) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadPrefixAllowed.Prefix(foot2) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWritePrefixDenied.Prefix(fo) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadDenied.Prefix(football) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWritePrefixDenied.Prefix(foot) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWritePrefixDenied.Prefix(for) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWriteAllowed.Prefix(foo) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadAllowed.Prefix(foo) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadPrefixAllowed.Prefix(for) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWritePrefixDenied.Prefix(fo) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(foot) (0.01s) -=== CONT TestACL_Enforce/event/foo/read/Allow - authorizer_test.go:618: PASS: EventRead(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/event/foo/write/Deny - authorizer_test.go:618: PASS: EventWrite(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/event/foo/write/Allow - authorizer_test.go:618: PASS: EventWrite(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/event/foo/list/Deny -=== RUN TestACL_Enforce/intention/foo/read/Deny - authorizer_test.go:618: PASS: IntentionRead(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/intention/foo/read/Allow - authorizer_test.go:618: PASS: IntentionRead(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/intention/foo/write/Deny - authorizer_test.go:618: PASS: IntentionWrite(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/intention/foo/write/Allow - authorizer_test.go:618: PASS: IntentionWrite(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/intention/foo/list/Deny -=== RUN TestACL_Enforce/node/foo/read/Deny - authorizer_test.go:618: PASS: NodeRead(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/node/foo/read/Allow - authorizer_test.go:618: PASS: NodeRead(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/node/foo/write/Deny - authorizer_test.go:618: PASS: NodeWrite(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/node/foo/write/Allow - authorizer_test.go:618: PASS: NodeWrite(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/node/foo/list/Deny -=== RUN TestACL_Enforce/query/foo/read/Deny - authorizer_test.go:618: PASS: PreparedQueryRead(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/query/foo/read/Allow - authorizer_test.go:618: PASS: PreparedQueryRead(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/query/foo/write/Deny - authorizer_test.go:618: PASS: PreparedQueryWrite(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/query/foo/write/Allow - authorizer_test.go:618: PASS: PreparedQueryWrite(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/query/foo/list/Deny -=== RUN TestACL_Enforce/service/foo/read/Deny - authorizer_test.go:618: PASS: ServiceRead(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/service/foo/read/Allow - authorizer_test.go:618: PASS: ServiceRead(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/service/foo/write/Deny - authorizer_test.go:618: PASS: ServiceWrite(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/service/foo/write/Allow - authorizer_test.go:618: PASS: ServiceWrite(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/session/foo/list/Deny -=== RUN TestACL_Enforce/session/foo/read/Deny - authorizer_test.go:618: PASS: SessionRead(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/session/foo/read/Allow - authorizer_test.go:618: PASS: SessionRead(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/session/foo/write/Deny - authorizer_test.go:618: PASS: SessionWrite(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/session/foo/write/Allow - authorizer_test.go:618: PASS: SessionWrite(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/session/foo/list/Deny#01 -=== RUN TestACL_Enforce/key/foo/read/Deny - authorizer_test.go:618: PASS: KeyRead(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/key/foo/read/Allow - authorizer_test.go:618: PASS: KeyRead(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/key/foo/write/Deny - authorizer_test.go:618: PASS: KeyWrite(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/key/foo/write/Allow - authorizer_test.go:618: PASS: KeyWrite(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/key/foo/list/Deny - authorizer_test.go:618: PASS: KeyList(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/key/foo/list/Allow - authorizer_test.go:618: PASS: KeyList(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/key/foo/deny/Deny -=== RUN TestACL_Enforce/not-a-real-resource/read/Deny ---- PASS: TestACL_Enforce (0.05s) - --- PASS: TestACL_Enforce/acl/read/Deny (0.01s) - --- PASS: TestACL_Enforce/acl/read/Allow (0.00s) - --- PASS: TestACL_Enforce/acl/write/Deny (0.01s) - --- PASS: TestACL_Enforce/acl/write/Allow (0.00s) - --- PASS: TestACL_Enforce/acl/list/Deny (0.00s) - --- PASS: TestACL_Enforce/operator/read/Deny (0.01s) - --- PASS: TestACL_Enforce/operator/read/Allow (0.00s) - --- PASS: TestACL_Enforce/operator/write/Deny (0.00s) - --- PASS: TestACL_Enforce/operator/write/Allow (0.00s) - --- PASS: TestACL_Enforce/operator/list/Deny (0.00s) - --- PASS: TestACL_Enforce/keyring/read/Deny (0.00s) - --- PASS: TestACL_Enforce/keyring/read/Allow (0.00s) - --- PASS: TestACL_Enforce/keyring/write/Deny (0.00s) - --- PASS: TestACL_Enforce/keyring/write/Allow (0.00s) - --- PASS: TestACL_Enforce/keyring/list/Deny (0.00s) - --- PASS: TestACL_Enforce/agent/foo/read/Deny (0.00s) - --- PASS: TestACL_Enforce/agent/foo/read/Allow (0.00s) - --- PASS: TestACL_Enforce/agent/foo/write/Deny (0.00s) - --- PASS: TestACL_Enforce/agent/foo/write/Allow (0.00s) - --- PASS: TestACL_Enforce/agent/foo/list/Deny (0.00s) - --- PASS: TestACL_Enforce/event/foo/read/Deny (0.00s) - --- PASS: TestACL_Enforce/event/foo/read/Allow (0.00s) - --- PASS: TestACL_Enforce/event/foo/write/Deny (0.00s) - --- PASS: TestACL_Enforce/event/foo/write/Allow (0.00s) - --- PASS: TestACL_Enforce/event/foo/list/Deny (0.00s) - --- PASS: TestACL_Enforce/intention/foo/read/Deny (0.00s) - --- PASS: TestACL_Enforce/intention/foo/read/Allow (0.00s) - --- PASS: TestACL_Enforce/intention/foo/write/Deny (0.00s) - --- PASS: TestACL_Enforce/intention/foo/write/Allow (0.00s) - --- PASS: TestACL_Enforce/intention/foo/list/Deny (0.00s) - --- PASS: TestACL_Enforce/node/foo/read/Deny (0.00s) - --- PASS: TestACL_Enforce/node/foo/read/Allow (0.00s) - --- PASS: TestACL_Enforce/node/foo/write/Deny (0.00s) - --- PASS: TestACL_Enforce/node/foo/write/Allow (0.00s) - --- PASS: TestACL_Enforce/node/foo/list/Deny (0.00s) - --- PASS: TestACL_Enforce/query/foo/read/Deny (0.00s) - --- PASS: TestACL_Enforce/query/foo/read/Allow (0.00s) - --- PASS: TestACL_Enforce/query/foo/write/Deny (0.00s) - --- PASS: TestACL_Enforce/query/foo/write/Allow (0.00s) - --- PASS: TestACL_Enforce/query/foo/list/Deny (0.00s) - --- PASS: TestACL_Enforce/service/foo/read/Deny (0.00s) - --- PASS: TestACL_Enforce/service/foo/read/Allow (0.00s) - --- PASS: TestACL_Enforce/service/foo/write/Deny (0.00s) - --- PASS: TestACL_Enforce/service/foo/write/Allow (0.00s) - --- PASS: TestACL_Enforce/session/foo/list/Deny (0.00s) - --- PASS: TestACL_Enforce/session/foo/read/Deny (0.00s) - --- PASS: TestACL_Enforce/session/foo/read/Allow (0.00s) - --- PASS: TestACL_Enforce/session/foo/write/Deny (0.00s) - --- PASS: TestACL_Enforce/session/foo/write/Allow (0.00s) - --- PASS: TestACL_Enforce/session/foo/list/Deny#01 (0.00s) - --- PASS: TestACL_Enforce/key/foo/read/Deny (0.00s) - --- PASS: TestACL_Enforce/key/foo/read/Allow (0.00s) - --- PASS: TestACL_Enforce/key/foo/write/Deny (0.00s) - --- PASS: TestACL_Enforce/key/foo/write/Allow (0.00s) - --- PASS: TestACL_Enforce/key/foo/list/Deny (0.00s) - --- PASS: TestACL_Enforce/key/foo/list/Allow (0.00s) - --- PASS: TestACL_Enforce/key/foo/deny/Deny (0.00s) - --- PASS: TestACL_Enforce/not-a-real-resource/read/Deny (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(foot) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadPrefixAllowed.Prefix(foot) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadPrefixAllowed.Prefix(food) (0.00s) PASS -ok github.com/hashicorp/consul/acl 0.145s +ok github.com/hashicorp/consul/acl 0.094s === RUN TestACL_Legacy_Disabled_Response === PAUSE TestACL_Legacy_Disabled_Response === RUN TestACL_Legacy_Update @@ -5777,80 +5856,69 @@ [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:40:49.238Z [WARN] TestAgent_ReloadDoesNotTriggerWatch: bootstrap = true: do not enable unless necessary -2022-09-15T21:40:47.203Z [INFO] TestAgent_ReloadDoesNotTriggerWatch: Consul is now ready -2022-09-15T21:40:46.795Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:40:46.828Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -40:46.833 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:225aad7f-39ef-0903-75cf-fc52d4bbdd97 Address:127.0.0.1:13069}]" -40:46.833 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13069 [Follower]" leader= -40:46.835 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-225aad7f-39ef-0903-75cf-fc52d4bbdd97.dc1 127.0.0.1 -40:46.836 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-225aad7f-39ef-0903-75cf-fc52d4bbdd97 127.0.0.1 -2022-09-15T21:40:46.836Z [INFO] agent.router: Initializing LAN area manager -40:46.837 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13064 network=udp -40:46.837 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-225aad7f-39ef-0903-75cf-fc52d4bbdd97.dc1 area=wan -40:46.837 [INFO] TestAgent.server: Adding LAN server: server="Node-225aad7f-39ef-0903-75cf-fc52d4bbdd97 (Addr: tcp/127.0.0.1:13069) (DC: dc1)" -40:46.837 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13064 network=tcp -40:46.838 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13065 network=tcp -40:46.839 [INFO] TestAgent: started state syncer -40:46.839 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13070 network=tcp -40:46.903 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -40:46.903 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13069 [Candidate]" term=2 -40:46.903 [DEBUG] TestAgent.server.raft: votes: needed=1 -40:46.903 [DEBUG] TestAgent.server.raft: vote granted: from=225aad7f-39ef-0903-75cf-fc52d4bbdd97 term=2 tally=1 -40:46.903 [INFO] TestAgent.server.raft: election won: tally=1 -40:46.903 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13069 [Leader]" -40:46.904 [INFO] TestAgent.server: cluster leadership acquired -40:46.904 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13069 -40:46.905 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -40:46.906 [INFO] TestAgent.server: New leader elected: payload=Node-225aad7f-39ef-0903-75cf-fc52d4bbdd97 -40:46.909 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -40:46.909 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -40:46.909 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -40:46.909 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -40:46.909 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -40:46.909 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-225aad7f-39ef-0903-75cf-fc52d4bbdd97 -40:46.909 [INFO] TestAgent.server: member joined, marking health alive: member=Node-225aad7f-39ef-0903-75cf-fc52d4bbdd97 -40:46.915 [INFO] TestAgent.server: federation state anti-entropy synced -40:47.015 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/health/service/redis?dc=dc1 from=127.0.0.1:44536 latency=174.03728ms -40:47.057 [DEBUG] TestAgent: watch handler output: watch_handler=[bash, -c, "(cat ; echo CONSUL_INDEX $CONSUL_INDEX) | tee '/tmp/rexec768994096.atomic' ; mv '/tmp/rexec768994096.atomic' '/tmp/rexec768994096'"] output="[] +2021-08-13T15:32:32.908Z [WARN] TestAgent_ReloadDoesNotTriggerWatch: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:32.892Z [INFO] TestAgent_ReloadDoesNotTriggerWatch: Watch not yet populated, retrying +2021-08-13T15:32:31.892Z [INFO] TestAgent_ReloadDoesNotTriggerWatch: Consul is now ready +2021-08-13T15:32:31.796Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:31.816Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:31.820 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:dcf72783-0ca8-144b-6d86-a4966e2f68f7 Address:127.0.0.1:24300}]" +32:31.820 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24300 [Follower]" leader= +32:31.821 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-dcf72783-0ca8-144b-6d86-a4966e2f68f7.dc1 127.0.0.1 +32:31.822 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-dcf72783-0ca8-144b-6d86-a4966e2f68f7 127.0.0.1 +2021-08-13T15:32:31.822Z [INFO] agent.router: Initializing LAN area manager +32:31.822 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-dcf72783-0ca8-144b-6d86-a4966e2f68f7.dc1 area=wan +32:31.822 [INFO] TestAgent.server: Adding LAN server: server="Node-dcf72783-0ca8-144b-6d86-a4966e2f68f7 (Addr: tcp/127.0.0.1:24300) (DC: dc1)" +32:31.823 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24295 network=udp +32:31.823 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24295 network=tcp +32:31.824 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24296 network=tcp +32:31.824 [INFO] TestAgent: started state syncer +32:31.824 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24301 network=tcp +32:31.871 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:31.871 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24300 [Candidate]" term=2 +32:31.871 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:31.871 [DEBUG] TestAgent.server.raft: vote granted: from=dcf72783-0ca8-144b-6d86-a4966e2f68f7 term=2 tally=1 +32:31.871 [INFO] TestAgent.server.raft: election won: tally=1 +32:31.871 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24300 [Leader]" +32:31.872 [INFO] TestAgent.server: cluster leadership acquired +32:31.872 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24300 +32:31.873 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:31.875 [INFO] TestAgent.server: New leader elected: payload=Node-dcf72783-0ca8-144b-6d86-a4966e2f68f7 +32:31.877 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:31.877 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:31.877 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:31.877 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:31.877 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:31.877 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-dcf72783-0ca8-144b-6d86-a4966e2f68f7 +32:31.877 [INFO] TestAgent.server: member joined, marking health alive: member=Node-dcf72783-0ca8-144b-6d86-a4966e2f68f7 +32:31.878 [INFO] TestAgent.server: federation state anti-entropy synced +32:31.892 [DEBUG] TestAgent: Check status updated: check=red-is-dead status=passing +32:31.980 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/health/service/redis?dc=dc1 from=127.0.0.1:33200 latency=154.908856ms +32:32.023 [DEBUG] TestAgent: watch handler output: watch_handler=[bash, -c, "(cat ; echo CONSUL_INDEX $CONSUL_INDEX) | tee '/tmp/rexec759295031.atomic' ; mv '/tmp/rexec759295031.atomic' '/tmp/rexec759295031'"] output="[] CONSUL_INDEX 10 " -40:47.103 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -40:47.103 [INFO] TestAgent: Synced node info -40:47.104 [INFO] TestAgent: Synced service: service=redis -40:47.104 [DEBUG] TestAgent: Check in sync: check=red-is-dead -40:47.105 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/health/service/redis?dc=dc1&index=10 from=127.0.0.1:44536 latency=47.79102ms -40:47.129 [DEBUG] TestAgent: watch handler output: watch_handler=[bash, -c, "(cat ; echo CONSUL_INDEX $CONSUL_INDEX) | tee '/tmp/rexec768994096.atomic' ; mv '/tmp/rexec768994096.atomic' '/tmp/rexec768994096'"] output="[{"Node":{"ID":"225aad7f-39ef-0903-75cf-fc52d4bbdd97","Node":"Node-225aad7f-39ef-0903-75cf-fc52d4bbdd97","Address":"127.0.0.1","Datacenter":"dc1","TaggedAddresses":{"lan":"127.0.0.1","lan_ipv4":"127.0.0.1","wan":"127.0.0.1","wan_ipv4":"127.0.0.1"},"Meta":{"consul-network-segment":""},"CreateIndex":10,"ModifyIndex":12},"Service":{"ID":"redis","Service":"redis","Tags":[],"Meta":null,"Port":0,"Address":"","Weights":{"Passing":1,"Warning":1},"EnableTagOverride":false,"CreateIndex":13,"ModifyIndex":13,"Proxy":{"MeshGateway":{},"Expose":{}},"Connect":{}},"Checks":[{"Node":"Node-225aad7f-39ef-0903-75cf-fc52d4bbdd97","CheckID":"serfHealth","Name":"Serf Health Status","Status":"passing","Notes":"","Output":"Agent alive and reachable","ServiceID":"","ServiceName":"","ServiceTags":[],"Type":"","Definition":{"Interval":"0s","Timeout":"0s","DeregisterCriticalServiceAfter":"0s","HTTP":"","Header":null,"Method":"","Body":"","TLSSkipVerify":false,"TCP":""},"CreateIndex":10,"ModifyIndex":10},{"Node":"Node-225aad7f-39ef-0903-75cf-fc52d4bbdd97","CheckID":"red-is-dead","Name":"Service 'redis' check","Status":"critical","Notes":"initial check","Output":"","ServiceID":"redis","ServiceName":"redis","ServiceTags":[],"Type":"ttl","Definition":{"Interval":"0s","Timeout":"0s","DeregisterCriticalServiceAfter":"0s","HTTP":"","Header":null,"Method":"","Body":"","TLSSkipVerify":false,"TCP":""},"CreateIndex":13,"ModifyIndex":13}]}] +32:32.096 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:32.096 [INFO] TestAgent: Synced node info +32:32.097 [INFO] TestAgent: Synced service: service=redis +32:32.097 [DEBUG] TestAgent: Check in sync: check=red-is-dead +32:32.098 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/health/service/redis?dc=dc1&index=10 from=127.0.0.1:33200 latency=75.129173ms +32:32.124 [DEBUG] TestAgent: watch handler output: watch_handler=[bash, -c, "(cat ; echo CONSUL_INDEX $CONSUL_INDEX) | tee '/tmp/rexec759295031.atomic' ; mv '/tmp/rexec759295031.atomic' '/tmp/rexec759295031'"] output="[{"Node":{"ID":"dcf72783-0ca8-144b-6d86-a4966e2f68f7","Node":"Node-dcf72783-0ca8-144b-6d86-a4966e2f68f7","Address":"127.0.0.1","Datacenter":"dc1","TaggedAddresses":{"lan":"127.0.0.1","lan_ipv4":"127.0.0.1","wan":"127.0.0.1","wan_ipv4":"127.0.0.1"},"Meta":{"consul-network-segment":""},"CreateIndex":10,"ModifyIndex":12},"Service":{"ID":"redis","Service":"redis","Tags":[],"Meta":null,"Port":0,"Address":"","Weights":{"Passing":1,"Warning":1},"EnableTagOverride":false,"CreateIndex":13,"ModifyIndex":13,"Proxy":{"MeshGateway":{},"Expose":{}},"Connect":{}},"Checks":[{"Node":"Node-dcf72783-0ca8-144b-6d86-a4966e2f68f7","CheckID":"serfHealth","Name":"Serf Health Status","Status":"passing","Notes":"","Output":"Agent alive and reachable","ServiceID":"","ServiceName":"","ServiceTags":[],"Type":"","Definition":{"Interval":"0s","Timeout":"0s","DeregisterCriticalServiceAfter":"0s","HTTP":"","Header":null,"Method":"","Body":"","TLSSkipVerify":false,"TCP":""},"CreateIndex":10,"ModifyIndex":10},{"Node":"Node-dcf72783-0ca8-144b-6d86-a4966e2f68f7","CheckID":"red-is-dead","Name":"Service 'redis' check","Status":"passing","Notes":"initial check","Output":"testing-agent-reload-001","ServiceID":"redis","ServiceName":"redis","ServiceTags":[],"Type":"ttl","Definition":{"Interval":"0s","Timeout":"0s","DeregisterCriticalServiceAfter":"0s","HTTP":"","Header":null,"Method":"","Body":"","TLSSkipVerify":false,"TCP":""},"CreateIndex":13,"ModifyIndex":13}]}] CONSUL_INDEX 13 " -40:47.203 [DEBUG] TestAgent: Check status updated: check=red-is-dead status=passing -40:49.090 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -40:49.090 [DEBUG] TestAgent: Node info in sync -40:49.090 [DEBUG] TestAgent: Service in sync: service=redis -40:49.091 [INFO] TestAgent: Synced check: check=red-is-dead -40:49.091 [DEBUG] TestAgent: Node info in sync -40:49.091 [DEBUG] TestAgent: Service in sync: service=redis -40:49.091 [DEBUG] TestAgent: Check in sync: check=red-is-dead -40:49.091 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/health/service/redis?dc=dc1&index=13 from=127.0.0.1:44536 latency=1.96180056s -40:49.108 [DEBUG] TestAgent: watch handler output: watch_handler=[bash, -c, "(cat ; echo CONSUL_INDEX $CONSUL_INDEX) | tee '/tmp/rexec768994096.atomic' ; mv '/tmp/rexec768994096.atomic' '/tmp/rexec768994096'"] output="[{"Node":{"ID":"225aad7f-39ef-0903-75cf-fc52d4bbdd97","Node":"Node-225aad7f-39ef-0903-75cf-fc52d4bbdd97","Address":"127.0.0.1","Datacenter":"dc1","TaggedAddresses":{"lan":"127.0.0.1","lan_ipv4":"127.0.0.1","wan":"127.0.0.1","wan_ipv4":"127.0.0.1"},"Meta":{"consul-network-segment":""},"CreateIndex":10,"ModifyIndex":12},"Service":{"ID":"redis","Service":"redis","Tags":[],"Meta":null,"Port":0,"Address":"","Weights":{"Passing":1,"Warning":1},"EnableTagOverride":false,"CreateIndex":13,"ModifyIndex":13,"Proxy":{"MeshGateway":{},"Expose":{}},"Connect":{}},"Checks":[{"Node":"Node-225aad7f-39ef-0903-75cf-fc52d4bbdd97","CheckID":"serfHealth","Name":"Serf Health Status","Status":"passing","Notes":"","Output":"Agent alive and reachable","ServiceID":"","ServiceName":"","ServiceTags":[],"Type":"","Definition":{"Interval":"0s","Timeout":"0s","DeregisterCriticalServiceAfter":"0s","HTTP":"","Header":null,"Method":"","Body":"","TLSSkipVerify":false,"TCP":""},"CreateIndex":10,"ModifyIndex":10},{"Node":"Node-225aad7f-39ef-0903-75cf-fc52d4bbdd97","CheckID":"red-is-dead","Name":"Service 'redis' check","Status":"passing","Notes":"initial check","Output":"testing-agent-reload-001","ServiceID":"redis","ServiceName":"redis","ServiceTags":[],"Type":"ttl","Definition":{"Interval":"0s","Timeout":"0s","DeregisterCriticalServiceAfter":"0s","HTTP":"","Header":null,"Method":"","Body":"","TLSSkipVerify":false,"TCP":""},"CreateIndex":13,"ModifyIndex":14}]}] -CONSUL_INDEX 14 -" -40:49.239 [INFO] TestAgent: Requesting shutdown -40:49.240 [INFO] TestAgent.server: shutting down server -40:49.240 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -40:49.240 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -40:49.240 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:40:49.240Z [INFO] agent.router.manager: shutting down -2022-09-15T21:40:49.240Z [INFO] agent.router.manager: shutting down -40:49.241 [INFO] TestAgent: Synced service: service=redis -40:49.241 [INFO] TestAgent: consul server down -40:49.241 [INFO] TestAgent: shutdown complete -40:49.241 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13064 network=tcp -40:49.241 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13064 network=udp -40:49.241 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13065 network=tcp -40:49.742 [INFO] TestAgent: Waiting for endpoints to shut down -40:49.742 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_ReloadDoesNotTriggerWatch (3.02s) +32:32.909 [INFO] TestAgent: Requesting shutdown +32:32.909 [INFO] TestAgent.server: shutting down server +32:32.909 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:32.910 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:32.910 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:32.910Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:32.910Z [INFO] agent.router.manager: shutting down +32:32.910 [INFO] TestAgent: consul server down +32:32.910 [INFO] TestAgent: shutdown complete +32:32.910 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24295 network=tcp +32:32.910 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24295 network=udp +32:32.910 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24296 network=tcp +32:33.410 [INFO] TestAgent: Waiting for endpoints to shut down +32:33.410 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_ReloadDoesNotTriggerWatch (1.68s) === RUN TestAgent_Reload_ACLDeny === PAUSE TestAgent_Reload_ACLDeny === RUN TestAgent_Members @@ -5921,133 +5989,132 @@ === PAUSE TestAgent_RegisterService/service_manager === CONT TestAgent_RegisterService/normal === CONT TestAgent_RegisterService/service_manager -2022-09-15T21:40:49.780Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:40:49.841Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -40:49.852 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:886b926a-9079-0b1d-44f3-bd604ea632f6 Address:127.0.0.1:13083}]" -40:49.853 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-886b926a-9079-0b1d-44f3-bd604ea632f6.dc1 127.0.0.1 -40:49.854 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13083 [Follower]" leader= -40:49.854 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-886b926a-9079-0b1d-44f3-bd604ea632f6 127.0.0.1 -2022-09-15T21:40:49.854Z [INFO] agent.router: Initializing LAN area manager -40:49.855 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13078 network=udp -40:49.868 [INFO] TestAgent.server: Adding LAN server: server="Node-886b926a-9079-0b1d-44f3-bd604ea632f6 (Addr: tcp/127.0.0.1:13083) (DC: dc1)" -40:49.872 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13078 network=tcp -40:49.873 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13079 network=tcp -40:49.874 [INFO] TestAgent: started state syncer -40:49.874 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13084 network=tcp -40:49.874 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-886b926a-9079-0b1d-44f3-bd604ea632f6.dc1 area=wan -40:49.911 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -40:49.912 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13083 [Candidate]" term=2 -40:49.912 [DEBUG] TestAgent.server.raft: votes: needed=1 -40:49.912 [DEBUG] TestAgent.server.raft: vote granted: from=886b926a-9079-0b1d-44f3-bd604ea632f6 term=2 tally=1 -40:49.912 [INFO] TestAgent.server.raft: election won: tally=1 -40:49.912 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13083 [Leader]" -40:49.912 [INFO] TestAgent.server: cluster leadership acquired -40:49.912 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13083 -40:49.914 [INFO] TestAgent.server: New leader elected: payload=Node-886b926a-9079-0b1d-44f3-bd604ea632f6 -40:49.914 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -40:49.920 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -40:49.921 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -40:49.921 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -40:49.921 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -40:49.921 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -40:49.921 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-886b926a-9079-0b1d-44f3-bd604ea632f6 -40:49.921 [INFO] TestAgent.server: member joined, marking health alive: member=Node-886b926a-9079-0b1d-44f3-bd604ea632f6 -40:49.926 [INFO] TestAgent.server: federation state anti-entropy synced -40:49.978 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -40:49.978 [INFO] TestAgent: Synced node info -40:49.978 [DEBUG] TestAgent: Node info in sync -40:50.093 [DEBUG] TestAgent: Node info in sync -40:50.095 [INFO] TestAgent: Synced service: service=test -40:50.095 [DEBUG] TestAgent: Check in sync: check=service:test:1 -40:50.095 [DEBUG] TestAgent: Check in sync: check=service:test:2 -40:50.095 [DEBUG] TestAgent: Check in sync: check=service:test:3 -40:50.095 [INFO] TestAgent: Requesting shutdown -40:50.096 [INFO] TestAgent.server: shutting down server -40:50.096 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -40:50.096 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -40:50.096 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -40:50.096 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -40:50.096 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -40:50.096 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -40:50.096 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -40:50.096 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -40:50.096 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -40:50.096 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -40:50.096 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:40:50.096Z [INFO] agent.router.manager: shutting down -2022-09-15T21:40:50.096Z [INFO] agent.router.manager: shutting down -40:50.097 [INFO] TestAgent: consul server down -40:50.097 [INFO] TestAgent: shutdown complete -40:50.097 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13078 network=tcp -40:50.097 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13078 network=udp -40:50.097 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13079 network=tcp -40:50.598 [INFO] TestAgent: Waiting for endpoints to shut down -40:50.598 [INFO] TestAgent: Endpoints down -2022-09-15T21:40:49.832Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:40:49.865Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -40:49.868 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:96890268-ad83-b9f0-794f-df5a2af3382d Address:127.0.0.1:13076}]" -40:49.868 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13076 [Follower]" leader= -40:49.869 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-96890268-ad83-b9f0-794f-df5a2af3382d.dc1 127.0.0.1 -40:49.870 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-96890268-ad83-b9f0-794f-df5a2af3382d 127.0.0.1 -2022-09-15T21:40:49.870Z [INFO] agent.router: Initializing LAN area manager -40:49.870 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13071 network=udp -40:49.871 [INFO] TestAgent.server: Adding LAN server: server="Node-96890268-ad83-b9f0-794f-df5a2af3382d (Addr: tcp/127.0.0.1:13076) (DC: dc1)" -40:49.871 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-96890268-ad83-b9f0-794f-df5a2af3382d.dc1 area=wan -40:49.871 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13071 network=tcp -40:49.872 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13072 network=tcp -40:49.872 [INFO] TestAgent: started state syncer -40:49.872 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13077 network=tcp -40:49.914 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -40:49.914 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13076 [Candidate]" term=2 -40:49.914 [DEBUG] TestAgent.server.raft: votes: needed=1 -40:49.914 [DEBUG] TestAgent.server.raft: vote granted: from=96890268-ad83-b9f0-794f-df5a2af3382d term=2 tally=1 -40:49.914 [INFO] TestAgent.server.raft: election won: tally=1 -40:49.914 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13076 [Leader]" -40:49.915 [INFO] TestAgent.server: cluster leadership acquired -40:49.922 [INFO] TestAgent.server: New leader elected: payload=Node-96890268-ad83-b9f0-794f-df5a2af3382d -40:49.922 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13076 -40:49.925 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -40:49.938 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -40:49.938 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -40:49.938 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -40:49.938 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -40:49.938 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -40:49.938 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-96890268-ad83-b9f0-794f-df5a2af3382d -40:49.938 [INFO] TestAgent.server: member joined, marking health alive: member=Node-96890268-ad83-b9f0-794f-df5a2af3382d -40:49.940 [INFO] TestAgent.server: federation state anti-entropy synced -40:50.246 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -40:50.247 [INFO] TestAgent: Synced node info -40:50.247 [DEBUG] TestAgent: Node info in sync -40:50.304 [DEBUG] TestAgent: Node info in sync -40:50.305 [INFO] TestAgent: Synced service: service=test -40:50.305 [DEBUG] TestAgent: Check in sync: check=service:test:1 -40:50.305 [DEBUG] TestAgent: Check in sync: check=service:test:2 -40:50.305 [DEBUG] TestAgent: Check in sync: check=service:test:3 -40:50.305 [INFO] TestAgent: Requesting shutdown -40:50.305 [INFO] TestAgent.server: shutting down server -40:50.305 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -40:50.305 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -40:50.305 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -40:50.305 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -40:50.305 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -40:50.306 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -40:50.306 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -40:50.306 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -40:50.306 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -40:50.306 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -40:50.310 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:40:50.310Z [INFO] agent.router.manager: shutting down -2022-09-15T21:40:50.310Z [INFO] agent.router.manager: shutting down -40:50.310 [INFO] TestAgent: consul server down -40:50.310 [INFO] TestAgent: shutdown complete -40:50.310 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13071 network=tcp -40:50.310 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13071 network=udp -40:50.310 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13072 network=tcp -40:50.814 [INFO] TestAgent: Waiting for endpoints to shut down -40:50.814 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:33.452Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:33.469Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:33.472 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:67918ea5-69e6-c757-4e84-8b0544db496a Address:127.0.0.1:24307}]" +32:33.473 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-67918ea5-69e6-c757-4e84-8b0544db496a.dc1 127.0.0.1 +32:33.473 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-67918ea5-69e6-c757-4e84-8b0544db496a 127.0.0.1 +2021-08-13T15:32:33.474Z [INFO] agent.router: Initializing LAN area manager +32:33.474 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24302 network=udp +32:33.474 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24307 [Follower]" leader= +32:33.475 [INFO] TestAgent.server: Adding LAN server: server="Node-67918ea5-69e6-c757-4e84-8b0544db496a (Addr: tcp/127.0.0.1:24307) (DC: dc1)" +32:33.475 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-67918ea5-69e6-c757-4e84-8b0544db496a.dc1 area=wan +32:33.479 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24302 network=tcp +32:33.481 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24303 network=tcp +32:33.481 [INFO] TestAgent: started state syncer +32:33.481 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24308 network=tcp +32:33.544 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:33.544 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24307 [Candidate]" term=2 +32:33.544 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:33.544 [DEBUG] TestAgent.server.raft: vote granted: from=67918ea5-69e6-c757-4e84-8b0544db496a term=2 tally=1 +32:33.544 [INFO] TestAgent.server.raft: election won: tally=1 +32:33.544 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24307 [Leader]" +32:33.545 [INFO] TestAgent.server: cluster leadership acquired +32:33.545 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24307 +32:33.546 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:33.547 [INFO] TestAgent.server: New leader elected: payload=Node-67918ea5-69e6-c757-4e84-8b0544db496a +32:33.549 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:33.549 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:33.549 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:33.549 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:33.549 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:33.549 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-67918ea5-69e6-c757-4e84-8b0544db496a +32:33.549 [INFO] TestAgent.server: member joined, marking health alive: member=Node-67918ea5-69e6-c757-4e84-8b0544db496a +32:33.551 [INFO] TestAgent.server: federation state anti-entropy synced +32:33.553 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:33.554 [INFO] TestAgent: Synced node info +32:33.554 [DEBUG] TestAgent: Node info in sync +32:33.745 [DEBUG] TestAgent: Node info in sync +32:33.747 [INFO] TestAgent: Synced service: service=test +32:33.747 [DEBUG] TestAgent: Check in sync: check=service:test:1 +32:33.747 [DEBUG] TestAgent: Check in sync: check=service:test:2 +32:33.747 [DEBUG] TestAgent: Check in sync: check=service:test:3 +32:33.747 [INFO] TestAgent: Requesting shutdown +32:33.747 [INFO] TestAgent.server: shutting down server +32:33.747 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:33.747 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:33.747 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:33.747 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:33.747 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:33.747 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:33.747 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:33.747 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:33.747 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:33.747 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:33.747 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:33.747Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:33.747Z [INFO] agent.router.manager: shutting down +32:33.747 [INFO] TestAgent: consul server down +32:33.747 [INFO] TestAgent: shutdown complete +32:33.747 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24302 network=tcp +32:33.747 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24302 network=udp +32:33.747 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24303 network=tcp +32:34.248 [INFO] TestAgent: Waiting for endpoints to shut down +32:34.248 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:33.452Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:33.470Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:33.473 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:38897338-0326-d7fe-1ae6-87648873cd85 Address:127.0.0.1:24314}]" +32:33.474 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-38897338-0326-d7fe-1ae6-87648873cd85.dc1 127.0.0.1 +32:33.474 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-38897338-0326-d7fe-1ae6-87648873cd85 127.0.0.1 +2021-08-13T15:32:33.474Z [INFO] agent.router: Initializing LAN area manager +32:33.475 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24314 [Follower]" leader= +32:33.475 [INFO] TestAgent.server: Adding LAN server: server="Node-38897338-0326-d7fe-1ae6-87648873cd85 (Addr: tcp/127.0.0.1:24314) (DC: dc1)" +32:33.475 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-38897338-0326-d7fe-1ae6-87648873cd85.dc1 area=wan +32:33.486 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24309 network=udp +32:33.486 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24309 network=tcp +32:33.487 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24310 network=tcp +32:33.487 [INFO] TestAgent: started state syncer +32:33.487 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24315 network=tcp +32:33.531 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:33.531 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24314 [Candidate]" term=2 +32:33.531 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:33.531 [DEBUG] TestAgent.server.raft: vote granted: from=38897338-0326-d7fe-1ae6-87648873cd85 term=2 tally=1 +32:33.531 [INFO] TestAgent.server.raft: election won: tally=1 +32:33.531 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24314 [Leader]" +32:33.532 [INFO] TestAgent.server: cluster leadership acquired +32:33.532 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24314 +32:33.533 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:33.535 [INFO] TestAgent.server: New leader elected: payload=Node-38897338-0326-d7fe-1ae6-87648873cd85 +32:33.536 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:33.536 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:33.536 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:33.536 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:33.536 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:33.536 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-38897338-0326-d7fe-1ae6-87648873cd85 +32:33.536 [INFO] TestAgent.server: member joined, marking health alive: member=Node-38897338-0326-d7fe-1ae6-87648873cd85 +32:33.538 [INFO] TestAgent.server: federation state anti-entropy synced +32:33.751 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:33.752 [INFO] TestAgent: Synced node info +32:33.865 [DEBUG] TestAgent: Node info in sync +32:33.866 [INFO] TestAgent: Synced service: service=test +32:33.866 [DEBUG] TestAgent: Check in sync: check=service:test:1 +32:33.866 [DEBUG] TestAgent: Check in sync: check=service:test:2 +32:33.866 [DEBUG] TestAgent: Check in sync: check=service:test:3 +32:33.866 [INFO] TestAgent: Requesting shutdown +32:33.866 [INFO] TestAgent.server: shutting down server +32:33.866 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:33.866 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:33.866 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:33.866 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:33.866 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:33.867 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:33.867 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:33.867 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:33.867 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:33.867 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:33.867Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:33.867Z [INFO] agent.router.manager: shutting down +32:33.867 [INFO] TestAgent: consul server down +32:33.867 [INFO] TestAgent: shutdown complete +32:33.867 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24309 network=tcp +32:33.867 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24309 network=udp +32:33.867 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24310 network=tcp +32:33.868 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:34.368 [INFO] TestAgent: Waiting for endpoints to shut down +32:34.368 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_RegisterService (0.00s) - --- PASS: TestAgent_RegisterService/service_manager (0.85s) - --- PASS: TestAgent_RegisterService/normal (1.07s) + --- PASS: TestAgent_RegisterService/normal (0.83s) + --- PASS: TestAgent_RegisterService/service_manager (0.95s) === RUN TestAgent_RegisterService_ReRegister === RUN TestAgent_RegisterService_ReRegister/normal === PAUSE TestAgent_RegisterService_ReRegister/normal @@ -6055,137 +6122,139 @@ === PAUSE TestAgent_RegisterService_ReRegister/service_manager === CONT TestAgent_RegisterService_ReRegister/normal === CONT TestAgent_RegisterService_ReRegister/service_manager -2022-09-15T21:40:50.859Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:40:50.892Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -40:50.895 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:34f197e7-71cf-19d6-2581-622727392598 Address:127.0.0.1:13090}]" -40:50.896 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-34f197e7-71cf-19d6-2581-622727392598.dc1 127.0.0.1 -40:50.897 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-34f197e7-71cf-19d6-2581-622727392598 127.0.0.1 -2022-09-15T21:40:50.897Z [INFO] agent.router: Initializing LAN area manager -40:50.897 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13085 network=udp -40:50.897 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13090 [Follower]" leader= -40:50.898 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-34f197e7-71cf-19d6-2581-622727392598.dc1 area=wan -40:50.898 [INFO] TestAgent.server: Adding LAN server: server="Node-34f197e7-71cf-19d6-2581-622727392598 (Addr: tcp/127.0.0.1:13090) (DC: dc1)" -40:50.898 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13085 network=tcp -40:50.899 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13086 network=tcp -40:50.899 [INFO] TestAgent: started state syncer -40:50.899 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13091 network=tcp -40:50.942 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -40:50.942 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13090 [Candidate]" term=2 -40:50.942 [DEBUG] TestAgent.server.raft: votes: needed=1 -40:50.942 [DEBUG] TestAgent.server.raft: vote granted: from=34f197e7-71cf-19d6-2581-622727392598 term=2 tally=1 -40:50.942 [INFO] TestAgent.server.raft: election won: tally=1 -40:50.942 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13090 [Leader]" -40:50.942 [INFO] TestAgent.server: cluster leadership acquired -40:50.943 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13090 -40:50.944 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -40:50.946 [INFO] TestAgent.server: New leader elected: payload=Node-34f197e7-71cf-19d6-2581-622727392598 -40:50.946 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -40:50.946 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -40:50.946 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -40:50.946 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -40:50.946 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -40:50.946 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-34f197e7-71cf-19d6-2581-622727392598 -40:50.946 [INFO] TestAgent.server: member joined, marking health alive: member=Node-34f197e7-71cf-19d6-2581-622727392598 -40:50.948 [INFO] TestAgent.server: federation state anti-entropy synced -40:51.017 [INFO] TestAgent: Synced node info -40:51.018 [INFO] TestAgent: Synced service: service=test -40:51.018 [DEBUG] TestAgent: Check in sync: check=check_1 -40:51.018 [DEBUG] TestAgent: Check in sync: check=check_2 -40:51.019 [DEBUG] TestAgent: Node info in sync -40:51.020 [INFO] TestAgent: Synced service: service=test -40:51.020 [DEBUG] TestAgent: Check in sync: check=check_1 -40:51.020 [DEBUG] TestAgent: Check in sync: check=check_2 -40:51.020 [DEBUG] TestAgent: Check in sync: check=check_3 -40:51.020 [INFO] TestAgent: Requesting shutdown -40:51.021 [INFO] TestAgent.server: shutting down server -40:51.021 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -40:51.021 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -40:51.021 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -40:51.021 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -40:51.021 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -40:51.021 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -40:51.021 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -40:51.021 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -40:51.021 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -40:51.021 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -40:51.021 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -40:51.021 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:40:51.021Z [INFO] agent.router.manager: shutting down -2022-09-15T21:40:51.021Z [INFO] agent.router.manager: shutting down -40:51.021 [INFO] TestAgent: consul server down -40:51.021 [INFO] TestAgent: shutdown complete -40:51.021 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13085 network=tcp -40:51.021 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13085 network=udp -40:51.021 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13086 network=tcp -40:51.522 [INFO] TestAgent: Waiting for endpoints to shut down -40:51.522 [INFO] TestAgent: Endpoints down -2022-09-15T21:40:50.839Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:40:50.883Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -40:50.899 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2933dffd-258e-249f-c01f-f01dff13099c Address:127.0.0.1:13097}]" -40:50.900 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2933dffd-258e-249f-c01f-f01dff13099c.dc1 127.0.0.1 -40:50.901 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2933dffd-258e-249f-c01f-f01dff13099c 127.0.0.1 -2022-09-15T21:40:50.901Z [INFO] agent.router: Initializing LAN area manager -40:50.901 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13092 network=udp -40:50.901 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13097 [Follower]" leader= -40:50.902 [INFO] TestAgent.server: Adding LAN server: server="Node-2933dffd-258e-249f-c01f-f01dff13099c (Addr: tcp/127.0.0.1:13097) (DC: dc1)" -40:50.902 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2933dffd-258e-249f-c01f-f01dff13099c.dc1 area=wan -40:50.902 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13092 network=tcp -40:50.903 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13093 network=tcp -40:50.903 [INFO] TestAgent: started state syncer -40:50.903 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13098 network=tcp -40:50.943 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -40:50.943 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13097 [Candidate]" term=2 -40:50.943 [DEBUG] TestAgent.server.raft: votes: needed=1 -40:50.943 [DEBUG] TestAgent.server.raft: vote granted: from=2933dffd-258e-249f-c01f-f01dff13099c term=2 tally=1 -40:50.943 [INFO] TestAgent.server.raft: election won: tally=1 -40:50.943 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13097 [Leader]" -40:50.943 [INFO] TestAgent.server: cluster leadership acquired -40:50.943 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13097 -40:50.944 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -40:50.946 [INFO] TestAgent.server: New leader elected: payload=Node-2933dffd-258e-249f-c01f-f01dff13099c -40:50.948 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -40:50.948 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -40:50.948 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -40:50.948 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -40:50.948 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -40:50.948 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2933dffd-258e-249f-c01f-f01dff13099c -40:50.948 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2933dffd-258e-249f-c01f-f01dff13099c -40:50.949 [INFO] TestAgent.server: federation state anti-entropy synced -40:51.230 [INFO] TestAgent: Synced node info -40:51.231 [INFO] TestAgent: Synced service: service=test -40:51.231 [DEBUG] TestAgent: Check in sync: check=check_2 -40:51.231 [DEBUG] TestAgent: Check in sync: check=check_1 -40:51.232 [DEBUG] TestAgent: Node info in sync -40:51.233 [INFO] TestAgent: Synced service: service=test -40:51.233 [DEBUG] TestAgent: Check in sync: check=check_2 -40:51.233 [DEBUG] TestAgent: Check in sync: check=check_3 -40:51.233 [DEBUG] TestAgent: Check in sync: check=check_1 -40:51.233 [INFO] TestAgent: Requesting shutdown -40:51.233 [INFO] TestAgent.server: shutting down server -40:51.233 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -40:51.233 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -40:51.233 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -40:51.233 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -40:51.233 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -40:51.233 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -40:51.233 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -40:51.233 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -40:51.233 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -40:51.233 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -40:51.233 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -40:51.233 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2022-09-15T21:40:51.234Z [INFO] agent.router.manager: shutting down -2022-09-15T21:40:51.234Z [INFO] agent.router.manager: shutting down -40:51.234 [INFO] TestAgent: consul server down -40:51.234 [INFO] TestAgent: shutdown complete -40:51.234 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13092 network=tcp -40:51.234 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13092 network=udp -40:51.234 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13093 network=tcp -40:51.738 [INFO] TestAgent: Waiting for endpoints to shut down -40:51.738 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:34.393Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:34.414Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:34.429 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:def5cbc5-fdc9-9644-9897-76c628e7ac73 Address:127.0.0.1:24321}]" +32:34.430 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-def5cbc5-fdc9-9644-9897-76c628e7ac73.dc1 127.0.0.1 +32:34.430 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24321 [Follower]" leader= +32:34.431 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-def5cbc5-fdc9-9644-9897-76c628e7ac73 127.0.0.1 +2021-08-13T15:32:34.431Z [INFO] agent.router: Initializing LAN area manager +32:34.431 [INFO] TestAgent.server: Adding LAN server: server="Node-def5cbc5-fdc9-9644-9897-76c628e7ac73 (Addr: tcp/127.0.0.1:24321) (DC: dc1)" +32:34.431 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-def5cbc5-fdc9-9644-9897-76c628e7ac73.dc1 area=wan +32:34.431 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24316 network=udp +32:34.431 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24316 network=tcp +32:34.432 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24317 network=tcp +32:34.432 [INFO] TestAgent: started state syncer +32:34.432 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24322 network=tcp +32:34.487 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:34.487 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24321 [Candidate]" term=2 +32:34.491 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:34.491 [DEBUG] TestAgent.server.raft: vote granted: from=def5cbc5-fdc9-9644-9897-76c628e7ac73 term=2 tally=1 +32:34.491 [INFO] TestAgent.server.raft: election won: tally=1 +32:34.491 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24321 [Leader]" +32:34.493 [INFO] TestAgent.server: cluster leadership acquired +32:34.493 [INFO] TestAgent.server: New leader elected: payload=Node-def5cbc5-fdc9-9644-9897-76c628e7ac73 +32:34.493 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24321 +32:34.495 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:34.498 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:34.498 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:34.498 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:34.498 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:34.498 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:34.498 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-def5cbc5-fdc9-9644-9897-76c628e7ac73 +32:34.498 [INFO] TestAgent.server: member joined, marking health alive: member=Node-def5cbc5-fdc9-9644-9897-76c628e7ac73 +32:34.500 [INFO] TestAgent.server: federation state anti-entropy synced +32:34.545 [INFO] TestAgent: Synced node info +32:34.546 [INFO] TestAgent: Synced service: service=test +32:34.546 [DEBUG] TestAgent: Check in sync: check=check_2 +32:34.546 [DEBUG] TestAgent: Check in sync: check=check_1 +32:34.548 [DEBUG] TestAgent: Node info in sync +32:34.548 [INFO] TestAgent: Synced service: service=test +32:34.548 [DEBUG] TestAgent: Check in sync: check=check_1 +32:34.548 [DEBUG] TestAgent: Check in sync: check=check_2 +32:34.548 [DEBUG] TestAgent: Check in sync: check=check_3 +32:34.549 [INFO] TestAgent: Requesting shutdown +32:34.549 [INFO] TestAgent.server: shutting down server +32:34.549 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:34.549 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:34.549 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:34.549 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:34.549 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:34.549 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:34.549 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +32:34.549 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:34.549 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:34.549 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:34.549 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:34.549 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:34.549Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:34.549Z [INFO] agent.router.manager: shutting down +32:34.549 [INFO] TestAgent: consul server down +32:34.549 [INFO] TestAgent: shutdown complete +32:34.549 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24316 network=tcp +32:34.549 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24316 network=udp +32:34.549 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24317 network=tcp +32:35.050 [INFO] TestAgent: Waiting for endpoints to shut down +32:35.050 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:34.393Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:34.412Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:34.415 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:794e1c7a-7109-94ae-3f0e-20ced0897f4d Address:127.0.0.1:24328}]" +32:34.415 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24328 [Follower]" leader= +32:34.424 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-794e1c7a-7109-94ae-3f0e-20ced0897f4d.dc1 127.0.0.1 +32:34.425 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-794e1c7a-7109-94ae-3f0e-20ced0897f4d 127.0.0.1 +2021-08-13T15:32:34.425Z [INFO] agent.router: Initializing LAN area manager +32:34.426 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24323 network=udp +32:34.426 [INFO] TestAgent.server: Adding LAN server: server="Node-794e1c7a-7109-94ae-3f0e-20ced0897f4d (Addr: tcp/127.0.0.1:24328) (DC: dc1)" +32:34.426 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-794e1c7a-7109-94ae-3f0e-20ced0897f4d.dc1 area=wan +32:34.427 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24323 network=tcp +32:34.429 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24324 network=tcp +32:34.429 [INFO] TestAgent: started state syncer +32:34.430 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24329 network=tcp +32:34.475 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:34.475 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24328 [Candidate]" term=2 +32:34.476 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:34.476 [DEBUG] TestAgent.server.raft: vote granted: from=794e1c7a-7109-94ae-3f0e-20ced0897f4d term=2 tally=1 +32:34.476 [INFO] TestAgent.server.raft: election won: tally=1 +32:34.476 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24328 [Leader]" +32:34.476 [INFO] TestAgent.server: cluster leadership acquired +32:34.476 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24328 +32:34.478 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:34.479 [INFO] TestAgent.server: New leader elected: payload=Node-794e1c7a-7109-94ae-3f0e-20ced0897f4d +32:34.480 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:34.480 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:34.480 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:34.480 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:34.480 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:34.481 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-794e1c7a-7109-94ae-3f0e-20ced0897f4d +32:34.481 [INFO] TestAgent.server: member joined, marking health alive: member=Node-794e1c7a-7109-94ae-3f0e-20ced0897f4d +32:34.482 [INFO] TestAgent.server: federation state anti-entropy synced +32:34.543 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:34.543 [INFO] TestAgent: Synced node info +32:34.543 [DEBUG] TestAgent: Node info in sync +32:34.611 [DEBUG] TestAgent: Node info in sync +32:34.612 [INFO] TestAgent: Synced service: service=test +32:34.612 [DEBUG] TestAgent: Check in sync: check=check_1 +32:34.612 [DEBUG] TestAgent: Check in sync: check=check_2 +32:34.614 [DEBUG] TestAgent: Node info in sync +32:34.614 [INFO] TestAgent: Synced service: service=test +32:34.614 [DEBUG] TestAgent: Check in sync: check=check_1 +32:34.614 [DEBUG] TestAgent: Check in sync: check=check_2 +32:34.614 [DEBUG] TestAgent: Check in sync: check=check_3 +32:34.615 [INFO] TestAgent: Requesting shutdown +32:34.615 [INFO] TestAgent.server: shutting down server +32:34.615 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:34.615 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:34.615 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:34.615 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:34.615 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:34.615 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:34.615 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:34.615 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:34.615 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:34.615 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:34.615 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:34.615Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:34.615Z [INFO] agent.router.manager: shutting down +32:34.615 [INFO] TestAgent: consul server down +32:34.615 [INFO] TestAgent: shutdown complete +32:34.615 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24323 network=tcp +32:34.615 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24323 network=udp +32:34.615 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24324 network=tcp +32:35.115 [INFO] TestAgent: Waiting for endpoints to shut down +32:35.115 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_RegisterService_ReRegister (0.00s) - --- PASS: TestAgent_RegisterService_ReRegister/normal (0.71s) - --- PASS: TestAgent_RegisterService_ReRegister/service_manager (0.92s) + --- PASS: TestAgent_RegisterService_ReRegister/normal (0.68s) + --- PASS: TestAgent_RegisterService_ReRegister/service_manager (0.75s) === RUN TestAgent_RegisterService_ReRegister_ReplaceExistingChecks === RUN TestAgent_RegisterService_ReRegister_ReplaceExistingChecks/normal === PAUSE TestAgent_RegisterService_ReRegister_ReplaceExistingChecks/normal @@ -6193,140 +6262,142 @@ === PAUSE TestAgent_RegisterService_ReRegister_ReplaceExistingChecks/service_manager === CONT TestAgent_RegisterService_ReRegister_ReplaceExistingChecks/normal === CONT TestAgent_RegisterService_ReRegister_ReplaceExistingChecks/service_manager -2022-09-15T21:40:51.835Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:40:51.851Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -40:51.856 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9de7739f-7caf-8776-0f63-bdcc1de7f212 Address:127.0.0.1:13104}]" -40:51.862 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13104 [Follower]" leader= -40:51.874 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9de7739f-7caf-8776-0f63-bdcc1de7f212.dc1 127.0.0.1 -40:51.875 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9de7739f-7caf-8776-0f63-bdcc1de7f212 127.0.0.1 -2022-09-15T21:40:51.875Z [INFO] agent.router: Initializing LAN area manager -40:51.875 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13099 network=udp -40:51.877 [INFO] TestAgent.server: Adding LAN server: server="Node-9de7739f-7caf-8776-0f63-bdcc1de7f212 (Addr: tcp/127.0.0.1:13104) (DC: dc1)" -40:51.877 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9de7739f-7caf-8776-0f63-bdcc1de7f212.dc1 area=wan -40:51.877 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13099 network=tcp -40:51.878 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13100 network=tcp -40:51.879 [INFO] TestAgent: started state syncer -40:51.879 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13105 network=tcp -40:51.919 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -40:51.919 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13104 [Candidate]" term=2 -40:51.919 [DEBUG] TestAgent.server.raft: votes: needed=1 -40:51.919 [DEBUG] TestAgent.server.raft: vote granted: from=9de7739f-7caf-8776-0f63-bdcc1de7f212 term=2 tally=1 -40:51.919 [INFO] TestAgent.server.raft: election won: tally=1 -40:51.919 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13104 [Leader]" -40:51.920 [INFO] TestAgent.server: cluster leadership acquired -40:51.920 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13104 -40:51.921 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -40:51.922 [INFO] TestAgent.server: New leader elected: payload=Node-9de7739f-7caf-8776-0f63-bdcc1de7f212 -40:51.924 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -40:51.924 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -40:51.924 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -40:51.924 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -40:51.924 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -40:51.924 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9de7739f-7caf-8776-0f63-bdcc1de7f212 -40:51.924 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9de7739f-7caf-8776-0f63-bdcc1de7f212 -40:51.925 [INFO] TestAgent.server: federation state anti-entropy synced -40:51.992 [INFO] TestAgent: Synced node info -40:51.998 [INFO] TestAgent: Synced service: service=test -40:51.998 [DEBUG] TestAgent: Check in sync: check=service:test:1 -40:51.998 [DEBUG] TestAgent: Check in sync: check=check_2 -40:51.999 [DEBUG] TestAgent: removed check: check=check_2 -40:51.999 [DEBUG] TestAgent: Node info in sync -40:52.000 [INFO] TestAgent: Synced service: service=test -40:52.000 [DEBUG] TestAgent: Check in sync: check=check_3 -40:52.000 [DEBUG] TestAgent: Check in sync: check=service:test:1 -40:52.001 [INFO] TestAgent: Deregistered check: check=check_2 -40:52.001 [INFO] TestAgent: Requesting shutdown -40:52.001 [INFO] TestAgent.server: shutting down server -40:52.001 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -40:52.001 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -40:52.001 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -40:52.001 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -40:52.001 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -40:52.002 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -40:52.002 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -40:52.002 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -40:52.002 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -40:52.002 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:40:52.002Z [INFO] agent.router.manager: shutting down -2022-09-15T21:40:52.002Z [INFO] agent.router.manager: shutting down -40:52.002 [INFO] TestAgent: consul server down -40:52.002 [INFO] TestAgent: shutdown complete -40:52.002 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13099 network=tcp -40:52.002 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13099 network=udp -40:52.002 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13100 network=tcp -40:52.006 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -40:52.006 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -40:52.503 [INFO] TestAgent: Waiting for endpoints to shut down -40:52.503 [INFO] TestAgent: Endpoints down -2022-09-15T21:40:51.816Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:40:51.852Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -40:51.860 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:237bad70-309e-1780-bfa0-be206b335b88 Address:127.0.0.1:13111}]" -40:51.862 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13111 [Follower]" leader= -40:51.872 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-237bad70-309e-1780-bfa0-be206b335b88.dc1 127.0.0.1 -40:51.873 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-237bad70-309e-1780-bfa0-be206b335b88 127.0.0.1 -2022-09-15T21:40:51.873Z [INFO] agent.router: Initializing LAN area manager -40:51.873 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13106 network=udp -40:51.875 [INFO] TestAgent.server: Adding LAN server: server="Node-237bad70-309e-1780-bfa0-be206b335b88 (Addr: tcp/127.0.0.1:13111) (DC: dc1)" -40:51.875 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-237bad70-309e-1780-bfa0-be206b335b88.dc1 area=wan -40:51.876 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13106 network=tcp -40:51.876 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13107 network=tcp -40:51.877 [INFO] TestAgent: started state syncer -40:51.878 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13112 network=tcp -40:51.912 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -40:51.912 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13111 [Candidate]" term=2 -40:51.913 [DEBUG] TestAgent.server.raft: votes: needed=1 -40:51.913 [DEBUG] TestAgent.server.raft: vote granted: from=237bad70-309e-1780-bfa0-be206b335b88 term=2 tally=1 -40:51.913 [INFO] TestAgent.server.raft: election won: tally=1 -40:51.913 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13111 [Leader]" -40:51.913 [INFO] TestAgent.server: cluster leadership acquired -40:51.913 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13111 -40:51.914 [INFO] TestAgent.server: New leader elected: payload=Node-237bad70-309e-1780-bfa0-be206b335b88 -40:51.914 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -40:51.917 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -40:51.917 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -40:51.917 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -40:51.917 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -40:51.917 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -40:51.917 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-237bad70-309e-1780-bfa0-be206b335b88 -40:51.917 [INFO] TestAgent.server: member joined, marking health alive: member=Node-237bad70-309e-1780-bfa0-be206b335b88 -40:51.926 [INFO] TestAgent.server: federation state anti-entropy synced -40:52.067 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -40:52.068 [INFO] TestAgent: Synced node info -40:52.296 [DEBUG] TestAgent: Node info in sync -40:52.297 [INFO] TestAgent: Synced service: service=test -40:52.297 [DEBUG] TestAgent: Check in sync: check=service:test:1 -40:52.297 [DEBUG] TestAgent: Check in sync: check=check_2 -40:52.298 [DEBUG] TestAgent: removed check: check=check_2 -40:52.298 [DEBUG] TestAgent: Node info in sync -40:52.299 [INFO] TestAgent: Synced service: service=test -40:52.299 [DEBUG] TestAgent: Check in sync: check=check_3 -40:52.299 [DEBUG] TestAgent: Check in sync: check=service:test:1 -40:52.299 [INFO] TestAgent: Deregistered check: check=check_2 -40:52.299 [INFO] TestAgent: Requesting shutdown -40:52.299 [INFO] TestAgent.server: shutting down server -40:52.300 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -40:52.300 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -40:52.300 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -40:52.300 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -40:52.300 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -40:52.300 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -40:52.300 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -40:52.300 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -40:52.300 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -40:52.300 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -40:52.300 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:40:52.300Z [INFO] agent.router.manager: shutting down -2022-09-15T21:40:52.300Z [INFO] agent.router.manager: shutting down -40:52.300 [INFO] TestAgent: consul server down -40:52.300 [INFO] TestAgent: shutdown complete -40:52.300 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13106 network=tcp -40:52.300 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13106 network=udp -40:52.300 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13107 network=tcp -40:52.800 [INFO] TestAgent: Waiting for endpoints to shut down -40:52.801 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:35.132Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:35.155Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:35.158 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8f54e2e1-aeb5-8c3c-96f9-790b34ebeaac Address:127.0.0.1:24335}]" +32:35.158 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24335 [Follower]" leader= +32:35.159 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8f54e2e1-aeb5-8c3c-96f9-790b34ebeaac.dc1 127.0.0.1 +32:35.160 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8f54e2e1-aeb5-8c3c-96f9-790b34ebeaac 127.0.0.1 +2021-08-13T15:32:35.160Z [INFO] agent.router: Initializing LAN area manager +32:35.160 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24330 network=udp +32:35.160 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8f54e2e1-aeb5-8c3c-96f9-790b34ebeaac.dc1 area=wan +32:35.160 [INFO] TestAgent.server: Adding LAN server: server="Node-8f54e2e1-aeb5-8c3c-96f9-790b34ebeaac (Addr: tcp/127.0.0.1:24335) (DC: dc1)" +32:35.160 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24330 network=tcp +32:35.161 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24331 network=tcp +32:35.161 [INFO] TestAgent: started state syncer +32:35.161 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24336 network=tcp +32:35.221 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:35.221 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24335 [Candidate]" term=2 +32:35.221 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:35.221 [DEBUG] TestAgent.server.raft: vote granted: from=8f54e2e1-aeb5-8c3c-96f9-790b34ebeaac term=2 tally=1 +32:35.221 [INFO] TestAgent.server.raft: election won: tally=1 +32:35.221 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24335 [Leader]" +32:35.221 [INFO] TestAgent.server: cluster leadership acquired +32:35.221 [INFO] TestAgent.server: New leader elected: payload=Node-8f54e2e1-aeb5-8c3c-96f9-790b34ebeaac +32:35.221 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24335 +32:35.223 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:35.225 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:35.225 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:35.225 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:35.225 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:35.225 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:35.225 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8f54e2e1-aeb5-8c3c-96f9-790b34ebeaac +32:35.225 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8f54e2e1-aeb5-8c3c-96f9-790b34ebeaac +32:35.226 [INFO] TestAgent.server: federation state anti-entropy synced +32:35.471 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:35.472 [INFO] TestAgent: Synced node info +32:35.535 [DEBUG] TestAgent: Node info in sync +32:35.537 [INFO] TestAgent: Synced service: service=test +32:35.537 [DEBUG] TestAgent: Check in sync: check=check_2 +32:35.537 [DEBUG] TestAgent: Check in sync: check=service:test:1 +32:35.538 [DEBUG] TestAgent: removed check: check=check_2 +32:35.538 [DEBUG] TestAgent: Node info in sync +32:35.540 [INFO] TestAgent: Synced service: service=test +32:35.540 [DEBUG] TestAgent: Check in sync: check=service:test:1 +32:35.540 [INFO] TestAgent: Deregistered check: check=check_2 +32:35.540 [DEBUG] TestAgent: Check in sync: check=check_3 +32:35.540 [INFO] TestAgent: Requesting shutdown +32:35.540 [INFO] TestAgent.server: shutting down server +32:35.540 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:35.540 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:35.540 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:35.540 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:35.540 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:35.541 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:35.541 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:35.541 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:35.541 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:35.541 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:35.541 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:35.541Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:35.541Z [INFO] agent.router.manager: shutting down +32:35.541 [INFO] TestAgent: consul server down +32:35.541 [INFO] TestAgent: shutdown complete +32:35.541 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24330 network=tcp +32:35.541 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24330 network=udp +32:35.541 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24331 network=tcp +32:36.041 [INFO] TestAgent: Waiting for endpoints to shut down +32:36.041 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:35.133Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:35.160Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:35.162 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6a4c52bb-9af5-5ad9-0653-bd95cb7d3e9c Address:127.0.0.1:24342}]" +32:35.163 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24342 [Follower]" leader= +32:35.163 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6a4c52bb-9af5-5ad9-0653-bd95cb7d3e9c.dc1 127.0.0.1 +32:35.164 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6a4c52bb-9af5-5ad9-0653-bd95cb7d3e9c 127.0.0.1 +2021-08-13T15:32:35.164Z [INFO] agent.router: Initializing LAN area manager +32:35.164 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6a4c52bb-9af5-5ad9-0653-bd95cb7d3e9c.dc1 area=wan +32:35.164 [INFO] TestAgent.server: Adding LAN server: server="Node-6a4c52bb-9af5-5ad9-0653-bd95cb7d3e9c (Addr: tcp/127.0.0.1:24342) (DC: dc1)" +32:35.165 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24337 network=tcp +32:35.165 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24337 network=udp +32:35.166 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24338 network=tcp +32:35.166 [INFO] TestAgent: started state syncer +32:35.166 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24343 network=tcp +32:35.207 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:35.207 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24342 [Candidate]" term=2 +32:35.207 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:35.207 [DEBUG] TestAgent.server.raft: vote granted: from=6a4c52bb-9af5-5ad9-0653-bd95cb7d3e9c term=2 tally=1 +32:35.207 [INFO] TestAgent.server.raft: election won: tally=1 +32:35.207 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24342 [Leader]" +32:35.207 [INFO] TestAgent.server: cluster leadership acquired +32:35.208 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24342 +32:35.209 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:35.211 [INFO] TestAgent.server: New leader elected: payload=Node-6a4c52bb-9af5-5ad9-0653-bd95cb7d3e9c +32:35.212 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:35.212 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:35.212 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:35.212 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:35.212 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:35.212 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6a4c52bb-9af5-5ad9-0653-bd95cb7d3e9c +32:35.212 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6a4c52bb-9af5-5ad9-0653-bd95cb7d3e9c +32:35.212 [INFO] TestAgent.server: federation state anti-entropy synced +32:35.358 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:35.359 [INFO] TestAgent: Synced node info +32:35.359 [DEBUG] TestAgent: Node info in sync +32:35.570 [DEBUG] TestAgent: Node info in sync +32:35.572 [INFO] TestAgent: Synced service: service=test +32:35.572 [DEBUG] TestAgent: Check in sync: check=service:test:1 +32:35.572 [DEBUG] TestAgent: Check in sync: check=check_2 +32:35.573 [DEBUG] TestAgent: removed check: check=check_2 +32:35.573 [DEBUG] TestAgent: Node info in sync +32:35.574 [INFO] TestAgent: Synced service: service=test +32:35.574 [DEBUG] TestAgent: Check in sync: check=check_3 +32:35.574 [DEBUG] TestAgent: Check in sync: check=service:test:1 +32:35.575 [INFO] TestAgent: Deregistered check: check=check_2 +32:35.575 [INFO] TestAgent: Requesting shutdown +32:35.575 [INFO] TestAgent.server: shutting down server +32:35.575 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:35.575 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:35.575 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:35.575 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:35.575 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:35.575 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:35.575 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:35.575 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:35.575 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:35.575 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:35.576 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:35.576Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:35.576Z [INFO] agent.router.manager: shutting down +32:35.576 [INFO] TestAgent: consul server down +32:35.576 [INFO] TestAgent: shutdown complete +32:35.576 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24337 network=tcp +32:35.576 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24337 network=udp +32:35.576 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24338 network=tcp +32:36.076 [INFO] TestAgent: Waiting for endpoints to shut down +32:36.076 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_RegisterService_ReRegister_ReplaceExistingChecks (0.00s) - --- PASS: TestAgent_RegisterService_ReRegister_ReplaceExistingChecks/normal (0.77s) - --- PASS: TestAgent_RegisterService_ReRegister_ReplaceExistingChecks/service_manager (1.06s) + --- PASS: TestAgent_RegisterService_ReRegister_ReplaceExistingChecks/normal (0.93s) + --- PASS: TestAgent_RegisterService_ReRegister_ReplaceExistingChecks/service_manager (0.96s) === RUN TestAgent_RegisterService_TranslateKeys === RUN TestAgent_RegisterService_TranslateKeys/normal === PAUSE TestAgent_RegisterService_TranslateKeys/normal @@ -6338,173 +6409,175 @@ === RUN TestAgent_RegisterService_TranslateKeys/service_manager/root_token === RUN TestAgent_RegisterService_TranslateKeys/normal/no_token === RUN TestAgent_RegisterService_TranslateKeys/normal/root_token -2022-09-15T21:40:52.836Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:40:52.836Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:40:52.853Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:40:52.853Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -40:52.865 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3d285e9e-47ec-9e52-0f2f-7b6ea74062f0 Address:127.0.0.1:13125}]" -40:52.868 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13125 [Follower]" leader= -40:52.872 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3d285e9e-47ec-9e52-0f2f-7b6ea74062f0.dc1 127.0.0.1 -40:52.873 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3d285e9e-47ec-9e52-0f2f-7b6ea74062f0 127.0.0.1 -2022-09-15T21:40:52.873Z [INFO] agent.router: Initializing LAN area manager -40:52.874 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3d285e9e-47ec-9e52-0f2f-7b6ea74062f0.dc1 area=wan -40:52.874 [INFO] TestAgent.server: Adding LAN server: server="Node-3d285e9e-47ec-9e52-0f2f-7b6ea74062f0 (Addr: tcp/127.0.0.1:13125) (DC: dc1)" -40:52.874 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13120 network=udp -40:52.874 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13120 network=tcp -40:52.875 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13121 network=tcp -40:52.875 [INFO] TestAgent: started state syncer -40:52.875 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13126 network=tcp -40:52.920 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -40:52.920 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13125 [Candidate]" term=2 -40:52.920 [DEBUG] TestAgent.server.raft: votes: needed=1 -40:52.920 [DEBUG] TestAgent.server.raft: vote granted: from=3d285e9e-47ec-9e52-0f2f-7b6ea74062f0 term=2 tally=1 -40:52.920 [INFO] TestAgent.server.raft: election won: tally=1 -40:52.920 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13125 [Leader]" -40:52.920 [INFO] TestAgent.server: cluster leadership acquired -40:52.920 [INFO] TestAgent.server: initializing acls -40:52.921 [INFO] TestAgent.server: Created ACL 'global-management' policy -40:52.921 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -40:52.921 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -40:52.922 [INFO] TestAgent.server: New leader elected: payload=Node-3d285e9e-47ec-9e52-0f2f-7b6ea74062f0 -40:52.922 [INFO] TestAgent.server: Created ACL anonymous token from configuration -40:52.922 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -40:52.922 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -40:52.922 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-3d285e9e-47ec-9e52-0f2f-7b6ea74062f0 -40:52.922 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-3d285e9e-47ec-9e52-0f2f-7b6ea74062f0.dc1 -40:52.923 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -40:52.923 [INFO] TestAgent.server: Updating LAN server: server="Node-3d285e9e-47ec-9e52-0f2f-7b6ea74062f0 (Addr: tcp/127.0.0.1:13125) (DC: dc1)" -40:52.924 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-3d285e9e-47ec-9e52-0f2f-7b6ea74062f0.dc1 area=wan -40:52.926 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -40:52.926 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -40:52.926 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -40:52.926 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -40:52.926 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -40:52.926 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3d285e9e-47ec-9e52-0f2f-7b6ea74062f0 -40:52.926 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3d285e9e-47ec-9e52-0f2f-7b6ea74062f0 -40:52.927 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3d285e9e-47ec-9e52-0f2f-7b6ea74062f0 -40:52.927 [INFO] TestAgent.server: federation state anti-entropy synced -40:53.191 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-3d285e9e-47ec-9e52-0f2f-7b6ea74062f0 -40:53.191 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-3d285e9e-47ec-9e52-0f2f-7b6ea74062f0 -40:53.194 [INFO] TestAgent: Synced node info -40:53.196 [INFO] TestAgent: Synced service: service=test -40:53.196 [DEBUG] TestAgent: Check in sync: check=service:test:1 -40:53.196 [DEBUG] TestAgent: Check in sync: check=service:test:2 -40:53.196 [DEBUG] TestAgent: Check in sync: check=service:test:3 -40:53.196 [INFO] TestAgent: Requesting shutdown -40:53.196 [INFO] TestAgent.server: shutting down server -40:53.196 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -40:53.196 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -40:53.196 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -40:53.196 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -40:53.196 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -40:53.196 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -40:53.196 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -40:53.196 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -40:53.196 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -40:53.196 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -40:53.196 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -40:53.196 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -40:53.196 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -40:53.196 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -40:53.196 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -40:53.196 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -2022-09-15T21:40:53.196Z [INFO] agent.router.manager: shutting down -2022-09-15T21:40:53.196Z [INFO] agent.router.manager: shutting down -40:53.196 [INFO] TestAgent: consul server down -40:53.196 [INFO] TestAgent: shutdown complete -40:53.196 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13120 network=tcp -40:53.197 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13120 network=udp -40:53.197 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13121 network=tcp -40:53.697 [INFO] TestAgent: Waiting for endpoints to shut down -40:53.697 [INFO] TestAgent: Endpoints down -2022-09-15T21:40:52.831Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:40:52.831Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:40:52.849Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:40:52.849Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -40:52.851 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6a8af7c3-6759-c37d-a361-a7220a3ca930 Address:127.0.0.1:13118}]" -40:52.852 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13118 [Follower]" leader= -40:52.852 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6a8af7c3-6759-c37d-a361-a7220a3ca930.dc1 127.0.0.1 -40:52.853 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6a8af7c3-6759-c37d-a361-a7220a3ca930 127.0.0.1 -2022-09-15T21:40:52.853Z [INFO] agent.router: Initializing LAN area manager -40:52.854 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13113 network=udp -40:52.854 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6a8af7c3-6759-c37d-a361-a7220a3ca930.dc1 area=wan -40:52.854 [INFO] TestAgent.server: Adding LAN server: server="Node-6a8af7c3-6759-c37d-a361-a7220a3ca930 (Addr: tcp/127.0.0.1:13118) (DC: dc1)" -40:52.854 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13113 network=tcp -40:52.855 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13114 network=tcp -40:52.855 [INFO] TestAgent: started state syncer -40:52.855 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13119 network=tcp -40:52.888 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -40:52.888 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13118 [Candidate]" term=2 -40:52.888 [DEBUG] TestAgent.server.raft: votes: needed=1 -40:52.888 [DEBUG] TestAgent.server.raft: vote granted: from=6a8af7c3-6759-c37d-a361-a7220a3ca930 term=2 tally=1 -40:52.888 [INFO] TestAgent.server.raft: election won: tally=1 -40:52.888 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13118 [Leader]" -40:52.888 [INFO] TestAgent.server: cluster leadership acquired -40:52.888 [INFO] TestAgent.server: New leader elected: payload=Node-6a8af7c3-6759-c37d-a361-a7220a3ca930 -40:52.889 [INFO] TestAgent.server: initializing acls -40:52.889 [INFO] TestAgent.server: Created ACL 'global-management' policy -40:52.889 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -40:52.890 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -40:52.890 [INFO] TestAgent.server: Created ACL anonymous token from configuration -40:52.890 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -40:52.890 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -40:52.891 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-6a8af7c3-6759-c37d-a361-a7220a3ca930 -40:52.891 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-6a8af7c3-6759-c37d-a361-a7220a3ca930.dc1 -40:52.891 [INFO] TestAgent.server: Updating LAN server: server="Node-6a8af7c3-6759-c37d-a361-a7220a3ca930 (Addr: tcp/127.0.0.1:13118) (DC: dc1)" -40:52.891 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-6a8af7c3-6759-c37d-a361-a7220a3ca930.dc1 area=wan -40:52.892 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -40:52.894 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -40:52.894 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -40:52.894 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -40:52.894 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -40:52.894 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -40:52.894 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6a8af7c3-6759-c37d-a361-a7220a3ca930 -40:52.894 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6a8af7c3-6759-c37d-a361-a7220a3ca930 -40:52.895 [INFO] TestAgent.server: federation state anti-entropy synced -40:52.895 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6a8af7c3-6759-c37d-a361-a7220a3ca930 -40:52.923 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -40:52.923 [INFO] TestAgent: Synced node info -40:52.923 [DEBUG] TestAgent: Node info in sync -40:53.250 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-6a8af7c3-6759-c37d-a361-a7220a3ca930 -40:53.250 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-6a8af7c3-6759-c37d-a361-a7220a3ca930 -40:53.256 [DEBUG] TestAgent: Node info in sync -40:53.257 [INFO] TestAgent: Synced service: service=test -40:53.258 [DEBUG] TestAgent: Check in sync: check=service:test:1 -40:53.258 [DEBUG] TestAgent: Check in sync: check=service:test:2 -40:53.258 [DEBUG] TestAgent: Check in sync: check=service:test:3 -40:53.258 [INFO] TestAgent: Requesting shutdown -40:53.258 [INFO] TestAgent.server: shutting down server -40:53.258 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -40:53.258 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -40:53.258 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -40:53.258 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -40:53.258 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -40:53.258 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -40:53.258 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -40:53.258 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -40:53.258 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -40:53.258 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -40:53.258 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -40:53.258 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -40:53.258 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -40:53.258 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -40:53.258 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:40:53.259Z [INFO] agent.router.manager: shutting down -2022-09-15T21:40:53.259Z [INFO] agent.router.manager: shutting down -40:53.259 [INFO] TestAgent: consul server down -40:53.259 [INFO] TestAgent: shutdown complete -40:53.259 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13113 network=tcp -40:53.259 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13113 network=udp -40:53.259 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13114 network=tcp -40:53.759 [INFO] TestAgent: Waiting for endpoints to shut down -40:53.759 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:36.100Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:32:36.100Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:36.116Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:32:36.116Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:36.123 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b4f79264-cf6b-9669-6a77-c2f8b6622ddb Address:127.0.0.1:24356}]" +32:36.124 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b4f79264-cf6b-9669-6a77-c2f8b6622ddb.dc1 127.0.0.1 +32:36.124 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b4f79264-cf6b-9669-6a77-c2f8b6622ddb 127.0.0.1 +2021-08-13T15:32:36.125Z [INFO] agent.router: Initializing LAN area manager +32:36.125 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24351 network=udp +32:36.125 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24356 [Follower]" leader= +32:36.125 [INFO] TestAgent.server: Adding LAN server: server="Node-b4f79264-cf6b-9669-6a77-c2f8b6622ddb (Addr: tcp/127.0.0.1:24356) (DC: dc1)" +32:36.125 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b4f79264-cf6b-9669-6a77-c2f8b6622ddb.dc1 area=wan +32:36.126 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24351 network=tcp +32:36.127 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24352 network=tcp +32:36.127 [INFO] TestAgent: started state syncer +32:36.127 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24357 network=tcp +32:36.176 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +32:36.185 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:36.185 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24356 [Candidate]" term=2 +32:36.186 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:36.186 [DEBUG] TestAgent.server.raft: vote granted: from=b4f79264-cf6b-9669-6a77-c2f8b6622ddb term=2 tally=1 +32:36.186 [INFO] TestAgent.server.raft: election won: tally=1 +32:36.186 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24356 [Leader]" +32:36.186 [INFO] TestAgent.server: cluster leadership acquired +32:36.186 [INFO] TestAgent.server: New leader elected: payload=Node-b4f79264-cf6b-9669-6a77-c2f8b6622ddb +32:36.186 [INFO] TestAgent.server: initializing acls +32:36.187 [INFO] TestAgent.server: Created ACL 'global-management' policy +32:36.187 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +32:36.188 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +32:36.188 [INFO] TestAgent.server: Created ACL anonymous token from configuration +32:36.188 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +32:36.188 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +32:36.188 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-b4f79264-cf6b-9669-6a77-c2f8b6622ddb +32:36.189 [INFO] TestAgent.server: Updating LAN server: server="Node-b4f79264-cf6b-9669-6a77-c2f8b6622ddb (Addr: tcp/127.0.0.1:24356) (DC: dc1)" +32:36.189 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-b4f79264-cf6b-9669-6a77-c2f8b6622ddb.dc1 +32:36.189 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-b4f79264-cf6b-9669-6a77-c2f8b6622ddb.dc1 area=wan +32:36.190 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:36.192 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:36.192 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:36.192 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:36.192 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:36.192 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:36.192 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b4f79264-cf6b-9669-6a77-c2f8b6622ddb +32:36.192 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b4f79264-cf6b-9669-6a77-c2f8b6622ddb +32:36.193 [INFO] TestAgent.server: federation state anti-entropy synced +32:36.193 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b4f79264-cf6b-9669-6a77-c2f8b6622ddb +32:36.284 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-b4f79264-cf6b-9669-6a77-c2f8b6622ddb +32:36.284 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-b4f79264-cf6b-9669-6a77-c2f8b6622ddb +32:36.288 [INFO] TestAgent: Synced node info +32:36.289 [INFO] TestAgent: Synced service: service=test +32:36.289 [DEBUG] TestAgent: Check in sync: check=service:test:1 +32:36.289 [DEBUG] TestAgent: Check in sync: check=service:test:2 +32:36.289 [DEBUG] TestAgent: Check in sync: check=service:test:3 +32:36.289 [INFO] TestAgent: Requesting shutdown +32:36.289 [INFO] TestAgent.server: shutting down server +32:36.289 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +32:36.289 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +32:36.289 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:36.289 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:36.289 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:36.289 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:36.289 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:36.289 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +32:36.289 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:36.289 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:36.289 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +32:36.289 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +32:36.289 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:36.289 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:36.289 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:36.290 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:36.290Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:36.290Z [INFO] agent.router.manager: shutting down +32:36.290 [INFO] TestAgent: consul server down +32:36.290 [INFO] TestAgent: shutdown complete +32:36.290 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24351 network=tcp +32:36.290 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24351 network=udp +32:36.290 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24352 network=tcp +32:36.790 [INFO] TestAgent: Waiting for endpoints to shut down +32:36.790 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:36.103Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:32:36.103Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:36.122Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:32:36.122Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:36.125 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:02938a12-f9cb-9060-6cb9-2f01c3e33931 Address:127.0.0.1:24349}]" +32:36.126 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24349 [Follower]" leader= +32:36.126 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-02938a12-f9cb-9060-6cb9-2f01c3e33931.dc1 127.0.0.1 +32:36.127 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-02938a12-f9cb-9060-6cb9-2f01c3e33931 127.0.0.1 +2021-08-13T15:32:36.127Z [INFO] agent.router: Initializing LAN area manager +32:36.128 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24344 network=udp +32:36.128 [INFO] TestAgent.server: Adding LAN server: server="Node-02938a12-f9cb-9060-6cb9-2f01c3e33931 (Addr: tcp/127.0.0.1:24349) (DC: dc1)" +32:36.128 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-02938a12-f9cb-9060-6cb9-2f01c3e33931.dc1 area=wan +32:36.128 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24344 network=tcp +32:36.129 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24345 network=tcp +32:36.130 [INFO] TestAgent: started state syncer +32:36.130 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24350 network=tcp +32:36.178 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +32:36.194 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:36.194 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24349 [Candidate]" term=2 +32:36.194 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:36.194 [DEBUG] TestAgent.server.raft: vote granted: from=02938a12-f9cb-9060-6cb9-2f01c3e33931 term=2 tally=1 +32:36.194 [INFO] TestAgent.server.raft: election won: tally=1 +32:36.194 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24349 [Leader]" +32:36.194 [INFO] TestAgent.server: cluster leadership acquired +32:36.194 [INFO] TestAgent.server: New leader elected: payload=Node-02938a12-f9cb-9060-6cb9-2f01c3e33931 +32:36.194 [INFO] TestAgent.server: initializing acls +32:36.195 [INFO] TestAgent.server: Created ACL 'global-management' policy +32:36.195 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +32:36.195 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +32:36.196 [INFO] TestAgent.server: Created ACL anonymous token from configuration +32:36.196 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +32:36.196 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +32:36.196 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-02938a12-f9cb-9060-6cb9-2f01c3e33931 +32:36.196 [INFO] TestAgent.server: Updating LAN server: server="Node-02938a12-f9cb-9060-6cb9-2f01c3e33931 (Addr: tcp/127.0.0.1:24349) (DC: dc1)" +32:36.196 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-02938a12-f9cb-9060-6cb9-2f01c3e33931.dc1 +32:36.196 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-02938a12-f9cb-9060-6cb9-2f01c3e33931.dc1 area=wan +32:36.197 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:36.199 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:36.199 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:36.199 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:36.199 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:36.199 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:36.199 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-02938a12-f9cb-9060-6cb9-2f01c3e33931 +32:36.199 [INFO] TestAgent.server: member joined, marking health alive: member=Node-02938a12-f9cb-9060-6cb9-2f01c3e33931 +32:36.201 [INFO] TestAgent.server: federation state anti-entropy synced +32:36.201 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-02938a12-f9cb-9060-6cb9-2f01c3e33931 +32:36.229 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:36.230 [INFO] TestAgent: Synced node info +32:36.230 [DEBUG] TestAgent: Node info in sync +32:36.343 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-02938a12-f9cb-9060-6cb9-2f01c3e33931 +32:36.344 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-02938a12-f9cb-9060-6cb9-2f01c3e33931 +32:36.346 [DEBUG] TestAgent: Node info in sync +32:36.347 [INFO] TestAgent: Synced service: service=test +32:36.347 [DEBUG] TestAgent: Check in sync: check=service:test:1 +32:36.347 [DEBUG] TestAgent: Check in sync: check=service:test:2 +32:36.347 [DEBUG] TestAgent: Check in sync: check=service:test:3 +32:36.347 [INFO] TestAgent: Requesting shutdown +32:36.348 [INFO] TestAgent.server: shutting down server +32:36.348 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +32:36.348 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:36.348 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:36.348 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:36.348 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:36.348 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +32:36.348 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:36.348 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:36.348 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:36.348 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +32:36.348 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:36.348 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:36.348 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:36.348 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +32:36.348 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:36.348Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:36.348Z [INFO] agent.router.manager: shutting down +32:36.348 [INFO] TestAgent: consul server down +32:36.348 [INFO] TestAgent: shutdown complete +32:36.348 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24344 network=tcp +32:36.348 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24344 network=udp +32:36.348 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24345 network=tcp +32:36.849 [INFO] TestAgent: Waiting for endpoints to shut down +32:36.849 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_RegisterService_TranslateKeys (0.00s) - --- PASS: TestAgent_RegisterService_TranslateKeys/service_manager (0.90s) + --- PASS: TestAgent_RegisterService_TranslateKeys/service_manager (0.71s) --- PASS: TestAgent_RegisterService_TranslateKeys/service_manager/no_token (0.00s) --- PASS: TestAgent_RegisterService_TranslateKeys/service_manager/root_token (0.00s) - --- PASS: TestAgent_RegisterService_TranslateKeys/normal (0.96s) + --- PASS: TestAgent_RegisterService_TranslateKeys/normal (0.77s) --- PASS: TestAgent_RegisterService_TranslateKeys/normal/no_token (0.00s) - --- PASS: TestAgent_RegisterService_TranslateKeys/normal/root_token (0.01s) + --- PASS: TestAgent_RegisterService_TranslateKeys/normal/root_token (0.00s) === RUN TestAgent_RegisterService_ACLDeny === RUN TestAgent_RegisterService_ACLDeny/normal === PAUSE TestAgent_RegisterService_ACLDeny/normal @@ -6512,175 +6585,179 @@ === PAUSE TestAgent_RegisterService_ACLDeny/service_manager === CONT TestAgent_RegisterService_ACLDeny/normal === CONT TestAgent_RegisterService_ACLDeny/service_manager -=== RUN TestAgent_RegisterService_ACLDeny/normal/no_token -=== RUN TestAgent_RegisterService_ACLDeny/normal/root_token === RUN TestAgent_RegisterService_ACLDeny/service_manager/no_token === RUN TestAgent_RegisterService_ACLDeny/service_manager/root_token -2022-09-15T21:40:53.791Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:40:53.792Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:40:53.810Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:40:53.810Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -40:53.813 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:707c6bf2-773d-b45d-a3d4-f6e816607018 Address:127.0.0.1:13132}]" -40:53.813 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13132 [Follower]" leader= -40:53.817 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-707c6bf2-773d-b45d-a3d4-f6e816607018.dc1 127.0.0.1 -40:53.821 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-707c6bf2-773d-b45d-a3d4-f6e816607018 127.0.0.1 -2022-09-15T21:40:53.822Z [INFO] agent.router: Initializing LAN area manager -40:53.822 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13127 network=udp -40:53.822 [INFO] TestAgent.server: Adding LAN server: server="Node-707c6bf2-773d-b45d-a3d4-f6e816607018 (Addr: tcp/127.0.0.1:13132) (DC: dc1)" -40:53.822 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-707c6bf2-773d-b45d-a3d4-f6e816607018.dc1 area=wan -40:53.823 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13127 network=tcp -40:53.824 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13128 network=tcp -40:53.824 [INFO] TestAgent: started state syncer -40:53.825 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13133 network=tcp -40:53.869 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -40:53.869 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13132 [Candidate]" term=2 -40:53.869 [DEBUG] TestAgent.server.raft: votes: needed=1 -40:53.869 [DEBUG] TestAgent.server.raft: vote granted: from=707c6bf2-773d-b45d-a3d4-f6e816607018 term=2 tally=1 -40:53.869 [INFO] TestAgent.server.raft: election won: tally=1 -40:53.869 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13132 [Leader]" -40:53.870 [INFO] TestAgent.server: cluster leadership acquired -40:53.870 [INFO] TestAgent.server: initializing acls -40:53.870 [INFO] TestAgent.server: Created ACL 'global-management' policy -40:53.870 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -40:53.871 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -40:53.872 [INFO] TestAgent.server: Created ACL anonymous token from configuration -40:53.872 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -40:53.872 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -40:53.872 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-707c6bf2-773d-b45d-a3d4-f6e816607018 -40:53.872 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-707c6bf2-773d-b45d-a3d4-f6e816607018.dc1 -40:53.873 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -40:53.874 [INFO] TestAgent.server: New leader elected: payload=Node-707c6bf2-773d-b45d-a3d4-f6e816607018 -40:53.874 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-707c6bf2-773d-b45d-a3d4-f6e816607018.dc1 area=wan -40:53.874 [INFO] TestAgent.server: Updating LAN server: server="Node-707c6bf2-773d-b45d-a3d4-f6e816607018 (Addr: tcp/127.0.0.1:13132) (DC: dc1)" -40:53.876 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -40:53.876 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -40:53.876 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -40:53.876 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -40:53.876 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -40:53.876 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-707c6bf2-773d-b45d-a3d4-f6e816607018 -40:53.876 [INFO] TestAgent.server: member joined, marking health alive: member=Node-707c6bf2-773d-b45d-a3d4-f6e816607018 -40:53.877 [INFO] TestAgent.server: federation state anti-entropy synced -40:53.877 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-707c6bf2-773d-b45d-a3d4-f6e816607018 -40:53.958 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-707c6bf2-773d-b45d-a3d4-f6e816607018 -40:53.959 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-707c6bf2-773d-b45d-a3d4-f6e816607018 -40:53.962 [INFO] TestAgent: Synced node info -40:53.966 [INFO] TestAgent: Synced service: service=test -40:53.966 [DEBUG] TestAgent: Check in sync: check=service:test:1 -40:53.966 [DEBUG] TestAgent: Check in sync: check=service:test:2 -40:53.966 [DEBUG] TestAgent: Check in sync: check=service:test:3 -40:53.966 [INFO] TestAgent: Requesting shutdown -40:53.967 [INFO] TestAgent.server: shutting down server -40:53.967 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -40:53.967 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -40:53.967 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -40:53.967 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -40:53.967 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -40:53.967 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -40:53.967 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -40:53.967 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -40:53.967 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -40:53.967 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -40:53.967 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -40:53.967 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -40:53.967 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -40:53.967 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -40:53.967 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -40:53.967 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:40:53.967Z [INFO] agent.router.manager: shutting down -2022-09-15T21:40:53.967Z [INFO] agent.router.manager: shutting down -40:53.967 [INFO] TestAgent: consul server down -40:53.967 [INFO] TestAgent: shutdown complete -40:53.967 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13127 network=tcp -40:53.967 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13127 network=udp -40:53.967 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13128 network=tcp -40:54.468 [INFO] TestAgent: Waiting for endpoints to shut down -40:54.468 [INFO] TestAgent: Endpoints down -2022-09-15T21:40:53.795Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:40:53.795Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:40:53.813Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:40:53.813Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -40:53.816 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:21f5e3aa-e0d9-70d5-7a52-e3559ece928a Address:127.0.0.1:13139}]" -40:53.816 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13139 [Follower]" leader= -40:53.817 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-21f5e3aa-e0d9-70d5-7a52-e3559ece928a.dc1 127.0.0.1 -40:53.817 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-21f5e3aa-e0d9-70d5-7a52-e3559ece928a 127.0.0.1 -2022-09-15T21:40:53.818Z [INFO] agent.router: Initializing LAN area manager -40:53.818 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-21f5e3aa-e0d9-70d5-7a52-e3559ece928a.dc1 area=wan -40:53.818 [INFO] TestAgent.server: Adding LAN server: server="Node-21f5e3aa-e0d9-70d5-7a52-e3559ece928a (Addr: tcp/127.0.0.1:13139) (DC: dc1)" -40:53.819 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13134 network=udp -40:53.819 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13134 network=tcp -40:53.821 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13135 network=tcp -40:53.821 [INFO] TestAgent: started state syncer -40:53.821 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13140 network=tcp -40:53.858 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -40:53.858 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13139 [Candidate]" term=2 -40:53.859 [DEBUG] TestAgent.server.raft: votes: needed=1 -40:53.859 [DEBUG] TestAgent.server.raft: vote granted: from=21f5e3aa-e0d9-70d5-7a52-e3559ece928a term=2 tally=1 -40:53.859 [INFO] TestAgent.server.raft: election won: tally=1 -40:53.859 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13139 [Leader]" -40:53.859 [INFO] TestAgent.server: cluster leadership acquired -40:53.859 [INFO] TestAgent.server: New leader elected: payload=Node-21f5e3aa-e0d9-70d5-7a52-e3559ece928a -40:53.859 [INFO] TestAgent.server: initializing acls -40:53.860 [INFO] TestAgent.server: Created ACL 'global-management' policy -40:53.860 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -40:53.860 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -40:53.861 [INFO] TestAgent.server: Created ACL anonymous token from configuration -40:53.861 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -40:53.861 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -40:53.861 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-21f5e3aa-e0d9-70d5-7a52-e3559ece928a -40:53.861 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-21f5e3aa-e0d9-70d5-7a52-e3559ece928a.dc1 -40:53.862 [INFO] TestAgent.server: Updating LAN server: server="Node-21f5e3aa-e0d9-70d5-7a52-e3559ece928a (Addr: tcp/127.0.0.1:13139) (DC: dc1)" -40:53.862 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-21f5e3aa-e0d9-70d5-7a52-e3559ece928a.dc1 area=wan -40:53.863 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -40:53.865 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -40:53.865 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -40:53.865 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -40:53.865 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -40:53.865 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -40:53.865 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-21f5e3aa-e0d9-70d5-7a52-e3559ece928a -40:53.865 [INFO] TestAgent.server: member joined, marking health alive: member=Node-21f5e3aa-e0d9-70d5-7a52-e3559ece928a -40:53.866 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-21f5e3aa-e0d9-70d5-7a52-e3559ece928a -40:53.867 [INFO] TestAgent.server: federation state anti-entropy synced -40:53.979 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-21f5e3aa-e0d9-70d5-7a52-e3559ece928a -40:53.979 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-21f5e3aa-e0d9-70d5-7a52-e3559ece928a -40:53.983 [INFO] TestAgent: Synced node info -40:53.984 [INFO] TestAgent: Synced service: service=test -40:53.984 [DEBUG] TestAgent: Check in sync: check=service:test:1 -40:53.984 [DEBUG] TestAgent: Check in sync: check=service:test:2 -40:53.984 [DEBUG] TestAgent: Check in sync: check=service:test:3 -40:53.984 [INFO] TestAgent: Requesting shutdown -40:53.984 [INFO] TestAgent.server: shutting down server -40:53.984 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -40:53.984 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -40:53.984 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -40:53.984 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -40:53.984 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -40:53.984 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -40:53.984 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -40:53.984 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -40:53.984 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -40:53.984 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -40:53.984 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -40:53.984 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -40:53.984 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -40:53.984 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -40:53.984 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -40:53.985 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:40:53.985Z [INFO] agent.router.manager: shutting down -2022-09-15T21:40:53.985Z [INFO] agent.router.manager: shutting down -40:53.985 [INFO] TestAgent: consul server down -40:53.985 [INFO] TestAgent: shutdown complete -40:53.985 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13134 network=tcp -40:53.985 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13134 network=udp -40:53.985 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13135 network=tcp -40:54.485 [INFO] TestAgent: Waiting for endpoints to shut down -40:54.485 [INFO] TestAgent: Endpoints down +=== RUN TestAgent_RegisterService_ACLDeny/normal/no_token +=== RUN TestAgent_RegisterService_ACLDeny/normal/root_token +2021-08-13T15:32:36.866Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:32:36.866Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:36.889Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:32:36.889Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:36.893 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:73a0b087-80b2-053d-8cc3-5d99287e2a8d Address:127.0.0.1:24370}]" +32:36.894 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24370 [Follower]" leader= +32:36.894 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-73a0b087-80b2-053d-8cc3-5d99287e2a8d.dc1 127.0.0.1 +32:36.895 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-73a0b087-80b2-053d-8cc3-5d99287e2a8d 127.0.0.1 +2021-08-13T15:32:36.895Z [INFO] agent.router: Initializing LAN area manager +32:36.895 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24365 network=udp +32:36.895 [INFO] TestAgent.server: Adding LAN server: server="Node-73a0b087-80b2-053d-8cc3-5d99287e2a8d (Addr: tcp/127.0.0.1:24370) (DC: dc1)" +32:36.895 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-73a0b087-80b2-053d-8cc3-5d99287e2a8d.dc1 area=wan +32:36.895 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24365 network=tcp +32:36.896 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24366 network=tcp +32:36.896 [INFO] TestAgent: started state syncer +32:36.896 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24371 network=tcp +32:36.946 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +32:36.961 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:36.961 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24370 [Candidate]" term=2 +32:36.962 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:36.962 [DEBUG] TestAgent.server.raft: vote granted: from=73a0b087-80b2-053d-8cc3-5d99287e2a8d term=2 tally=1 +32:36.962 [INFO] TestAgent.server.raft: election won: tally=1 +32:36.962 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24370 [Leader]" +32:36.962 [INFO] TestAgent.server: cluster leadership acquired +32:36.962 [INFO] TestAgent.server: initializing acls +32:36.962 [INFO] TestAgent.server: Created ACL 'global-management' policy +32:36.962 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +32:36.962 [INFO] TestAgent.server: New leader elected: payload=Node-73a0b087-80b2-053d-8cc3-5d99287e2a8d +32:36.970 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +32:36.971 [INFO] TestAgent.server: Created ACL anonymous token from configuration +32:36.971 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +32:36.971 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +32:36.971 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-73a0b087-80b2-053d-8cc3-5d99287e2a8d +32:36.971 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-73a0b087-80b2-053d-8cc3-5d99287e2a8d.dc1 +32:36.971 [INFO] TestAgent.server: Updating LAN server: server="Node-73a0b087-80b2-053d-8cc3-5d99287e2a8d (Addr: tcp/127.0.0.1:24370) (DC: dc1)" +32:36.971 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-73a0b087-80b2-053d-8cc3-5d99287e2a8d.dc1 area=wan +32:36.972 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:36.975 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:36.975 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:36.975 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:36.975 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:36.975 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:36.975 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-73a0b087-80b2-053d-8cc3-5d99287e2a8d +32:36.975 [INFO] TestAgent.server: member joined, marking health alive: member=Node-73a0b087-80b2-053d-8cc3-5d99287e2a8d +32:36.976 [INFO] TestAgent.server: federation state anti-entropy synced +32:36.976 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-73a0b087-80b2-053d-8cc3-5d99287e2a8d +32:37.020 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-73a0b087-80b2-053d-8cc3-5d99287e2a8d +32:37.021 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-73a0b087-80b2-053d-8cc3-5d99287e2a8d +32:37.024 [INFO] TestAgent: Synced node info +32:37.025 [INFO] TestAgent: Synced service: service=test +32:37.025 [DEBUG] TestAgent: Check in sync: check=service:test:1 +32:37.025 [DEBUG] TestAgent: Check in sync: check=service:test:2 +32:37.025 [DEBUG] TestAgent: Check in sync: check=service:test:3 +32:37.025 [INFO] TestAgent: Requesting shutdown +32:37.025 [INFO] TestAgent.server: shutting down server +32:37.025 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +32:37.025 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +32:37.025 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:37.025 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:37.025 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:37.025 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:37.025 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:37.025 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:37.025 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:37.025 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:37.025 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +32:37.025 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:37.025 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:37.025 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +32:37.025 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +32:37.025 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:37.026Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:37.026Z [INFO] agent.router.manager: shutting down +32:37.026 [INFO] TestAgent: consul server down +32:37.026 [INFO] TestAgent: shutdown complete +32:37.026 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24365 network=tcp +32:37.026 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24365 network=udp +32:37.026 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24366 network=tcp +32:37.526 [INFO] TestAgent: Waiting for endpoints to shut down +32:37.526 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:36.866Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:32:36.866Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:36.891Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:32:36.891Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:36.893 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:38c12864-9c24-90f7-24d6-316a7c57967f Address:127.0.0.1:24363}]" +32:36.893 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24363 [Follower]" leader= +32:36.894 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-38c12864-9c24-90f7-24d6-316a7c57967f.dc1 127.0.0.1 +32:36.895 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-38c12864-9c24-90f7-24d6-316a7c57967f 127.0.0.1 +2021-08-13T15:32:36.895Z [INFO] agent.router: Initializing LAN area manager +32:36.895 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-38c12864-9c24-90f7-24d6-316a7c57967f.dc1 area=wan +32:36.895 [INFO] TestAgent.server: Adding LAN server: server="Node-38c12864-9c24-90f7-24d6-316a7c57967f (Addr: tcp/127.0.0.1:24363) (DC: dc1)" +32:36.895 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24358 network=udp +32:36.895 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24358 network=tcp +32:36.896 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24359 network=tcp +32:36.896 [INFO] TestAgent: started state syncer +32:36.896 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24364 network=tcp +32:36.945 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +32:36.955 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:36.955 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24363 [Candidate]" term=2 +32:36.955 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:36.955 [DEBUG] TestAgent.server.raft: vote granted: from=38c12864-9c24-90f7-24d6-316a7c57967f term=2 tally=1 +32:36.955 [INFO] TestAgent.server.raft: election won: tally=1 +32:36.955 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24363 [Leader]" +32:36.955 [INFO] TestAgent.server: cluster leadership acquired +32:36.956 [INFO] TestAgent.server: New leader elected: payload=Node-38c12864-9c24-90f7-24d6-316a7c57967f +32:36.956 [INFO] TestAgent.server: initializing acls +32:36.956 [INFO] TestAgent.server: Created ACL 'global-management' policy +32:36.956 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +32:36.957 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +32:36.957 [INFO] TestAgent.server: Created ACL anonymous token from configuration +32:36.957 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +32:36.957 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +32:36.958 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-38c12864-9c24-90f7-24d6-316a7c57967f +32:36.958 [INFO] TestAgent.server: Updating LAN server: server="Node-38c12864-9c24-90f7-24d6-316a7c57967f (Addr: tcp/127.0.0.1:24363) (DC: dc1)" +32:36.958 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-38c12864-9c24-90f7-24d6-316a7c57967f.dc1 +32:36.958 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-38c12864-9c24-90f7-24d6-316a7c57967f.dc1 area=wan +32:36.959 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:36.962 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:36.962 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:36.962 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:36.962 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:36.962 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:36.962 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-38c12864-9c24-90f7-24d6-316a7c57967f +32:36.962 [INFO] TestAgent.server: member joined, marking health alive: member=Node-38c12864-9c24-90f7-24d6-316a7c57967f +32:36.964 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-38c12864-9c24-90f7-24d6-316a7c57967f +32:36.965 [INFO] TestAgent.server: federation state anti-entropy synced +32:37.001 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:37.001 [INFO] TestAgent: Synced node info +32:37.001 [DEBUG] TestAgent: Node info in sync +32:37.160 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-38c12864-9c24-90f7-24d6-316a7c57967f +32:37.161 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-38c12864-9c24-90f7-24d6-316a7c57967f +32:37.163 [DEBUG] TestAgent: Node info in sync +32:37.164 [INFO] TestAgent: Synced service: service=test +32:37.164 [DEBUG] TestAgent: Check in sync: check=service:test:1 +32:37.164 [DEBUG] TestAgent: Check in sync: check=service:test:2 +32:37.164 [DEBUG] TestAgent: Check in sync: check=service:test:3 +32:37.164 [INFO] TestAgent: Requesting shutdown +32:37.164 [INFO] TestAgent.server: shutting down server +32:37.164 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:37.164 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:37.164 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:37.164 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:37.164 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +32:37.164 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +32:37.164 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:37.164 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:37.164 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:37.164 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:37.164 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:37.164 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +32:37.164 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +32:37.164 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +32:37.164 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2021-08-13T15:32:37.165Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:37.165Z [INFO] agent.router.manager: shutting down +32:37.165 [INFO] TestAgent: consul server down +32:37.165 [INFO] TestAgent: shutdown complete +32:37.165 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24358 network=tcp +32:37.165 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24358 network=udp +32:37.165 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24359 network=tcp +32:37.665 [INFO] TestAgent: Waiting for endpoints to shut down +32:37.665 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_RegisterService_ACLDeny (0.00s) - --- PASS: TestAgent_RegisterService_ACLDeny/normal (0.71s) - --- PASS: TestAgent_RegisterService_ACLDeny/normal/no_token (0.00s) - --- PASS: TestAgent_RegisterService_ACLDeny/normal/root_token (0.01s) - --- PASS: TestAgent_RegisterService_ACLDeny/service_manager (0.73s) + --- PASS: TestAgent_RegisterService_ACLDeny/service_manager (0.68s) --- PASS: TestAgent_RegisterService_ACLDeny/service_manager/no_token (0.00s) --- PASS: TestAgent_RegisterService_ACLDeny/service_manager/root_token (0.00s) + --- PASS: TestAgent_RegisterService_ACLDeny/normal (0.82s) + --- PASS: TestAgent_RegisterService_ACLDeny/normal/no_token (0.00s) + --- PASS: TestAgent_RegisterService_ACLDeny/normal/root_token (0.00s) === RUN TestAgent_RegisterService_InvalidAddress === RUN TestAgent_RegisterService_InvalidAddress/normal === PAUSE TestAgent_RegisterService_InvalidAddress/normal @@ -6691,132 +6768,124 @@ === RUN TestAgent_RegisterService_InvalidAddress/service_manager/addr_0.0.0.0 === RUN TestAgent_RegisterService_InvalidAddress/service_manager/addr_:: === RUN TestAgent_RegisterService_InvalidAddress/service_manager/addr_[::] -2022-09-15T21:40:54.538Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:40:54.595Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -40:54.597 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f9bba66a-9fb7-0285-f66e-de67d0d942dc Address:127.0.0.1:13153}]" -40:54.604 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f9bba66a-9fb7-0285-f66e-de67d0d942dc.dc1 127.0.0.1 -40:54.605 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f9bba66a-9fb7-0285-f66e-de67d0d942dc 127.0.0.1 -2022-09-15T21:40:54.605Z [INFO] agent.router: Initializing LAN area manager -40:54.605 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13148 network=udp -40:54.605 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13153 [Follower]" leader= -40:54.606 [INFO] TestAgent.server: Adding LAN server: server="Node-f9bba66a-9fb7-0285-f66e-de67d0d942dc (Addr: tcp/127.0.0.1:13153) (DC: dc1)" -40:54.606 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f9bba66a-9fb7-0285-f66e-de67d0d942dc.dc1 area=wan -40:54.606 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13148 network=tcp -40:54.607 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13149 network=tcp -40:54.607 [INFO] TestAgent: started state syncer -40:54.607 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13154 network=tcp -40:54.666 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -40:54.666 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13153 [Candidate]" term=2 -40:54.666 [DEBUG] TestAgent.server.raft: votes: needed=1 -40:54.666 [DEBUG] TestAgent.server.raft: vote granted: from=f9bba66a-9fb7-0285-f66e-de67d0d942dc term=2 tally=1 -40:54.666 [INFO] TestAgent.server.raft: election won: tally=1 -40:54.666 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13153 [Leader]" -40:54.670 [INFO] TestAgent.server: cluster leadership acquired -40:54.670 [INFO] TestAgent.server: New leader elected: payload=Node-f9bba66a-9fb7-0285-f66e-de67d0d942dc -40:54.671 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13153 -40:54.673 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -40:54.681 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -40:54.681 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -40:54.681 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -40:54.681 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -40:54.681 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -40:54.682 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f9bba66a-9fb7-0285-f66e-de67d0d942dc -40:54.682 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f9bba66a-9fb7-0285-f66e-de67d0d942dc -40:54.683 [INFO] TestAgent.server: federation state anti-entropy synced -40:54.795 [INFO] TestAgent: Requesting shutdown -40:54.795 [INFO] TestAgent.server: shutting down server -40:54.795 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -40:54.795 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -40:54.795 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -40:54.795 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -40:54.795 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -40:54.796 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -40:54.796 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -40:54.796 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -40:54.796 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -40:54.796 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -40:54.796 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -40:54.796 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:40:54.796Z [INFO] agent.router.manager: shutting down -2022-09-15T21:40:54.796Z [INFO] agent.router.manager: shutting down -40:54.796 [INFO] TestAgent: consul server down -40:54.796 [INFO] TestAgent: shutdown complete -40:54.796 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13148 network=tcp -40:54.796 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13148 network=udp -40:54.796 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13149 network=tcp -40:55.297 [INFO] TestAgent: Waiting for endpoints to shut down -40:55.297 [INFO] TestAgent: Endpoints down -2022-09-15T21:40:54.545Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:40:54.592Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -40:54.599 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:62299685-f6a5-0a78-bdef-9f389f4756d2 Address:127.0.0.1:13146}]" -40:54.600 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-62299685-f6a5-0a78-bdef-9f389f4756d2.dc1 127.0.0.1 -40:54.601 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-62299685-f6a5-0a78-bdef-9f389f4756d2 127.0.0.1 -2022-09-15T21:40:54.601Z [INFO] agent.router: Initializing LAN area manager -40:54.602 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13146 [Follower]" leader= -40:54.602 [INFO] TestAgent.server: Adding LAN server: server="Node-62299685-f6a5-0a78-bdef-9f389f4756d2 (Addr: tcp/127.0.0.1:13146) (DC: dc1)" -40:54.602 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-62299685-f6a5-0a78-bdef-9f389f4756d2.dc1 area=wan -40:54.602 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13141 network=tcp -40:54.607 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13141 network=udp -40:54.608 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13142 network=tcp -40:54.608 [INFO] TestAgent: started state syncer -40:54.608 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13147 network=tcp -40:54.649 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -40:54.650 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13146 [Candidate]" term=2 -40:54.650 [DEBUG] TestAgent.server.raft: votes: needed=1 -40:54.650 [DEBUG] TestAgent.server.raft: vote granted: from=62299685-f6a5-0a78-bdef-9f389f4756d2 term=2 tally=1 -40:54.650 [INFO] TestAgent.server.raft: election won: tally=1 -40:54.650 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13146 [Leader]" -40:54.650 [INFO] TestAgent.server: cluster leadership acquired -40:54.651 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13146 -40:54.652 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -40:54.654 [INFO] TestAgent.server: New leader elected: payload=Node-62299685-f6a5-0a78-bdef-9f389f4756d2 -40:54.677 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -40:54.677 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -40:54.677 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -40:54.677 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -40:54.677 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -40:54.677 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-62299685-f6a5-0a78-bdef-9f389f4756d2 -40:54.677 [INFO] TestAgent.server: member joined, marking health alive: member=Node-62299685-f6a5-0a78-bdef-9f389f4756d2 -40:54.680 [INFO] TestAgent.server: federation state anti-entropy synced -40:54.738 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -40:54.739 [INFO] TestAgent: Synced node info -40:54.992 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -40:54.992 [DEBUG] TestAgent: Node info in sync -40:54.992 [DEBUG] TestAgent: Node info in sync -40:55.001 [DEBUG] TestAgent: Node info in sync -40:55.002 [INFO] TestAgent: Synced service: service=connect-proxy -40:55.002 [INFO] TestAgent: Requesting shutdown -40:55.002 [DEBUG] TestAgent: Node info in sync -40:55.002 [DEBUG] TestAgent: Service in sync: service=connect-proxy -40:55.002 [DEBUG] TestAgent: Node info in sync -40:55.002 [DEBUG] TestAgent: Service in sync: service=connect-proxy -40:55.003 [INFO] TestAgent.server: shutting down server -40:55.003 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -40:55.003 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -40:55.003 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -40:55.003 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -40:55.003 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -40:55.003 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -40:55.003 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -40:55.003 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -40:55.003 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -40:55.003 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -40:55.003 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:40:55.003Z [INFO] agent.router.manager: shutting down -2022-09-15T21:40:55.003Z [INFO] agent.router.manager: shutting down -40:55.003 [INFO] TestAgent: consul server down -40:55.003 [INFO] TestAgent: shutdown complete -40:55.003 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13141 network=tcp -40:55.003 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13141 network=udp -40:55.003 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13142 network=tcp -40:55.504 [INFO] TestAgent: Waiting for endpoints to shut down -40:55.504 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:37.699Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:37.739Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:37.742 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e52273dd-4e24-d650-2af4-227822632804 Address:127.0.0.1:24384}]" +32:37.742 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e52273dd-4e24-d650-2af4-227822632804.dc1 127.0.0.1 +32:37.743 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24384 [Follower]" leader= +32:37.743 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e52273dd-4e24-d650-2af4-227822632804 127.0.0.1 +2021-08-13T15:32:37.744Z [INFO] agent.router: Initializing LAN area manager +32:37.744 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24379 network=udp +32:37.744 [INFO] TestAgent.server: Adding LAN server: server="Node-e52273dd-4e24-d650-2af4-227822632804 (Addr: tcp/127.0.0.1:24384) (DC: dc1)" +32:37.744 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e52273dd-4e24-d650-2af4-227822632804.dc1 area=wan +32:37.744 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24379 network=tcp +32:37.745 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24380 network=tcp +32:37.745 [INFO] TestAgent: started state syncer +32:37.745 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24385 network=tcp +32:37.795 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:37.795 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24384 [Candidate]" term=2 +32:37.795 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:37.795 [DEBUG] TestAgent.server.raft: vote granted: from=e52273dd-4e24-d650-2af4-227822632804 term=2 tally=1 +32:37.795 [INFO] TestAgent.server.raft: election won: tally=1 +32:37.795 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24384 [Leader]" +32:37.796 [INFO] TestAgent.server: cluster leadership acquired +32:37.796 [INFO] TestAgent.server: New leader elected: payload=Node-e52273dd-4e24-d650-2af4-227822632804 +32:37.796 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24384 +32:37.797 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:37.799 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:37.799 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:37.799 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:37.799 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:37.799 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:37.799 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e52273dd-4e24-d650-2af4-227822632804 +32:37.799 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e52273dd-4e24-d650-2af4-227822632804 +32:37.801 [INFO] TestAgent.server: federation state anti-entropy synced +32:38.047 [INFO] TestAgent: Requesting shutdown +32:38.047 [INFO] TestAgent.server: shutting down server +32:38.047 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:38.047 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:38.047 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:38.047 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:38.047 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:38.048 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:38.048 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:38.048 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +32:38.048 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:38.048 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:38.048 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:38.048 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:38.048Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:38.048Z [INFO] agent.router.manager: shutting down +32:38.048 [INFO] TestAgent: consul server down +32:38.048 [INFO] TestAgent: shutdown complete +32:38.048 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24379 network=tcp +32:38.048 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24379 network=udp +32:38.048 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24380 network=tcp +32:38.551 [INFO] TestAgent: Waiting for endpoints to shut down +32:38.551 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:37.697Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:37.739Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:37.742 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:583cc183-9cc7-7149-5ba5-86380f548a15 Address:127.0.0.1:24377}]" +32:37.742 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-583cc183-9cc7-7149-5ba5-86380f548a15.dc1 127.0.0.1 +32:37.743 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24377 [Follower]" leader= +32:37.748 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-583cc183-9cc7-7149-5ba5-86380f548a15 127.0.0.1 +2021-08-13T15:32:37.748Z [INFO] agent.router: Initializing LAN area manager +32:37.748 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24372 network=udp +32:37.748 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-583cc183-9cc7-7149-5ba5-86380f548a15.dc1 area=wan +32:37.748 [INFO] TestAgent.server: Adding LAN server: server="Node-583cc183-9cc7-7149-5ba5-86380f548a15 (Addr: tcp/127.0.0.1:24377) (DC: dc1)" +32:37.748 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24372 network=tcp +32:37.749 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24373 network=tcp +32:37.749 [INFO] TestAgent: started state syncer +32:37.749 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24378 network=tcp +32:37.796 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:37.796 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24377 [Candidate]" term=2 +32:37.797 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:37.797 [DEBUG] TestAgent.server.raft: vote granted: from=583cc183-9cc7-7149-5ba5-86380f548a15 term=2 tally=1 +32:37.797 [INFO] TestAgent.server.raft: election won: tally=1 +32:37.797 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24377 [Leader]" +32:37.797 [INFO] TestAgent.server: cluster leadership acquired +32:37.797 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24377 +32:37.798 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:37.799 [INFO] TestAgent.server: New leader elected: payload=Node-583cc183-9cc7-7149-5ba5-86380f548a15 +32:37.800 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:37.800 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:37.800 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:37.800 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:37.800 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:37.800 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-583cc183-9cc7-7149-5ba5-86380f548a15 +32:37.800 [INFO] TestAgent.server: member joined, marking health alive: member=Node-583cc183-9cc7-7149-5ba5-86380f548a15 +32:37.801 [INFO] TestAgent.server: federation state anti-entropy synced +32:38.155 [INFO] TestAgent: Synced node info +32:38.156 [INFO] TestAgent: Synced service: service=connect-proxy +32:38.156 [INFO] TestAgent: Requesting shutdown +32:38.157 [INFO] TestAgent.server: shutting down server +32:38.157 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:38.157 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:38.157 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:38.157 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:38.157 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:38.157 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:38.157 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:38.157 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +32:38.157 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:38.157 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:38.157 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:38.157 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:38.157Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:38.157Z [INFO] agent.router.manager: shutting down +32:38.158 [INFO] TestAgent: consul server down +32:38.158 [INFO] TestAgent: shutdown complete +32:38.158 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24372 network=tcp +32:38.158 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24372 network=udp +32:38.158 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24373 network=tcp +32:38.658 [INFO] TestAgent: Waiting for endpoints to shut down +32:38.658 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_RegisterService_InvalidAddress (0.00s) - --- PASS: TestAgent_RegisterService_InvalidAddress/service_manager (0.81s) + --- PASS: TestAgent_RegisterService_InvalidAddress/service_manager (0.88s) --- PASS: TestAgent_RegisterService_InvalidAddress/service_manager/addr_0.0.0.0 (0.00s) --- PASS: TestAgent_RegisterService_InvalidAddress/service_manager/addr_:: (0.00s) --- PASS: TestAgent_RegisterService_InvalidAddress/service_manager/addr_[::] (0.00s) - --- PASS: TestAgent_RegisterService_InvalidAddress/normal (1.02s) + --- PASS: TestAgent_RegisterService_InvalidAddress/normal (0.99s) === RUN TestAgent_RegisterService_UnmanagedConnectProxy === RUN TestAgent_RegisterService_UnmanagedConnectProxy/normal === PAUSE TestAgent_RegisterService_UnmanagedConnectProxy/normal @@ -6824,2241 +6893,2264 @@ === PAUSE TestAgent_RegisterService_UnmanagedConnectProxy/service_manager === CONT TestAgent_RegisterService_UnmanagedConnectProxy/normal === CONT TestAgent_RegisterService_UnmanagedConnectProxy/service_manager -2022-09-15T21:40:55.576Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:40:55.603Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -40:55.607 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e964b791-a033-6530-8ede-64759cf182b7 Address:127.0.0.1:13167}]" -40:55.608 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e964b791-a033-6530-8ede-64759cf182b7.dc1 127.0.0.1 -40:55.609 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e964b791-a033-6530-8ede-64759cf182b7 127.0.0.1 -2022-09-15T21:40:55.609Z [INFO] agent.router: Initializing LAN area manager -40:55.610 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13167 [Follower]" leader= -40:55.610 [INFO] TestAgent.server: Adding LAN server: server="Node-e964b791-a033-6530-8ede-64759cf182b7 (Addr: tcp/127.0.0.1:13167) (DC: dc1)" -40:55.610 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e964b791-a033-6530-8ede-64759cf182b7.dc1 area=wan -40:55.610 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13162 network=tcp -40:55.611 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13162 network=udp -40:55.612 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13163 network=tcp -40:55.612 [INFO] TestAgent: started state syncer -40:55.612 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13168 network=tcp -40:55.666 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -40:55.666 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13167 [Candidate]" term=2 -40:55.666 [DEBUG] TestAgent.server.raft: votes: needed=1 -40:55.666 [DEBUG] TestAgent.server.raft: vote granted: from=e964b791-a033-6530-8ede-64759cf182b7 term=2 tally=1 -40:55.666 [INFO] TestAgent.server.raft: election won: tally=1 -40:55.666 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13167 [Leader]" -40:55.666 [INFO] TestAgent.server: cluster leadership acquired -40:55.666 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13167 -40:55.668 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -40:55.669 [INFO] TestAgent.server: New leader elected: payload=Node-e964b791-a033-6530-8ede-64759cf182b7 -40:55.671 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -40:55.671 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -40:55.671 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -40:55.671 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -40:55.671 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -40:55.671 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e964b791-a033-6530-8ede-64759cf182b7 -40:55.671 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e964b791-a033-6530-8ede-64759cf182b7 -40:55.672 [INFO] TestAgent.server: federation state anti-entropy synced -40:55.890 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -40:55.890 [INFO] TestAgent: Synced node info -40:55.890 [DEBUG] TestAgent: Node info in sync -40:55.894 [DEBUG] TestAgent: Node info in sync -40:55.895 [INFO] TestAgent: Synced service: service=connect-proxy -40:55.895 [INFO] TestAgent: Requesting shutdown -40:55.895 [INFO] TestAgent.server: shutting down server -40:55.895 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -40:55.895 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -40:55.895 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -40:55.895 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -40:55.895 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -40:55.896 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -40:55.896 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -40:55.896 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -40:55.896 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -40:55.896 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -40:55.896 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:40:55.896Z [INFO] agent.router.manager: shutting down -2022-09-15T21:40:55.896Z [INFO] agent.router.manager: shutting down -40:55.896 [INFO] TestAgent: consul server down -40:55.896 [INFO] TestAgent: shutdown complete -40:55.896 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13162 network=tcp -40:55.896 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13162 network=udp -40:55.896 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13163 network=tcp -40:56.397 [INFO] TestAgent: Waiting for endpoints to shut down -40:56.397 [INFO] TestAgent: Endpoints down -2022-09-15T21:40:55.525Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:40:55.568Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -40:55.573 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:320127c4-d69c-a635-0da2-b4780dada313 Address:127.0.0.1:13160}]" -40:55.573 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13160 [Follower]" leader= -40:55.575 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-320127c4-d69c-a635-0da2-b4780dada313.dc1 127.0.0.1 -40:55.583 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-320127c4-d69c-a635-0da2-b4780dada313 127.0.0.1 -2022-09-15T21:40:55.583Z [INFO] agent.router: Initializing LAN area manager -40:55.584 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13155 network=udp -40:55.584 [INFO] TestAgent.server: Adding LAN server: server="Node-320127c4-d69c-a635-0da2-b4780dada313 (Addr: tcp/127.0.0.1:13160) (DC: dc1)" -40:55.584 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-320127c4-d69c-a635-0da2-b4780dada313.dc1 area=wan -40:55.584 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13155 network=tcp -40:55.585 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13156 network=tcp -40:55.586 [INFO] TestAgent: started state syncer -40:55.586 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13161 network=tcp -40:55.626 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -40:55.626 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13160 [Candidate]" term=2 -40:55.626 [DEBUG] TestAgent.server.raft: votes: needed=1 -40:55.626 [DEBUG] TestAgent.server.raft: vote granted: from=320127c4-d69c-a635-0da2-b4780dada313 term=2 tally=1 -40:55.626 [INFO] TestAgent.server.raft: election won: tally=1 -40:55.627 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13160 [Leader]" -40:55.627 [INFO] TestAgent.server: cluster leadership acquired -40:55.627 [INFO] TestAgent.server: New leader elected: payload=Node-320127c4-d69c-a635-0da2-b4780dada313 -40:55.627 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13160 -40:55.628 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -40:55.631 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -40:55.631 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -40:55.631 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -40:55.631 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -40:55.631 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -40:55.631 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-320127c4-d69c-a635-0da2-b4780dada313 -40:55.631 [INFO] TestAgent.server: member joined, marking health alive: member=Node-320127c4-d69c-a635-0da2-b4780dada313 -40:55.633 [INFO] TestAgent.server: federation state anti-entropy synced -40:55.994 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -40:55.995 [INFO] TestAgent: Synced node info -40:56.002 [DEBUG] TestAgent: Node info in sync -40:56.003 [INFO] TestAgent: Synced service: service=connect-proxy -40:56.003 [INFO] TestAgent: Requesting shutdown -40:56.003 [INFO] TestAgent.server: shutting down server -40:56.003 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -40:56.003 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -40:56.003 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -40:56.003 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -40:56.003 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -40:56.004 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -40:56.004 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -40:56.004 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -40:56.004 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -40:56.004 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -40:56.004 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:40:56.005Z [INFO] agent.router.manager: shutting down -2022-09-15T21:40:56.005Z [INFO] agent.router.manager: shutting down -40:56.005 [INFO] TestAgent: consul server down -40:56.005 [INFO] TestAgent: shutdown complete -40:56.005 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13155 network=tcp -40:56.005 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13155 network=udp -40:56.005 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13156 network=tcp -40:56.505 [INFO] TestAgent: Waiting for endpoints to shut down -40:56.505 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:38.692Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:38.707Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:38.712 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:490815b9-b7fc-ddb2-03e1-fafa9d68a0db Address:127.0.0.1:24391}]" +32:38.713 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-490815b9-b7fc-ddb2-03e1-fafa9d68a0db.dc1 127.0.0.1 +32:38.714 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-490815b9-b7fc-ddb2-03e1-fafa9d68a0db 127.0.0.1 +2021-08-13T15:32:38.714Z [INFO] agent.router: Initializing LAN area manager +32:38.714 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24386 network=udp +32:38.714 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24391 [Follower]" leader= +32:38.717 [INFO] TestAgent.server: Adding LAN server: server="Node-490815b9-b7fc-ddb2-03e1-fafa9d68a0db (Addr: tcp/127.0.0.1:24391) (DC: dc1)" +32:38.717 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-490815b9-b7fc-ddb2-03e1-fafa9d68a0db.dc1 area=wan +32:38.717 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24386 network=tcp +32:38.718 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24387 network=tcp +32:38.718 [INFO] TestAgent: started state syncer +32:38.718 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24392 network=tcp +32:38.780 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:38.780 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24391 [Candidate]" term=2 +32:38.780 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:38.780 [DEBUG] TestAgent.server.raft: vote granted: from=490815b9-b7fc-ddb2-03e1-fafa9d68a0db term=2 tally=1 +32:38.780 [INFO] TestAgent.server.raft: election won: tally=1 +32:38.780 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24391 [Leader]" +32:38.780 [INFO] TestAgent.server: cluster leadership acquired +32:38.780 [INFO] TestAgent.server: New leader elected: payload=Node-490815b9-b7fc-ddb2-03e1-fafa9d68a0db +32:38.780 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24391 +32:38.781 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:38.783 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:38.783 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:38.783 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:38.783 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:38.783 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:38.784 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-490815b9-b7fc-ddb2-03e1-fafa9d68a0db +32:38.784 [INFO] TestAgent.server: member joined, marking health alive: member=Node-490815b9-b7fc-ddb2-03e1-fafa9d68a0db +32:38.784 [INFO] TestAgent.server: federation state anti-entropy synced +32:38.877 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:38.878 [INFO] TestAgent: Synced node info +32:38.878 [DEBUG] TestAgent: Node info in sync +32:38.896 [DEBUG] TestAgent: Node info in sync +32:38.896 [INFO] TestAgent: Synced service: service=connect-proxy +32:38.896 [INFO] TestAgent: Requesting shutdown +32:38.897 [INFO] TestAgent.server: shutting down server +32:38.897 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:38.897 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:38.897 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:38.897 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:38.897 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:38.897 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:38.897 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:38.897 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:38.897 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:38.897 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:38.897 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:38.898Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:38.898Z [INFO] agent.router.manager: shutting down +32:38.898 [INFO] TestAgent: consul server down +32:38.898 [INFO] TestAgent: shutdown complete +32:38.898 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24386 network=tcp +32:38.898 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24386 network=udp +32:38.898 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24387 network=tcp +32:39.398 [INFO] TestAgent: Waiting for endpoints to shut down +32:39.398 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:38.687Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:38.703Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:38.706 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bd9f9060-7a3a-bdf9-cfd2-738ef39be4f6 Address:127.0.0.1:24398}]" +32:38.706 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24398 [Follower]" leader= +32:38.707 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bd9f9060-7a3a-bdf9-cfd2-738ef39be4f6.dc1 127.0.0.1 +32:38.708 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bd9f9060-7a3a-bdf9-cfd2-738ef39be4f6 127.0.0.1 +2021-08-13T15:32:38.708Z [INFO] agent.router: Initializing LAN area manager +32:38.708 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24393 network=udp +32:38.708 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bd9f9060-7a3a-bdf9-cfd2-738ef39be4f6.dc1 area=wan +32:38.708 [INFO] TestAgent.server: Adding LAN server: server="Node-bd9f9060-7a3a-bdf9-cfd2-738ef39be4f6 (Addr: tcp/127.0.0.1:24398) (DC: dc1)" +32:38.708 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24393 network=tcp +32:38.711 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24394 network=tcp +32:38.711 [INFO] TestAgent: started state syncer +32:38.711 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24399 network=tcp +32:38.764 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:38.764 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24398 [Candidate]" term=2 +32:38.764 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:38.764 [DEBUG] TestAgent.server.raft: vote granted: from=bd9f9060-7a3a-bdf9-cfd2-738ef39be4f6 term=2 tally=1 +32:38.764 [INFO] TestAgent.server.raft: election won: tally=1 +32:38.764 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24398 [Leader]" +32:38.765 [INFO] TestAgent.server: cluster leadership acquired +32:38.765 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24398 +32:38.766 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:38.768 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:38.768 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:38.768 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:38.768 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:38.768 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:38.768 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bd9f9060-7a3a-bdf9-cfd2-738ef39be4f6 +32:38.768 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bd9f9060-7a3a-bdf9-cfd2-738ef39be4f6 +32:38.765 [INFO] TestAgent.server: New leader elected: payload=Node-bd9f9060-7a3a-bdf9-cfd2-738ef39be4f6 +32:38.770 [INFO] TestAgent.server: federation state anti-entropy synced +32:39.000 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:39.001 [INFO] TestAgent: Synced node info +32:39.001 [DEBUG] TestAgent: Node info in sync +32:39.138 [DEBUG] TestAgent: Node info in sync +32:39.139 [INFO] TestAgent: Synced service: service=connect-proxy +32:39.139 [INFO] TestAgent: Requesting shutdown +32:39.140 [INFO] TestAgent.server: shutting down server +32:39.140 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:39.140 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:39.140 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:39.140 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:39.140 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:39.140 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:39.140 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:39.140 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:39.140 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:39.140 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:39.140 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:39.140Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:39.140Z [INFO] agent.router.manager: shutting down +32:39.141 [INFO] TestAgent: consul server down +32:39.141 [INFO] TestAgent: shutdown complete +32:39.141 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24393 network=tcp +32:39.141 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24393 network=udp +32:39.141 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24394 network=tcp +32:39.641 [INFO] TestAgent: Waiting for endpoints to shut down +32:39.641 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_RegisterService_UnmanagedConnectProxy (0.00s) - --- PASS: TestAgent_RegisterService_UnmanagedConnectProxy/service_manager (0.89s) - --- PASS: TestAgent_RegisterService_UnmanagedConnectProxy/normal (1.00s) + --- PASS: TestAgent_RegisterService_UnmanagedConnectProxy/normal (0.74s) + --- PASS: TestAgent_RegisterService_UnmanagedConnectProxy/service_manager (0.98s) === RUN TestAgent_RegisterServiceDeregisterService_Sidecar === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/normal === PAUSE TestAgent_RegisterServiceDeregisterService_Sidecar/normal === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager === PAUSE TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager === CONT TestAgent_RegisterServiceDeregisterService_Sidecar/normal -=== CONT TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/normal/sanity_check_no_sidecar_case +=== CONT TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/sanity_check_no_sidecar_case -2022-09-15T21:40:56.548Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:40:56.568Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -40:56.572 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8f9dd079-b534-581c-71a8-1da47af83654 Address:127.0.0.1:13181}]" -40:56.575 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8f9dd079-b534-581c-71a8-1da47af83654.dc1 127.0.0.1 -40:56.577 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8f9dd079-b534-581c-71a8-1da47af83654 127.0.0.1 -2022-09-15T21:40:56.577Z [INFO] agent.router: Initializing LAN area manager -40:56.577 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13176 network=udp -40:56.578 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13181 [Follower]" leader= -40:56.580 [INFO] TestAgent.server: Adding LAN server: server="Node-8f9dd079-b534-581c-71a8-1da47af83654 (Addr: tcp/127.0.0.1:13181) (DC: dc1)" -40:56.580 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8f9dd079-b534-581c-71a8-1da47af83654.dc1 area=wan -40:56.583 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13176 network=tcp -40:56.598 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13177 network=tcp -40:56.598 [INFO] TestAgent: started state syncer -40:56.599 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13182 network=tcp -40:56.618 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -40:56.618 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13181 [Candidate]" term=2 -40:56.618 [DEBUG] TestAgent.server.raft: votes: needed=1 -40:56.618 [DEBUG] TestAgent.server.raft: vote granted: from=8f9dd079-b534-581c-71a8-1da47af83654 term=2 tally=1 -40:56.618 [INFO] TestAgent.server.raft: election won: tally=1 -40:56.618 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13181 [Leader]" -40:56.618 [INFO] TestAgent.server: cluster leadership acquired -40:56.619 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13181 -40:56.619 [INFO] TestAgent.server: New leader elected: payload=Node-8f9dd079-b534-581c-71a8-1da47af83654 -40:56.621 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -40:56.623 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -40:56.623 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -40:56.623 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -40:56.623 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -40:56.623 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -40:56.623 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8f9dd079-b534-581c-71a8-1da47af83654 -40:56.624 [INFO] TestAgent.server: federation state anti-entropy synced -40:56.624 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8f9dd079-b534-581c-71a8-1da47af83654 -40:56.724 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -40:56.724 [INFO] TestAgent: Synced node info -40:56.724 [DEBUG] TestAgent: Node info in sync -40:56.791 [DEBUG] TestAgent: Node info in sync -40:56.792 [INFO] TestAgent: Synced service: service=web -40:56.793 [INFO] TestAgent: Requesting shutdown -40:56.793 [INFO] TestAgent.server: shutting down server -40:56.793 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -40:56.793 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -40:56.793 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -40:56.793 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -40:56.793 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -40:56.793 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -40:56.793 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -40:56.793 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -40:56.793 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -40:56.793 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -40:56.793 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:40:56.793Z [INFO] agent.router.manager: shutting down -2022-09-15T21:40:56.793Z [INFO] agent.router.manager: shutting down -40:56.793 [INFO] TestAgent: consul server down -40:56.793 [INFO] TestAgent: shutdown complete -40:56.793 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13176 network=tcp -40:56.793 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13176 network=udp -40:56.793 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13177 network=tcp -40:57.294 [INFO] TestAgent: Waiting for endpoints to shut down -40:57.294 [INFO] TestAgent: Endpoints down -=== RUN TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/default_sidecar -2022-09-15T21:40:56.549Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:40:56.580Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -40:56.597 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6079602d-7aec-7bec-6dde-9812f3b5479e Address:127.0.0.1:13174}]" -40:56.598 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13174 [Follower]" leader= -40:56.600 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6079602d-7aec-7bec-6dde-9812f3b5479e.dc1 127.0.0.1 -40:56.601 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6079602d-7aec-7bec-6dde-9812f3b5479e 127.0.0.1 -2022-09-15T21:40:56.601Z [INFO] agent.router: Initializing LAN area manager -40:56.601 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13169 network=udp -40:56.602 [INFO] TestAgent.server: Adding LAN server: server="Node-6079602d-7aec-7bec-6dde-9812f3b5479e (Addr: tcp/127.0.0.1:13174) (DC: dc1)" -40:56.602 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6079602d-7aec-7bec-6dde-9812f3b5479e.dc1 area=wan -40:56.602 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13169 network=tcp -40:56.604 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13170 network=tcp -40:56.604 [INFO] TestAgent: started state syncer -40:56.604 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13175 network=tcp -40:56.655 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -40:56.655 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13174 [Candidate]" term=2 -40:56.655 [DEBUG] TestAgent.server.raft: votes: needed=1 -40:56.655 [DEBUG] TestAgent.server.raft: vote granted: from=6079602d-7aec-7bec-6dde-9812f3b5479e term=2 tally=1 -40:56.655 [INFO] TestAgent.server.raft: election won: tally=1 -40:56.655 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13174 [Leader]" -40:56.655 [INFO] TestAgent.server: cluster leadership acquired -40:56.656 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13174 -40:56.656 [INFO] TestAgent.server: New leader elected: payload=Node-6079602d-7aec-7bec-6dde-9812f3b5479e -40:56.657 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -40:56.660 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -40:56.660 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -40:56.660 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -40:56.660 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -40:56.660 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -40:56.660 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6079602d-7aec-7bec-6dde-9812f3b5479e -40:56.660 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6079602d-7aec-7bec-6dde-9812f3b5479e -40:56.661 [INFO] TestAgent.server: federation state anti-entropy synced -40:56.984 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -40:56.985 [INFO] TestAgent: Synced node info -40:57.004 [DEBUG] TestAgent: Node info in sync -40:57.005 [INFO] TestAgent: Synced service: service=web -40:57.005 [INFO] TestAgent: Requesting shutdown -40:57.005 [INFO] TestAgent.server: shutting down server -40:57.005 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -40:57.005 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -40:57.005 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -40:57.005 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -40:57.005 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -40:57.005 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -40:57.005 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -40:57.005 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -40:57.005 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -40:57.005 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -40:57.005 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -2022-09-15T21:40:57.005Z [INFO] agent.router.manager: shutting down -2022-09-15T21:40:57.005Z [INFO] agent.router.manager: shutting down -40:57.006 [INFO] TestAgent: consul server down -40:57.006 [INFO] TestAgent: shutdown complete -40:57.006 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13169 network=tcp -40:57.006 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13169 network=udp -40:57.006 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13170 network=tcp -40:57.510 [INFO] TestAgent: Waiting for endpoints to shut down -40:57.510 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:39.667Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:39.684Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:39.686 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c171690f-fd9d-33e1-793a-e68f2ea77a66 Address:127.0.0.1:24405}]" +32:39.687 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24405 [Follower]" leader= +32:39.687 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c171690f-fd9d-33e1-793a-e68f2ea77a66.dc1 127.0.0.1 +32:39.688 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c171690f-fd9d-33e1-793a-e68f2ea77a66 127.0.0.1 +2021-08-13T15:32:39.688Z [INFO] agent.router: Initializing LAN area manager +32:39.688 [INFO] TestAgent.server: Adding LAN server: server="Node-c171690f-fd9d-33e1-793a-e68f2ea77a66 (Addr: tcp/127.0.0.1:24405) (DC: dc1)" +32:39.688 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c171690f-fd9d-33e1-793a-e68f2ea77a66.dc1 area=wan +32:39.688 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24400 network=udp +32:39.688 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24400 network=tcp +32:39.689 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24401 network=tcp +32:39.689 [INFO] TestAgent: started state syncer +32:39.689 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24406 network=tcp +32:39.736 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:39.736 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24405 [Candidate]" term=2 +32:39.736 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:39.736 [DEBUG] TestAgent.server.raft: vote granted: from=c171690f-fd9d-33e1-793a-e68f2ea77a66 term=2 tally=1 +32:39.736 [INFO] TestAgent.server.raft: election won: tally=1 +32:39.736 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24405 [Leader]" +32:39.736 [INFO] TestAgent.server: cluster leadership acquired +32:39.736 [INFO] TestAgent.server: New leader elected: payload=Node-c171690f-fd9d-33e1-793a-e68f2ea77a66 +32:39.737 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24405 +32:39.738 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:39.740 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:39.740 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:39.740 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:39.740 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:39.740 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:39.740 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c171690f-fd9d-33e1-793a-e68f2ea77a66 +32:39.740 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c171690f-fd9d-33e1-793a-e68f2ea77a66 +32:39.741 [INFO] TestAgent.server: federation state anti-entropy synced +32:40.003 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:40.003 [INFO] TestAgent: Synced node info +32:40.003 [DEBUG] TestAgent: Node info in sync +32:40.085 [DEBUG] TestAgent: Node info in sync +32:40.086 [INFO] TestAgent: Synced service: service=web +32:40.086 [INFO] TestAgent: Requesting shutdown +32:40.086 [INFO] TestAgent.server: shutting down server +32:40.086 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:40.086 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:40.086 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:40.086 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:40.086 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:40.086 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:40.086 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:40.086 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:40.086 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:40.086 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:40.087 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:40.087Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:40.087Z [INFO] agent.router.manager: shutting down +32:40.087 [INFO] TestAgent: consul server down +32:40.087 [INFO] TestAgent: shutdown complete +32:40.087 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24400 network=tcp +32:40.087 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24400 network=udp +32:40.087 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24401 network=tcp +32:40.587 [INFO] TestAgent: Waiting for endpoints to shut down +32:40.587 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/normal/default_sidecar -2022-09-15T21:40:57.536Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:40:57.557Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -40:57.560 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:df0856b2-262f-f777-edc7-3d7c0b47d778 Address:127.0.0.1:13195}]" -40:57.561 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13195 [Follower]" leader= -40:57.561 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-df0856b2-262f-f777-edc7-3d7c0b47d778.dc1 127.0.0.1 -40:57.562 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-df0856b2-262f-f777-edc7-3d7c0b47d778 127.0.0.1 -2022-09-15T21:40:57.563Z [INFO] agent.router: Initializing LAN area manager -40:57.563 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13190 network=udp -40:57.563 [INFO] TestAgent.server: Adding LAN server: server="Node-df0856b2-262f-f777-edc7-3d7c0b47d778 (Addr: tcp/127.0.0.1:13195) (DC: dc1)" -40:57.563 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-df0856b2-262f-f777-edc7-3d7c0b47d778.dc1 area=wan -40:57.563 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13190 network=tcp -40:57.564 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13191 network=tcp -40:57.565 [INFO] TestAgent: started state syncer -40:57.565 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13196 network=tcp -40:57.611 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -40:57.611 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13195 [Candidate]" term=2 -40:57.611 [DEBUG] TestAgent.server.raft: votes: needed=1 -40:57.611 [DEBUG] TestAgent.server.raft: vote granted: from=df0856b2-262f-f777-edc7-3d7c0b47d778 term=2 tally=1 -40:57.611 [INFO] TestAgent.server.raft: election won: tally=1 -40:57.611 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13195 [Leader]" -40:57.612 [INFO] TestAgent.server: cluster leadership acquired -40:57.612 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13195 -40:57.613 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -40:57.613 [INFO] TestAgent.server: New leader elected: payload=Node-df0856b2-262f-f777-edc7-3d7c0b47d778 -40:57.616 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -40:57.616 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -40:57.616 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -40:57.616 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -40:57.616 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -40:57.616 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-df0856b2-262f-f777-edc7-3d7c0b47d778 -40:57.616 [INFO] TestAgent.server: member joined, marking health alive: member=Node-df0856b2-262f-f777-edc7-3d7c0b47d778 -40:57.618 [INFO] TestAgent.server: federation state anti-entropy synced -40:57.625 [INFO] TestAgent: Synced node info -40:57.626 [INFO] TestAgent: Synced service: service=web -40:57.627 [INFO] TestAgent: Synced service: service=web-sidecar-proxy -40:57.627 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:1 -40:57.627 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:2 -40:57.628 [DEBUG] TestAgent: removed service: service=web -40:57.628 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:2 -40:57.629 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:1 -40:57.629 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy -40:57.629 [DEBUG] TestAgent: Node info in sync -40:57.629 [INFO] TestAgent: Deregistered service: service=web -40:57.631 [INFO] TestAgent: Deregistered service: service=web-sidecar-proxy -40:57.631 [INFO] TestAgent: Requesting shutdown -40:57.631 [INFO] TestAgent.server: shutting down server -40:57.631 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -40:57.631 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -40:57.631 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -40:57.631 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -40:57.631 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -40:57.631 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -40:57.631 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -40:57.631 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -40:57.631 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -40:57.631 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -40:57.631 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -40:57.631 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2022-09-15T21:40:57.632Z [INFO] agent.router.manager: shutting down -2022-09-15T21:40:57.632Z [INFO] agent.router.manager: shutting down -40:57.632 [INFO] TestAgent: consul server down -40:57.632 [INFO] TestAgent: shutdown complete -40:57.632 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13190 network=tcp -40:57.632 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13190 network=udp -40:57.632 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13191 network=tcp -40:58.132 [INFO] TestAgent: Waiting for endpoints to shut down -40:58.132 [INFO] TestAgent: Endpoints down -=== RUN TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_OK_defaults -2022-09-15T21:40:57.316Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:40:57.355Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -40:57.363 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6360c69e-8c11-0c63-bb0f-e18037e02974 Address:127.0.0.1:13188}]" -40:57.365 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6360c69e-8c11-0c63-bb0f-e18037e02974.dc1 127.0.0.1 -40:57.366 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13188 [Follower]" leader= -40:57.382 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6360c69e-8c11-0c63-bb0f-e18037e02974 127.0.0.1 -2022-09-15T21:40:57.383Z [INFO] agent.router: Initializing LAN area manager -40:57.383 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13183 network=udp -40:57.383 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6360c69e-8c11-0c63-bb0f-e18037e02974.dc1 area=wan -40:57.383 [INFO] TestAgent.server: Adding LAN server: server="Node-6360c69e-8c11-0c63-bb0f-e18037e02974 (Addr: tcp/127.0.0.1:13188) (DC: dc1)" -40:57.383 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13183 network=tcp -40:57.384 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13184 network=tcp -40:57.384 [INFO] TestAgent: started state syncer -40:57.385 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13189 network=tcp -40:57.423 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -40:57.423 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13188 [Candidate]" term=2 -40:57.424 [DEBUG] TestAgent.server.raft: votes: needed=1 -40:57.424 [DEBUG] TestAgent.server.raft: vote granted: from=6360c69e-8c11-0c63-bb0f-e18037e02974 term=2 tally=1 -40:57.424 [INFO] TestAgent.server.raft: election won: tally=1 -40:57.424 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13188 [Leader]" -40:57.424 [INFO] TestAgent.server: cluster leadership acquired -40:57.424 [INFO] TestAgent.server: New leader elected: payload=Node-6360c69e-8c11-0c63-bb0f-e18037e02974 -40:57.424 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13188 -40:57.425 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -40:57.428 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -40:57.428 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -40:57.428 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -40:57.428 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -40:57.428 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -40:57.428 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6360c69e-8c11-0c63-bb0f-e18037e02974 -40:57.428 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6360c69e-8c11-0c63-bb0f-e18037e02974 -40:57.430 [INFO] TestAgent.server: federation state anti-entropy synced -40:57.561 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -40:57.562 [INFO] TestAgent: Synced node info -40:57.714 [DEBUG] TestAgent: added local registration for service: service=web-sidecar-proxy -40:57.714 [DEBUG] TestAgent: Node info in sync -40:57.723 [INFO] TestAgent: Synced service: service=web-sidecar-proxy -40:57.724 [INFO] TestAgent: Synced service: service=web -40:57.724 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:1 -40:57.724 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:2 -40:57.727 [DEBUG] TestAgent: removed service: service=web -40:57.727 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:2 -40:57.727 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:1 -40:57.727 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy -40:57.727 [DEBUG] TestAgent: Node info in sync -40:57.728 [INFO] TestAgent: Deregistered service: service=web -40:57.730 [INFO] TestAgent: Deregistered service: service=web-sidecar-proxy -40:57.730 [INFO] TestAgent: Requesting shutdown -40:57.730 [INFO] TestAgent.server: shutting down server -40:57.730 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -40:57.730 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -40:57.730 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -40:57.730 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -40:57.730 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -40:57.730 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -40:57.730 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -40:57.730 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -40:57.730 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -40:57.730 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -40:57.730 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:40:57.731Z [INFO] agent.router.manager: shutting down -2022-09-15T21:40:57.731Z [INFO] agent.router.manager: shutting down -40:57.731 [INFO] TestAgent: consul server down -40:57.731 [INFO] TestAgent: shutdown complete -40:57.731 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13183 network=tcp -40:57.731 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13183 network=udp -40:57.731 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13184 network=tcp -40:58.231 [INFO] TestAgent: Waiting for endpoints to shut down -40:58.231 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:39.661Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:39.684Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:39.686 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:846a21f9-306b-a29f-77d3-9fe2170b7519 Address:127.0.0.1:24412}]" +32:39.687 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24412 [Follower]" leader= +32:39.689 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-846a21f9-306b-a29f-77d3-9fe2170b7519.dc1 127.0.0.1 +32:39.690 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-846a21f9-306b-a29f-77d3-9fe2170b7519 127.0.0.1 +2021-08-13T15:32:39.690Z [INFO] agent.router: Initializing LAN area manager +32:39.690 [INFO] TestAgent.server: Adding LAN server: server="Node-846a21f9-306b-a29f-77d3-9fe2170b7519 (Addr: tcp/127.0.0.1:24412) (DC: dc1)" +32:39.690 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-846a21f9-306b-a29f-77d3-9fe2170b7519.dc1 area=wan +32:39.695 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24407 network=udp +32:39.695 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24407 network=tcp +32:39.697 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24408 network=tcp +32:39.697 [INFO] TestAgent: started state syncer +32:39.697 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24413 network=tcp +32:39.725 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:39.725 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24412 [Candidate]" term=2 +32:39.726 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:39.726 [DEBUG] TestAgent.server.raft: vote granted: from=846a21f9-306b-a29f-77d3-9fe2170b7519 term=2 tally=1 +32:39.726 [INFO] TestAgent.server.raft: election won: tally=1 +32:39.726 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24412 [Leader]" +32:39.726 [INFO] TestAgent.server: cluster leadership acquired +32:39.726 [INFO] TestAgent.server: New leader elected: payload=Node-846a21f9-306b-a29f-77d3-9fe2170b7519 +32:39.726 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24412 +32:39.727 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:39.730 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:39.730 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:39.730 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:39.730 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:39.730 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:39.730 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-846a21f9-306b-a29f-77d3-9fe2170b7519 +32:39.730 [INFO] TestAgent.server: member joined, marking health alive: member=Node-846a21f9-306b-a29f-77d3-9fe2170b7519 +32:39.731 [INFO] TestAgent.server: federation state anti-entropy synced +32:39.826 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:39.827 [INFO] TestAgent: Synced node info +32:40.095 [DEBUG] TestAgent: Node info in sync +32:40.096 [INFO] TestAgent: Synced service: service=web +32:40.096 [INFO] TestAgent: Requesting shutdown +32:40.096 [INFO] TestAgent.server: shutting down server +32:40.096 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:40.096 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:40.096 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:40.096 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:40.096 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:40.096 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:40.096 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:40.097 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:40.097 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:40.097 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:40.097 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:40.097Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:40.097Z [INFO] agent.router.manager: shutting down +32:40.097 [INFO] TestAgent: consul server down +32:40.097 [INFO] TestAgent: shutdown complete +32:40.097 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24407 network=tcp +32:40.097 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24407 network=udp +32:40.097 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24408 network=tcp +32:40.599 [INFO] TestAgent: Waiting for endpoints to shut down +32:40.599 [INFO] TestAgent: Endpoints down +=== RUN TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/default_sidecar +2021-08-13T15:32:40.628Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:40.667Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:40.670 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:009eb30c-9537-4b88-4039-07eedd1f7363 Address:127.0.0.1:24426}]" +32:40.670 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24426 [Follower]" leader= +32:40.671 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-009eb30c-9537-4b88-4039-07eedd1f7363.dc1 127.0.0.1 +32:40.672 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-009eb30c-9537-4b88-4039-07eedd1f7363 127.0.0.1 +2021-08-13T15:32:40.672Z [INFO] agent.router: Initializing LAN area manager +32:40.672 [INFO] TestAgent.server: Adding LAN server: server="Node-009eb30c-9537-4b88-4039-07eedd1f7363 (Addr: tcp/127.0.0.1:24426) (DC: dc1)" +32:40.672 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-009eb30c-9537-4b88-4039-07eedd1f7363.dc1 area=wan +32:40.672 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24421 network=udp +32:40.672 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24421 network=tcp +32:40.673 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24422 network=tcp +32:40.673 [INFO] TestAgent: started state syncer +32:40.673 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24427 network=tcp +32:40.721 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:40.721 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24426 [Candidate]" term=2 +32:40.721 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:40.721 [DEBUG] TestAgent.server.raft: vote granted: from=009eb30c-9537-4b88-4039-07eedd1f7363 term=2 tally=1 +32:40.721 [INFO] TestAgent.server.raft: election won: tally=1 +32:40.721 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24426 [Leader]" +32:40.721 [INFO] TestAgent.server: cluster leadership acquired +32:40.721 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24426 +32:40.722 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:40.724 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:40.725 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:40.725 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:40.725 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:40.725 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:40.725 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-009eb30c-9537-4b88-4039-07eedd1f7363 +32:40.725 [INFO] TestAgent.server: member joined, marking health alive: member=Node-009eb30c-9537-4b88-4039-07eedd1f7363 +32:40.725 [INFO] TestAgent.server: New leader elected: payload=Node-009eb30c-9537-4b88-4039-07eedd1f7363 +32:40.726 [INFO] TestAgent.server: federation state anti-entropy synced +32:40.728 [DEBUG] TestAgent: added local registration for service: service=web-sidecar-proxy +32:40.729 [INFO] TestAgent: Synced node info +32:40.729 [INFO] TestAgent: Synced service: service=web +32:40.730 [INFO] TestAgent: Synced service: service=web-sidecar-proxy +32:40.730 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:1 +32:40.730 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:2 +32:40.731 [DEBUG] TestAgent: removed service: service=web +32:40.731 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:2 +32:40.731 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:1 +32:40.731 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy +32:40.731 [DEBUG] TestAgent: Node info in sync +32:40.732 [INFO] TestAgent: Deregistered service: service=web +32:40.732 [INFO] TestAgent: Deregistered service: service=web-sidecar-proxy +32:40.732 [INFO] TestAgent: Requesting shutdown +32:40.733 [INFO] TestAgent.server: shutting down server +32:40.733 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:40.733 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:40.733 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:40.733 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:40.733 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:40.733 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:40.733 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:40.733 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +32:40.733 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:40.733 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:40.733 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:40.733 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:40.733Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:40.733Z [INFO] agent.router.manager: shutting down +32:40.733 [INFO] TestAgent: consul server down +32:40.733 [INFO] TestAgent: shutdown complete +32:40.733 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24421 network=tcp +32:40.733 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24421 network=udp +32:40.733 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24422 network=tcp +32:41.234 [INFO] TestAgent: Waiting for endpoints to shut down +32:41.234 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/ACL_OK_defaults -2022-09-15T21:40:58.265Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:40:58.266Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:40:58.284Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:40:58.284Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -40:58.287 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:348ce35d-725d-8434-5f83-3b3aa4b0c9e3 Address:127.0.0.1:13209}]" -40:58.288 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-348ce35d-725d-8434-5f83-3b3aa4b0c9e3.dc1 127.0.0.1 -40:58.288 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13209 [Follower]" leader= -40:58.289 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-348ce35d-725d-8434-5f83-3b3aa4b0c9e3 127.0.0.1 -2022-09-15T21:40:58.289Z [INFO] agent.router: Initializing LAN area manager -40:58.289 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13204 network=udp -40:58.289 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-348ce35d-725d-8434-5f83-3b3aa4b0c9e3.dc1 area=wan -40:58.289 [INFO] TestAgent.server: Adding LAN server: server="Node-348ce35d-725d-8434-5f83-3b3aa4b0c9e3 (Addr: tcp/127.0.0.1:13209) (DC: dc1)" -40:58.289 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13204 network=tcp -40:58.291 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13205 network=tcp -40:58.291 [INFO] TestAgent: started state syncer -40:58.291 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13210 network=tcp -40:58.339 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -40:58.346 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -40:58.346 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13209 [Candidate]" term=2 -40:58.346 [DEBUG] TestAgent.server.raft: votes: needed=1 -40:58.346 [DEBUG] TestAgent.server.raft: vote granted: from=348ce35d-725d-8434-5f83-3b3aa4b0c9e3 term=2 tally=1 -40:58.346 [INFO] TestAgent.server.raft: election won: tally=1 -40:58.346 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13209 [Leader]" -40:58.347 [INFO] TestAgent.server: cluster leadership acquired -40:58.347 [INFO] TestAgent.server: initializing acls -40:58.347 [INFO] TestAgent.server: Created ACL 'global-management' policy -40:58.347 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -40:58.348 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -40:58.349 [INFO] TestAgent.server: Created ACL anonymous token from configuration -40:58.349 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -40:58.349 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -40:58.349 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-348ce35d-725d-8434-5f83-3b3aa4b0c9e3 -40:58.349 [INFO] TestAgent.server: New leader elected: payload=Node-348ce35d-725d-8434-5f83-3b3aa4b0c9e3 -40:58.349 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-348ce35d-725d-8434-5f83-3b3aa4b0c9e3.dc1 -40:58.349 [INFO] TestAgent.server: Updating LAN server: server="Node-348ce35d-725d-8434-5f83-3b3aa4b0c9e3 (Addr: tcp/127.0.0.1:13209) (DC: dc1)" -40:58.349 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-348ce35d-725d-8434-5f83-3b3aa4b0c9e3.dc1 area=wan -40:58.350 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -40:58.353 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -40:58.353 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -40:58.353 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -40:58.353 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -40:58.353 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -40:58.353 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-348ce35d-725d-8434-5f83-3b3aa4b0c9e3 -40:58.353 [INFO] TestAgent.server: member joined, marking health alive: member=Node-348ce35d-725d-8434-5f83-3b3aa4b0c9e3 -40:58.354 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-348ce35d-725d-8434-5f83-3b3aa4b0c9e3 -40:58.354 [INFO] TestAgent.server: federation state anti-entropy synced -40:58.524 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-348ce35d-725d-8434-5f83-3b3aa4b0c9e3 -40:58.525 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-348ce35d-725d-8434-5f83-3b3aa4b0c9e3 -40:58.534 [DEBUG] TestAgent: added local registration for service: service=web-sidecar-proxy -40:58.534 [INFO] TestAgent: Synced node info -40:58.535 [INFO] TestAgent: Synced service: service=web -40:58.536 [INFO] TestAgent: Synced service: service=web-sidecar-proxy -40:58.536 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:2 -40:58.536 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:1 -40:58.543 [DEBUG] TestAgent: removed service: service=web -40:58.544 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" -40:58.544 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" -40:58.544 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:1 -40:58.544 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" -40:58.544 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:2 -40:58.544 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy -40:58.544 [DEBUG] TestAgent: Node info in sync -40:58.545 [INFO] TestAgent: Deregistered service: service=web -40:58.545 [INFO] TestAgent: Deregistered service: service=web-sidecar-proxy -40:58.546 [INFO] TestAgent: Requesting shutdown -40:58.546 [INFO] TestAgent.server: shutting down server -40:58.546 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -40:58.546 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -40:58.546 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -40:58.546 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -40:58.546 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -40:58.546 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -40:58.546 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -40:58.546 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -40:58.546 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -40:58.546 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -40:58.546 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -40:58.546 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -40:58.546 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -40:58.546 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -40:58.546 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -40:58.546 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -2022-09-15T21:40:58.546Z [INFO] agent.router.manager: shutting down -2022-09-15T21:40:58.546Z [INFO] agent.router.manager: shutting down -40:58.546 [INFO] TestAgent: consul server down -40:58.546 [INFO] TestAgent: shutdown complete -40:58.546 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13204 network=tcp -40:58.546 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13204 network=udp -40:58.546 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13205 network=tcp -40:59.047 [INFO] TestAgent: Waiting for endpoints to shut down -40:59.047 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:40.617Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:40.660Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:40.663 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4ffd0ff2-d018-a5ae-5d70-8b6906069d7b Address:127.0.0.1:24419}]" +32:40.663 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24419 [Follower]" leader= +32:40.664 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4ffd0ff2-d018-a5ae-5d70-8b6906069d7b.dc1 127.0.0.1 +32:40.665 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4ffd0ff2-d018-a5ae-5d70-8b6906069d7b 127.0.0.1 +2021-08-13T15:32:40.665Z [INFO] agent.router: Initializing LAN area manager +32:40.665 [INFO] TestAgent.server: Adding LAN server: server="Node-4ffd0ff2-d018-a5ae-5d70-8b6906069d7b (Addr: tcp/127.0.0.1:24419) (DC: dc1)" +32:40.665 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4ffd0ff2-d018-a5ae-5d70-8b6906069d7b.dc1 area=wan +32:40.665 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24414 network=udp +32:40.666 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24414 network=tcp +32:40.666 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24415 network=tcp +32:40.667 [INFO] TestAgent: started state syncer +32:40.667 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24420 network=tcp +32:40.712 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:40.712 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24419 [Candidate]" term=2 +32:40.712 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:40.713 [DEBUG] TestAgent.server.raft: vote granted: from=4ffd0ff2-d018-a5ae-5d70-8b6906069d7b term=2 tally=1 +32:40.713 [INFO] TestAgent.server.raft: election won: tally=1 +32:40.713 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24419 [Leader]" +32:40.713 [INFO] TestAgent.server: cluster leadership acquired +32:40.713 [INFO] TestAgent.server: New leader elected: payload=Node-4ffd0ff2-d018-a5ae-5d70-8b6906069d7b +32:40.713 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24419 +32:40.714 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:40.717 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:40.717 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:40.717 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:40.717 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:40.717 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:40.717 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4ffd0ff2-d018-a5ae-5d70-8b6906069d7b +32:40.717 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4ffd0ff2-d018-a5ae-5d70-8b6906069d7b +32:40.718 [INFO] TestAgent.server: federation state anti-entropy synced +32:40.777 [INFO] TestAgent: Synced node info +32:40.777 [INFO] TestAgent: Synced service: service=web +32:40.778 [INFO] TestAgent: Synced service: service=web-sidecar-proxy +32:40.778 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:1 +32:40.778 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:2 +32:40.779 [DEBUG] TestAgent: removed service: service=web +32:40.779 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:1 +32:40.779 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:2 +32:40.779 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy +32:40.779 [DEBUG] TestAgent: Node info in sync +32:40.780 [INFO] TestAgent: Deregistered service: service=web +32:40.780 [INFO] TestAgent: Deregistered service: service=web-sidecar-proxy +32:40.780 [INFO] TestAgent: Requesting shutdown +32:40.781 [INFO] TestAgent.server: shutting down server +32:40.781 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:40.781 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:40.781 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:40.781 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:40.781 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:40.781 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:40.781 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:40.781 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +32:40.781 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:40.781 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:40.781 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:40.781 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:40.781Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:40.781Z [INFO] agent.router.manager: shutting down +32:40.781 [INFO] TestAgent: consul server down +32:40.781 [INFO] TestAgent: shutdown complete +32:40.781 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24414 network=tcp +32:40.781 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24414 network=udp +32:40.781 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24415 network=tcp +32:41.282 [INFO] TestAgent: Waiting for endpoints to shut down +32:41.282 [INFO] TestAgent: Endpoints down +=== RUN TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_OK_defaults +2021-08-13T15:32:41.269Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:32:41.269Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:41.289Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:32:41.289Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:41.292 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b21147e9-2d1b-6a7a-9ccb-13d2db935bcb Address:127.0.0.1:24433}]" +32:41.293 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b21147e9-2d1b-6a7a-9ccb-13d2db935bcb.dc1 127.0.0.1 +32:41.294 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b21147e9-2d1b-6a7a-9ccb-13d2db935bcb 127.0.0.1 +2021-08-13T15:32:41.294Z [INFO] agent.router: Initializing LAN area manager +32:41.294 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24428 network=udp +32:41.294 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24433 [Follower]" leader= +32:41.295 [INFO] TestAgent.server: Adding LAN server: server="Node-b21147e9-2d1b-6a7a-9ccb-13d2db935bcb (Addr: tcp/127.0.0.1:24433) (DC: dc1)" +32:41.295 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b21147e9-2d1b-6a7a-9ccb-13d2db935bcb.dc1 area=wan +32:41.295 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24428 network=tcp +32:41.296 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24429 network=tcp +32:41.296 [INFO] TestAgent: started state syncer +32:41.296 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24434 network=tcp +32:41.332 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:41.332 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24433 [Candidate]" term=2 +32:41.333 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:41.333 [DEBUG] TestAgent.server.raft: vote granted: from=b21147e9-2d1b-6a7a-9ccb-13d2db935bcb term=2 tally=1 +32:41.333 [INFO] TestAgent.server.raft: election won: tally=1 +32:41.333 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24433 [Leader]" +32:41.333 [INFO] TestAgent.server: cluster leadership acquired +32:41.333 [INFO] TestAgent.server: New leader elected: payload=Node-b21147e9-2d1b-6a7a-9ccb-13d2db935bcb +32:41.333 [INFO] TestAgent.server: initializing acls +32:41.334 [INFO] TestAgent.server: Created ACL 'global-management' policy +32:41.334 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +32:41.335 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +32:41.340 [INFO] TestAgent.server: Created ACL anonymous token from configuration +32:41.340 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +32:41.340 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +32:41.340 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-b21147e9-2d1b-6a7a-9ccb-13d2db935bcb +32:41.340 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-b21147e9-2d1b-6a7a-9ccb-13d2db935bcb.dc1 +32:41.341 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:41.341 [INFO] TestAgent.server: Updating LAN server: server="Node-b21147e9-2d1b-6a7a-9ccb-13d2db935bcb (Addr: tcp/127.0.0.1:24433) (DC: dc1)" +32:41.341 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-b21147e9-2d1b-6a7a-9ccb-13d2db935bcb.dc1 area=wan +32:41.346 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:41.346 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:41.346 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:41.346 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:41.346 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:41.346 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b21147e9-2d1b-6a7a-9ccb-13d2db935bcb +32:41.346 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b21147e9-2d1b-6a7a-9ccb-13d2db935bcb +32:41.347 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b21147e9-2d1b-6a7a-9ccb-13d2db935bcb +32:41.347 [INFO] TestAgent.server: federation state anti-entropy synced +32:41.603 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-b21147e9-2d1b-6a7a-9ccb-13d2db935bcb +32:41.603 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-b21147e9-2d1b-6a7a-9ccb-13d2db935bcb +32:41.608 [DEBUG] TestAgent: added local registration for service: service=web-sidecar-proxy +32:41.608 [INFO] TestAgent: Synced node info +32:41.609 [INFO] TestAgent: Synced service: service=web +32:41.610 [INFO] TestAgent: Synced service: service=web-sidecar-proxy +32:41.610 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:2 +32:41.610 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:1 +32:41.610 [DEBUG] TestAgent: removed service: service=web +32:41.611 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:1 +32:41.611 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:2 +32:41.611 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy +32:41.611 [DEBUG] TestAgent: Node info in sync +32:41.615 [INFO] TestAgent: Deregistered service: service=web +32:41.616 [INFO] TestAgent: Deregistered service: service=web-sidecar-proxy +32:41.616 [INFO] TestAgent: Requesting shutdown +32:41.616 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" +32:41.616 [INFO] TestAgent.server: shutting down server +32:41.616 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +32:41.616 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:41.616 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:41.616 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:41.616 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:41.616 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +32:41.616 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:41.616 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +32:41.616 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +32:41.616 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:41.616 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:41.616 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:41.616 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +32:41.616 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:41.617Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:41.617Z [INFO] agent.router.manager: shutting down +32:41.617 [INFO] TestAgent: consul server down +32:41.617 [INFO] TestAgent: shutdown complete +32:41.617 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24428 network=tcp +32:41.617 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24428 network=udp +32:41.617 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24429 network=tcp +32:41.617 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:41.617 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:42.117 [INFO] TestAgent: Waiting for endpoints to shut down +32:42.117 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/ACL_denied -2022-09-15T21:40:58.151Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:40:58.151Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:40:58.171Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:40:58.171Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -40:58.175 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:31de0349-88b8-a1be-78a7-d0e746519acc Address:127.0.0.1:13202}]" -40:58.176 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-31de0349-88b8-a1be-78a7-d0e746519acc.dc1 127.0.0.1 -40:58.176 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13202 [Follower]" leader= -40:58.176 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-31de0349-88b8-a1be-78a7-d0e746519acc 127.0.0.1 -2022-09-15T21:40:58.176Z [INFO] agent.router: Initializing LAN area manager -40:58.177 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-31de0349-88b8-a1be-78a7-d0e746519acc.dc1 area=wan -40:58.177 [INFO] TestAgent.server: Adding LAN server: server="Node-31de0349-88b8-a1be-78a7-d0e746519acc (Addr: tcp/127.0.0.1:13202) (DC: dc1)" -40:58.177 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13197 network=udp -40:58.177 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13197 network=tcp -40:58.178 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13198 network=tcp -40:58.178 [INFO] TestAgent: started state syncer -40:58.178 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13203 network=tcp -40:58.222 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -40:58.222 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13202 [Candidate]" term=2 -40:58.222 [DEBUG] TestAgent.server.raft: votes: needed=1 -40:58.222 [DEBUG] TestAgent.server.raft: vote granted: from=31de0349-88b8-a1be-78a7-d0e746519acc term=2 tally=1 -40:58.222 [INFO] TestAgent.server.raft: election won: tally=1 -40:58.222 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13202 [Leader]" -40:58.222 [INFO] TestAgent.server: cluster leadership acquired -40:58.223 [INFO] TestAgent.server: initializing acls -40:58.223 [INFO] TestAgent.server: New leader elected: payload=Node-31de0349-88b8-a1be-78a7-d0e746519acc -40:58.223 [INFO] TestAgent.server: Created ACL 'global-management' policy -40:58.223 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -40:58.224 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -40:58.225 [INFO] TestAgent.server: Created ACL anonymous token from configuration -40:58.225 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -40:58.225 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -40:58.225 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-31de0349-88b8-a1be-78a7-d0e746519acc -40:58.225 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-31de0349-88b8-a1be-78a7-d0e746519acc.dc1 -40:58.226 [INFO] TestAgent.server: Updating LAN server: server="Node-31de0349-88b8-a1be-78a7-d0e746519acc (Addr: tcp/127.0.0.1:13202) (DC: dc1)" -40:58.226 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-31de0349-88b8-a1be-78a7-d0e746519acc.dc1 area=wan -40:58.226 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -40:58.228 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -40:58.229 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -40:58.229 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -40:58.229 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -40:58.229 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -40:58.229 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-31de0349-88b8-a1be-78a7-d0e746519acc -40:58.229 [INFO] TestAgent.server: member joined, marking health alive: member=Node-31de0349-88b8-a1be-78a7-d0e746519acc -40:58.230 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-31de0349-88b8-a1be-78a7-d0e746519acc -40:58.230 [INFO] TestAgent.server: federation state anti-entropy synced -40:58.533 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -40:58.534 [INFO] TestAgent: Synced node info -40:58.534 [DEBUG] TestAgent: Node info in sync -40:58.544 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-31de0349-88b8-a1be-78a7-d0e746519acc -40:58.544 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-31de0349-88b8-a1be-78a7-d0e746519acc -40:58.547 [DEBUG] TestAgent: Node info in sync -40:58.549 [INFO] TestAgent: Synced service: service=web-sidecar-proxy -40:58.550 [INFO] TestAgent: Synced service: service=web -40:58.550 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:1 -40:58.550 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:2 -40:58.551 [DEBUG] TestAgent: removed service: service=web -40:58.551 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:2 -40:58.551 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:1 -40:58.551 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy -40:58.551 [DEBUG] TestAgent: Node info in sync -40:58.552 [INFO] TestAgent: Deregistered service: service=web -40:58.552 [INFO] TestAgent: Deregistered service: service=web-sidecar-proxy -40:58.553 [INFO] TestAgent: Requesting shutdown -40:58.553 [INFO] TestAgent.server: shutting down server -40:58.553 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -40:58.553 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -40:58.553 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -40:58.553 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -40:58.553 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -40:58.553 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -40:58.553 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -40:58.553 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -40:58.553 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -40:58.553 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -40:58.553 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -40:58.553 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -40:58.553 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -40:58.553 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -40:58.553 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:40:58.553Z [INFO] agent.router.manager: shutting down -40:58.556 [INFO] TestAgent: consul server down -40:58.556 [INFO] TestAgent: shutdown complete -40:58.556 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13197 network=tcp -40:58.556 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13197 network=udp -40:58.556 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13198 network=tcp -2022-09-15T21:40:58.556Z [INFO] agent.router.manager: shutting down -40:59.057 [INFO] TestAgent: Waiting for endpoints to shut down -40:59.057 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:41.305Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:32:41.305Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:41.351Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:32:41.351Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:41.354 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4de2d199-6e22-5b74-ca73-a867c185a71b Address:127.0.0.1:24440}]" +32:41.354 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24440 [Follower]" leader= +32:41.355 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4de2d199-6e22-5b74-ca73-a867c185a71b.dc1 127.0.0.1 +32:41.356 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4de2d199-6e22-5b74-ca73-a867c185a71b 127.0.0.1 +2021-08-13T15:32:41.356Z [INFO] agent.router: Initializing LAN area manager +32:41.356 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24435 network=udp +32:41.356 [INFO] TestAgent.server: Adding LAN server: server="Node-4de2d199-6e22-5b74-ca73-a867c185a71b (Addr: tcp/127.0.0.1:24440) (DC: dc1)" +32:41.356 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4de2d199-6e22-5b74-ca73-a867c185a71b.dc1 area=wan +32:41.357 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24435 network=tcp +32:41.357 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24436 network=tcp +32:41.358 [INFO] TestAgent: started state syncer +32:41.358 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24441 network=tcp +32:41.407 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +32:41.415 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:41.415 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24440 [Candidate]" term=2 +32:41.415 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:41.415 [DEBUG] TestAgent.server.raft: vote granted: from=4de2d199-6e22-5b74-ca73-a867c185a71b term=2 tally=1 +32:41.415 [INFO] TestAgent.server.raft: election won: tally=1 +32:41.415 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24440 [Leader]" +32:41.415 [INFO] TestAgent.server: cluster leadership acquired +32:41.416 [INFO] TestAgent.server: initializing acls +32:41.416 [INFO] TestAgent.server: New leader elected: payload=Node-4de2d199-6e22-5b74-ca73-a867c185a71b +32:41.416 [INFO] TestAgent.server: Created ACL 'global-management' policy +32:41.416 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +32:41.417 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +32:41.417 [INFO] TestAgent.server: Created ACL anonymous token from configuration +32:41.417 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +32:41.417 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +32:41.418 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-4de2d199-6e22-5b74-ca73-a867c185a71b +32:41.418 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-4de2d199-6e22-5b74-ca73-a867c185a71b.dc1 +32:41.419 [INFO] TestAgent.server: Updating LAN server: server="Node-4de2d199-6e22-5b74-ca73-a867c185a71b (Addr: tcp/127.0.0.1:24440) (DC: dc1)" +32:41.419 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-4de2d199-6e22-5b74-ca73-a867c185a71b.dc1 area=wan +32:41.419 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:41.421 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:41.421 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:41.421 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:41.421 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:41.421 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:41.422 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4de2d199-6e22-5b74-ca73-a867c185a71b +32:41.422 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4de2d199-6e22-5b74-ca73-a867c185a71b +32:41.423 [INFO] TestAgent.server: federation state anti-entropy synced +32:41.423 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4de2d199-6e22-5b74-ca73-a867c185a71b +32:41.779 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-4de2d199-6e22-5b74-ca73-a867c185a71b +32:41.779 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-4de2d199-6e22-5b74-ca73-a867c185a71b +32:41.783 [INFO] TestAgent: Synced node info +32:41.784 [INFO] TestAgent: Synced service: service=web-sidecar-proxy +32:41.785 [INFO] TestAgent: Synced service: service=web +32:41.785 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:2 +32:41.785 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:1 +32:41.785 [DEBUG] TestAgent: removed service: service=web +32:41.787 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:1 +32:41.787 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:2 +32:41.787 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy +32:41.787 [DEBUG] TestAgent: Node info in sync +32:41.788 [INFO] TestAgent: Deregistered service: service=web +32:41.788 [INFO] TestAgent: Deregistered service: service=web-sidecar-proxy +32:41.788 [INFO] TestAgent: Requesting shutdown +32:41.789 [INFO] TestAgent.server: shutting down server +32:41.789 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:41.789 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:41.789 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:41.789 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +32:41.789 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +32:41.789 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:41.789 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:41.789 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:41.789 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:41.789 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:41.789 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:41.789 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:41.789 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +32:41.789 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +32:41.789 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:41.789Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:41.789Z [INFO] agent.router.manager: shutting down +32:41.789 [INFO] TestAgent: consul server down +32:41.789 [INFO] TestAgent: shutdown complete +32:41.789 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24435 network=tcp +32:41.789 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24435 network=udp +32:41.789 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24436 network=tcp +32:41.790 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:41.790 [WARN] TestAgent: Syncing node info failed.: error="No cluster leader" +32:41.790 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +32:42.290 [INFO] TestAgent: Waiting for endpoints to shut down +32:42.290 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_denied -2022-09-15T21:40:59.150Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:40:59.150Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:40:59.202Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:40:59.202Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -40:59.205 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0ecb062c-50eb-4b1e-296f-df202017f679 Address:127.0.0.1:13223}]" -40:59.206 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13223 [Follower]" leader= -40:59.207 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0ecb062c-50eb-4b1e-296f-df202017f679.dc1 127.0.0.1 -40:59.208 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0ecb062c-50eb-4b1e-296f-df202017f679 127.0.0.1 -2022-09-15T21:40:59.208Z [INFO] agent.router: Initializing LAN area manager -40:59.208 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13218 network=udp -40:59.208 [INFO] TestAgent.server: Adding LAN server: server="Node-0ecb062c-50eb-4b1e-296f-df202017f679 (Addr: tcp/127.0.0.1:13223) (DC: dc1)" -40:59.209 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0ecb062c-50eb-4b1e-296f-df202017f679.dc1 area=wan -40:59.209 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13218 network=tcp -40:59.210 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13219 network=tcp -40:59.210 [INFO] TestAgent: started state syncer -40:59.210 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13224 network=tcp -40:59.259 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -40:59.270 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -40:59.270 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13223 [Candidate]" term=2 -40:59.270 [DEBUG] TestAgent.server.raft: votes: needed=1 -40:59.270 [DEBUG] TestAgent.server.raft: vote granted: from=0ecb062c-50eb-4b1e-296f-df202017f679 term=2 tally=1 -40:59.270 [INFO] TestAgent.server.raft: election won: tally=1 -40:59.270 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13223 [Leader]" -40:59.270 [INFO] TestAgent.server: cluster leadership acquired -40:59.271 [INFO] TestAgent.server: initializing acls -40:59.277 [INFO] TestAgent.server: Created ACL 'global-management' policy -40:59.277 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -40:59.279 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -40:59.280 [INFO] TestAgent.server: Created ACL anonymous token from configuration -40:59.280 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -40:59.280 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -40:59.281 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-0ecb062c-50eb-4b1e-296f-df202017f679 -40:59.281 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-0ecb062c-50eb-4b1e-296f-df202017f679.dc1 -40:59.282 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-0ecb062c-50eb-4b1e-296f-df202017f679.dc1 area=wan -40:59.298 [INFO] TestAgent.server: New leader elected: payload=Node-0ecb062c-50eb-4b1e-296f-df202017f679 -40:59.298 [INFO] TestAgent.server: Updating LAN server: server="Node-0ecb062c-50eb-4b1e-296f-df202017f679 (Addr: tcp/127.0.0.1:13223) (DC: dc1)" -40:59.299 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -40:59.303 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -40:59.303 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -40:59.303 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -40:59.303 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -40:59.303 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -40:59.303 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0ecb062c-50eb-4b1e-296f-df202017f679 -40:59.303 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0ecb062c-50eb-4b1e-296f-df202017f679 -40:59.305 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0ecb062c-50eb-4b1e-296f-df202017f679 -40:59.327 [INFO] TestAgent.server: federation state anti-entropy synced -40:59.451 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-0ecb062c-50eb-4b1e-296f-df202017f679 -40:59.451 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-0ecb062c-50eb-4b1e-296f-df202017f679 -40:59.452 [INFO] TestAgent: Requesting shutdown -40:59.452 [INFO] TestAgent.server: shutting down server -40:59.452 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -40:59.452 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -40:59.452 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -40:59.452 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -40:59.452 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -40:59.452 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -40:59.452 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -40:59.452 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -40:59.452 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -40:59.452 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -40:59.452 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -40:59.452 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -40:59.452 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -40:59.452 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -40:59.452 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -40:59.452 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:40:59.453Z [INFO] agent.router.manager: shutting down -2022-09-15T21:40:59.453Z [INFO] agent.router.manager: shutting down -40:59.453 [INFO] TestAgent: consul server down -40:59.453 [INFO] TestAgent: shutdown complete -40:59.453 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13218 network=tcp -40:59.453 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13218 network=udp -40:59.453 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13219 network=tcp -40:59.954 [INFO] TestAgent: Waiting for endpoints to shut down -40:59.954 [INFO] TestAgent: Endpoints down -=== RUN TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_OK_for_service_but_not_for_sidecar -2022-09-15T21:40:59.133Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:40:59.133Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:40:59.190Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:40:59.190Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -40:59.193 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b23ad7fc-6098-5983-a260-7a45e21f045b Address:127.0.0.1:13216}]" -40:59.194 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13216 [Follower]" leader= -40:59.194 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b23ad7fc-6098-5983-a260-7a45e21f045b.dc1 127.0.0.1 -40:59.195 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b23ad7fc-6098-5983-a260-7a45e21f045b 127.0.0.1 -2022-09-15T21:40:59.195Z [INFO] agent.router: Initializing LAN area manager -40:59.196 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13211 network=udp -40:59.196 [INFO] TestAgent.server: Adding LAN server: server="Node-b23ad7fc-6098-5983-a260-7a45e21f045b (Addr: tcp/127.0.0.1:13216) (DC: dc1)" -40:59.196 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b23ad7fc-6098-5983-a260-7a45e21f045b.dc1 area=wan -40:59.197 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13211 network=tcp -40:59.202 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13212 network=tcp -40:59.202 [INFO] TestAgent: started state syncer -40:59.202 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13217 network=tcp -40:59.248 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -40:59.249 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -40:59.249 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13216 [Candidate]" term=2 -40:59.249 [DEBUG] TestAgent.server.raft: votes: needed=1 -40:59.249 [DEBUG] TestAgent.server.raft: vote granted: from=b23ad7fc-6098-5983-a260-7a45e21f045b term=2 tally=1 -40:59.249 [INFO] TestAgent.server.raft: election won: tally=1 -40:59.249 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13216 [Leader]" -40:59.250 [INFO] TestAgent.server: cluster leadership acquired -40:59.250 [INFO] TestAgent.server: initializing acls -40:59.250 [INFO] TestAgent.server: Created ACL 'global-management' policy -40:59.250 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -40:59.251 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -40:59.252 [INFO] TestAgent.server: Created ACL anonymous token from configuration -40:59.252 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -40:59.252 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -40:59.252 [INFO] TestAgent.server: New leader elected: payload=Node-b23ad7fc-6098-5983-a260-7a45e21f045b -40:59.252 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-b23ad7fc-6098-5983-a260-7a45e21f045b -40:59.252 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-b23ad7fc-6098-5983-a260-7a45e21f045b.dc1 -40:59.252 [INFO] TestAgent.server: Updating LAN server: server="Node-b23ad7fc-6098-5983-a260-7a45e21f045b (Addr: tcp/127.0.0.1:13216) (DC: dc1)" -40:59.253 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -40:59.254 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-b23ad7fc-6098-5983-a260-7a45e21f045b.dc1 area=wan -40:59.259 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -40:59.259 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -40:59.259 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -40:59.259 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -40:59.259 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -40:59.259 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b23ad7fc-6098-5983-a260-7a45e21f045b -40:59.259 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b23ad7fc-6098-5983-a260-7a45e21f045b -40:59.260 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b23ad7fc-6098-5983-a260-7a45e21f045b -40:59.261 [INFO] TestAgent.server: federation state anti-entropy synced -40:59.383 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -40:59.384 [INFO] TestAgent: Synced node info -40:59.384 [DEBUG] TestAgent: Node info in sync -40:59.580 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-b23ad7fc-6098-5983-a260-7a45e21f045b -40:59.581 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-b23ad7fc-6098-5983-a260-7a45e21f045b -40:59.581 [INFO] TestAgent: Requesting shutdown -40:59.581 [INFO] TestAgent.server: shutting down server -40:59.581 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -40:59.581 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -40:59.581 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -40:59.581 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -40:59.581 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -40:59.581 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -40:59.581 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -40:59.581 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -40:59.581 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -40:59.581 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -40:59.581 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -40:59.581 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -40:59.581 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -40:59.581 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -40:59.581 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:40:59.582Z [INFO] agent.router.manager: shutting down -2022-09-15T21:40:59.582Z [INFO] agent.router.manager: shutting down -40:59.582 [INFO] TestAgent: consul server down -40:59.582 [INFO] TestAgent: shutdown complete -40:59.582 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13211 network=tcp -40:59.582 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13211 network=udp -40:59.582 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13212 network=tcp -41:00.082 [INFO] TestAgent: Waiting for endpoints to shut down -41:00.082 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:42.150Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:32:42.150Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:42.167Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:32:42.167Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:42.170 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7709c32c-753e-5362-0c5b-7d004d6edf1e Address:127.0.0.1:24447}]" +32:42.170 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24447 [Follower]" leader= +32:42.171 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7709c32c-753e-5362-0c5b-7d004d6edf1e.dc1 127.0.0.1 +32:42.172 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7709c32c-753e-5362-0c5b-7d004d6edf1e 127.0.0.1 +2021-08-13T15:32:42.172Z [INFO] agent.router: Initializing LAN area manager +32:42.172 [INFO] TestAgent.server: Adding LAN server: server="Node-7709c32c-753e-5362-0c5b-7d004d6edf1e (Addr: tcp/127.0.0.1:24447) (DC: dc1)" +32:42.172 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7709c32c-753e-5362-0c5b-7d004d6edf1e.dc1 area=wan +32:42.172 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24442 network=udp +32:42.172 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24442 network=tcp +32:42.173 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24443 network=tcp +32:42.173 [INFO] TestAgent: started state syncer +32:42.173 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24448 network=tcp +32:42.222 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +32:42.237 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:42.237 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24447 [Candidate]" term=2 +32:42.238 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:42.238 [DEBUG] TestAgent.server.raft: vote granted: from=7709c32c-753e-5362-0c5b-7d004d6edf1e term=2 tally=1 +32:42.238 [INFO] TestAgent.server.raft: election won: tally=1 +32:42.238 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24447 [Leader]" +32:42.238 [INFO] TestAgent.server: cluster leadership acquired +32:42.238 [INFO] TestAgent.server: initializing acls +32:42.238 [INFO] TestAgent.server: Created ACL 'global-management' policy +32:42.238 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +32:42.239 [INFO] TestAgent.server: New leader elected: payload=Node-7709c32c-753e-5362-0c5b-7d004d6edf1e +32:42.239 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +32:42.240 [INFO] TestAgent.server: Created ACL anonymous token from configuration +32:42.240 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +32:42.240 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +32:42.240 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-7709c32c-753e-5362-0c5b-7d004d6edf1e +32:42.240 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-7709c32c-753e-5362-0c5b-7d004d6edf1e.dc1 +32:42.241 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:42.243 [INFO] TestAgent.server: Updating LAN server: server="Node-7709c32c-753e-5362-0c5b-7d004d6edf1e (Addr: tcp/127.0.0.1:24447) (DC: dc1)" +32:42.243 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-7709c32c-753e-5362-0c5b-7d004d6edf1e.dc1 area=wan +32:42.245 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:42.245 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:42.245 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:42.245 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:42.245 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:42.246 [INFO] TestAgent.server: federation state anti-entropy synced +32:42.246 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7709c32c-753e-5362-0c5b-7d004d6edf1e +32:42.246 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7709c32c-753e-5362-0c5b-7d004d6edf1e +32:42.251 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7709c32c-753e-5362-0c5b-7d004d6edf1e +32:42.421 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:42.421 [INFO] TestAgent: Synced node info +32:42.526 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-7709c32c-753e-5362-0c5b-7d004d6edf1e +32:42.526 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-7709c32c-753e-5362-0c5b-7d004d6edf1e +32:42.526 [INFO] TestAgent: Requesting shutdown +32:42.526 [INFO] TestAgent.server: shutting down server +32:42.526 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:42.526 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:42.526 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:42.526 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:42.526 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +32:42.526 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +32:42.526 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:42.527 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:42.527 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:42.527 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:42.527 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:42.527 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:42.527 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +32:42.527 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +32:42.527 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:42.527Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:42.527Z [INFO] agent.router.manager: shutting down +32:42.527 [INFO] TestAgent: consul server down +32:42.527 [INFO] TestAgent: shutdown complete +32:42.527 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24442 network=tcp +32:42.527 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24442 network=udp +32:42.527 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24443 network=tcp +32:43.027 [INFO] TestAgent: Waiting for endpoints to shut down +32:43.027 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/ACL_OK_for_service_but_not_for_sidecar -2022-09-15T21:40:59.993Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:40:59.993Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:00.017Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:41:00.017Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:00.020 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:910c743a-e8fe-6066-7ab3-ac5f2a19bcbf Address:127.0.0.1:13230}]" -41:00.021 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-910c743a-e8fe-6066-7ab3-ac5f2a19bcbf.dc1 127.0.0.1 -41:00.022 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13230 [Follower]" leader= -41:00.022 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-910c743a-e8fe-6066-7ab3-ac5f2a19bcbf 127.0.0.1 -2022-09-15T21:41:00.022Z [INFO] agent.router: Initializing LAN area manager -41:00.023 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13225 network=udp -41:00.023 [INFO] TestAgent.server: Adding LAN server: server="Node-910c743a-e8fe-6066-7ab3-ac5f2a19bcbf (Addr: tcp/127.0.0.1:13230) (DC: dc1)" -41:00.023 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-910c743a-e8fe-6066-7ab3-ac5f2a19bcbf.dc1 area=wan -41:00.023 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13225 network=tcp -41:00.024 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13226 network=tcp -41:00.024 [INFO] TestAgent: started state syncer -41:00.024 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13231 network=tcp -41:00.073 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -41:00.091 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:00.091 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13230 [Candidate]" term=2 -41:00.092 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:00.092 [DEBUG] TestAgent.server.raft: vote granted: from=910c743a-e8fe-6066-7ab3-ac5f2a19bcbf term=2 tally=1 -41:00.092 [INFO] TestAgent.server.raft: election won: tally=1 -41:00.092 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13230 [Leader]" -41:00.092 [INFO] TestAgent.server: cluster leadership acquired -41:00.092 [INFO] TestAgent.server: initializing acls -41:00.093 [INFO] TestAgent.server: Created ACL 'global-management' policy -41:00.093 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -41:00.093 [INFO] TestAgent.server: New leader elected: payload=Node-910c743a-e8fe-6066-7ab3-ac5f2a19bcbf -41:00.093 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -41:00.094 [INFO] TestAgent.server: Created ACL anonymous token from configuration -41:00.094 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -41:00.094 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -41:00.095 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-910c743a-e8fe-6066-7ab3-ac5f2a19bcbf -41:00.095 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-910c743a-e8fe-6066-7ab3-ac5f2a19bcbf.dc1 -41:00.096 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:00.098 [INFO] TestAgent.server: Updating LAN server: server="Node-910c743a-e8fe-6066-7ab3-ac5f2a19bcbf (Addr: tcp/127.0.0.1:13230) (DC: dc1)" -41:00.098 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-910c743a-e8fe-6066-7ab3-ac5f2a19bcbf.dc1 area=wan -41:00.099 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:00.099 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:00.099 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:00.099 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:00.099 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:00.099 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-910c743a-e8fe-6066-7ab3-ac5f2a19bcbf -41:00.099 [INFO] TestAgent.server: member joined, marking health alive: member=Node-910c743a-e8fe-6066-7ab3-ac5f2a19bcbf -41:00.099 [INFO] TestAgent.server: federation state anti-entropy synced -41:00.100 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-910c743a-e8fe-6066-7ab3-ac5f2a19bcbf -41:00.128 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-910c743a-e8fe-6066-7ab3-ac5f2a19bcbf -41:00.128 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-910c743a-e8fe-6066-7ab3-ac5f2a19bcbf -41:00.132 [INFO] TestAgent: Requesting shutdown -41:00.132 [INFO] TestAgent.server: shutting down server -41:00.132 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -41:00.132 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -41:00.132 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:00.132 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:00.132 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:00.132 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:00.132 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:00.132 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -41:00.132 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:00.132 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:00.132 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:00.132 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -41:00.132 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:00.132 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:00.132 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:00.133 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:00.133Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:00.133Z [INFO] agent.router.manager: shutting down -41:00.133 [INFO] TestAgent: consul server down -41:00.133 [INFO] TestAgent: shutdown complete -41:00.133 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13225 network=tcp -41:00.133 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13225 network=udp -41:00.133 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13226 network=tcp -41:00.633 [INFO] TestAgent: Waiting for endpoints to shut down -41:00.633 [INFO] TestAgent: Endpoints down -=== RUN TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_OK_for_service_and_sidecar_but_not_sidecar's_overridden_destination -2022-09-15T21:41:00.101Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:41:00.101Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:00.129Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:41:00.129Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:00.133 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a4838a39-fbfd-2055-fdc0-8fe6f50ba026 Address:127.0.0.1:13237}]" -41:00.133 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13237 [Follower]" leader= -41:00.134 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a4838a39-fbfd-2055-fdc0-8fe6f50ba026.dc1 127.0.0.1 -41:00.135 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a4838a39-fbfd-2055-fdc0-8fe6f50ba026 127.0.0.1 -2022-09-15T21:41:00.136Z [INFO] agent.router: Initializing LAN area manager -41:00.136 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13232 network=udp -41:00.138 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a4838a39-fbfd-2055-fdc0-8fe6f50ba026.dc1 area=wan -41:00.139 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13232 network=tcp -41:00.139 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13233 network=tcp -41:00.140 [INFO] TestAgent: started state syncer -41:00.140 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13238 network=tcp -41:00.140 [INFO] TestAgent.server: Adding LAN server: server="Node-a4838a39-fbfd-2055-fdc0-8fe6f50ba026 (Addr: tcp/127.0.0.1:13237) (DC: dc1)" -41:00.189 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -41:00.193 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:00.193 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13237 [Candidate]" term=2 -41:00.194 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:00.194 [DEBUG] TestAgent.server.raft: vote granted: from=a4838a39-fbfd-2055-fdc0-8fe6f50ba026 term=2 tally=1 -41:00.194 [INFO] TestAgent.server.raft: election won: tally=1 -41:00.194 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13237 [Leader]" -41:00.194 [INFO] TestAgent.server: cluster leadership acquired -41:00.194 [INFO] TestAgent.server: initializing acls -41:00.195 [INFO] TestAgent.server: Created ACL 'global-management' policy -41:00.195 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -41:00.195 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -41:00.196 [INFO] TestAgent.server: Created ACL anonymous token from configuration -41:00.196 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -41:00.196 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -41:00.196 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-a4838a39-fbfd-2055-fdc0-8fe6f50ba026 -41:00.196 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-a4838a39-fbfd-2055-fdc0-8fe6f50ba026.dc1 -41:00.197 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:00.198 [INFO] TestAgent.server: New leader elected: payload=Node-a4838a39-fbfd-2055-fdc0-8fe6f50ba026 -41:00.198 [INFO] TestAgent.server: Updating LAN server: server="Node-a4838a39-fbfd-2055-fdc0-8fe6f50ba026 (Addr: tcp/127.0.0.1:13237) (DC: dc1)" -41:00.198 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-a4838a39-fbfd-2055-fdc0-8fe6f50ba026.dc1 area=wan -41:00.201 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:00.201 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:00.201 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:00.201 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:00.201 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:00.201 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a4838a39-fbfd-2055-fdc0-8fe6f50ba026 -41:00.201 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a4838a39-fbfd-2055-fdc0-8fe6f50ba026 -41:00.202 [INFO] TestAgent.server: federation state anti-entropy synced -41:00.202 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a4838a39-fbfd-2055-fdc0-8fe6f50ba026 -41:00.230 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-a4838a39-fbfd-2055-fdc0-8fe6f50ba026 -41:00.230 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-a4838a39-fbfd-2055-fdc0-8fe6f50ba026 -41:00.233 [INFO] TestAgent: Requesting shutdown -41:00.233 [INFO] TestAgent.server: shutting down server -41:00.233 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:00.233 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -41:00.233 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -41:00.233 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:00.233 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:00.233 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:00.233 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:00.233 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:00.233 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:00.233 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:00.233 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:00.233 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -41:00.233 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:00.233 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -41:00.233 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:00.233 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:00.233Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:00.233Z [INFO] agent.router.manager: shutting down -41:00.234 [INFO] TestAgent: consul server down -41:00.234 [INFO] TestAgent: shutdown complete -41:00.234 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13232 network=tcp -41:00.234 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13232 network=udp -41:00.234 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13233 network=tcp -41:00.734 [INFO] TestAgent: Waiting for endpoints to shut down -41:00.734 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:42.308Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:32:42.308Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:42.324Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:32:42.324Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:42.327 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:61f3cb57-d1bf-3bd7-75a0-3f5a46b34cca Address:127.0.0.1:24454}]" +32:42.327 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24454 [Follower]" leader= +32:42.331 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-61f3cb57-d1bf-3bd7-75a0-3f5a46b34cca.dc1 127.0.0.1 +32:42.333 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-61f3cb57-d1bf-3bd7-75a0-3f5a46b34cca 127.0.0.1 +2021-08-13T15:32:42.333Z [INFO] agent.router: Initializing LAN area manager +32:42.334 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24449 network=udp +32:42.335 [INFO] TestAgent.server: Adding LAN server: server="Node-61f3cb57-d1bf-3bd7-75a0-3f5a46b34cca (Addr: tcp/127.0.0.1:24454) (DC: dc1)" +32:42.336 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-61f3cb57-d1bf-3bd7-75a0-3f5a46b34cca.dc1 area=wan +32:42.338 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24449 network=tcp +32:42.342 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24450 network=tcp +32:42.342 [INFO] TestAgent: started state syncer +32:42.342 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24455 network=tcp +32:42.383 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:42.383 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24454 [Candidate]" term=2 +32:42.383 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:42.383 [DEBUG] TestAgent.server.raft: vote granted: from=61f3cb57-d1bf-3bd7-75a0-3f5a46b34cca term=2 tally=1 +32:42.383 [INFO] TestAgent.server.raft: election won: tally=1 +32:42.383 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24454 [Leader]" +32:42.383 [INFO] TestAgent.server: cluster leadership acquired +32:42.384 [INFO] TestAgent.server: initializing acls +32:42.384 [INFO] TestAgent.server: Created ACL 'global-management' policy +32:42.384 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +32:42.385 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +32:42.385 [INFO] TestAgent.server: Created ACL anonymous token from configuration +32:42.385 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +32:42.385 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +32:42.385 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-61f3cb57-d1bf-3bd7-75a0-3f5a46b34cca +32:42.386 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-61f3cb57-d1bf-3bd7-75a0-3f5a46b34cca.dc1 +32:42.386 [INFO] TestAgent.server: New leader elected: payload=Node-61f3cb57-d1bf-3bd7-75a0-3f5a46b34cca +32:42.386 [INFO] TestAgent.server: Updating LAN server: server="Node-61f3cb57-d1bf-3bd7-75a0-3f5a46b34cca (Addr: tcp/127.0.0.1:24454) (DC: dc1)" +32:42.386 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-61f3cb57-d1bf-3bd7-75a0-3f5a46b34cca.dc1 area=wan +32:42.387 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:42.389 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:42.389 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:42.389 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:42.389 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:42.389 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:42.389 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-61f3cb57-d1bf-3bd7-75a0-3f5a46b34cca +32:42.389 [INFO] TestAgent.server: member joined, marking health alive: member=Node-61f3cb57-d1bf-3bd7-75a0-3f5a46b34cca +32:42.390 [INFO] TestAgent.server: federation state anti-entropy synced +32:42.390 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-61f3cb57-d1bf-3bd7-75a0-3f5a46b34cca +32:42.667 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:42.667 [INFO] TestAgent: Synced node info +32:42.667 [DEBUG] TestAgent: Node info in sync +32:42.771 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-61f3cb57-d1bf-3bd7-75a0-3f5a46b34cca +32:42.772 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-61f3cb57-d1bf-3bd7-75a0-3f5a46b34cca +32:42.772 [INFO] TestAgent: Requesting shutdown +32:42.772 [INFO] TestAgent.server: shutting down server +32:42.772 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:42.772 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +32:42.772 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +32:42.772 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:42.772 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:42.772 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:42.772 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:42.772 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:42.772 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:42.772 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:42.772 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +32:42.772 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +32:42.772 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:42.772 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:42.772 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:42.773Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:42.773Z [INFO] agent.router.manager: shutting down +32:42.773 [INFO] TestAgent: consul server down +32:42.773 [INFO] TestAgent: shutdown complete +32:42.773 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24449 network=tcp +32:42.773 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24449 network=udp +32:42.773 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24450 network=tcp +32:43.273 [INFO] TestAgent: Waiting for endpoints to shut down +32:43.273 [INFO] TestAgent: Endpoints down +=== RUN TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_OK_for_service_but_not_for_sidecar +2021-08-13T15:32:43.045Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:32:43.045Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:43.061Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:32:43.061Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:43.064 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:99729434-2eba-0f26-f752-cf46eb4eb8e6 Address:127.0.0.1:24461}]" +32:43.065 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-99729434-2eba-0f26-f752-cf46eb4eb8e6.dc1 127.0.0.1 +32:43.066 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-99729434-2eba-0f26-f752-cf46eb4eb8e6 127.0.0.1 +2021-08-13T15:32:43.066Z [INFO] agent.router: Initializing LAN area manager +32:43.066 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24456 network=udp +32:43.066 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24461 [Follower]" leader= +32:43.066 [INFO] TestAgent.server: Adding LAN server: server="Node-99729434-2eba-0f26-f752-cf46eb4eb8e6 (Addr: tcp/127.0.0.1:24461) (DC: dc1)" +32:43.066 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-99729434-2eba-0f26-f752-cf46eb4eb8e6.dc1 area=wan +32:43.067 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24456 network=tcp +32:43.068 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24457 network=tcp +32:43.068 [INFO] TestAgent: started state syncer +32:43.068 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24462 network=tcp +32:43.117 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +32:43.121 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:43.121 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24461 [Candidate]" term=2 +32:43.121 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:43.121 [DEBUG] TestAgent.server.raft: vote granted: from=99729434-2eba-0f26-f752-cf46eb4eb8e6 term=2 tally=1 +32:43.121 [INFO] TestAgent.server.raft: election won: tally=1 +32:43.121 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24461 [Leader]" +32:43.121 [INFO] TestAgent.server: cluster leadership acquired +32:43.122 [INFO] TestAgent.server: initializing acls +32:43.122 [INFO] TestAgent.server: Created ACL 'global-management' policy +32:43.122 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +32:43.123 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +32:43.123 [INFO] TestAgent.server: New leader elected: payload=Node-99729434-2eba-0f26-f752-cf46eb4eb8e6 +32:43.123 [INFO] TestAgent.server: Created ACL anonymous token from configuration +32:43.123 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +32:43.123 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +32:43.123 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-99729434-2eba-0f26-f752-cf46eb4eb8e6 +32:43.124 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-99729434-2eba-0f26-f752-cf46eb4eb8e6.dc1 +32:43.124 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:43.127 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:43.127 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:43.127 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:43.127 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:43.127 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:43.127 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-99729434-2eba-0f26-f752-cf46eb4eb8e6 +32:43.127 [INFO] TestAgent.server: member joined, marking health alive: member=Node-99729434-2eba-0f26-f752-cf46eb4eb8e6 +32:43.127 [INFO] TestAgent.server: Updating LAN server: server="Node-99729434-2eba-0f26-f752-cf46eb4eb8e6 (Addr: tcp/127.0.0.1:24461) (DC: dc1)" +32:43.127 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-99729434-2eba-0f26-f752-cf46eb4eb8e6.dc1 area=wan +32:43.128 [INFO] TestAgent.server: federation state anti-entropy synced +32:43.129 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-99729434-2eba-0f26-f752-cf46eb4eb8e6 +32:43.188 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-99729434-2eba-0f26-f752-cf46eb4eb8e6 +32:43.188 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-99729434-2eba-0f26-f752-cf46eb4eb8e6 +32:43.190 [INFO] TestAgent: Requesting shutdown +32:43.190 [INFO] TestAgent.server: shutting down server +32:43.190 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +32:43.190 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +32:43.190 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:43.190 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:43.190 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:43.190 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:43.190 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:43.191 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +32:43.191 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:43.191 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:43.191 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +32:43.191 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +32:43.191 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:43.191 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:43.191 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:43.191 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:43.191Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:43.191Z [INFO] agent.router.manager: shutting down +32:43.191 [INFO] TestAgent: consul server down +32:43.191 [INFO] TestAgent: shutdown complete +32:43.191 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24456 network=tcp +32:43.191 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24456 network=udp +32:43.191 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24457 network=tcp +32:43.691 [INFO] TestAgent: Waiting for endpoints to shut down +32:43.691 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/ACL_OK_for_service_and_sidecar_but_not_sidecar's_overridden_destination -2022-09-15T21:41:00.661Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:41:00.661Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:00.680Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:41:00.680Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:00.683 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7ed8e7b1-fdac-9e56-9148-a5ec785e242e Address:127.0.0.1:13244}]" -41:00.684 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13244 [Follower]" leader= -41:00.684 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7ed8e7b1-fdac-9e56-9148-a5ec785e242e.dc1 127.0.0.1 -41:00.685 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7ed8e7b1-fdac-9e56-9148-a5ec785e242e 127.0.0.1 -2022-09-15T21:41:00.685Z [INFO] agent.router: Initializing LAN area manager -41:00.686 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7ed8e7b1-fdac-9e56-9148-a5ec785e242e.dc1 area=wan -41:00.686 [INFO] TestAgent.server: Adding LAN server: server="Node-7ed8e7b1-fdac-9e56-9148-a5ec785e242e (Addr: tcp/127.0.0.1:13244) (DC: dc1)" -41:00.686 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13239 network=tcp -41:00.689 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13239 network=udp -41:00.690 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13240 network=tcp -41:00.690 [INFO] TestAgent: started state syncer -41:00.690 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13245 network=tcp -41:00.729 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:00.729 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13244 [Candidate]" term=2 -41:00.729 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:00.729 [DEBUG] TestAgent.server.raft: vote granted: from=7ed8e7b1-fdac-9e56-9148-a5ec785e242e term=2 tally=1 -41:00.729 [INFO] TestAgent.server.raft: election won: tally=1 -41:00.729 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13244 [Leader]" -41:00.729 [INFO] TestAgent.server: cluster leadership acquired -41:00.730 [INFO] TestAgent.server: initializing acls -41:00.730 [INFO] TestAgent.server: Created ACL 'global-management' policy -41:00.730 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -41:00.731 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -41:00.731 [INFO] TestAgent.server: Created ACL anonymous token from configuration -41:00.731 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -41:00.731 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -41:00.732 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-7ed8e7b1-fdac-9e56-9148-a5ec785e242e -41:00.732 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-7ed8e7b1-fdac-9e56-9148-a5ec785e242e.dc1 -41:00.733 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:00.734 [INFO] TestAgent.server: New leader elected: payload=Node-7ed8e7b1-fdac-9e56-9148-a5ec785e242e -41:00.734 [INFO] TestAgent.server: Updating LAN server: server="Node-7ed8e7b1-fdac-9e56-9148-a5ec785e242e (Addr: tcp/127.0.0.1:13244) (DC: dc1)" -41:00.735 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-7ed8e7b1-fdac-9e56-9148-a5ec785e242e.dc1 area=wan -41:00.743 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:00.743 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:00.743 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:00.743 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:00.743 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:00.743 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7ed8e7b1-fdac-9e56-9148-a5ec785e242e -41:00.743 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7ed8e7b1-fdac-9e56-9148-a5ec785e242e -41:00.744 [INFO] TestAgent.server: federation state anti-entropy synced -41:00.745 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7ed8e7b1-fdac-9e56-9148-a5ec785e242e -41:00.886 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-7ed8e7b1-fdac-9e56-9148-a5ec785e242e -41:00.886 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-7ed8e7b1-fdac-9e56-9148-a5ec785e242e -41:00.889 [INFO] TestAgent: Requesting shutdown -41:00.889 [INFO] TestAgent.server: shutting down server -41:00.889 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:00.889 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -41:00.889 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -41:00.889 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:00.889 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:00.889 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:00.889 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:00.889 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:00.889 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:00.889 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:00.889 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:00.889 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:00.889 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -41:00.889 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -41:00.889 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:00.889 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:00.890Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:00.890Z [INFO] agent.router.manager: shutting down -41:00.890 [INFO] TestAgent: consul server down -41:00.890 [INFO] TestAgent: shutdown complete -41:00.890 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13239 network=tcp -41:00.890 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13239 network=udp -41:00.890 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13240 network=tcp -41:01.391 [INFO] TestAgent: Waiting for endpoints to shut down -41:01.391 [INFO] TestAgent: Endpoints down -=== RUN TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_OK_for_service_but_not_for_overridden_sidecar -2022-09-15T21:41:00.757Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:41:00.757Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:00.780Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:41:00.780Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:00.795 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:08acec1e-590f-6058-66d0-c6ed083b604c Address:127.0.0.1:13251}]" -41:00.796 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-08acec1e-590f-6058-66d0-c6ed083b604c.dc1 127.0.0.1 -41:00.797 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-08acec1e-590f-6058-66d0-c6ed083b604c 127.0.0.1 -2022-09-15T21:41:00.797Z [INFO] agent.router: Initializing LAN area manager -41:00.797 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13246 network=udp -41:00.797 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13251 [Follower]" leader= -41:00.797 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-08acec1e-590f-6058-66d0-c6ed083b604c.dc1 area=wan -41:00.798 [INFO] TestAgent.server: Adding LAN server: server="Node-08acec1e-590f-6058-66d0-c6ed083b604c (Addr: tcp/127.0.0.1:13251) (DC: dc1)" -41:00.798 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13246 network=tcp -41:00.799 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13247 network=tcp -41:00.799 [INFO] TestAgent: started state syncer -41:00.799 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13252 network=tcp -41:00.838 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:00.838 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13251 [Candidate]" term=2 -41:00.838 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:00.838 [DEBUG] TestAgent.server.raft: vote granted: from=08acec1e-590f-6058-66d0-c6ed083b604c term=2 tally=1 -41:00.838 [INFO] TestAgent.server.raft: election won: tally=1 -41:00.838 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13251 [Leader]" -41:00.839 [INFO] TestAgent.server: cluster leadership acquired -41:00.839 [INFO] TestAgent.server: initializing acls -41:00.839 [INFO] TestAgent.server: New leader elected: payload=Node-08acec1e-590f-6058-66d0-c6ed083b604c -41:00.839 [INFO] TestAgent.server: Created ACL 'global-management' policy -41:00.839 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -41:00.840 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -41:00.841 [INFO] TestAgent.server: Created ACL anonymous token from configuration -41:00.841 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -41:00.841 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -41:00.841 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-08acec1e-590f-6058-66d0-c6ed083b604c -41:00.841 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-08acec1e-590f-6058-66d0-c6ed083b604c.dc1 -41:00.841 [INFO] TestAgent.server: Updating LAN server: server="Node-08acec1e-590f-6058-66d0-c6ed083b604c (Addr: tcp/127.0.0.1:13251) (DC: dc1)" -41:00.841 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-08acec1e-590f-6058-66d0-c6ed083b604c.dc1 area=wan -41:00.842 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:00.845 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:00.845 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:00.845 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:00.845 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:00.845 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:00.845 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-08acec1e-590f-6058-66d0-c6ed083b604c -41:00.845 [INFO] TestAgent.server: member joined, marking health alive: member=Node-08acec1e-590f-6058-66d0-c6ed083b604c -41:00.846 [INFO] TestAgent.server: federation state anti-entropy synced -41:00.846 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-08acec1e-590f-6058-66d0-c6ed083b604c -41:01.106 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-08acec1e-590f-6058-66d0-c6ed083b604c -41:01.106 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-08acec1e-590f-6058-66d0-c6ed083b604c -41:01.110 [INFO] TestAgent: Requesting shutdown -41:01.110 [INFO] TestAgent.server: shutting down server -41:01.110 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -41:01.110 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -41:01.110 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:01.110 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:01.110 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:01.110 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:01.110 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:01.110 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:01.110 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:01.111 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:01.111 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -41:01.111 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -41:01.111 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:01.111 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:01.111 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:01.111 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:01.111Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:01.111Z [INFO] agent.router.manager: shutting down -41:01.111 [INFO] TestAgent: consul server down -41:01.111 [INFO] TestAgent: shutdown complete -41:01.111 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13246 network=tcp -41:01.111 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13246 network=udp -41:01.111 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13247 network=tcp -41:01.614 [INFO] TestAgent: Waiting for endpoints to shut down -41:01.614 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:43.290Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:32:43.290Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:43.342Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:32:43.342Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:43.344 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:48a5d431-a151-3508-c2b4-57e608637c3d Address:127.0.0.1:24468}]" +32:43.345 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24468 [Follower]" leader= +32:43.347 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-48a5d431-a151-3508-c2b4-57e608637c3d.dc1 127.0.0.1 +32:43.348 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-48a5d431-a151-3508-c2b4-57e608637c3d 127.0.0.1 +2021-08-13T15:32:43.348Z [INFO] agent.router: Initializing LAN area manager +32:43.348 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24463 network=udp +32:43.349 [INFO] TestAgent.server: Adding LAN server: server="Node-48a5d431-a151-3508-c2b4-57e608637c3d (Addr: tcp/127.0.0.1:24468) (DC: dc1)" +32:43.349 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-48a5d431-a151-3508-c2b4-57e608637c3d.dc1 area=wan +32:43.349 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24463 network=tcp +32:43.350 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24464 network=tcp +32:43.350 [INFO] TestAgent: started state syncer +32:43.350 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24469 network=tcp +32:43.399 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +32:43.400 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:43.400 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24468 [Candidate]" term=2 +32:43.401 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:43.401 [DEBUG] TestAgent.server.raft: vote granted: from=48a5d431-a151-3508-c2b4-57e608637c3d term=2 tally=1 +32:43.401 [INFO] TestAgent.server.raft: election won: tally=1 +32:43.401 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24468 [Leader]" +32:43.401 [INFO] TestAgent.server: cluster leadership acquired +32:43.401 [INFO] TestAgent.server: initializing acls +32:43.401 [INFO] TestAgent.server: Created ACL 'global-management' policy +32:43.401 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +32:43.402 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +32:43.402 [INFO] TestAgent.server: Created ACL anonymous token from configuration +32:43.402 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +32:43.402 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +32:43.403 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-48a5d431-a151-3508-c2b4-57e608637c3d +32:43.403 [INFO] TestAgent.server: New leader elected: payload=Node-48a5d431-a151-3508-c2b4-57e608637c3d +32:43.403 [INFO] TestAgent.server: Updating LAN server: server="Node-48a5d431-a151-3508-c2b4-57e608637c3d (Addr: tcp/127.0.0.1:24468) (DC: dc1)" +32:43.403 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-48a5d431-a151-3508-c2b4-57e608637c3d.dc1 +32:43.404 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:43.406 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:43.406 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:43.406 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:43.406 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:43.406 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:43.406 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-48a5d431-a151-3508-c2b4-57e608637c3d +32:43.406 [INFO] TestAgent.server: member joined, marking health alive: member=Node-48a5d431-a151-3508-c2b4-57e608637c3d +32:43.407 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-48a5d431-a151-3508-c2b4-57e608637c3d.dc1 area=wan +32:43.408 [INFO] TestAgent.server: federation state anti-entropy synced +32:43.408 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-48a5d431-a151-3508-c2b4-57e608637c3d +32:43.414 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-48a5d431-a151-3508-c2b4-57e608637c3d +32:43.415 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-48a5d431-a151-3508-c2b4-57e608637c3d +32:43.417 [INFO] TestAgent: Requesting shutdown +32:43.417 [INFO] TestAgent.server: shutting down server +32:43.417 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:43.417 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:43.417 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:43.417 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +32:43.417 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +32:43.417 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:43.417 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:43.417 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +32:43.417 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:43.417 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:43.417 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +32:43.417 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:43.417 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:43.418 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:43.418 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +32:43.418 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:43.418Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:43.418Z [INFO] agent.router.manager: shutting down +32:43.418 [INFO] TestAgent: consul server down +32:43.418 [INFO] TestAgent: shutdown complete +32:43.418 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24463 network=tcp +32:43.418 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24463 network=udp +32:43.418 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24464 network=tcp +32:43.918 [INFO] TestAgent: Waiting for endpoints to shut down +32:43.918 [INFO] TestAgent: Endpoints down +=== RUN TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_OK_for_service_and_sidecar_but_not_sidecar's_overridden_destination +2021-08-13T15:32:43.709Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:32:43.709Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:43.725Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:32:43.725Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:43.728 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b8636c6e-e4b6-fc0b-3989-9e698d7c0cab Address:127.0.0.1:24475}]" +32:43.729 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b8636c6e-e4b6-fc0b-3989-9e698d7c0cab.dc1 127.0.0.1 +32:43.730 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b8636c6e-e4b6-fc0b-3989-9e698d7c0cab 127.0.0.1 +2021-08-13T15:32:43.730Z [INFO] agent.router: Initializing LAN area manager +32:43.730 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24470 network=udp +32:43.730 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24475 [Follower]" leader= +32:43.731 [INFO] TestAgent.server: Adding LAN server: server="Node-b8636c6e-e4b6-fc0b-3989-9e698d7c0cab (Addr: tcp/127.0.0.1:24475) (DC: dc1)" +32:43.731 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b8636c6e-e4b6-fc0b-3989-9e698d7c0cab.dc1 area=wan +32:43.731 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24470 network=tcp +32:43.732 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24471 network=tcp +32:43.732 [INFO] TestAgent: started state syncer +32:43.732 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24476 network=tcp +32:43.781 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +32:43.784 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:43.784 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24475 [Candidate]" term=2 +32:43.784 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:43.784 [DEBUG] TestAgent.server.raft: vote granted: from=b8636c6e-e4b6-fc0b-3989-9e698d7c0cab term=2 tally=1 +32:43.784 [INFO] TestAgent.server.raft: election won: tally=1 +32:43.784 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24475 [Leader]" +32:43.785 [INFO] TestAgent.server: cluster leadership acquired +32:43.785 [INFO] TestAgent.server: initializing acls +32:43.785 [INFO] TestAgent.server: Created ACL 'global-management' policy +32:43.785 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +32:43.786 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +32:43.786 [INFO] TestAgent.server: Created ACL anonymous token from configuration +32:43.786 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +32:43.786 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +32:43.786 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-b8636c6e-e4b6-fc0b-3989-9e698d7c0cab +32:43.787 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-b8636c6e-e4b6-fc0b-3989-9e698d7c0cab.dc1 +32:43.787 [INFO] TestAgent.server: New leader elected: payload=Node-b8636c6e-e4b6-fc0b-3989-9e698d7c0cab +32:43.787 [INFO] TestAgent.server: Updating LAN server: server="Node-b8636c6e-e4b6-fc0b-3989-9e698d7c0cab (Addr: tcp/127.0.0.1:24475) (DC: dc1)" +32:43.787 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-b8636c6e-e4b6-fc0b-3989-9e698d7c0cab.dc1 area=wan +32:43.788 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:43.790 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:43.790 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:43.790 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:43.790 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:43.790 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:43.790 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b8636c6e-e4b6-fc0b-3989-9e698d7c0cab +32:43.790 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b8636c6e-e4b6-fc0b-3989-9e698d7c0cab +32:43.791 [INFO] TestAgent.server: federation state anti-entropy synced +32:43.792 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b8636c6e-e4b6-fc0b-3989-9e698d7c0cab +32:44.002 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:44.003 [INFO] TestAgent: Synced node info +32:44.133 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-b8636c6e-e4b6-fc0b-3989-9e698d7c0cab +32:44.133 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-b8636c6e-e4b6-fc0b-3989-9e698d7c0cab +32:44.135 [INFO] TestAgent: Requesting shutdown +32:44.135 [INFO] TestAgent.server: shutting down server +32:44.135 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:44.135 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:44.135 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +32:44.135 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +32:44.135 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:44.135 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:44.135 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:44.135 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:44.136 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:44.136 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:44.136 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:44.136 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +32:44.136 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +32:44.136 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:44.136 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:44.136Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:44.136Z [INFO] agent.router.manager: shutting down +32:44.136 [INFO] TestAgent: consul server down +32:44.136 [INFO] TestAgent: shutdown complete +32:44.136 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24470 network=tcp +32:44.136 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24470 network=udp +32:44.136 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24471 network=tcp +32:44.636 [INFO] TestAgent: Waiting for endpoints to shut down +32:44.636 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/ACL_OK_for_service_but_not_for_overridden_sidecar -2022-09-15T21:41:01.411Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:41:01.411Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:01.429Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:41:01.429Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:01.432 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8440d76c-f279-5a95-bfe8-5df6ba71330e Address:127.0.0.1:13258}]" -41:01.433 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8440d76c-f279-5a95-bfe8-5df6ba71330e.dc1 127.0.0.1 -41:01.434 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13258 [Follower]" leader= -41:01.434 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8440d76c-f279-5a95-bfe8-5df6ba71330e 127.0.0.1 -2022-09-15T21:41:01.434Z [INFO] agent.router: Initializing LAN area manager -41:01.435 [INFO] TestAgent.server: Adding LAN server: server="Node-8440d76c-f279-5a95-bfe8-5df6ba71330e (Addr: tcp/127.0.0.1:13258) (DC: dc1)" -41:01.435 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13253 network=udp -41:01.435 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8440d76c-f279-5a95-bfe8-5df6ba71330e.dc1 area=wan -41:01.435 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13253 network=tcp -41:01.436 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13254 network=tcp -41:01.436 [INFO] TestAgent: started state syncer -41:01.436 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13259 network=tcp -41:01.486 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -41:01.497 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:01.497 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13258 [Candidate]" term=2 -41:01.498 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:01.498 [DEBUG] TestAgent.server.raft: vote granted: from=8440d76c-f279-5a95-bfe8-5df6ba71330e term=2 tally=1 -41:01.498 [INFO] TestAgent.server.raft: election won: tally=1 -41:01.498 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13258 [Leader]" -41:01.498 [INFO] TestAgent.server: cluster leadership acquired -41:01.498 [INFO] TestAgent.server: initializing acls -41:01.499 [INFO] TestAgent.server: Created ACL 'global-management' policy -41:01.499 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -41:01.499 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -41:01.500 [INFO] TestAgent.server: Created ACL anonymous token from configuration -41:01.500 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -41:01.500 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -41:01.500 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-8440d76c-f279-5a95-bfe8-5df6ba71330e -41:01.500 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-8440d76c-f279-5a95-bfe8-5df6ba71330e.dc1 -41:01.502 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:01.502 [INFO] TestAgent.server: New leader elected: payload=Node-8440d76c-f279-5a95-bfe8-5df6ba71330e -41:01.502 [INFO] TestAgent.server: Updating LAN server: server="Node-8440d76c-f279-5a95-bfe8-5df6ba71330e (Addr: tcp/127.0.0.1:13258) (DC: dc1)" -41:01.502 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-8440d76c-f279-5a95-bfe8-5df6ba71330e.dc1 area=wan -41:01.504 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:01.504 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:01.504 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:01.504 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:01.504 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:01.504 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8440d76c-f279-5a95-bfe8-5df6ba71330e -41:01.504 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8440d76c-f279-5a95-bfe8-5df6ba71330e -41:01.505 [INFO] TestAgent.server: federation state anti-entropy synced -41:01.506 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8440d76c-f279-5a95-bfe8-5df6ba71330e -41:01.686 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-8440d76c-f279-5a95-bfe8-5df6ba71330e -41:01.687 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-8440d76c-f279-5a95-bfe8-5df6ba71330e -41:01.690 [INFO] TestAgent: Requesting shutdown -41:01.690 [INFO] TestAgent.server: shutting down server -41:01.690 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:01.690 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:01.690 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -41:01.690 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -41:01.690 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:01.690 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:01.690 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:01.690 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:01.690 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:01.690 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:01.690 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:01.690 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:01.690 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -41:01.690 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -41:01.690 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:01.690 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:01.691Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:01.691Z [INFO] agent.router.manager: shutting down -41:01.691 [INFO] TestAgent: consul server down -41:01.691 [INFO] TestAgent: shutdown complete -41:01.691 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13253 network=tcp -41:01.691 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13253 network=udp -41:01.691 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13254 network=tcp -41:02.191 [INFO] TestAgent: Waiting for endpoints to shut down -41:02.191 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:43.936Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:32:43.936Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:43.982Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:32:43.982Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:43.985 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1059e517-309d-3a08-db15-48a0605f98e6 Address:127.0.0.1:24482}]" +32:43.986 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1059e517-309d-3a08-db15-48a0605f98e6.dc1 127.0.0.1 +32:43.987 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24482 [Follower]" leader= +32:43.991 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1059e517-309d-3a08-db15-48a0605f98e6 127.0.0.1 +2021-08-13T15:32:43.991Z [INFO] agent.router: Initializing LAN area manager +32:43.991 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24477 network=udp +32:43.992 [INFO] TestAgent.server: Adding LAN server: server="Node-1059e517-309d-3a08-db15-48a0605f98e6 (Addr: tcp/127.0.0.1:24482) (DC: dc1)" +32:43.992 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1059e517-309d-3a08-db15-48a0605f98e6.dc1 area=wan +32:43.992 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24477 network=tcp +32:43.993 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24478 network=tcp +32:43.993 [INFO] TestAgent: started state syncer +32:43.993 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24483 network=tcp +32:44.042 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +32:44.043 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:44.043 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24482 [Candidate]" term=2 +32:44.044 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:44.044 [DEBUG] TestAgent.server.raft: vote granted: from=1059e517-309d-3a08-db15-48a0605f98e6 term=2 tally=1 +32:44.044 [INFO] TestAgent.server.raft: election won: tally=1 +32:44.044 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24482 [Leader]" +32:44.044 [INFO] TestAgent.server: cluster leadership acquired +32:44.044 [INFO] TestAgent.server: initializing acls +32:44.044 [INFO] TestAgent.server: Created ACL 'global-management' policy +32:44.044 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +32:44.045 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +32:44.045 [INFO] TestAgent.server: Created ACL anonymous token from configuration +32:44.045 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +32:44.045 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +32:44.046 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-1059e517-309d-3a08-db15-48a0605f98e6 +32:44.046 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-1059e517-309d-3a08-db15-48a0605f98e6.dc1 +32:44.047 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:44.047 [INFO] TestAgent.server: New leader elected: payload=Node-1059e517-309d-3a08-db15-48a0605f98e6 +32:44.047 [INFO] TestAgent.server: Updating LAN server: server="Node-1059e517-309d-3a08-db15-48a0605f98e6 (Addr: tcp/127.0.0.1:24482) (DC: dc1)" +32:44.047 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-1059e517-309d-3a08-db15-48a0605f98e6.dc1 area=wan +32:44.049 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:44.049 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:44.049 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:44.049 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:44.049 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:44.049 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1059e517-309d-3a08-db15-48a0605f98e6 +32:44.050 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1059e517-309d-3a08-db15-48a0605f98e6 +32:44.050 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1059e517-309d-3a08-db15-48a0605f98e6 +32:44.051 [INFO] TestAgent.server: federation state anti-entropy synced +32:44.297 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-1059e517-309d-3a08-db15-48a0605f98e6 +32:44.297 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-1059e517-309d-3a08-db15-48a0605f98e6 +32:44.299 [INFO] TestAgent: Requesting shutdown +32:44.300 [INFO] TestAgent.server: shutting down server +32:44.300 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:44.300 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +32:44.300 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +32:44.300 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:44.300 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:44.300 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:44.300 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:44.300 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +32:44.300 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:44.300 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:44.300 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +32:44.300 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:44.300 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +32:44.300 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:44.300 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:44.300 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:44.300Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:44.300Z [INFO] agent.router.manager: shutting down +32:44.300 [INFO] TestAgent: consul server down +32:44.300 [INFO] TestAgent: shutdown complete +32:44.300 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24477 network=tcp +32:44.300 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24477 network=udp +32:44.300 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24478 network=tcp +32:44.801 [INFO] TestAgent: Waiting for endpoints to shut down +32:44.801 [INFO] TestAgent: Endpoints down +=== RUN TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_OK_for_service_but_not_for_overridden_sidecar +2021-08-13T15:32:44.820Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:32:44.820Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:44.838Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:32:44.838Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:44.845 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f5091f4d-26ed-0e8a-f5ef-b81d9df0a23b Address:127.0.0.1:24496}]" +32:44.846 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f5091f4d-26ed-0e8a-f5ef-b81d9df0a23b.dc1 127.0.0.1 +32:44.846 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f5091f4d-26ed-0e8a-f5ef-b81d9df0a23b 127.0.0.1 +2021-08-13T15:32:44.846Z [INFO] agent.router: Initializing LAN area manager +32:44.847 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24491 network=udp +32:44.847 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24496 [Follower]" leader= +32:44.847 [INFO] TestAgent.server: Adding LAN server: server="Node-f5091f4d-26ed-0e8a-f5ef-b81d9df0a23b (Addr: tcp/127.0.0.1:24496) (DC: dc1)" +32:44.847 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f5091f4d-26ed-0e8a-f5ef-b81d9df0a23b.dc1 area=wan +32:44.848 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24491 network=tcp +32:44.849 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24492 network=tcp +32:44.849 [INFO] TestAgent: started state syncer +32:44.849 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24497 network=tcp +32:44.898 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +32:44.899 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:44.900 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24496 [Candidate]" term=2 +32:44.900 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:44.900 [DEBUG] TestAgent.server.raft: vote granted: from=f5091f4d-26ed-0e8a-f5ef-b81d9df0a23b term=2 tally=1 +32:44.900 [INFO] TestAgent.server.raft: election won: tally=1 +32:44.900 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24496 [Leader]" +32:44.900 [INFO] TestAgent.server: cluster leadership acquired +32:44.900 [INFO] TestAgent.server: initializing acls +32:44.901 [INFO] TestAgent.server: Created ACL 'global-management' policy +32:44.901 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +32:44.901 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +32:44.902 [INFO] TestAgent.server: Created ACL anonymous token from configuration +32:44.902 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +32:44.902 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +32:44.902 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-f5091f4d-26ed-0e8a-f5ef-b81d9df0a23b +32:44.902 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-f5091f4d-26ed-0e8a-f5ef-b81d9df0a23b.dc1 +32:44.903 [INFO] TestAgent.server: New leader elected: payload=Node-f5091f4d-26ed-0e8a-f5ef-b81d9df0a23b +32:44.903 [INFO] TestAgent.server: Updating LAN server: server="Node-f5091f4d-26ed-0e8a-f5ef-b81d9df0a23b (Addr: tcp/127.0.0.1:24496) (DC: dc1)" +32:44.903 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-f5091f4d-26ed-0e8a-f5ef-b81d9df0a23b.dc1 area=wan +32:44.904 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:44.906 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:44.906 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:44.906 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:44.906 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:44.906 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:44.906 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f5091f4d-26ed-0e8a-f5ef-b81d9df0a23b +32:44.906 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f5091f4d-26ed-0e8a-f5ef-b81d9df0a23b +32:44.908 [INFO] TestAgent.server: federation state anti-entropy synced +32:44.908 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f5091f4d-26ed-0e8a-f5ef-b81d9df0a23b +32:44.989 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-f5091f4d-26ed-0e8a-f5ef-b81d9df0a23b +32:44.989 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-f5091f4d-26ed-0e8a-f5ef-b81d9df0a23b +32:44.992 [INFO] TestAgent: Requesting shutdown +32:44.992 [INFO] TestAgent.server: shutting down server +32:44.992 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:44.992 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:44.992 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:44.992 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:44.992 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +32:44.992 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +32:44.992 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:44.992 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +32:44.992 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:44.992 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:44.992 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +32:44.992 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:44.992 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:44.992 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:44.992 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +32:44.992 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:44.993Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:44.993Z [INFO] agent.router.manager: shutting down +32:44.993 [INFO] TestAgent: consul server down +32:44.993 [INFO] TestAgent: shutdown complete +32:44.993 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24491 network=tcp +32:44.993 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24491 network=udp +32:44.993 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24492 network=tcp +32:45.493 [INFO] TestAgent: Waiting for endpoints to shut down +32:45.493 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_OK_for_service_but_and_overridden_for_sidecar -2022-09-15T21:41:01.633Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:41:01.634Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:01.667Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:41:01.667Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:01.670 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a40004c7-3996-fb5b-8ac0-01b1324b3dcc Address:127.0.0.1:13265}]" -41:01.671 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a40004c7-3996-fb5b-8ac0-01b1324b3dcc.dc1 127.0.0.1 -41:01.672 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13265 [Follower]" leader= -41:01.673 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a40004c7-3996-fb5b-8ac0-01b1324b3dcc 127.0.0.1 -2022-09-15T21:41:01.673Z [INFO] agent.router: Initializing LAN area manager -41:01.674 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13260 network=udp -41:01.674 [INFO] TestAgent.server: Adding LAN server: server="Node-a40004c7-3996-fb5b-8ac0-01b1324b3dcc (Addr: tcp/127.0.0.1:13265) (DC: dc1)" -41:01.674 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a40004c7-3996-fb5b-8ac0-01b1324b3dcc.dc1 area=wan -41:01.674 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13260 network=tcp -41:01.675 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13261 network=tcp -41:01.675 [INFO] TestAgent: started state syncer -41:01.675 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13266 network=tcp -41:01.723 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:01.723 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13265 [Candidate]" term=2 -41:01.724 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:01.724 [DEBUG] TestAgent.server.raft: vote granted: from=a40004c7-3996-fb5b-8ac0-01b1324b3dcc term=2 tally=1 -41:01.724 [INFO] TestAgent.server.raft: election won: tally=1 -41:01.724 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13265 [Leader]" -41:01.724 [INFO] TestAgent.server: cluster leadership acquired -41:01.724 [INFO] TestAgent.server: initializing acls -41:01.724 [INFO] TestAgent.server: New leader elected: payload=Node-a40004c7-3996-fb5b-8ac0-01b1324b3dcc -41:01.725 [INFO] TestAgent.server: Created ACL 'global-management' policy -41:01.725 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -41:01.725 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -41:01.726 [INFO] TestAgent.server: Created ACL anonymous token from configuration -41:01.726 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -41:01.726 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -41:01.726 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode -41:01.726 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-a40004c7-3996-fb5b-8ac0-01b1324b3dcc -41:01.726 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-a40004c7-3996-fb5b-8ac0-01b1324b3dcc.dc1 -41:01.726 [INFO] TestAgent.server: initializing acls -41:01.726 [INFO] TestAgent.server: Updating LAN server: server="Node-a40004c7-3996-fb5b-8ac0-01b1324b3dcc (Addr: tcp/127.0.0.1:13265) (DC: dc1)" -41:01.726 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -41:01.726 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-a40004c7-3996-fb5b-8ac0-01b1324b3dcc.dc1 area=wan -41:01.727 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:01.730 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:01.730 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:01.730 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:01.730 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:01.730 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:01.730 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a40004c7-3996-fb5b-8ac0-01b1324b3dcc -41:01.730 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a40004c7-3996-fb5b-8ac0-01b1324b3dcc -41:01.731 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a40004c7-3996-fb5b-8ac0-01b1324b3dcc -41:01.732 [INFO] TestAgent.server: federation state anti-entropy synced -41:01.870 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-a40004c7-3996-fb5b-8ac0-01b1324b3dcc -41:01.870 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-a40004c7-3996-fb5b-8ac0-01b1324b3dcc -41:01.873 [INFO] TestAgent: Requesting shutdown -41:01.874 [INFO] TestAgent.server: shutting down server -41:01.874 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:01.874 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:01.874 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -41:01.874 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -41:01.874 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:01.874 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:01.874 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:01.874 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:01.874 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:01.874 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:01.874 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:01.874 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:01.874 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -41:01.874 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -41:01.874 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:01.874 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:01.875Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:01.875Z [INFO] agent.router.manager: shutting down -41:01.875 [INFO] TestAgent: consul server down -41:01.875 [INFO] TestAgent: shutdown complete -41:01.875 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13260 network=tcp -41:01.875 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13260 network=udp -41:01.875 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13261 network=tcp -41:02.378 [INFO] TestAgent: Waiting for endpoints to shut down -41:02.378 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:44.654Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:32:44.654Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:44.670Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:32:44.670Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:44.672 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1ea5257d-34fe-213b-b5a7-6a01480183ce Address:127.0.0.1:24489}]" +32:44.673 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1ea5257d-34fe-213b-b5a7-6a01480183ce.dc1 127.0.0.1 +32:44.674 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1ea5257d-34fe-213b-b5a7-6a01480183ce 127.0.0.1 +2021-08-13T15:32:44.674Z [INFO] agent.router: Initializing LAN area manager +32:44.674 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24484 network=udp +32:44.674 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24489 [Follower]" leader= +32:44.675 [INFO] TestAgent.server: Adding LAN server: server="Node-1ea5257d-34fe-213b-b5a7-6a01480183ce (Addr: tcp/127.0.0.1:24489) (DC: dc1)" +32:44.675 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1ea5257d-34fe-213b-b5a7-6a01480183ce.dc1 area=wan +32:44.675 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24484 network=tcp +32:44.676 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24485 network=tcp +32:44.676 [INFO] TestAgent: started state syncer +32:44.676 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24490 network=tcp +32:44.720 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:44.720 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24489 [Candidate]" term=2 +32:44.721 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:44.721 [DEBUG] TestAgent.server.raft: vote granted: from=1ea5257d-34fe-213b-b5a7-6a01480183ce term=2 tally=1 +32:44.721 [INFO] TestAgent.server.raft: election won: tally=1 +32:44.721 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24489 [Leader]" +32:44.721 [INFO] TestAgent.server: cluster leadership acquired +32:44.721 [INFO] TestAgent.server: initializing acls +32:44.721 [INFO] TestAgent.server: Created ACL 'global-management' policy +32:44.721 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +32:44.722 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +32:44.731 [INFO] TestAgent.server: New leader elected: payload=Node-1ea5257d-34fe-213b-b5a7-6a01480183ce +32:44.731 [INFO] TestAgent.server: initializing acls +32:44.731 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +32:44.735 [INFO] TestAgent.server: Created ACL anonymous token from configuration +32:44.735 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +32:44.735 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +32:44.735 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode +32:44.735 [INFO] TestAgent.server: Created ACL anonymous token from configuration +32:44.735 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-1ea5257d-34fe-213b-b5a7-6a01480183ce +32:44.735 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-1ea5257d-34fe-213b-b5a7-6a01480183ce.dc1 +32:44.735 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-1ea5257d-34fe-213b-b5a7-6a01480183ce.dc1 area=wan +32:44.736 [INFO] TestAgent.server: Updating LAN server: server="Node-1ea5257d-34fe-213b-b5a7-6a01480183ce (Addr: tcp/127.0.0.1:24489) (DC: dc1)" +32:44.736 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-1ea5257d-34fe-213b-b5a7-6a01480183ce +32:44.736 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-1ea5257d-34fe-213b-b5a7-6a01480183ce.dc1 +32:44.737 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:44.739 [INFO] TestAgent.server: Updating LAN server: server="Node-1ea5257d-34fe-213b-b5a7-6a01480183ce (Addr: tcp/127.0.0.1:24489) (DC: dc1)" +32:44.739 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-1ea5257d-34fe-213b-b5a7-6a01480183ce.dc1 area=wan +32:44.749 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:44.749 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:44.749 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:44.749 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:44.749 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:44.749 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1ea5257d-34fe-213b-b5a7-6a01480183ce +32:44.749 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1ea5257d-34fe-213b-b5a7-6a01480183ce +32:44.751 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1ea5257d-34fe-213b-b5a7-6a01480183ce +32:44.751 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1ea5257d-34fe-213b-b5a7-6a01480183ce +32:44.752 [INFO] TestAgent.server: federation state anti-entropy synced +32:45.010 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-1ea5257d-34fe-213b-b5a7-6a01480183ce +32:45.011 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-1ea5257d-34fe-213b-b5a7-6a01480183ce +32:45.013 [INFO] TestAgent: Requesting shutdown +32:45.013 [INFO] TestAgent.server: shutting down server +32:45.013 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +32:45.013 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +32:45.013 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:45.013 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:45.013 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:45.013 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:45.013 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:45.013 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +32:45.014 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:45.014 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:45.014 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +32:45.014 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +32:45.014 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:45.014 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:45.014 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:45.014 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:45.014Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:45.014Z [INFO] agent.router.manager: shutting down +32:45.014 [INFO] TestAgent: consul server down +32:45.014 [INFO] TestAgent: shutdown complete +32:45.014 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24484 network=tcp +32:45.014 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24484 network=udp +32:45.014 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24485 network=tcp +32:45.515 [INFO] TestAgent: Waiting for endpoints to shut down +32:45.515 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/ACL_OK_for_service_but_and_overridden_for_sidecar -2022-09-15T21:41:02.414Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:41:02.414Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:02.436Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:41:02.436Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:02.440 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:da2e14be-8c22-1561-c034-9fc563be5ee9 Address:127.0.0.1:13279}]" -41:02.440 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13279 [Follower]" leader= -41:02.441 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-da2e14be-8c22-1561-c034-9fc563be5ee9.dc1 127.0.0.1 -41:02.443 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-da2e14be-8c22-1561-c034-9fc563be5ee9 127.0.0.1 -2022-09-15T21:41:02.443Z [INFO] agent.router: Initializing LAN area manager -41:02.443 [INFO] TestAgent.server: Adding LAN server: server="Node-da2e14be-8c22-1561-c034-9fc563be5ee9 (Addr: tcp/127.0.0.1:13279) (DC: dc1)" -41:02.443 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13274 network=udp -41:02.443 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-da2e14be-8c22-1561-c034-9fc563be5ee9.dc1 area=wan -41:02.444 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13274 network=tcp -41:02.444 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13275 network=tcp -41:02.445 [INFO] TestAgent: started state syncer -41:02.445 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13280 network=tcp -41:02.494 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -41:02.506 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:02.507 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13279 [Candidate]" term=2 -41:02.507 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:02.507 [DEBUG] TestAgent.server.raft: vote granted: from=da2e14be-8c22-1561-c034-9fc563be5ee9 term=2 tally=1 -41:02.507 [INFO] TestAgent.server.raft: election won: tally=1 -41:02.507 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13279 [Leader]" -41:02.507 [INFO] TestAgent.server: cluster leadership acquired -41:02.507 [INFO] TestAgent.server: initializing acls -41:02.508 [INFO] TestAgent.server: Created ACL 'global-management' policy -41:02.508 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -41:02.508 [INFO] TestAgent.server: New leader elected: payload=Node-da2e14be-8c22-1561-c034-9fc563be5ee9 -41:02.509 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -41:02.509 [INFO] TestAgent.server: Created ACL anonymous token from configuration -41:02.509 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -41:02.509 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -41:02.510 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-da2e14be-8c22-1561-c034-9fc563be5ee9 -41:02.510 [INFO] TestAgent.server: Updating LAN server: server="Node-da2e14be-8c22-1561-c034-9fc563be5ee9 (Addr: tcp/127.0.0.1:13279) (DC: dc1)" -41:02.510 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-da2e14be-8c22-1561-c034-9fc563be5ee9.dc1 -41:02.511 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:02.513 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:02.513 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:02.513 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:02.513 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:02.513 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:02.513 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-da2e14be-8c22-1561-c034-9fc563be5ee9 -41:02.513 [INFO] TestAgent.server: member joined, marking health alive: member=Node-da2e14be-8c22-1561-c034-9fc563be5ee9 -41:02.514 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-da2e14be-8c22-1561-c034-9fc563be5ee9.dc1 area=wan -41:02.516 [INFO] TestAgent.server: federation state anti-entropy synced -41:02.516 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-da2e14be-8c22-1561-c034-9fc563be5ee9 -41:02.551 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-da2e14be-8c22-1561-c034-9fc563be5ee9 -41:02.552 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-da2e14be-8c22-1561-c034-9fc563be5ee9 -41:02.557 [DEBUG] TestAgent: added local registration for service: service=web-sidecar-proxy -41:02.558 [INFO] TestAgent: Synced node info -41:02.558 [INFO] TestAgent: Synced service: service=web -41:02.560 [INFO] TestAgent: Synced service: service=web-sidecar-proxy -41:02.560 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:1 -41:02.560 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:2 -41:02.561 [DEBUG] TestAgent: removed service: service=web -41:02.562 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:1 -41:02.562 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:2 -41:02.562 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy -41:02.562 [DEBUG] TestAgent: Node info in sync -41:02.564 [INFO] TestAgent: Deregistered service: service=web -41:02.565 [INFO] TestAgent: Deregistered service: service=web-sidecar-proxy -41:02.565 [INFO] TestAgent: Requesting shutdown -41:02.565 [INFO] TestAgent.server: shutting down server -41:02.565 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:02.565 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:02.565 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:02.565 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -41:02.565 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -41:02.565 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:02.565 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:02.565 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:02.565 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:02.565 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:02.565 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:02.565 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:02.565 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -41:02.565 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:02.565 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -41:02.565 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:02.566Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:02.566Z [INFO] agent.router.manager: shutting down -41:02.566 [INFO] TestAgent: consul server down -41:02.566 [INFO] TestAgent: shutdown complete -41:02.566 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13274 network=tcp -41:02.566 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13274 network=udp -41:02.566 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13275 network=tcp -41:03.067 [INFO] TestAgent: Waiting for endpoints to shut down -41:03.067 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:45.570Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:32:45.570Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:45.593Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:32:45.593Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:45.596 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:97d555dd-90c7-565d-a17d-731676328461 Address:127.0.0.1:24510}]" +32:45.596 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24510 [Follower]" leader= +32:45.597 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-97d555dd-90c7-565d-a17d-731676328461.dc1 127.0.0.1 +32:45.598 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-97d555dd-90c7-565d-a17d-731676328461 127.0.0.1 +2021-08-13T15:32:45.598Z [INFO] agent.router: Initializing LAN area manager +32:45.598 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24505 network=udp +32:45.598 [INFO] TestAgent.server: Adding LAN server: server="Node-97d555dd-90c7-565d-a17d-731676328461 (Addr: tcp/127.0.0.1:24510) (DC: dc1)" +32:45.599 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-97d555dd-90c7-565d-a17d-731676328461.dc1 area=wan +32:45.599 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24505 network=tcp +32:45.600 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24506 network=tcp +32:45.600 [INFO] TestAgent: started state syncer +32:45.600 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24511 network=tcp +32:45.649 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +32:45.661 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:45.661 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24510 [Candidate]" term=2 +32:45.662 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:45.662 [DEBUG] TestAgent.server.raft: vote granted: from=97d555dd-90c7-565d-a17d-731676328461 term=2 tally=1 +32:45.662 [INFO] TestAgent.server.raft: election won: tally=1 +32:45.662 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24510 [Leader]" +32:45.662 [INFO] TestAgent.server: cluster leadership acquired +32:45.662 [INFO] TestAgent.server: initializing acls +32:45.663 [INFO] TestAgent.server: Created ACL 'global-management' policy +32:45.663 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +32:45.663 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +32:45.664 [INFO] TestAgent.server: Created ACL anonymous token from configuration +32:45.664 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +32:45.664 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +32:45.664 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-97d555dd-90c7-565d-a17d-731676328461 +32:45.664 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-97d555dd-90c7-565d-a17d-731676328461.dc1 +32:45.665 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:45.667 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-97d555dd-90c7-565d-a17d-731676328461.dc1 area=wan +32:45.667 [INFO] TestAgent.server: New leader elected: payload=Node-97d555dd-90c7-565d-a17d-731676328461 +32:45.667 [INFO] TestAgent.server: Updating LAN server: server="Node-97d555dd-90c7-565d-a17d-731676328461 (Addr: tcp/127.0.0.1:24510) (DC: dc1)" +32:45.668 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:45.668 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:45.668 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:45.668 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:45.668 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:45.668 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-97d555dd-90c7-565d-a17d-731676328461 +32:45.668 [INFO] TestAgent.server: member joined, marking health alive: member=Node-97d555dd-90c7-565d-a17d-731676328461 +32:45.669 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-97d555dd-90c7-565d-a17d-731676328461 +32:45.669 [INFO] TestAgent.server: federation state anti-entropy synced +32:45.762 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-97d555dd-90c7-565d-a17d-731676328461 +32:45.763 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-97d555dd-90c7-565d-a17d-731676328461 +32:45.771 [DEBUG] TestAgent: added local registration for service: service=web-sidecar-proxy +32:45.772 [INFO] TestAgent: Synced node info +32:45.773 [INFO] TestAgent: Synced service: service=web +32:45.773 [INFO] TestAgent: Synced service: service=web-sidecar-proxy +32:45.773 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:1 +32:45.773 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:2 +32:45.775 [DEBUG] TestAgent: removed service: service=web +32:45.775 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" +32:45.775 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:1 +32:45.775 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" +32:45.775 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:2 +32:45.775 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy +32:45.775 [DEBUG] TestAgent: Node info in sync +32:45.775 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" +32:45.775 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" +32:45.776 [INFO] TestAgent: Deregistered service: service=web +32:45.777 [INFO] TestAgent: Deregistered service: service=web-sidecar-proxy +32:45.777 [INFO] TestAgent: Requesting shutdown +32:45.777 [INFO] TestAgent.server: shutting down server +32:45.777 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +32:45.777 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +32:45.777 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:45.777 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:45.777 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:45.777 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:45.777 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:45.777 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:45.777 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:45.777 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +32:45.777 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +32:45.777 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +32:45.777 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:45.777 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:45.777 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:45.777 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:45.777Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:45.777Z [INFO] agent.router.manager: shutting down +32:45.777 [INFO] TestAgent: consul server down +32:45.777 [INFO] TestAgent: shutdown complete +32:45.777 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24505 network=tcp +32:45.777 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24505 network=udp +32:45.778 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24506 network=tcp +32:46.278 [INFO] TestAgent: Waiting for endpoints to shut down +32:46.278 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/invalid_check_definition_in_sidecar -2022-09-15T21:41:02.213Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:41:02.213Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:02.232Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:41:02.232Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:02.235 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7923958d-38c1-562e-0154-e913fccc2a6e Address:127.0.0.1:13272}]" -41:02.236 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7923958d-38c1-562e-0154-e913fccc2a6e.dc1 127.0.0.1 -41:02.237 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7923958d-38c1-562e-0154-e913fccc2a6e 127.0.0.1 -2022-09-15T21:41:02.237Z [INFO] agent.router: Initializing LAN area manager -41:02.238 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13267 network=udp -41:02.238 [INFO] TestAgent.server: Adding LAN server: server="Node-7923958d-38c1-562e-0154-e913fccc2a6e (Addr: tcp/127.0.0.1:13272) (DC: dc1)" -41:02.238 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7923958d-38c1-562e-0154-e913fccc2a6e.dc1 area=wan -41:02.238 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13267 network=tcp -41:02.239 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13268 network=tcp -41:02.239 [INFO] TestAgent: started state syncer -41:02.239 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13273 network=tcp -41:02.240 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13272 [Follower]" leader= -41:02.282 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:02.282 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13272 [Candidate]" term=2 -41:02.282 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:02.282 [DEBUG] TestAgent.server.raft: vote granted: from=7923958d-38c1-562e-0154-e913fccc2a6e term=2 tally=1 -41:02.282 [INFO] TestAgent.server.raft: election won: tally=1 -41:02.282 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13272 [Leader]" -41:02.282 [INFO] TestAgent.server: cluster leadership acquired -41:02.283 [INFO] TestAgent.server: initializing acls -41:02.283 [INFO] TestAgent.server: Created ACL 'global-management' policy -41:02.283 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -41:02.284 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -41:02.285 [INFO] TestAgent.server: Created ACL anonymous token from configuration -41:02.285 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -41:02.285 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -41:02.285 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-7923958d-38c1-562e-0154-e913fccc2a6e -41:02.285 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-7923958d-38c1-562e-0154-e913fccc2a6e.dc1 -41:02.286 [INFO] TestAgent.server: New leader elected: payload=Node-7923958d-38c1-562e-0154-e913fccc2a6e -41:02.286 [INFO] TestAgent.server: Updating LAN server: server="Node-7923958d-38c1-562e-0154-e913fccc2a6e (Addr: tcp/127.0.0.1:13272) (DC: dc1)" -41:02.286 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-7923958d-38c1-562e-0154-e913fccc2a6e.dc1 area=wan -41:02.287 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:02.295 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:02.295 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:02.295 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:02.295 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:02.295 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:02.295 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7923958d-38c1-562e-0154-e913fccc2a6e -41:02.295 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7923958d-38c1-562e-0154-e913fccc2a6e -41:02.296 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7923958d-38c1-562e-0154-e913fccc2a6e -41:02.297 [INFO] TestAgent.server: federation state anti-entropy synced -41:02.465 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:02.466 [INFO] TestAgent: Synced node info -41:02.575 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-7923958d-38c1-562e-0154-e913fccc2a6e -41:02.575 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-7923958d-38c1-562e-0154-e913fccc2a6e -41:02.599 [DEBUG] TestAgent: Node info in sync -41:02.600 [INFO] TestAgent: Synced service: service=web -41:02.601 [INFO] TestAgent: Synced service: service=web-sidecar-proxy -41:02.601 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:1 -41:02.601 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:2 -41:02.602 [DEBUG] TestAgent: removed service: service=web -41:02.602 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:1 -41:02.602 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:2 -41:02.602 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy -41:02.602 [DEBUG] TestAgent: Node info in sync -41:02.603 [INFO] TestAgent: Deregistered service: service=web-sidecar-proxy -41:02.604 [INFO] TestAgent: Deregistered service: service=web -41:02.604 [INFO] TestAgent: Requesting shutdown -41:02.622 [INFO] TestAgent.server: shutting down server -41:02.622 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -41:02.622 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -41:02.622 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:02.622 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:02.622 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:02.622 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:02.622 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:02.622 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:02.622 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:02.622 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -41:02.622 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -41:02.622 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:02.622 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:02.622 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:02.622 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:02.623Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:02.623Z [INFO] agent.router.manager: shutting down -41:02.623 [INFO] TestAgent: consul server down -41:02.623 [INFO] TestAgent: shutdown complete -41:02.623 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13267 network=tcp -41:02.623 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13267 network=udp -41:02.623 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13268 network=tcp -41:03.124 [INFO] TestAgent: Waiting for endpoints to shut down -41:03.124 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:45.512Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:32:45.512Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:45.576Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:32:45.576Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:45.579 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1b53286a-e8a8-1cdc-26a0-a0bba5baf550 Address:127.0.0.1:24503}]" +32:45.579 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24503 [Follower]" leader= +32:45.584 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1b53286a-e8a8-1cdc-26a0-a0bba5baf550.dc1 127.0.0.1 +32:45.584 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1b53286a-e8a8-1cdc-26a0-a0bba5baf550 127.0.0.1 +2021-08-13T15:32:45.584Z [INFO] agent.router: Initializing LAN area manager +32:45.585 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24498 network=udp +32:45.585 [INFO] TestAgent.server: Adding LAN server: server="Node-1b53286a-e8a8-1cdc-26a0-a0bba5baf550 (Addr: tcp/127.0.0.1:24503) (DC: dc1)" +32:45.585 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1b53286a-e8a8-1cdc-26a0-a0bba5baf550.dc1 area=wan +32:45.585 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24498 network=tcp +32:45.586 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24499 network=tcp +32:45.586 [INFO] TestAgent: started state syncer +32:45.586 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24504 network=tcp +32:45.627 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:45.627 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24503 [Candidate]" term=2 +32:45.627 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:45.627 [DEBUG] TestAgent.server.raft: vote granted: from=1b53286a-e8a8-1cdc-26a0-a0bba5baf550 term=2 tally=1 +32:45.627 [INFO] TestAgent.server.raft: election won: tally=1 +32:45.627 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24503 [Leader]" +32:45.627 [INFO] TestAgent.server: cluster leadership acquired +32:45.627 [INFO] TestAgent.server: New leader elected: payload=Node-1b53286a-e8a8-1cdc-26a0-a0bba5baf550 +32:45.628 [INFO] TestAgent.server: initializing acls +32:45.628 [INFO] TestAgent.server: Created ACL 'global-management' policy +32:45.628 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +32:45.629 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +32:45.629 [INFO] TestAgent.server: Created ACL anonymous token from configuration +32:45.629 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +32:45.629 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +32:45.629 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-1b53286a-e8a8-1cdc-26a0-a0bba5baf550 +32:45.630 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-1b53286a-e8a8-1cdc-26a0-a0bba5baf550.dc1 +32:45.630 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:45.631 [INFO] TestAgent.server: Updating LAN server: server="Node-1b53286a-e8a8-1cdc-26a0-a0bba5baf550 (Addr: tcp/127.0.0.1:24503) (DC: dc1)" +32:45.631 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-1b53286a-e8a8-1cdc-26a0-a0bba5baf550.dc1 area=wan +32:45.633 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:45.633 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:45.633 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:45.633 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:45.633 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:45.633 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1b53286a-e8a8-1cdc-26a0-a0bba5baf550 +32:45.633 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1b53286a-e8a8-1cdc-26a0-a0bba5baf550 +32:45.634 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1b53286a-e8a8-1cdc-26a0-a0bba5baf550 +32:45.634 [INFO] TestAgent.server: federation state anti-entropy synced +32:45.859 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:45.859 [INFO] TestAgent: Synced node info +32:45.919 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-1b53286a-e8a8-1cdc-26a0-a0bba5baf550 +32:45.919 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-1b53286a-e8a8-1cdc-26a0-a0bba5baf550 +32:45.922 [DEBUG] TestAgent: Node info in sync +32:45.923 [INFO] TestAgent: Synced service: service=web +32:45.924 [INFO] TestAgent: Synced service: service=web-sidecar-proxy +32:45.924 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:1 +32:45.924 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:2 +32:45.925 [DEBUG] TestAgent: removed service: service=web +32:45.927 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:1 +32:45.927 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:2 +32:45.927 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy +32:45.927 [DEBUG] TestAgent: Node info in sync +32:45.938 [INFO] TestAgent: Deregistered service: service=web +32:45.940 [INFO] TestAgent: Deregistered service: service=web-sidecar-proxy +32:45.940 [INFO] TestAgent: Requesting shutdown +32:45.940 [INFO] TestAgent.server: shutting down server +32:45.940 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +32:45.940 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +32:45.940 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:45.940 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:45.940 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:45.940 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:45.940 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:45.941 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +32:45.941 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +32:45.941 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:45.941 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:45.941 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:45.941 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:45.942Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:45.942Z [INFO] agent.router.manager: shutting down +32:45.942 [INFO] TestAgent: consul server down +32:45.942 [INFO] TestAgent: shutdown complete +32:45.942 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24498 network=tcp +32:45.942 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24498 network=udp +32:45.942 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24499 network=tcp +32:45.945 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="raft is already shutdown" +32:45.945 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:46.442 [INFO] TestAgent: Waiting for endpoints to shut down +32:46.442 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/normal/invalid_check_definition_in_sidecar -2022-09-15T21:41:03.085Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:03.159Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:03.166 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:96acad01-bad6-ee30-49ef-7a6c1947a670 Address:127.0.0.1:13286}]" -41:03.167 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-96acad01-bad6-ee30-49ef-7a6c1947a670.dc1 127.0.0.1 -41:03.168 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-96acad01-bad6-ee30-49ef-7a6c1947a670 127.0.0.1 -2022-09-15T21:41:03.168Z [INFO] agent.router: Initializing LAN area manager -41:03.169 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13281 network=udp -41:03.169 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13286 [Follower]" leader= -41:03.169 [INFO] TestAgent.server: Adding LAN server: server="Node-96acad01-bad6-ee30-49ef-7a6c1947a670 (Addr: tcp/127.0.0.1:13286) (DC: dc1)" -41:03.169 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-96acad01-bad6-ee30-49ef-7a6c1947a670.dc1 area=wan -41:03.173 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13281 network=tcp -41:03.174 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13282 network=tcp -41:03.174 [INFO] TestAgent: started state syncer -41:03.175 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13287 network=tcp -41:03.206 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:03.206 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13286 [Candidate]" term=2 -41:03.206 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:03.206 [DEBUG] TestAgent.server.raft: vote granted: from=96acad01-bad6-ee30-49ef-7a6c1947a670 term=2 tally=1 -41:03.206 [INFO] TestAgent.server.raft: election won: tally=1 -41:03.206 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13286 [Leader]" -41:03.206 [INFO] TestAgent.server: cluster leadership acquired -41:03.207 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13286 -41:03.208 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:03.210 [INFO] TestAgent.server: New leader elected: payload=Node-96acad01-bad6-ee30-49ef-7a6c1947a670 -41:03.211 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:03.212 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:03.212 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:03.212 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:03.212 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:03.212 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-96acad01-bad6-ee30-49ef-7a6c1947a670 -41:03.212 [INFO] TestAgent.server: member joined, marking health alive: member=Node-96acad01-bad6-ee30-49ef-7a6c1947a670 -41:03.213 [INFO] TestAgent.server: federation state anti-entropy synced -41:03.356 [INFO] TestAgent: Requesting shutdown -41:03.357 [INFO] TestAgent.server: shutting down server -41:03.357 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:03.357 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:03.357 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:03.357 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:03.357 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:03.357 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:03.357 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:03.357 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:03.357 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:03.357 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:03.357 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:03.357 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:03.357Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:03.357Z [INFO] agent.router.manager: shutting down -41:03.357 [INFO] TestAgent: consul server down -41:03.357 [INFO] TestAgent: shutdown complete -41:03.357 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13281 network=tcp -41:03.357 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13281 network=udp -41:03.357 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13282 network=tcp -41:03.858 [INFO] TestAgent: Waiting for endpoints to shut down -41:03.858 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:46.296Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:46.314Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:46.317 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3e0ba48d-b861-b317-ec43-e945656e3b36 Address:127.0.0.1:24517}]" +32:46.318 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3e0ba48d-b861-b317-ec43-e945656e3b36.dc1 127.0.0.1 +32:46.318 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24517 [Follower]" leader= +32:46.319 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3e0ba48d-b861-b317-ec43-e945656e3b36 127.0.0.1 +2021-08-13T15:32:46.319Z [INFO] agent.router: Initializing LAN area manager +32:46.320 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24512 network=udp +32:46.320 [INFO] TestAgent.server: Adding LAN server: server="Node-3e0ba48d-b861-b317-ec43-e945656e3b36 (Addr: tcp/127.0.0.1:24517) (DC: dc1)" +32:46.320 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3e0ba48d-b861-b317-ec43-e945656e3b36.dc1 area=wan +32:46.320 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24512 network=tcp +32:46.321 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24513 network=tcp +32:46.321 [INFO] TestAgent: started state syncer +32:46.321 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24518 network=tcp +32:46.364 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:46.364 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24517 [Candidate]" term=2 +32:46.364 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:46.364 [DEBUG] TestAgent.server.raft: vote granted: from=3e0ba48d-b861-b317-ec43-e945656e3b36 term=2 tally=1 +32:46.364 [INFO] TestAgent.server.raft: election won: tally=1 +32:46.364 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24517 [Leader]" +32:46.364 [INFO] TestAgent.server: cluster leadership acquired +32:46.364 [INFO] TestAgent.server: New leader elected: payload=Node-3e0ba48d-b861-b317-ec43-e945656e3b36 +32:46.364 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24517 +32:46.366 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:46.368 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:46.368 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:46.368 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:46.368 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:46.368 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:46.368 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3e0ba48d-b861-b317-ec43-e945656e3b36 +32:46.368 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3e0ba48d-b861-b317-ec43-e945656e3b36 +32:46.369 [INFO] TestAgent.server: federation state anti-entropy synced +32:46.656 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:46.656 [INFO] TestAgent: Synced node info +32:46.656 [DEBUG] TestAgent: Node info in sync +32:46.724 [INFO] TestAgent: Requesting shutdown +32:46.724 [INFO] TestAgent.server: shutting down server +32:46.724 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:46.724 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:46.724 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:46.724 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:46.724 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:46.724 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:46.724 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:46.724 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:46.724 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:46.724 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:46.724 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:46.725Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:46.725Z [INFO] agent.router.manager: shutting down +32:46.725 [INFO] TestAgent: consul server down +32:46.725 [INFO] TestAgent: shutdown complete +32:46.725 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24512 network=tcp +32:46.725 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24512 network=udp +32:46.725 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24513 network=tcp +32:47.225 [INFO] TestAgent: Waiting for endpoints to shut down +32:47.225 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/invalid_checks_definitions_in_sidecar -2022-09-15T21:41:03.168Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:03.225Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:03.228 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:48dd0e10-d8c0-cdba-e234-c22c2b34011c Address:127.0.0.1:13293}]" -41:03.230 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-48dd0e10-d8c0-cdba-e234-c22c2b34011c.dc1 127.0.0.1 -41:03.230 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13293 [Follower]" leader= -41:03.230 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-48dd0e10-d8c0-cdba-e234-c22c2b34011c 127.0.0.1 -2022-09-15T21:41:03.231Z [INFO] agent.router: Initializing LAN area manager -41:03.231 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13288 network=udp -41:03.231 [INFO] TestAgent.server: Adding LAN server: server="Node-48dd0e10-d8c0-cdba-e234-c22c2b34011c (Addr: tcp/127.0.0.1:13293) (DC: dc1)" -41:03.231 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-48dd0e10-d8c0-cdba-e234-c22c2b34011c.dc1 area=wan -41:03.231 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13288 network=tcp -41:03.232 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13289 network=tcp -41:03.232 [INFO] TestAgent: started state syncer -41:03.233 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13294 network=tcp -41:03.282 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:03.282 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13293 [Candidate]" term=2 -41:03.282 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:03.282 [DEBUG] TestAgent.server.raft: vote granted: from=48dd0e10-d8c0-cdba-e234-c22c2b34011c term=2 tally=1 -41:03.282 [INFO] TestAgent.server.raft: election won: tally=1 -41:03.282 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13293 [Leader]" -41:03.282 [INFO] TestAgent.server: cluster leadership acquired -41:03.283 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13293 -41:03.284 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:03.286 [INFO] TestAgent.server: New leader elected: payload=Node-48dd0e10-d8c0-cdba-e234-c22c2b34011c -41:03.288 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:03.288 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:03.288 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:03.288 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:03.288 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:03.288 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-48dd0e10-d8c0-cdba-e234-c22c2b34011c -41:03.288 [INFO] TestAgent.server: member joined, marking health alive: member=Node-48dd0e10-d8c0-cdba-e234-c22c2b34011c -41:03.294 [INFO] TestAgent.server: federation state anti-entropy synced -41:03.506 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:03.507 [INFO] TestAgent: Synced node info -41:03.547 [INFO] TestAgent: Requesting shutdown -41:03.547 [INFO] TestAgent.server: shutting down server -41:03.547 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:03.547 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:03.547 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:03.547 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:03.547 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:03.547 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:03.547 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:03.547 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:03.547 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:03.547 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:03.547 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:03.547Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:03.547Z [INFO] agent.router.manager: shutting down -41:03.548 [INFO] TestAgent: consul server down -41:03.548 [INFO] TestAgent: shutdown complete -41:03.548 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13288 network=tcp -41:03.548 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13288 network=udp -41:03.548 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13289 network=tcp -41:04.048 [INFO] TestAgent: Waiting for endpoints to shut down -41:04.048 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:46.461Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:46.488Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:46.491 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:82aa3110-44b7-ac9f-0e6f-f5c9f842382b Address:127.0.0.1:24524}]" +32:46.491 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24524 [Follower]" leader= +32:46.492 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-82aa3110-44b7-ac9f-0e6f-f5c9f842382b.dc1 127.0.0.1 +32:46.493 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-82aa3110-44b7-ac9f-0e6f-f5c9f842382b 127.0.0.1 +2021-08-13T15:32:46.493Z [INFO] agent.router: Initializing LAN area manager +32:46.493 [INFO] TestAgent.server: Adding LAN server: server="Node-82aa3110-44b7-ac9f-0e6f-f5c9f842382b (Addr: tcp/127.0.0.1:24524) (DC: dc1)" +32:46.493 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-82aa3110-44b7-ac9f-0e6f-f5c9f842382b.dc1 area=wan +32:46.493 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24519 network=udp +32:46.493 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24519 network=tcp +32:46.494 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24520 network=tcp +32:46.494 [INFO] TestAgent: started state syncer +32:46.494 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24525 network=tcp +32:46.540 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:46.541 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24524 [Candidate]" term=2 +32:46.541 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:46.541 [DEBUG] TestAgent.server.raft: vote granted: from=82aa3110-44b7-ac9f-0e6f-f5c9f842382b term=2 tally=1 +32:46.541 [INFO] TestAgent.server.raft: election won: tally=1 +32:46.541 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24524 [Leader]" +32:46.541 [INFO] TestAgent.server: cluster leadership acquired +32:46.541 [INFO] TestAgent.server: New leader elected: payload=Node-82aa3110-44b7-ac9f-0e6f-f5c9f842382b +32:46.541 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24524 +32:46.543 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:46.545 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:46.545 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:46.545 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:46.545 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:46.545 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:46.545 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-82aa3110-44b7-ac9f-0e6f-f5c9f842382b +32:46.545 [INFO] TestAgent.server: member joined, marking health alive: member=Node-82aa3110-44b7-ac9f-0e6f-f5c9f842382b +32:46.546 [INFO] TestAgent.server: federation state anti-entropy synced +32:46.594 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:46.594 [INFO] TestAgent: Synced node info +32:46.595 [DEBUG] TestAgent: Node info in sync +32:46.823 [INFO] TestAgent: Requesting shutdown +32:46.824 [INFO] TestAgent.server: shutting down server +32:46.824 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:46.824 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:46.824 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:46.824 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:46.824 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:46.824 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:46.824 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:46.824 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:46.824 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:46.824 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:46.824 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:46.824Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:46.824Z [INFO] agent.router.manager: shutting down +32:46.824 [INFO] TestAgent: consul server down +32:46.824 [INFO] TestAgent: shutdown complete +32:46.824 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24519 network=tcp +32:46.824 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24519 network=udp +32:46.824 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24520 network=tcp +32:47.324 [INFO] TestAgent: Waiting for endpoints to shut down +32:47.324 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/normal/invalid_checks_definitions_in_sidecar -2022-09-15T21:41:03.877Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:03.928Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:03.932 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9c4b2f1a-4177-6567-c057-f84668e4a23b Address:127.0.0.1:13300}]" -41:03.933 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9c4b2f1a-4177-6567-c057-f84668e4a23b.dc1 127.0.0.1 -41:03.934 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13300 [Follower]" leader= -41:03.934 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9c4b2f1a-4177-6567-c057-f84668e4a23b 127.0.0.1 -2022-09-15T21:41:03.934Z [INFO] agent.router: Initializing LAN area manager -41:03.935 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13295 network=udp -41:03.935 [INFO] TestAgent.server: Adding LAN server: server="Node-9c4b2f1a-4177-6567-c057-f84668e4a23b (Addr: tcp/127.0.0.1:13300) (DC: dc1)" -41:03.935 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9c4b2f1a-4177-6567-c057-f84668e4a23b.dc1 area=wan -41:03.935 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13295 network=tcp -41:03.936 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13296 network=tcp -41:03.936 [INFO] TestAgent: started state syncer -41:03.936 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13301 network=tcp -41:03.996 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:03.996 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13300 [Candidate]" term=2 -41:03.997 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:03.997 [DEBUG] TestAgent.server.raft: vote granted: from=9c4b2f1a-4177-6567-c057-f84668e4a23b term=2 tally=1 -41:03.997 [INFO] TestAgent.server.raft: election won: tally=1 -41:03.997 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13300 [Leader]" -41:03.997 [INFO] TestAgent.server: cluster leadership acquired -41:03.997 [INFO] TestAgent.server: New leader elected: payload=Node-9c4b2f1a-4177-6567-c057-f84668e4a23b -41:03.997 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13300 -41:03.999 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:04.001 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:04.001 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:04.001 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:04.001 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:04.001 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:04.001 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9c4b2f1a-4177-6567-c057-f84668e4a23b -41:04.001 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9c4b2f1a-4177-6567-c057-f84668e4a23b -41:04.003 [INFO] TestAgent.server: federation state anti-entropy synced -41:04.051 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:04.052 [INFO] TestAgent: Synced node info -41:04.052 [DEBUG] TestAgent: Node info in sync -41:04.269 [INFO] TestAgent: Requesting shutdown -41:04.269 [INFO] TestAgent.server: shutting down server -41:04.269 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:04.269 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:04.269 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:04.269 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:04.269 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:04.270 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:04.270 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:04.270 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:04.270 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:04.270 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:04.270 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:04.270Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:04.270Z [INFO] agent.router.manager: shutting down -41:04.270 [INFO] TestAgent: consul server down -41:04.270 [INFO] TestAgent: shutdown complete -41:04.270 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13295 network=tcp -41:04.270 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13295 network=udp -41:04.270 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13296 network=tcp -41:04.771 [INFO] TestAgent: Waiting for endpoints to shut down -41:04.771 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:47.242Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:47.258Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:47.261 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:db1aa8ab-628f-ade2-255a-294fecec7909 Address:127.0.0.1:24531}]" +32:47.261 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24531 [Follower]" leader= +32:47.262 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-db1aa8ab-628f-ade2-255a-294fecec7909.dc1 127.0.0.1 +32:47.263 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-db1aa8ab-628f-ade2-255a-294fecec7909 127.0.0.1 +2021-08-13T15:32:47.263Z [INFO] agent.router: Initializing LAN area manager +32:47.263 [INFO] TestAgent.server: Adding LAN server: server="Node-db1aa8ab-628f-ade2-255a-294fecec7909 (Addr: tcp/127.0.0.1:24531) (DC: dc1)" +32:47.263 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-db1aa8ab-628f-ade2-255a-294fecec7909.dc1 area=wan +32:47.263 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24526 network=udp +32:47.263 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24526 network=tcp +32:47.264 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24527 network=tcp +32:47.264 [INFO] TestAgent: started state syncer +32:47.264 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24532 network=tcp +32:47.300 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:47.300 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24531 [Candidate]" term=2 +32:47.301 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:47.301 [DEBUG] TestAgent.server.raft: vote granted: from=db1aa8ab-628f-ade2-255a-294fecec7909 term=2 tally=1 +32:47.301 [INFO] TestAgent.server.raft: election won: tally=1 +32:47.301 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24531 [Leader]" +32:47.301 [INFO] TestAgent.server: cluster leadership acquired +32:47.301 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24531 +32:47.302 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:47.303 [INFO] TestAgent.server: New leader elected: payload=Node-db1aa8ab-628f-ade2-255a-294fecec7909 +32:47.305 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:47.305 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:47.305 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:47.305 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:47.305 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:47.305 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-db1aa8ab-628f-ade2-255a-294fecec7909 +32:47.305 [INFO] TestAgent.server: member joined, marking health alive: member=Node-db1aa8ab-628f-ade2-255a-294fecec7909 +32:47.306 [INFO] TestAgent.server: federation state anti-entropy synced +32:47.347 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:47.347 [INFO] TestAgent: Synced node info +32:47.347 [DEBUG] TestAgent: Node info in sync +32:47.611 [INFO] TestAgent: Requesting shutdown +32:47.611 [INFO] TestAgent.server: shutting down server +32:47.611 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:47.611 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:47.611 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:47.611 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:47.611 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:47.611 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:47.611 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:47.611 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:47.611 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:47.611 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:47.611 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:47.611Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:47.611Z [INFO] agent.router.manager: shutting down +32:47.612 [INFO] TestAgent: consul server down +32:47.612 [INFO] TestAgent: shutdown complete +32:47.612 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24526 network=tcp +32:47.612 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24526 network=udp +32:47.612 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24527 network=tcp +32:48.112 [INFO] TestAgent: Waiting for endpoints to shut down +32:48.112 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/invalid_check_status_in_sidecar -2022-09-15T21:41:04.066Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:04.084Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:04.087 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a0afb7ce-5d81-26ff-e15d-8ff56123de34 Address:127.0.0.1:13307}]" -41:04.088 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a0afb7ce-5d81-26ff-e15d-8ff56123de34.dc1 127.0.0.1 -41:04.089 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a0afb7ce-5d81-26ff-e15d-8ff56123de34 127.0.0.1 -2022-09-15T21:41:04.089Z [INFO] agent.router: Initializing LAN area manager -41:04.089 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13302 network=udp -41:04.089 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13307 [Follower]" leader= -41:04.090 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a0afb7ce-5d81-26ff-e15d-8ff56123de34.dc1 area=wan -41:04.090 [INFO] TestAgent.server: Adding LAN server: server="Node-a0afb7ce-5d81-26ff-e15d-8ff56123de34 (Addr: tcp/127.0.0.1:13307) (DC: dc1)" -41:04.090 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13302 network=tcp -41:04.091 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13303 network=tcp -41:04.091 [INFO] TestAgent: started state syncer -41:04.091 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13308 network=tcp -41:04.135 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:04.135 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13307 [Candidate]" term=2 -41:04.136 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:04.136 [DEBUG] TestAgent.server.raft: vote granted: from=a0afb7ce-5d81-26ff-e15d-8ff56123de34 term=2 tally=1 -41:04.136 [INFO] TestAgent.server.raft: election won: tally=1 -41:04.136 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13307 [Leader]" -41:04.136 [INFO] TestAgent.server: cluster leadership acquired -41:04.136 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13307 -41:04.137 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:04.139 [INFO] TestAgent.server: New leader elected: payload=Node-a0afb7ce-5d81-26ff-e15d-8ff56123de34 -41:04.140 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:04.140 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:04.140 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:04.140 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:04.140 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:04.140 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a0afb7ce-5d81-26ff-e15d-8ff56123de34 -41:04.140 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a0afb7ce-5d81-26ff-e15d-8ff56123de34 -41:04.141 [INFO] TestAgent.server: federation state anti-entropy synced -41:04.283 [INFO] TestAgent: Requesting shutdown -41:04.283 [INFO] TestAgent.server: shutting down server -41:04.283 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:04.283 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:04.283 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:04.283 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:04.283 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:04.283 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:04.283 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:04.283 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:04.283 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:04.283 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:04.283 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:04.283 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:04.283Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:04.283Z [INFO] agent.router.manager: shutting down -41:04.283 [INFO] TestAgent: consul server down -41:04.283 [INFO] TestAgent: shutdown complete -41:04.283 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13302 network=tcp -41:04.284 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13302 network=udp -41:04.284 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13303 network=tcp -41:04.784 [INFO] TestAgent: Waiting for endpoints to shut down -41:04.784 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:47.350Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:47.366Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:47.369 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0d24a105-6430-3aad-35f1-43b031dc9c50 Address:127.0.0.1:24538}]" +32:47.369 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24538 [Follower]" leader= +32:47.370 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0d24a105-6430-3aad-35f1-43b031dc9c50.dc1 127.0.0.1 +32:47.370 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0d24a105-6430-3aad-35f1-43b031dc9c50 127.0.0.1 +2021-08-13T15:32:47.370Z [INFO] agent.router: Initializing LAN area manager +32:47.371 [INFO] TestAgent.server: Adding LAN server: server="Node-0d24a105-6430-3aad-35f1-43b031dc9c50 (Addr: tcp/127.0.0.1:24538) (DC: dc1)" +32:47.371 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0d24a105-6430-3aad-35f1-43b031dc9c50.dc1 area=wan +32:47.371 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24533 network=tcp +32:47.371 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24533 network=udp +32:47.372 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24534 network=tcp +32:47.372 [INFO] TestAgent: started state syncer +32:47.372 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24539 network=tcp +32:47.407 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:47.407 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24538 [Candidate]" term=2 +32:47.407 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:47.407 [DEBUG] TestAgent.server.raft: vote granted: from=0d24a105-6430-3aad-35f1-43b031dc9c50 term=2 tally=1 +32:47.407 [INFO] TestAgent.server.raft: election won: tally=1 +32:47.407 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24538 [Leader]" +32:47.407 [INFO] TestAgent.server: cluster leadership acquired +32:47.407 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24538 +32:47.409 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:47.411 [INFO] TestAgent.server: New leader elected: payload=Node-0d24a105-6430-3aad-35f1-43b031dc9c50 +32:47.411 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:47.411 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:47.411 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:47.411 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:47.411 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:47.411 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0d24a105-6430-3aad-35f1-43b031dc9c50 +32:47.411 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0d24a105-6430-3aad-35f1-43b031dc9c50 +32:47.412 [INFO] TestAgent.server: federation state anti-entropy synced +32:47.413 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:47.413 [INFO] TestAgent: Synced node info +32:47.760 [INFO] TestAgent: Requesting shutdown +32:47.760 [INFO] TestAgent.server: shutting down server +32:47.760 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:47.760 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:47.760 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:47.760 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:47.760 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:47.761 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:47.761 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:47.761 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:47.761 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:47.761 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:47.761 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:47.761Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:47.761Z [INFO] agent.router.manager: shutting down +32:47.761 [INFO] TestAgent: consul server down +32:47.761 [INFO] TestAgent: shutdown complete +32:47.761 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24533 network=tcp +32:47.761 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24533 network=udp +32:47.761 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24534 network=tcp +32:48.261 [INFO] TestAgent: Waiting for endpoints to shut down +32:48.261 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/normal/invalid_check_status_in_sidecar -2022-09-15T21:41:04.791Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:04.850Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:04.853 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:38adf1c8-8991-66eb-db87-6329db3d69e9 Address:127.0.0.1:13314}]" -41:04.853 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13314 [Follower]" leader= -41:04.854 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-38adf1c8-8991-66eb-db87-6329db3d69e9.dc1 127.0.0.1 -41:04.855 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-38adf1c8-8991-66eb-db87-6329db3d69e9 127.0.0.1 -2022-09-15T21:41:04.855Z [INFO] agent.router: Initializing LAN area manager -41:04.855 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13309 network=udp -41:04.855 [INFO] TestAgent.server: Adding LAN server: server="Node-38adf1c8-8991-66eb-db87-6329db3d69e9 (Addr: tcp/127.0.0.1:13314) (DC: dc1)" -41:04.855 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-38adf1c8-8991-66eb-db87-6329db3d69e9.dc1 area=wan -41:04.856 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13309 network=tcp -41:04.856 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13310 network=tcp -41:04.857 [INFO] TestAgent: started state syncer -41:04.857 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13315 network=tcp -41:04.918 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:04.918 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13314 [Candidate]" term=2 -41:04.918 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:04.918 [DEBUG] TestAgent.server.raft: vote granted: from=38adf1c8-8991-66eb-db87-6329db3d69e9 term=2 tally=1 -41:04.918 [INFO] TestAgent.server.raft: election won: tally=1 -41:04.918 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13314 [Leader]" -41:04.919 [INFO] TestAgent.server: cluster leadership acquired -41:04.919 [INFO] TestAgent.server: New leader elected: payload=Node-38adf1c8-8991-66eb-db87-6329db3d69e9 -41:04.919 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13314 -41:04.921 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:04.924 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:04.924 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:04.924 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:04.924 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:04.924 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:04.924 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-38adf1c8-8991-66eb-db87-6329db3d69e9 -41:04.924 [INFO] TestAgent.server: member joined, marking health alive: member=Node-38adf1c8-8991-66eb-db87-6329db3d69e9 -41:04.925 [INFO] TestAgent.server: federation state anti-entropy synced -41:04.928 [INFO] TestAgent: Requesting shutdown -41:04.929 [INFO] TestAgent.server: shutting down server -41:04.929 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:04.929 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:04.929 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:04.929 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:04.929 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:04.929 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:04.929 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:04.929 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:04.929 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:04.929 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:04.929 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:04.929 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:04.929Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:04.929Z [INFO] agent.router.manager: shutting down -41:04.929 [INFO] TestAgent: consul server down -41:04.929 [INFO] TestAgent: shutdown complete -41:04.929 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13309 network=tcp -41:04.929 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13309 network=udp -41:04.930 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13310 network=tcp -41:05.430 [INFO] TestAgent: Waiting for endpoints to shut down -41:05.430 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:48.129Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:48.145Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:48.147 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b715e9f9-df94-660d-2b9d-e53a73bc122f Address:127.0.0.1:24545}]" +32:48.148 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24545 [Follower]" leader= +32:48.148 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b715e9f9-df94-660d-2b9d-e53a73bc122f.dc1 127.0.0.1 +32:48.149 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b715e9f9-df94-660d-2b9d-e53a73bc122f 127.0.0.1 +2021-08-13T15:32:48.149Z [INFO] agent.router: Initializing LAN area manager +32:48.149 [INFO] TestAgent.server: Adding LAN server: server="Node-b715e9f9-df94-660d-2b9d-e53a73bc122f (Addr: tcp/127.0.0.1:24545) (DC: dc1)" +32:48.150 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b715e9f9-df94-660d-2b9d-e53a73bc122f.dc1 area=wan +32:48.150 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24540 network=udp +32:48.150 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24540 network=tcp +32:48.151 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24541 network=tcp +32:48.151 [INFO] TestAgent: started state syncer +32:48.151 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24546 network=tcp +32:48.209 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:48.209 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24545 [Candidate]" term=2 +32:48.209 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:48.209 [DEBUG] TestAgent.server.raft: vote granted: from=b715e9f9-df94-660d-2b9d-e53a73bc122f term=2 tally=1 +32:48.209 [INFO] TestAgent.server.raft: election won: tally=1 +32:48.209 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24545 [Leader]" +32:48.209 [INFO] TestAgent.server: cluster leadership acquired +32:48.209 [INFO] TestAgent.server: New leader elected: payload=Node-b715e9f9-df94-660d-2b9d-e53a73bc122f +32:48.210 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24545 +32:48.211 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:48.213 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:48.213 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:48.213 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:48.213 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:48.213 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:48.213 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b715e9f9-df94-660d-2b9d-e53a73bc122f +32:48.213 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b715e9f9-df94-660d-2b9d-e53a73bc122f +32:48.214 [INFO] TestAgent.server: federation state anti-entropy synced +32:48.250 [INFO] TestAgent: Requesting shutdown +32:48.250 [INFO] TestAgent.server: shutting down server +32:48.250 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:48.250 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:48.250 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:48.250 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:48.250 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:48.250 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:48.250 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:48.250 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +32:48.250 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:48.250 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:48.250 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:48.250 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:48.251Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:48.251Z [INFO] agent.router.manager: shutting down +32:48.251 [INFO] TestAgent: consul server down +32:48.251 [INFO] TestAgent: shutdown complete +32:48.251 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24540 network=tcp +32:48.251 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24540 network=udp +32:48.251 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24541 network=tcp +32:48.751 [INFO] TestAgent: Waiting for endpoints to shut down +32:48.751 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/invalid_checks_status_in_sidecar -2022-09-15T21:41:04.841Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:04.866Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:04.869 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0a71b542-9277-d402-06d0-059da21d54e9 Address:127.0.0.1:13321}]" -41:04.869 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13321 [Follower]" leader= -41:04.870 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0a71b542-9277-d402-06d0-059da21d54e9.dc1 127.0.0.1 -41:04.871 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0a71b542-9277-d402-06d0-059da21d54e9 127.0.0.1 -2022-09-15T21:41:04.871Z [INFO] agent.router: Initializing LAN area manager -41:04.872 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13316 network=udp -41:04.872 [INFO] TestAgent.server: Adding LAN server: server="Node-0a71b542-9277-d402-06d0-059da21d54e9 (Addr: tcp/127.0.0.1:13321) (DC: dc1)" -41:04.872 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0a71b542-9277-d402-06d0-059da21d54e9.dc1 area=wan -41:04.872 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13316 network=tcp -41:04.873 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13317 network=tcp -41:04.873 [INFO] TestAgent: started state syncer -41:04.873 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13322 network=tcp -41:04.927 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:04.927 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13321 [Candidate]" term=2 -41:04.928 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:04.928 [DEBUG] TestAgent.server.raft: vote granted: from=0a71b542-9277-d402-06d0-059da21d54e9 term=2 tally=1 -41:04.928 [INFO] TestAgent.server.raft: election won: tally=1 -41:04.928 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13321 [Leader]" -41:04.928 [INFO] TestAgent.server: cluster leadership acquired -41:04.928 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13321 -41:04.929 [INFO] TestAgent.server: New leader elected: payload=Node-0a71b542-9277-d402-06d0-059da21d54e9 -41:04.930 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:04.932 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:04.932 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:04.932 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:04.932 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:04.932 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:04.932 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0a71b542-9277-d402-06d0-059da21d54e9 -41:04.932 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0a71b542-9277-d402-06d0-059da21d54e9 -41:04.935 [INFO] TestAgent.server: federation state anti-entropy synced -41:05.080 [INFO] TestAgent: Requesting shutdown -41:05.080 [INFO] TestAgent.server: shutting down server -41:05.080 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:05.080 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:05.080 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:05.080 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:05.080 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:05.080 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:05.080 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:05.080 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:05.080 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:05.080 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:05.080 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:05.080 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:05.081Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:05.081Z [INFO] agent.router.manager: shutting down -41:05.081 [INFO] TestAgent: consul server down -41:05.081 [INFO] TestAgent: shutdown complete -41:05.081 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13316 network=tcp -41:05.081 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13316 network=udp -41:05.081 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13317 network=tcp -41:05.584 [INFO] TestAgent: Waiting for endpoints to shut down -41:05.584 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:48.291Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:48.307Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:48.310 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fbad79c3-6440-07b6-bb15-3c10c944fc80 Address:127.0.0.1:24552}]" +32:48.310 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24552 [Follower]" leader= +32:48.311 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fbad79c3-6440-07b6-bb15-3c10c944fc80.dc1 127.0.0.1 +32:48.311 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fbad79c3-6440-07b6-bb15-3c10c944fc80 127.0.0.1 +2021-08-13T15:32:48.311Z [INFO] agent.router: Initializing LAN area manager +32:48.312 [INFO] TestAgent.server: Adding LAN server: server="Node-fbad79c3-6440-07b6-bb15-3c10c944fc80 (Addr: tcp/127.0.0.1:24552) (DC: dc1)" +32:48.312 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fbad79c3-6440-07b6-bb15-3c10c944fc80.dc1 area=wan +32:48.312 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24547 network=udp +32:48.312 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24547 network=tcp +32:48.313 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24548 network=tcp +32:48.313 [INFO] TestAgent: started state syncer +32:48.313 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24553 network=tcp +32:48.361 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:48.361 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24552 [Candidate]" term=2 +32:48.362 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:48.362 [DEBUG] TestAgent.server.raft: vote granted: from=fbad79c3-6440-07b6-bb15-3c10c944fc80 term=2 tally=1 +32:48.362 [INFO] TestAgent.server.raft: election won: tally=1 +32:48.362 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24552 [Leader]" +32:48.362 [INFO] TestAgent.server: cluster leadership acquired +32:48.362 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24552 +32:48.362 [INFO] TestAgent.server: New leader elected: payload=Node-fbad79c3-6440-07b6-bb15-3c10c944fc80 +32:48.363 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:48.366 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:48.366 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:48.366 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:48.366 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:48.366 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:48.366 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fbad79c3-6440-07b6-bb15-3c10c944fc80 +32:48.366 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fbad79c3-6440-07b6-bb15-3c10c944fc80 +32:48.367 [INFO] TestAgent.server: federation state anti-entropy synced +32:48.389 [INFO] TestAgent: Requesting shutdown +32:48.389 [INFO] TestAgent.server: shutting down server +32:48.389 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:48.389 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:48.389 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:48.389 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:48.389 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:48.389 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:48.389 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:48.389 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +32:48.389 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:48.389 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:48.389 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:48.389 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:48.390Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:48.390Z [INFO] agent.router.manager: shutting down +32:48.390 [INFO] TestAgent: consul server down +32:48.390 [INFO] TestAgent: shutdown complete +32:48.390 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24547 network=tcp +32:48.390 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24547 network=udp +32:48.390 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24548 network=tcp +32:48.890 [INFO] TestAgent: Waiting for endpoints to shut down +32:48.890 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/normal/invalid_checks_status_in_sidecar -2022-09-15T21:41:05.456Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:05.488Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:05.499 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a60e92b5-6cc6-e698-033c-5ac3ff3be81f Address:127.0.0.1:13328}]" -41:05.500 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a60e92b5-6cc6-e698-033c-5ac3ff3be81f.dc1 127.0.0.1 -41:05.501 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a60e92b5-6cc6-e698-033c-5ac3ff3be81f 127.0.0.1 -2022-09-15T21:41:05.501Z [INFO] agent.router: Initializing LAN area manager -41:05.501 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13323 network=udp -41:05.501 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13328 [Follower]" leader= -41:05.502 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a60e92b5-6cc6-e698-033c-5ac3ff3be81f.dc1 area=wan -41:05.502 [INFO] TestAgent.server: Adding LAN server: server="Node-a60e92b5-6cc6-e698-033c-5ac3ff3be81f (Addr: tcp/127.0.0.1:13328) (DC: dc1)" -41:05.503 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13323 network=tcp -41:05.509 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13324 network=tcp -41:05.509 [INFO] TestAgent: started state syncer -41:05.509 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13329 network=tcp -41:05.573 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:05.573 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13328 [Candidate]" term=2 -41:05.573 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:05.573 [DEBUG] TestAgent.server.raft: vote granted: from=a60e92b5-6cc6-e698-033c-5ac3ff3be81f term=2 tally=1 -41:05.573 [INFO] TestAgent.server.raft: election won: tally=1 -41:05.573 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13328 [Leader]" -41:05.573 [INFO] TestAgent.server: cluster leadership acquired -41:05.573 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13328 -41:05.574 [INFO] TestAgent.server: New leader elected: payload=Node-a60e92b5-6cc6-e698-033c-5ac3ff3be81f -41:05.577 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:05.590 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:05.590 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:05.591 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:05.591 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:05.591 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:05.591 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a60e92b5-6cc6-e698-033c-5ac3ff3be81f -41:05.591 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a60e92b5-6cc6-e698-033c-5ac3ff3be81f -41:05.591 [INFO] TestAgent.server: federation state anti-entropy synced -41:05.884 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:05.885 [INFO] TestAgent: Synced node info -41:05.912 [INFO] TestAgent: Requesting shutdown -41:05.912 [INFO] TestAgent.server: shutting down server -41:05.912 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:05.912 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:05.912 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:05.912 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:05.912 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:05.912 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:05.912 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:05.912 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:05.912 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:05.912 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:05.913 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:05.913Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:05.913Z [INFO] agent.router.manager: shutting down -41:05.913 [INFO] TestAgent: consul server down -41:05.913 [INFO] TestAgent: shutdown complete -41:05.913 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13323 network=tcp -41:05.913 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13323 network=udp -41:05.913 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13324 network=tcp -41:06.413 [INFO] TestAgent: Waiting for endpoints to shut down -41:06.413 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:48.772Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:48.793Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:48.796 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:81e632fc-a356-4103-0d58-b34f12277556 Address:127.0.0.1:24559}]" +32:48.797 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-81e632fc-a356-4103-0d58-b34f12277556.dc1 127.0.0.1 +32:48.798 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-81e632fc-a356-4103-0d58-b34f12277556 127.0.0.1 +2021-08-13T15:32:48.798Z [INFO] agent.router: Initializing LAN area manager +32:48.798 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24554 network=udp +32:48.798 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24559 [Follower]" leader= +32:48.799 [INFO] TestAgent.server: Adding LAN server: server="Node-81e632fc-a356-4103-0d58-b34f12277556 (Addr: tcp/127.0.0.1:24559) (DC: dc1)" +32:48.799 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-81e632fc-a356-4103-0d58-b34f12277556.dc1 area=wan +32:48.799 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24554 network=tcp +32:48.800 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24555 network=tcp +32:48.800 [INFO] TestAgent: started state syncer +32:48.800 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24560 network=tcp +32:48.853 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:48.853 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24559 [Candidate]" term=2 +32:48.854 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:48.854 [DEBUG] TestAgent.server.raft: vote granted: from=81e632fc-a356-4103-0d58-b34f12277556 term=2 tally=1 +32:48.854 [INFO] TestAgent.server.raft: election won: tally=1 +32:48.854 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24559 [Leader]" +32:48.854 [INFO] TestAgent.server: cluster leadership acquired +32:48.854 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24559 +32:48.855 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:48.858 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:48.858 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:48.858 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:48.858 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:48.858 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:48.858 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-81e632fc-a356-4103-0d58-b34f12277556 +32:48.858 [INFO] TestAgent.server: member joined, marking health alive: member=Node-81e632fc-a356-4103-0d58-b34f12277556 +32:48.859 [INFO] TestAgent.server: New leader elected: payload=Node-81e632fc-a356-4103-0d58-b34f12277556 +32:48.860 [INFO] TestAgent.server: federation state anti-entropy synced +32:49.005 [INFO] TestAgent: Requesting shutdown +32:49.005 [INFO] TestAgent.server: shutting down server +32:49.005 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:49.005 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:49.005 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:49.005 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:49.005 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:49.005 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:49.005 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:49.005 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:49.005 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:49.005 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +32:49.005 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:49.005 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:49.006Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:49.006Z [INFO] agent.router.manager: shutting down +32:49.006 [INFO] TestAgent: consul server down +32:49.006 [INFO] TestAgent: shutdown complete +32:49.006 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24554 network=tcp +32:49.006 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24554 network=udp +32:49.006 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24555 network=tcp +32:49.506 [INFO] TestAgent: Waiting for endpoints to shut down +32:49.506 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/another_service_registered_with_same_ID_as_a_sidecar_should_not_be_deregistered -2022-09-15T21:41:05.623Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:05.657Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:05.668 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cd7f0728-c27f-89b6-67be-87d668870eb8 Address:127.0.0.1:13335}]" -41:05.669 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-cd7f0728-c27f-89b6-67be-87d668870eb8.dc1 127.0.0.1 -41:05.669 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13335 [Follower]" leader= -41:05.674 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-cd7f0728-c27f-89b6-67be-87d668870eb8 127.0.0.1 -2022-09-15T21:41:05.674Z [INFO] agent.router: Initializing LAN area manager -41:05.674 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13330 network=udp -41:05.674 [INFO] TestAgent.server: Adding LAN server: server="Node-cd7f0728-c27f-89b6-67be-87d668870eb8 (Addr: tcp/127.0.0.1:13335) (DC: dc1)" -41:05.675 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-cd7f0728-c27f-89b6-67be-87d668870eb8.dc1 area=wan -41:05.675 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13330 network=tcp -41:05.676 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13331 network=tcp -41:05.676 [INFO] TestAgent: started state syncer -41:05.676 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13336 network=tcp -41:05.719 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:05.719 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13335 [Candidate]" term=2 -41:05.719 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:05.719 [DEBUG] TestAgent.server.raft: vote granted: from=cd7f0728-c27f-89b6-67be-87d668870eb8 term=2 tally=1 -41:05.719 [INFO] TestAgent.server.raft: election won: tally=1 -41:05.719 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13335 [Leader]" -41:05.719 [INFO] TestAgent.server: cluster leadership acquired -41:05.720 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13335 -41:05.721 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:05.722 [INFO] TestAgent.server: New leader elected: payload=Node-cd7f0728-c27f-89b6-67be-87d668870eb8 -41:05.724 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:05.724 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:05.724 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:05.724 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:05.724 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:05.724 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cd7f0728-c27f-89b6-67be-87d668870eb8 -41:05.724 [INFO] TestAgent.server: member joined, marking health alive: member=Node-cd7f0728-c27f-89b6-67be-87d668870eb8 -41:05.725 [INFO] TestAgent.server: federation state anti-entropy synced -41:05.993 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:05.994 [INFO] TestAgent: Synced node info -41:06.005 [INFO] TestAgent: Requesting shutdown -41:06.005 [INFO] TestAgent.server: shutting down server -41:06.005 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:06.005 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:06.005 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:06.005 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:06.005 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:06.005 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:06.005 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:06.005 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:06.005 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:06.005 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:06.005 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:06.006Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:06.006Z [INFO] agent.router.manager: shutting down -41:06.006 [INFO] TestAgent: consul server down -41:06.006 [INFO] TestAgent: shutdown complete -41:06.006 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13330 network=tcp -41:06.006 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13330 network=udp -41:06.006 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13331 network=tcp -41:06.506 [INFO] TestAgent: Waiting for endpoints to shut down -41:06.506 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:48.930Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:48.946Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:48.948 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7e28e65a-e27a-650a-54a5-e84770473433 Address:127.0.0.1:24566}]" +32:48.949 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24566 [Follower]" leader= +32:48.949 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7e28e65a-e27a-650a-54a5-e84770473433.dc1 127.0.0.1 +32:48.950 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7e28e65a-e27a-650a-54a5-e84770473433 127.0.0.1 +2021-08-13T15:32:48.950Z [INFO] agent.router: Initializing LAN area manager +32:48.950 [INFO] TestAgent.server: Adding LAN server: server="Node-7e28e65a-e27a-650a-54a5-e84770473433 (Addr: tcp/127.0.0.1:24566) (DC: dc1)" +32:48.950 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7e28e65a-e27a-650a-54a5-e84770473433.dc1 area=wan +32:48.951 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24561 network=udp +32:48.951 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24561 network=tcp +32:48.952 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24562 network=tcp +32:48.952 [INFO] TestAgent: started state syncer +32:48.952 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24567 network=tcp +32:49.012 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:49.012 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24566 [Candidate]" term=2 +32:49.012 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:49.012 [DEBUG] TestAgent.server.raft: vote granted: from=7e28e65a-e27a-650a-54a5-e84770473433 term=2 tally=1 +32:49.012 [INFO] TestAgent.server.raft: election won: tally=1 +32:49.012 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24566 [Leader]" +32:49.012 [INFO] TestAgent.server: cluster leadership acquired +32:49.013 [INFO] TestAgent.server: New leader elected: payload=Node-7e28e65a-e27a-650a-54a5-e84770473433 +32:49.013 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24566 +32:49.014 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:49.017 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:49.017 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:49.017 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:49.017 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:49.017 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:49.017 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7e28e65a-e27a-650a-54a5-e84770473433 +32:49.017 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7e28e65a-e27a-650a-54a5-e84770473433 +32:49.018 [INFO] TestAgent.server: federation state anti-entropy synced +32:49.085 [INFO] TestAgent: Requesting shutdown +32:49.086 [INFO] TestAgent.server: shutting down server +32:49.086 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:49.086 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:49.086 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:49.086 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:49.086 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:49.086 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:49.086 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:49.086 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +32:49.086 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:49.086 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:49.086 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:49.086 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:49.086Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:49.086Z [INFO] agent.router.manager: shutting down +32:49.086 [INFO] TestAgent: consul server down +32:49.086 [INFO] TestAgent: shutdown complete +32:49.086 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24561 network=tcp +32:49.086 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24561 network=udp +32:49.086 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24562 network=tcp +32:49.587 [INFO] TestAgent: Waiting for endpoints to shut down +32:49.587 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/normal/another_service_registered_with_same_ID_as_a_sidecar_should_not_be_deregistered -2022-09-15T21:41:06.432Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:06.449Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:06.452 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4b2b1c61-24ee-8d9f-4359-277ac253e24f Address:127.0.0.1:13342}]" -41:06.452 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13342 [Follower]" leader= -41:06.455 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4b2b1c61-24ee-8d9f-4359-277ac253e24f.dc1 127.0.0.1 -41:06.457 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4b2b1c61-24ee-8d9f-4359-277ac253e24f 127.0.0.1 -2022-09-15T21:41:06.457Z [INFO] agent.router: Initializing LAN area manager -41:06.460 [INFO] TestAgent.server: Adding LAN server: server="Node-4b2b1c61-24ee-8d9f-4359-277ac253e24f (Addr: tcp/127.0.0.1:13342) (DC: dc1)" -41:06.460 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4b2b1c61-24ee-8d9f-4359-277ac253e24f.dc1 area=wan -41:06.460 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13337 network=tcp -41:06.461 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13337 network=udp -41:06.465 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13338 network=tcp -41:06.465 [INFO] TestAgent: started state syncer -41:06.465 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13343 network=tcp -41:06.503 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:06.503 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13342 [Candidate]" term=2 -41:06.503 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:06.503 [DEBUG] TestAgent.server.raft: vote granted: from=4b2b1c61-24ee-8d9f-4359-277ac253e24f term=2 tally=1 -41:06.503 [INFO] TestAgent.server.raft: election won: tally=1 -41:06.503 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13342 [Leader]" -41:06.503 [INFO] TestAgent.server: cluster leadership acquired -41:06.504 [INFO] TestAgent.server: New leader elected: payload=Node-4b2b1c61-24ee-8d9f-4359-277ac253e24f -41:06.504 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13342 -41:06.505 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:06.507 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:06.508 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:06.508 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:06.508 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:06.508 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:06.508 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4b2b1c61-24ee-8d9f-4359-277ac253e24f -41:06.508 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4b2b1c61-24ee-8d9f-4359-277ac253e24f -41:06.509 [INFO] TestAgent.server: federation state anti-entropy synced -41:06.713 [INFO] TestAgent: Synced node info -41:06.713 [INFO] TestAgent: Synced service: service=web -41:06.714 [INFO] TestAgent: Synced service: service=web-sidecar-proxy -41:06.714 [DEBUG] TestAgent: removed service: service=web -41:06.714 [DEBUG] TestAgent: Node info in sync -41:06.714 [DEBUG] TestAgent: Service in sync: service=web-sidecar-proxy -41:06.715 [INFO] TestAgent: Deregistered service: service=web -41:06.715 [INFO] TestAgent: Requesting shutdown -41:06.715 [INFO] TestAgent.server: shutting down server -41:06.715 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:06.715 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:06.715 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:06.715 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:06.715 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:06.715 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:06.715 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:06.715 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:06.715 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:06.715 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:06.715 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:06.715 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:06.715Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:06.715Z [INFO] agent.router.manager: shutting down -41:06.715 [INFO] TestAgent: consul server down -41:06.715 [INFO] TestAgent: shutdown complete -41:06.715 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13337 network=tcp -41:06.716 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13337 network=udp -41:06.716 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13338 network=tcp -41:07.216 [INFO] TestAgent: Waiting for endpoints to shut down -41:07.216 [INFO] TestAgent: Endpoints down -=== RUN TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/updates_to_sidecar_should_work -2022-09-15T21:41:06.525Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:06.542Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:06.545 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:61323377-1e09-4f50-41cb-fe9362a8b149 Address:127.0.0.1:13349}]" -41:06.545 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13349 [Follower]" leader= -41:06.546 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-61323377-1e09-4f50-41cb-fe9362a8b149.dc1 127.0.0.1 -41:06.547 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-61323377-1e09-4f50-41cb-fe9362a8b149 127.0.0.1 -2022-09-15T21:41:06.547Z [INFO] agent.router: Initializing LAN area manager -41:06.547 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13344 network=udp -41:06.547 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-61323377-1e09-4f50-41cb-fe9362a8b149.dc1 area=wan -41:06.547 [INFO] TestAgent.server: Adding LAN server: server="Node-61323377-1e09-4f50-41cb-fe9362a8b149 (Addr: tcp/127.0.0.1:13349) (DC: dc1)" -41:06.548 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13344 network=tcp -41:06.549 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13345 network=tcp -41:06.549 [INFO] TestAgent: started state syncer -41:06.549 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13350 network=tcp -41:06.592 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:06.592 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13349 [Candidate]" term=2 -41:06.592 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:06.592 [DEBUG] TestAgent.server.raft: vote granted: from=61323377-1e09-4f50-41cb-fe9362a8b149 term=2 tally=1 -41:06.592 [INFO] TestAgent.server.raft: election won: tally=1 -41:06.592 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13349 [Leader]" -41:06.593 [INFO] TestAgent.server: cluster leadership acquired -41:06.593 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13349 -41:06.594 [INFO] TestAgent.server: New leader elected: payload=Node-61323377-1e09-4f50-41cb-fe9362a8b149 -41:06.594 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:06.597 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:06.597 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:06.597 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:06.597 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:06.597 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:06.597 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-61323377-1e09-4f50-41cb-fe9362a8b149 -41:06.597 [INFO] TestAgent.server: member joined, marking health alive: member=Node-61323377-1e09-4f50-41cb-fe9362a8b149 -41:06.598 [INFO] TestAgent.server: federation state anti-entropy synced -41:06.807 [INFO] TestAgent: Synced node info -41:06.808 [INFO] TestAgent: Synced service: service=web-sidecar-proxy -41:06.808 [INFO] TestAgent: Synced service: service=web -41:06.809 [DEBUG] TestAgent: removed service: service=web -41:06.809 [DEBUG] TestAgent: Node info in sync -41:06.809 [DEBUG] TestAgent: Service in sync: service=web-sidecar-proxy -41:06.809 [INFO] TestAgent: Deregistered service: service=web -41:06.809 [INFO] TestAgent: Requesting shutdown -41:06.809 [INFO] TestAgent.server: shutting down server -41:06.809 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:06.809 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:06.809 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:06.809 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:06.809 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:06.809 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:06.809 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:06.809 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:06.809 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:06.809 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:06.809 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:06.810 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:06.810Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:06.810Z [INFO] agent.router.manager: shutting down -41:06.810 [INFO] TestAgent: consul server down -41:06.810 [INFO] TestAgent: shutdown complete -41:06.810 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13344 network=tcp -41:06.810 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13344 network=udp -41:06.810 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13345 network=tcp -41:07.311 [INFO] TestAgent: Waiting for endpoints to shut down -41:07.311 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:49.622Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:49.645Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:49.648 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d59d37de-d6ab-0280-d180-c4bc7520efbe Address:127.0.0.1:24580}]" +32:49.648 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24580 [Follower]" leader= +32:49.649 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d59d37de-d6ab-0280-d180-c4bc7520efbe.dc1 127.0.0.1 +32:49.649 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d59d37de-d6ab-0280-d180-c4bc7520efbe 127.0.0.1 +2021-08-13T15:32:49.649Z [INFO] agent.router: Initializing LAN area manager +32:49.650 [INFO] TestAgent.server: Adding LAN server: server="Node-d59d37de-d6ab-0280-d180-c4bc7520efbe (Addr: tcp/127.0.0.1:24580) (DC: dc1)" +32:49.650 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d59d37de-d6ab-0280-d180-c4bc7520efbe.dc1 area=wan +32:49.650 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24575 network=udp +32:49.650 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24575 network=tcp +32:49.651 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24576 network=tcp +32:49.651 [INFO] TestAgent: started state syncer +32:49.651 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24581 network=tcp +32:49.697 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:49.697 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24580 [Candidate]" term=2 +32:49.698 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:49.698 [DEBUG] TestAgent.server.raft: vote granted: from=d59d37de-d6ab-0280-d180-c4bc7520efbe term=2 tally=1 +32:49.698 [INFO] TestAgent.server.raft: election won: tally=1 +32:49.698 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24580 [Leader]" +32:49.698 [INFO] TestAgent.server: cluster leadership acquired +32:49.698 [INFO] TestAgent.server: New leader elected: payload=Node-d59d37de-d6ab-0280-d180-c4bc7520efbe +32:49.698 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24580 +32:49.699 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:49.702 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:49.702 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:49.702 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:49.702 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:49.702 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:49.702 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d59d37de-d6ab-0280-d180-c4bc7520efbe +32:49.702 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d59d37de-d6ab-0280-d180-c4bc7520efbe +32:49.703 [INFO] TestAgent.server: federation state anti-entropy synced +32:49.861 [INFO] TestAgent: Synced node info +32:49.861 [INFO] TestAgent: Synced service: service=web +32:49.862 [INFO] TestAgent: Synced service: service=web-sidecar-proxy +32:49.862 [DEBUG] TestAgent: removed service: service=web +32:49.862 [DEBUG] TestAgent: Node info in sync +32:49.862 [DEBUG] TestAgent: Service in sync: service=web-sidecar-proxy +32:49.863 [INFO] TestAgent: Deregistered service: service=web +32:49.863 [INFO] TestAgent: Requesting shutdown +32:49.863 [INFO] TestAgent.server: shutting down server +32:49.863 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:49.863 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:49.863 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:49.863 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:49.863 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:49.863 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:49.863 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:49.863 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +32:49.863 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:49.863 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:49.863 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:49.863 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:49.863Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:49.863Z [INFO] agent.router.manager: shutting down +32:49.863 [INFO] TestAgent: consul server down +32:49.863 [INFO] TestAgent: shutdown complete +32:49.863 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24575 network=tcp +32:49.863 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24575 network=udp +32:49.863 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24576 network=tcp +32:50.364 [INFO] TestAgent: Waiting for endpoints to shut down +32:50.364 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/normal/updates_to_sidecar_should_work -2022-09-15T21:41:07.236Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:07.267Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:07.271 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ff7cc37c-1e13-ebe8-bb82-717e9cdc0d22 Address:127.0.0.1:13356}]" -41:07.274 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13356 [Follower]" leader= -41:07.287 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ff7cc37c-1e13-ebe8-bb82-717e9cdc0d22.dc1 127.0.0.1 -41:07.288 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ff7cc37c-1e13-ebe8-bb82-717e9cdc0d22 127.0.0.1 -2022-09-15T21:41:07.288Z [INFO] agent.router: Initializing LAN area manager -41:07.288 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13351 network=udp -41:07.289 [INFO] TestAgent.server: Adding LAN server: server="Node-ff7cc37c-1e13-ebe8-bb82-717e9cdc0d22 (Addr: tcp/127.0.0.1:13356) (DC: dc1)" -41:07.289 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ff7cc37c-1e13-ebe8-bb82-717e9cdc0d22.dc1 area=wan -41:07.289 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13351 network=tcp -41:07.291 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13352 network=tcp -41:07.291 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13357 network=tcp -41:07.291 [INFO] TestAgent: started state syncer -41:07.338 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:07.338 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13356 [Candidate]" term=2 -41:07.338 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:07.338 [DEBUG] TestAgent.server.raft: vote granted: from=ff7cc37c-1e13-ebe8-bb82-717e9cdc0d22 term=2 tally=1 -41:07.338 [INFO] TestAgent.server.raft: election won: tally=1 -41:07.338 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13356 [Leader]" -41:07.338 [INFO] TestAgent.server: cluster leadership acquired -41:07.338 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13356 -41:07.340 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:07.342 [INFO] TestAgent.server: New leader elected: payload=Node-ff7cc37c-1e13-ebe8-bb82-717e9cdc0d22 -41:07.343 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:07.343 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:07.343 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:07.343 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:07.343 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:07.343 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ff7cc37c-1e13-ebe8-bb82-717e9cdc0d22 -41:07.343 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ff7cc37c-1e13-ebe8-bb82-717e9cdc0d22 -41:07.344 [INFO] TestAgent.server: federation state anti-entropy synced -41:07.398 [DEBUG] TestAgent: added local registration for service: service=web-sidecar-proxy -41:07.398 [INFO] TestAgent: Synced node info -41:07.400 [INFO] TestAgent: Synced service: service=web-sidecar-proxy -41:07.400 [INFO] TestAgent: Synced service: service=web -41:07.400 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:1 -41:07.400 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:2 -41:07.402 [DEBUG] TestAgent: removed service: service=web -41:07.403 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:1 -41:07.403 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:2 -41:07.403 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy -41:07.403 [DEBUG] TestAgent: Node info in sync -41:07.404 [INFO] TestAgent: Deregistered service: service=web-sidecar-proxy -41:07.404 [INFO] TestAgent: Deregistered service: service=web -41:07.404 [INFO] TestAgent: Requesting shutdown -41:07.404 [INFO] TestAgent.server: shutting down server -41:07.404 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:07.404 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:07.404 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:07.404 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:07.404 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:07.405 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:07.405 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:07.405 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:07.405 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:07.405 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:07.405 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:07.405 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:07.405Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:07.405Z [INFO] agent.router.manager: shutting down -41:07.405 [INFO] TestAgent: consul server down -41:07.405 [INFO] TestAgent: shutdown complete -41:07.405 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13351 network=tcp -41:07.405 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13351 network=udp -41:07.405 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13352 network=tcp -41:07.907 [INFO] TestAgent: Waiting for endpoints to shut down -41:07.907 [INFO] TestAgent: Endpoints down -=== RUN TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/update_that_removes_sidecar_should_NOT_deregister_it -2022-09-15T21:41:07.330Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:07.352Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:07.355 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e1161907-3e1e-3fca-1b05-7704be5d5250 Address:127.0.0.1:13363}]" -41:07.356 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e1161907-3e1e-3fca-1b05-7704be5d5250.dc1 127.0.0.1 -41:07.357 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e1161907-3e1e-3fca-1b05-7704be5d5250 127.0.0.1 -2022-09-15T21:41:07.357Z [INFO] agent.router: Initializing LAN area manager -41:07.357 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13358 network=udp -41:07.357 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13363 [Follower]" leader= -41:07.358 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e1161907-3e1e-3fca-1b05-7704be5d5250.dc1 area=wan -41:07.358 [INFO] TestAgent.server: Adding LAN server: server="Node-e1161907-3e1e-3fca-1b05-7704be5d5250 (Addr: tcp/127.0.0.1:13363) (DC: dc1)" -41:07.358 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13358 network=tcp -41:07.359 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13359 network=tcp -41:07.359 [INFO] TestAgent: started state syncer -41:07.360 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13364 network=tcp -41:07.400 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:07.400 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13363 [Candidate]" term=2 -41:07.400 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:07.400 [DEBUG] TestAgent.server.raft: vote granted: from=e1161907-3e1e-3fca-1b05-7704be5d5250 term=2 tally=1 -41:07.400 [INFO] TestAgent.server.raft: election won: tally=1 -41:07.400 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13363 [Leader]" -41:07.400 [INFO] TestAgent.server: cluster leadership acquired -41:07.400 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13363 -41:07.401 [INFO] TestAgent.server: New leader elected: payload=Node-e1161907-3e1e-3fca-1b05-7704be5d5250 -41:07.402 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:07.404 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:07.404 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:07.404 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:07.404 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:07.404 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:07.404 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e1161907-3e1e-3fca-1b05-7704be5d5250 -41:07.404 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e1161907-3e1e-3fca-1b05-7704be5d5250 -41:07.408 [INFO] TestAgent.server: federation state anti-entropy synced -41:07.667 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:07.668 [INFO] TestAgent: Synced node info -41:07.668 [DEBUG] TestAgent: Node info in sync -41:07.784 [DEBUG] TestAgent: Node info in sync -41:07.785 [INFO] TestAgent: Synced service: service=web-sidecar-proxy -41:07.786 [INFO] TestAgent: Synced service: service=web -41:07.786 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:1 -41:07.786 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:2 -41:07.787 [DEBUG] TestAgent: removed service: service=web -41:07.787 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:1 -41:07.787 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:2 -41:07.787 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy -41:07.787 [DEBUG] TestAgent: Node info in sync -41:07.788 [INFO] TestAgent: Deregistered service: service=web-sidecar-proxy -41:07.790 [INFO] TestAgent: Deregistered service: service=web -41:07.790 [INFO] TestAgent: Requesting shutdown -41:07.790 [INFO] TestAgent.server: shutting down server -41:07.790 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:07.790 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:07.790 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:07.790 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:07.790 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:07.790 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:07.790 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:07.790 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:07.790 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:07.790 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -41:07.790 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2022-09-15T21:41:07.790Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:07.790Z [INFO] agent.router.manager: shutting down -41:07.790 [INFO] TestAgent: consul server down -41:07.790 [INFO] TestAgent: shutdown complete -41:07.790 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13358 network=tcp -41:07.790 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13358 network=udp -41:07.790 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13359 network=tcp -41:08.291 [INFO] TestAgent: Waiting for endpoints to shut down -41:08.291 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:49.524Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:49.541Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:49.544 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:314a0ed3-1be7-7c52-987a-e37256175ec9 Address:127.0.0.1:24573}]" +32:49.544 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24573 [Follower]" leader= +32:49.545 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-314a0ed3-1be7-7c52-987a-e37256175ec9.dc1 127.0.0.1 +32:49.546 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-314a0ed3-1be7-7c52-987a-e37256175ec9 127.0.0.1 +2021-08-13T15:32:49.546Z [INFO] agent.router: Initializing LAN area manager +32:49.546 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24568 network=udp +32:49.546 [INFO] TestAgent.server: Adding LAN server: server="Node-314a0ed3-1be7-7c52-987a-e37256175ec9 (Addr: tcp/127.0.0.1:24573) (DC: dc1)" +32:49.546 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-314a0ed3-1be7-7c52-987a-e37256175ec9.dc1 area=wan +32:49.547 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24568 network=tcp +32:49.547 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24569 network=tcp +32:49.548 [INFO] TestAgent: started state syncer +32:49.548 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24574 network=tcp +32:49.606 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:49.606 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24573 [Candidate]" term=2 +32:49.607 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:49.607 [DEBUG] TestAgent.server.raft: vote granted: from=314a0ed3-1be7-7c52-987a-e37256175ec9 term=2 tally=1 +32:49.607 [INFO] TestAgent.server.raft: election won: tally=1 +32:49.607 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24573 [Leader]" +32:49.609 [INFO] TestAgent.server: cluster leadership acquired +32:49.610 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24573 +32:49.611 [INFO] TestAgent.server: New leader elected: payload=Node-314a0ed3-1be7-7c52-987a-e37256175ec9 +32:49.616 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:49.618 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:49.618 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:49.618 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:49.618 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:49.619 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:49.619 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-314a0ed3-1be7-7c52-987a-e37256175ec9 +32:49.619 [INFO] TestAgent.server: member joined, marking health alive: member=Node-314a0ed3-1be7-7c52-987a-e37256175ec9 +32:49.624 [INFO] TestAgent.server: federation state anti-entropy synced +32:49.724 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:49.724 [INFO] TestAgent: Synced node info +32:49.724 [DEBUG] TestAgent: Node info in sync +32:49.952 [DEBUG] TestAgent: Node info in sync +32:49.953 [INFO] TestAgent: Synced service: service=web-sidecar-proxy +32:49.953 [INFO] TestAgent: Synced service: service=web +32:49.954 [DEBUG] TestAgent: removed service: service=web +32:49.954 [DEBUG] TestAgent: Node info in sync +32:49.954 [DEBUG] TestAgent: Service in sync: service=web-sidecar-proxy +32:49.954 [INFO] TestAgent: Deregistered service: service=web +32:49.954 [INFO] TestAgent: Requesting shutdown +32:49.954 [INFO] TestAgent.server: shutting down server +32:49.954 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:49.954 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:49.954 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:49.954 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:49.954 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:49.954 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:49.954 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:49.955 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:49.955 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:49.955 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:49.955 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:49.955Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:49.955Z [INFO] agent.router.manager: shutting down +32:49.955 [INFO] TestAgent: consul server down +32:49.955 [INFO] TestAgent: shutdown complete +32:49.955 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24568 network=tcp +32:49.955 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24568 network=udp +32:49.955 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24569 network=tcp +32:50.455 [INFO] TestAgent: Waiting for endpoints to shut down +32:50.455 [INFO] TestAgent: Endpoints down +=== RUN TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/updates_to_sidecar_should_work +2021-08-13T15:32:50.381Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:50.397Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:50.400 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3b1f9988-1e4a-cdcb-35e1-16f65ef6d714 Address:127.0.0.1:24587}]" +32:50.400 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24587 [Follower]" leader= +32:50.401 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3b1f9988-1e4a-cdcb-35e1-16f65ef6d714.dc1 127.0.0.1 +32:50.402 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3b1f9988-1e4a-cdcb-35e1-16f65ef6d714 127.0.0.1 +2021-08-13T15:32:50.402Z [INFO] agent.router: Initializing LAN area manager +32:50.402 [INFO] TestAgent.server: Adding LAN server: server="Node-3b1f9988-1e4a-cdcb-35e1-16f65ef6d714 (Addr: tcp/127.0.0.1:24587) (DC: dc1)" +32:50.402 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3b1f9988-1e4a-cdcb-35e1-16f65ef6d714.dc1 area=wan +32:50.402 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24582 network=tcp +32:50.402 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24582 network=udp +32:50.403 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24583 network=tcp +32:50.403 [INFO] TestAgent: started state syncer +32:50.403 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24588 network=tcp +32:50.467 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:50.467 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24587 [Candidate]" term=2 +32:50.475 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:50.475 [DEBUG] TestAgent.server.raft: vote granted: from=3b1f9988-1e4a-cdcb-35e1-16f65ef6d714 term=2 tally=1 +32:50.475 [INFO] TestAgent.server.raft: election won: tally=1 +32:50.475 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24587 [Leader]" +32:50.475 [INFO] TestAgent.server: cluster leadership acquired +32:50.476 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24587 +32:50.478 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:50.478 [INFO] TestAgent.server: New leader elected: payload=Node-3b1f9988-1e4a-cdcb-35e1-16f65ef6d714 +32:50.484 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:50.484 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:50.484 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:50.484 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:50.484 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:50.484 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3b1f9988-1e4a-cdcb-35e1-16f65ef6d714 +32:50.484 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3b1f9988-1e4a-cdcb-35e1-16f65ef6d714 +32:50.486 [INFO] TestAgent.server: federation state anti-entropy synced +32:50.752 [INFO] TestAgent: Synced node info +32:50.753 [INFO] TestAgent: Synced service: service=web-sidecar-proxy +32:50.753 [INFO] TestAgent: Synced service: service=web +32:50.753 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:1 +32:50.753 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:2 +32:50.754 [DEBUG] TestAgent: removed service: service=web +32:50.754 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:1 +32:50.754 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:2 +32:50.754 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy +32:50.754 [DEBUG] TestAgent: Node info in sync +32:50.755 [INFO] TestAgent: Deregistered service: service=web-sidecar-proxy +32:50.755 [INFO] TestAgent: Deregistered service: service=web +32:50.755 [INFO] TestAgent: Requesting shutdown +32:50.756 [INFO] TestAgent.server: shutting down server +32:50.756 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:50.756 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:50.756 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:50.756 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:50.756 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:50.756 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:50.756 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:50.756 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:50.756 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +32:50.756 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:50.756 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:50.756 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:50.756Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:50.756Z [INFO] agent.router.manager: shutting down +32:50.756 [INFO] TestAgent: consul server down +32:50.756 [INFO] TestAgent: shutdown complete +32:50.756 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24582 network=tcp +32:50.756 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24582 network=udp +32:50.756 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24583 network=tcp +32:51.257 [INFO] TestAgent: Waiting for endpoints to shut down +32:51.257 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/normal/update_that_removes_sidecar_should_NOT_deregister_it -2022-09-15T21:41:07.929Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:07.974Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:07.977 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5e167a9c-b2b1-d596-bb64-60beb2951965 Address:127.0.0.1:13370}]" -41:07.977 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13370 [Follower]" leader= -41:07.978 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5e167a9c-b2b1-d596-bb64-60beb2951965.dc1 127.0.0.1 -41:07.979 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5e167a9c-b2b1-d596-bb64-60beb2951965 127.0.0.1 -2022-09-15T21:41:07.979Z [INFO] agent.router: Initializing LAN area manager -41:07.979 [INFO] TestAgent.server: Adding LAN server: server="Node-5e167a9c-b2b1-d596-bb64-60beb2951965 (Addr: tcp/127.0.0.1:13370) (DC: dc1)" -41:07.979 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5e167a9c-b2b1-d596-bb64-60beb2951965.dc1 area=wan -41:07.979 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13365 network=udp -41:07.980 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13365 network=tcp -41:07.980 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13366 network=tcp -41:07.981 [INFO] TestAgent: started state syncer -41:07.981 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13371 network=tcp -41:08.046 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:08.046 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13370 [Candidate]" term=2 -41:08.046 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:08.046 [DEBUG] TestAgent.server.raft: vote granted: from=5e167a9c-b2b1-d596-bb64-60beb2951965 term=2 tally=1 -41:08.046 [INFO] TestAgent.server.raft: election won: tally=1 -41:08.046 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13370 [Leader]" -41:08.046 [INFO] TestAgent.server: cluster leadership acquired -41:08.047 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13370 -41:08.047 [INFO] TestAgent.server: New leader elected: payload=Node-5e167a9c-b2b1-d596-bb64-60beb2951965 -41:08.048 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:08.050 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:08.050 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:08.050 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:08.050 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:08.050 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:08.051 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5e167a9c-b2b1-d596-bb64-60beb2951965 -41:08.051 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5e167a9c-b2b1-d596-bb64-60beb2951965 -41:08.051 [INFO] TestAgent.server: federation state anti-entropy synced -41:08.366 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:08.366 [INFO] TestAgent: Synced node info -41:08.366 [DEBUG] TestAgent: Node info in sync -41:08.416 [DEBUG] TestAgent: added local registration for service: service=web-sidecar-proxy -41:08.417 [DEBUG] TestAgent: Node info in sync -41:08.418 [INFO] TestAgent: Synced service: service=web -41:08.418 [INFO] TestAgent: Synced service: service=web-sidecar-proxy -41:08.419 [DEBUG] TestAgent: removed service: service=web -41:08.419 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy -41:08.419 [DEBUG] TestAgent: Node info in sync -41:08.420 [INFO] TestAgent: Deregistered service: service=web-sidecar-proxy -41:08.420 [INFO] TestAgent: Deregistered service: service=web -41:08.420 [INFO] TestAgent: Requesting shutdown -41:08.420 [INFO] TestAgent.server: shutting down server -41:08.420 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:08.420 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:08.420 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:08.420 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:08.420 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:08.420 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:08.421 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:08.421 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:08.421 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:08.421 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -41:08.421 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2022-09-15T21:41:08.421Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:08.421Z [INFO] agent.router.manager: shutting down -41:08.421 [INFO] TestAgent: consul server down -41:08.421 [INFO] TestAgent: shutdown complete -41:08.421 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13365 network=tcp -41:08.421 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13365 network=udp -41:08.421 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13366 network=tcp -41:08.921 [INFO] TestAgent: Waiting for endpoints to shut down -41:08.921 [INFO] TestAgent: Endpoints down -2022-09-15T21:41:08.314Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:08.331Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:08.334 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:afa0c7b1-5dc1-2be3-a449-73e22be8f58d Address:127.0.0.1:13377}]" -41:08.334 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13377 [Follower]" leader= -41:08.335 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-afa0c7b1-5dc1-2be3-a449-73e22be8f58d.dc1 127.0.0.1 -41:08.336 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-afa0c7b1-5dc1-2be3-a449-73e22be8f58d 127.0.0.1 -2022-09-15T21:41:08.336Z [INFO] agent.router: Initializing LAN area manager -41:08.336 [INFO] TestAgent.server: Adding LAN server: server="Node-afa0c7b1-5dc1-2be3-a449-73e22be8f58d (Addr: tcp/127.0.0.1:13377) (DC: dc1)" -41:08.336 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-afa0c7b1-5dc1-2be3-a449-73e22be8f58d.dc1 area=wan -41:08.336 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13372 network=udp -41:08.336 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13372 network=tcp -41:08.337 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13373 network=tcp -41:08.337 [INFO] TestAgent: started state syncer -41:08.337 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13378 network=tcp -41:08.376 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:08.377 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13377 [Candidate]" term=2 -41:08.377 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:08.377 [DEBUG] TestAgent.server.raft: vote granted: from=afa0c7b1-5dc1-2be3-a449-73e22be8f58d term=2 tally=1 -41:08.377 [INFO] TestAgent.server.raft: election won: tally=1 -41:08.377 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13377 [Leader]" -41:08.377 [INFO] TestAgent.server: cluster leadership acquired -41:08.377 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13377 -41:08.378 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:08.380 [INFO] TestAgent.server: New leader elected: payload=Node-afa0c7b1-5dc1-2be3-a449-73e22be8f58d -41:08.381 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:08.381 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:08.381 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:08.381 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:08.381 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:08.381 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-afa0c7b1-5dc1-2be3-a449-73e22be8f58d -41:08.381 [INFO] TestAgent.server: member joined, marking health alive: member=Node-afa0c7b1-5dc1-2be3-a449-73e22be8f58d -41:08.382 [INFO] TestAgent.server: federation state anti-entropy synced -41:08.440 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:08.441 [INFO] TestAgent: Synced node info -41:08.760 [DEBUG] TestAgent: Node info in sync -41:08.761 [INFO] TestAgent: Synced service: service=web -41:08.761 [INFO] TestAgent: Synced service: service=web-sidecar-proxy -41:08.761 [DEBUG] TestAgent: removed service: service=web -41:08.762 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy -41:08.762 [DEBUG] TestAgent: Node info in sync -41:08.763 [INFO] TestAgent: Deregistered service: service=web -41:08.763 [INFO] TestAgent: Deregistered service: service=web-sidecar-proxy -41:08.763 [INFO] TestAgent: Requesting shutdown -41:08.763 [INFO] TestAgent.server: shutting down server -41:08.763 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:08.763 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:08.763 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:08.763 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:08.763 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:08.763 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:08.763 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:08.763 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:08.763 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:08.764 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:08.764 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:08.764Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:08.764Z [INFO] agent.router.manager: shutting down -41:08.764 [INFO] TestAgent: consul server down -41:08.764 [INFO] TestAgent: shutdown complete -41:08.764 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13372 network=tcp -41:08.764 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13372 network=udp -41:08.764 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13373 network=tcp -41:09.264 [INFO] TestAgent: Waiting for endpoints to shut down -41:09.264 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:50.479Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:50.500Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:50.502 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:63c33fa7-9ff7-557c-be90-62411d81398b Address:127.0.0.1:24594}]" +32:50.502 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24594 [Follower]" leader= +32:50.503 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-63c33fa7-9ff7-557c-be90-62411d81398b.dc1 127.0.0.1 +32:50.504 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-63c33fa7-9ff7-557c-be90-62411d81398b 127.0.0.1 +2021-08-13T15:32:50.504Z [INFO] agent.router: Initializing LAN area manager +32:50.504 [INFO] TestAgent.server: Adding LAN server: server="Node-63c33fa7-9ff7-557c-be90-62411d81398b (Addr: tcp/127.0.0.1:24594) (DC: dc1)" +32:50.504 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-63c33fa7-9ff7-557c-be90-62411d81398b.dc1 area=wan +32:50.505 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24589 network=udp +32:50.505 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24589 network=tcp +32:50.505 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24590 network=tcp +32:50.506 [INFO] TestAgent: started state syncer +32:50.506 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24595 network=tcp +32:50.541 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:50.541 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24594 [Candidate]" term=2 +32:50.541 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:50.542 [DEBUG] TestAgent.server.raft: vote granted: from=63c33fa7-9ff7-557c-be90-62411d81398b term=2 tally=1 +32:50.542 [INFO] TestAgent.server.raft: election won: tally=1 +32:50.542 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24594 [Leader]" +32:50.542 [INFO] TestAgent.server: cluster leadership acquired +32:50.542 [INFO] TestAgent.server: New leader elected: payload=Node-63c33fa7-9ff7-557c-be90-62411d81398b +32:50.542 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24594 +32:50.543 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:50.546 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:50.546 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:50.546 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:50.546 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:50.546 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:50.546 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-63c33fa7-9ff7-557c-be90-62411d81398b +32:50.546 [INFO] TestAgent.server: member joined, marking health alive: member=Node-63c33fa7-9ff7-557c-be90-62411d81398b +32:50.546 [INFO] TestAgent.server: federation state anti-entropy synced +32:50.886 [DEBUG] TestAgent: added local registration for service: service=web-sidecar-proxy +32:50.887 [INFO] TestAgent: Synced node info +32:50.888 [INFO] TestAgent: Synced service: service=web-sidecar-proxy +32:50.889 [INFO] TestAgent: Synced service: service=web +32:50.889 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:1 +32:50.889 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:2 +32:50.889 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:50.889 [DEBUG] TestAgent: Node info in sync +32:50.890 [INFO] TestAgent: Synced service: service=web-sidecar-proxy +32:50.890 [INFO] TestAgent: Synced service: service=web +32:50.890 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:1 +32:50.890 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:2 +32:50.892 [DEBUG] TestAgent: removed service: service=web +32:50.892 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:2 +32:50.892 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:1 +32:50.892 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy +32:50.892 [DEBUG] TestAgent: Node info in sync +32:50.893 [INFO] TestAgent: Deregistered service: service=web-sidecar-proxy +32:50.893 [INFO] TestAgent: Deregistered service: service=web +32:50.894 [INFO] TestAgent: Requesting shutdown +32:50.894 [INFO] TestAgent.server: shutting down server +32:50.894 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:50.894 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:50.894 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:50.894 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:50.894 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:50.894 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:50.894 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:50.894 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:50.894 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:50.894 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:50.894 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:50.894Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:50.894Z [INFO] agent.router.manager: shutting down +32:50.895 [INFO] TestAgent: consul server down +32:50.895 [INFO] TestAgent: shutdown complete +32:50.895 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24589 network=tcp +32:50.895 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24589 network=udp +32:50.895 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24590 network=tcp +32:51.395 [INFO] TestAgent: Waiting for endpoints to shut down +32:51.395 [INFO] TestAgent: Endpoints down +=== RUN TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/update_that_removes_sidecar_should_NOT_deregister_it +2021-08-13T15:32:51.276Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:51.292Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:51.295 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:64d41230-663c-0598-5808-d98e03f00cba Address:127.0.0.1:24601}]" +32:51.295 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24601 [Follower]" leader= +32:51.296 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-64d41230-663c-0598-5808-d98e03f00cba.dc1 127.0.0.1 +32:51.297 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-64d41230-663c-0598-5808-d98e03f00cba 127.0.0.1 +2021-08-13T15:32:51.297Z [INFO] agent.router: Initializing LAN area manager +32:51.297 [INFO] TestAgent.server: Adding LAN server: server="Node-64d41230-663c-0598-5808-d98e03f00cba (Addr: tcp/127.0.0.1:24601) (DC: dc1)" +32:51.297 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-64d41230-663c-0598-5808-d98e03f00cba.dc1 area=wan +32:51.297 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24596 network=udp +32:51.297 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24596 network=tcp +32:51.298 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24597 network=tcp +32:51.298 [INFO] TestAgent: started state syncer +32:51.298 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24602 network=tcp +32:51.340 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:51.340 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24601 [Candidate]" term=2 +32:51.341 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:51.341 [DEBUG] TestAgent.server.raft: vote granted: from=64d41230-663c-0598-5808-d98e03f00cba term=2 tally=1 +32:51.341 [INFO] TestAgent.server.raft: election won: tally=1 +32:51.341 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24601 [Leader]" +32:51.341 [INFO] TestAgent.server: cluster leadership acquired +32:51.341 [INFO] TestAgent.server: New leader elected: payload=Node-64d41230-663c-0598-5808-d98e03f00cba +32:51.341 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24601 +32:51.342 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:51.345 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:51.345 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:51.345 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:51.345 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:51.345 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:51.345 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-64d41230-663c-0598-5808-d98e03f00cba +32:51.345 [INFO] TestAgent.server: member joined, marking health alive: member=Node-64d41230-663c-0598-5808-d98e03f00cba +32:51.346 [INFO] TestAgent.server: federation state anti-entropy synced +32:51.347 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:51.348 [INFO] TestAgent: Synced node info +32:51.495 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:51.495 [DEBUG] TestAgent: Node info in sync +32:51.495 [DEBUG] TestAgent: Node info in sync +32:51.564 [DEBUG] TestAgent: Node info in sync +32:51.565 [INFO] TestAgent: Synced service: service=web +32:51.566 [INFO] TestAgent: Synced service: service=web-sidecar-proxy +32:51.566 [DEBUG] TestAgent: removed service: service=web +32:51.566 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy +32:51.566 [DEBUG] TestAgent: Node info in sync +32:51.567 [INFO] TestAgent: Deregistered service: service=web +32:51.567 [INFO] TestAgent: Deregistered service: service=web-sidecar-proxy +32:51.567 [INFO] TestAgent: Requesting shutdown +32:51.568 [INFO] TestAgent.server: shutting down server +32:51.568 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:51.568 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:51.568 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:51.568 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:51.568 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:51.568 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:51.568 [DEBUG] TestAgent: Node info in sync +32:51.568 [DEBUG] TestAgent: Node info in sync +32:51.568 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:51.568 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:51.568 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:51.568 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:51.568 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:51.568Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:51.568Z [INFO] agent.router.manager: shutting down +32:51.568 [INFO] TestAgent: consul server down +32:51.568 [INFO] TestAgent: shutdown complete +32:51.568 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24596 network=tcp +32:51.568 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24596 network=udp +32:51.568 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24597 network=tcp +32:52.068 [INFO] TestAgent: Waiting for endpoints to shut down +32:52.068 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:51.452Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:51.473Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:51.476 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:efed76a7-d2c2-903f-b1f1-0968abd4ba07 Address:127.0.0.1:24608}]" +32:51.477 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-efed76a7-d2c2-903f-b1f1-0968abd4ba07.dc1 127.0.0.1 +32:51.478 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-efed76a7-d2c2-903f-b1f1-0968abd4ba07 127.0.0.1 +2021-08-13T15:32:51.478Z [INFO] agent.router: Initializing LAN area manager +32:51.478 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24603 network=udp +32:51.478 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24608 [Follower]" leader= +32:51.479 [INFO] TestAgent.server: Adding LAN server: server="Node-efed76a7-d2c2-903f-b1f1-0968abd4ba07 (Addr: tcp/127.0.0.1:24608) (DC: dc1)" +32:51.479 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-efed76a7-d2c2-903f-b1f1-0968abd4ba07.dc1 area=wan +32:51.479 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24603 network=tcp +32:51.480 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24604 network=tcp +32:51.480 [INFO] TestAgent: started state syncer +32:51.481 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24609 network=tcp +32:51.540 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:51.540 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24608 [Candidate]" term=2 +32:51.540 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:51.540 [DEBUG] TestAgent.server.raft: vote granted: from=efed76a7-d2c2-903f-b1f1-0968abd4ba07 term=2 tally=1 +32:51.540 [INFO] TestAgent.server.raft: election won: tally=1 +32:51.540 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24608 [Leader]" +32:51.541 [INFO] TestAgent.server: cluster leadership acquired +32:51.541 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24608 +32:51.542 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:51.543 [INFO] TestAgent.server: New leader elected: payload=Node-efed76a7-d2c2-903f-b1f1-0968abd4ba07 +32:51.545 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:51.545 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:51.545 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:51.545 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:51.545 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:51.545 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-efed76a7-d2c2-903f-b1f1-0968abd4ba07 +32:51.545 [INFO] TestAgent.server: member joined, marking health alive: member=Node-efed76a7-d2c2-903f-b1f1-0968abd4ba07 +32:51.546 [INFO] TestAgent.server: federation state anti-entropy synced +32:51.913 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:51.914 [INFO] TestAgent: Synced node info +32:51.914 [DEBUG] TestAgent: Node info in sync +32:51.935 [DEBUG] TestAgent: added local registration for service: service=web-sidecar-proxy +32:51.935 [DEBUG] TestAgent: Node info in sync +32:51.936 [INFO] TestAgent: Synced service: service=web +32:51.937 [INFO] TestAgent: Synced service: service=web-sidecar-proxy +32:51.937 [DEBUG] TestAgent: removed service: service=web +32:51.938 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy +32:51.938 [DEBUG] TestAgent: Node info in sync +32:51.938 [INFO] TestAgent: Deregistered service: service=web +32:51.938 [INFO] TestAgent: Deregistered service: service=web-sidecar-proxy +32:51.938 [INFO] TestAgent: Requesting shutdown +32:51.939 [INFO] TestAgent.server: shutting down server +32:51.939 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:51.939 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:51.939 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:51.939 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:51.939 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:51.939 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:51.939 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:51.939 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:51.939 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:51.939 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:51.939 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:51.939Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:51.939Z [INFO] agent.router.manager: shutting down +32:51.939 [INFO] TestAgent: consul server down +32:51.939 [INFO] TestAgent: shutdown complete +32:51.939 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24603 network=tcp +32:51.939 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24603 network=udp +32:51.939 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24604 network=tcp +32:52.440 [INFO] TestAgent: Waiting for endpoints to shut down +32:52.440 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar (0.00s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager (12.42s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/sanity_check_no_sidecar_case (0.79s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/default_sidecar (0.94s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/ACL_OK_defaults (0.82s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/ACL_denied (1.03s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/ACL_OK_for_service_but_not_for_sidecar (0.65s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/ACL_OK_for_service_and_sidecar_but_not_sidecar's_overridden_destination (0.88s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/ACL_OK_for_service_but_not_for_overridden_sidecar (0.76s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/ACL_OK_for_service_but_and_overridden_for_sidecar (0.69s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/invalid_check_definition_in_sidecar (0.79s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/invalid_checks_definitions_in_sidecar (0.91s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/invalid_check_status_in_sidecar (0.66s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/invalid_checks_status_in_sidecar (0.98s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/another_service_registered_with_same_ID_as_a_sidecar_should_not_be_deregistered (0.80s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/updates_to_sidecar_should_work (0.69s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/update_that_removes_sidecar_should_NOT_deregister_it (1.01s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal (12.76s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/sanity_check_no_sidecar_case (1.00s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/default_sidecar (0.62s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_OK_defaults (0.93s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_denied (0.90s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_OK_for_service_but_not_for_sidecar (0.68s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_OK_for_service_and_sidecar_but_not_sidecar's_overridden_destination (0.76s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_OK_for_service_but_not_for_overridden_sidecar (0.80s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_OK_for_service_but_and_overridden_for_sidecar (0.93s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/invalid_check_definition_in_sidecar (0.92s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/invalid_checks_definitions_in_sidecar (0.74s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/invalid_check_status_in_sidecar (0.80s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/invalid_checks_status_in_sidecar (0.92s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/another_service_registered_with_same_ID_as_a_sidecar_should_not_be_deregistered (0.81s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/updates_to_sidecar_should_work (0.98s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/update_that_removes_sidecar_should_NOT_deregister_it (0.97s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal (12.43s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/sanity_check_no_sidecar_case (0.95s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/default_sidecar (0.70s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_OK_defaults (1.01s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_denied (0.98s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_OK_for_service_but_not_for_sidecar (0.64s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_OK_for_service_and_sidecar_but_not_sidecar's_overridden_destination (0.88s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_OK_for_service_but_not_for_overridden_sidecar (0.69s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_OK_for_service_but_and_overridden_for_sidecar (0.95s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/invalid_check_definition_in_sidecar (0.88s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/invalid_checks_definitions_in_sidecar (0.94s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/invalid_check_status_in_sidecar (0.63s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/invalid_checks_status_in_sidecar (0.69s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/another_service_registered_with_same_ID_as_a_sidecar_should_not_be_deregistered (0.78s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/updates_to_sidecar_should_work (0.89s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/update_that_removes_sidecar_should_NOT_deregister_it (0.81s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager (12.80s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/sanity_check_no_sidecar_case (0.96s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/default_sidecar (0.63s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/ACL_OK_defaults (0.88s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/ACL_denied (0.91s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/ACL_OK_for_service_but_not_for_sidecar (0.66s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/ACL_OK_for_service_and_sidecar_but_not_sidecar's_overridden_destination (0.94s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/ACL_OK_for_service_but_not_for_overridden_sidecar (0.88s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/ACL_OK_for_service_but_and_overridden_for_sidecar (0.76s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/invalid_check_definition_in_sidecar (0.95s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/invalid_checks_definitions_in_sidecar (0.89s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/invalid_check_status_in_sidecar (0.64s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/invalid_checks_status_in_sidecar (0.75s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/another_service_registered_with_same_ID_as_a_sidecar_should_not_be_deregistered (0.95s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/updates_to_sidecar_should_work (0.94s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/update_that_removes_sidecar_should_NOT_deregister_it (1.04s) === RUN TestAgent_RegisterService_UnmanagedConnectProxyInvalid === RUN TestAgent_RegisterService_UnmanagedConnectProxyInvalid/normal === PAUSE TestAgent_RegisterService_UnmanagedConnectProxyInvalid/normal @@ -9066,121 +9158,119 @@ === PAUSE TestAgent_RegisterService_UnmanagedConnectProxyInvalid/service_manager === CONT TestAgent_RegisterService_UnmanagedConnectProxyInvalid/normal === CONT TestAgent_RegisterService_UnmanagedConnectProxyInvalid/service_manager -2022-09-15T21:41:09.358Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:09.378Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:09.381 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5fb7239e-5179-a838-79bb-0390bfb64db1 Address:127.0.0.1:13384}]" -41:09.382 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13384 [Follower]" leader= -41:09.383 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5fb7239e-5179-a838-79bb-0390bfb64db1.dc1 127.0.0.1 -41:09.384 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5fb7239e-5179-a838-79bb-0390bfb64db1 127.0.0.1 -2022-09-15T21:41:09.384Z [INFO] agent.router: Initializing LAN area manager -41:09.385 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13379 network=udp -41:09.385 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5fb7239e-5179-a838-79bb-0390bfb64db1.dc1 area=wan -41:09.385 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13379 network=tcp -41:09.386 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13380 network=tcp -41:09.386 [INFO] TestAgent: started state syncer -41:09.385 [INFO] TestAgent.server: Adding LAN server: server="Node-5fb7239e-5179-a838-79bb-0390bfb64db1 (Addr: tcp/127.0.0.1:13384) (DC: dc1)" -41:09.386 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13385 network=tcp -41:09.452 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:09.452 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13384 [Candidate]" term=2 -41:09.452 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:09.452 [DEBUG] TestAgent.server.raft: vote granted: from=5fb7239e-5179-a838-79bb-0390bfb64db1 term=2 tally=1 -41:09.452 [INFO] TestAgent.server.raft: election won: tally=1 -41:09.452 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13384 [Leader]" -41:09.452 [INFO] TestAgent.server: cluster leadership acquired -41:09.453 [INFO] TestAgent.server: New leader elected: payload=Node-5fb7239e-5179-a838-79bb-0390bfb64db1 -41:09.453 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13384 -41:09.454 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:09.457 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:09.457 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:09.457 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:09.457 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:09.457 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:09.457 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5fb7239e-5179-a838-79bb-0390bfb64db1 -41:09.457 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5fb7239e-5179-a838-79bb-0390bfb64db1 -41:09.458 [INFO] TestAgent.server: federation state anti-entropy synced -41:09.605 [INFO] TestAgent: Requesting shutdown -41:09.605 [INFO] TestAgent.server: shutting down server -41:09.605 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:09.605 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:09.605 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:09.605 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:09.605 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:09.605 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:09.605 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:09.605 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:09.605 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:09.605 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:09.605 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:09.605 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:09.606Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:09.606Z [INFO] agent.router.manager: shutting down -41:09.606 [INFO] TestAgent: consul server down -41:09.606 [INFO] TestAgent: shutdown complete -41:09.606 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13379 network=tcp -41:09.606 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13379 network=udp -41:09.606 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13380 network=tcp -41:10.106 [INFO] TestAgent: Waiting for endpoints to shut down -41:10.107 [INFO] TestAgent: Endpoints down -2022-09-15T21:41:09.364Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:09.387Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:09.390 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2869006e-a57f-8f36-e6b7-bd086d3e4b54 Address:127.0.0.1:13391}]" -41:09.391 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13391 [Follower]" leader= -41:09.392 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2869006e-a57f-8f36-e6b7-bd086d3e4b54.dc1 127.0.0.1 -41:09.394 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2869006e-a57f-8f36-e6b7-bd086d3e4b54 127.0.0.1 -2022-09-15T21:41:09.394Z [INFO] agent.router: Initializing LAN area manager -41:09.394 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13386 network=udp -41:09.394 [INFO] TestAgent.server: Adding LAN server: server="Node-2869006e-a57f-8f36-e6b7-bd086d3e4b54 (Addr: tcp/127.0.0.1:13391) (DC: dc1)" -41:09.395 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2869006e-a57f-8f36-e6b7-bd086d3e4b54.dc1 area=wan -41:09.395 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13386 network=tcp -41:09.396 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13387 network=tcp -41:09.396 [INFO] TestAgent: started state syncer -41:09.396 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13392 network=tcp -41:09.453 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:09.453 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13391 [Candidate]" term=2 -41:09.453 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:09.453 [DEBUG] TestAgent.server.raft: vote granted: from=2869006e-a57f-8f36-e6b7-bd086d3e4b54 term=2 tally=1 -41:09.453 [INFO] TestAgent.server.raft: election won: tally=1 -41:09.453 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13391 [Leader]" -41:09.453 [INFO] TestAgent.server: cluster leadership acquired -41:09.453 [INFO] TestAgent.server: New leader elected: payload=Node-2869006e-a57f-8f36-e6b7-bd086d3e4b54 -41:09.453 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13391 -41:09.455 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:09.457 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:09.457 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:09.457 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:09.457 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:09.457 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:09.457 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2869006e-a57f-8f36-e6b7-bd086d3e4b54 -41:09.457 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2869006e-a57f-8f36-e6b7-bd086d3e4b54 -41:09.458 [INFO] TestAgent.server: federation state anti-entropy synced -41:09.597 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:09.598 [INFO] TestAgent: Synced node info -41:09.598 [DEBUG] TestAgent: Node info in sync -41:09.615 [INFO] TestAgent: Requesting shutdown -41:09.615 [INFO] TestAgent.server: shutting down server -41:09.615 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:09.615 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:09.615 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:09.615 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:09.615 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:09.615 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:09.615 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:09.615 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:09.615 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:09.615 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:09.615 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:09.616Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:09.616Z [INFO] agent.router.manager: shutting down -41:09.616 [INFO] TestAgent: consul server down -41:09.616 [INFO] TestAgent: shutdown complete -41:09.616 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13386 network=tcp -41:09.616 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13386 network=udp -41:09.616 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13387 network=tcp -41:10.117 [INFO] TestAgent: Waiting for endpoints to shut down -41:10.117 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:52.477Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:52.505Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:52.508 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:29193a7d-93db-eb1b-057a-4cae29043c6b Address:127.0.0.1:24615}]" +32:52.509 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24615 [Follower]" leader= +32:52.509 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-29193a7d-93db-eb1b-057a-4cae29043c6b.dc1 127.0.0.1 +32:52.510 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-29193a7d-93db-eb1b-057a-4cae29043c6b 127.0.0.1 +2021-08-13T15:32:52.510Z [INFO] agent.router: Initializing LAN area manager +32:52.510 [INFO] TestAgent.server: Adding LAN server: server="Node-29193a7d-93db-eb1b-057a-4cae29043c6b (Addr: tcp/127.0.0.1:24615) (DC: dc1)" +32:52.510 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-29193a7d-93db-eb1b-057a-4cae29043c6b.dc1 area=wan +32:52.510 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24610 network=udp +32:52.511 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24610 network=tcp +32:52.511 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24611 network=tcp +32:52.512 [INFO] TestAgent: started state syncer +32:52.512 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24616 network=tcp +32:52.549 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:52.549 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24615 [Candidate]" term=2 +32:52.549 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:52.549 [DEBUG] TestAgent.server.raft: vote granted: from=29193a7d-93db-eb1b-057a-4cae29043c6b term=2 tally=1 +32:52.549 [INFO] TestAgent.server.raft: election won: tally=1 +32:52.549 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24615 [Leader]" +32:52.549 [INFO] TestAgent.server: cluster leadership acquired +32:52.550 [INFO] TestAgent.server: New leader elected: payload=Node-29193a7d-93db-eb1b-057a-4cae29043c6b +32:52.550 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24615 +32:52.551 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:52.553 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:52.553 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:52.553 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:52.553 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:52.553 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:52.553 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-29193a7d-93db-eb1b-057a-4cae29043c6b +32:52.553 [INFO] TestAgent.server: member joined, marking health alive: member=Node-29193a7d-93db-eb1b-057a-4cae29043c6b +32:52.555 [INFO] TestAgent.server: federation state anti-entropy synced +32:52.699 [INFO] TestAgent: Requesting shutdown +32:52.699 [INFO] TestAgent.server: shutting down server +32:52.699 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:52.699 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:52.699 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:52.699 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:52.699 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:52.699 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:52.699 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:52.699 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:52.699 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:52.699 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:52.699 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +32:52.699 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:52.699Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:52.699Z [INFO] agent.router.manager: shutting down +32:52.699 [INFO] TestAgent: consul server down +32:52.699 [INFO] TestAgent: shutdown complete +32:52.699 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24610 network=tcp +32:52.699 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24610 network=udp +32:52.699 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24611 network=tcp +32:53.200 [INFO] TestAgent: Waiting for endpoints to shut down +32:53.200 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:52.472Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:52.512Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:52.514 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d7e7161b-bc06-c6c1-1064-d67776c9be01 Address:127.0.0.1:24622}]" +32:52.514 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24622 [Follower]" leader= +32:52.515 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d7e7161b-bc06-c6c1-1064-d67776c9be01.dc1 127.0.0.1 +32:52.516 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d7e7161b-bc06-c6c1-1064-d67776c9be01 127.0.0.1 +2021-08-13T15:32:52.516Z [INFO] agent.router: Initializing LAN area manager +32:52.516 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d7e7161b-bc06-c6c1-1064-d67776c9be01.dc1 area=wan +32:52.516 [INFO] TestAgent.server: Adding LAN server: server="Node-d7e7161b-bc06-c6c1-1064-d67776c9be01 (Addr: tcp/127.0.0.1:24622) (DC: dc1)" +32:52.516 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24617 network=udp +32:52.517 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24617 network=tcp +32:52.517 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24618 network=tcp +32:52.518 [INFO] TestAgent: started state syncer +32:52.518 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24623 network=tcp +32:52.569 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:52.569 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24622 [Candidate]" term=2 +32:52.570 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:52.570 [DEBUG] TestAgent.server.raft: vote granted: from=d7e7161b-bc06-c6c1-1064-d67776c9be01 term=2 tally=1 +32:52.570 [INFO] TestAgent.server.raft: election won: tally=1 +32:52.570 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24622 [Leader]" +32:52.570 [INFO] TestAgent.server: cluster leadership acquired +32:52.570 [INFO] TestAgent.server: New leader elected: payload=Node-d7e7161b-bc06-c6c1-1064-d67776c9be01 +32:52.570 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24622 +32:52.571 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:52.574 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:52.574 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:52.574 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:52.574 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:52.574 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:52.574 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d7e7161b-bc06-c6c1-1064-d67776c9be01 +32:52.574 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d7e7161b-bc06-c6c1-1064-d67776c9be01 +32:52.575 [INFO] TestAgent.server: federation state anti-entropy synced +32:52.805 [INFO] TestAgent: Requesting shutdown +32:52.805 [INFO] TestAgent.server: shutting down server +32:52.805 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:52.805 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:52.805 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:52.805 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:52.805 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:52.805 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +32:52.805 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:52.805 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:52.805 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:52.805 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:52.806 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:52.806 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:52.806Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:52.806Z [INFO] agent.router.manager: shutting down +32:52.806 [INFO] TestAgent: consul server down +32:52.806 [INFO] TestAgent: shutdown complete +32:52.806 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24617 network=tcp +32:52.806 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24617 network=udp +32:52.806 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24618 network=tcp +32:53.306 [INFO] TestAgent: Waiting for endpoints to shut down +32:53.306 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_RegisterService_UnmanagedConnectProxyInvalid (0.00s) - --- PASS: TestAgent_RegisterService_UnmanagedConnectProxyInvalid/normal (0.84s) - --- PASS: TestAgent_RegisterService_UnmanagedConnectProxyInvalid/service_manager (0.85s) + --- PASS: TestAgent_RegisterService_UnmanagedConnectProxyInvalid/normal (0.76s) + --- PASS: TestAgent_RegisterService_UnmanagedConnectProxyInvalid/service_manager (0.86s) === RUN TestAgent_RegisterService_ConnectNative === RUN TestAgent_RegisterService_ConnectNative/normal === PAUSE TestAgent_RegisterService_ConnectNative/normal @@ -9188,125 +9278,125 @@ === PAUSE TestAgent_RegisterService_ConnectNative/service_manager === CONT TestAgent_RegisterService_ConnectNative/normal === CONT TestAgent_RegisterService_ConnectNative/service_manager -2022-09-15T21:41:10.146Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:10.180Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:10.184 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bbb8bb69-6dbf-efdd-0217-3eaa96f294be Address:127.0.0.1:13405}]" -41:10.185 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bbb8bb69-6dbf-efdd-0217-3eaa96f294be.dc1 127.0.0.1 -41:10.185 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13405 [Follower]" leader= -41:10.186 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bbb8bb69-6dbf-efdd-0217-3eaa96f294be 127.0.0.1 -2022-09-15T21:41:10.186Z [INFO] agent.router: Initializing LAN area manager -41:10.187 [INFO] TestAgent.server: Adding LAN server: server="Node-bbb8bb69-6dbf-efdd-0217-3eaa96f294be (Addr: tcp/127.0.0.1:13405) (DC: dc1)" -41:10.187 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bbb8bb69-6dbf-efdd-0217-3eaa96f294be.dc1 area=wan -41:10.187 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13400 network=udp -41:10.187 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13400 network=tcp -41:10.188 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13401 network=tcp -41:10.188 [INFO] TestAgent: started state syncer -41:10.188 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13406 network=tcp -41:10.241 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:10.241 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13405 [Candidate]" term=2 -41:10.241 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:10.241 [DEBUG] TestAgent.server.raft: vote granted: from=bbb8bb69-6dbf-efdd-0217-3eaa96f294be term=2 tally=1 -41:10.241 [INFO] TestAgent.server.raft: election won: tally=1 -41:10.241 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13405 [Leader]" -41:10.241 [INFO] TestAgent.server: cluster leadership acquired -41:10.242 [INFO] TestAgent.server: New leader elected: payload=Node-bbb8bb69-6dbf-efdd-0217-3eaa96f294be -41:10.242 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13405 -41:10.243 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:10.246 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:10.246 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:10.246 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:10.246 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:10.246 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:10.246 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bbb8bb69-6dbf-efdd-0217-3eaa96f294be -41:10.246 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bbb8bb69-6dbf-efdd-0217-3eaa96f294be -41:10.247 [INFO] TestAgent.server: federation state anti-entropy synced -41:10.279 [INFO] TestAgent: Synced node info -41:10.280 [INFO] TestAgent: Synced service: service=web -41:10.280 [DEBUG] TestAgent: Check in sync: check=service:web -41:10.280 [INFO] TestAgent: Requesting shutdown -41:10.280 [INFO] TestAgent.server: shutting down server -41:10.280 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:10.280 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:10.280 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:10.280 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:10.280 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:10.281 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:10.281 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:10.281 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:10.281 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:10.281 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:10.281 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:10.281 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:10.281Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:10.281Z [INFO] agent.router.manager: shutting down -41:10.281 [INFO] TestAgent: consul server down -41:10.281 [INFO] TestAgent: shutdown complete -41:10.281 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13400 network=tcp -41:10.281 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13400 network=udp -41:10.281 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13401 network=tcp -41:10.781 [INFO] TestAgent: Waiting for endpoints to shut down -41:10.782 [INFO] TestAgent: Endpoints down -2022-09-15T21:41:10.160Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:10.191Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:10.195 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c3557dc3-fdda-feb7-02dd-73b78800dc48 Address:127.0.0.1:13398}]" -41:10.196 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c3557dc3-fdda-feb7-02dd-73b78800dc48.dc1 127.0.0.1 -41:10.197 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c3557dc3-fdda-feb7-02dd-73b78800dc48 127.0.0.1 -2022-09-15T21:41:10.197Z [INFO] agent.router: Initializing LAN area manager -41:10.197 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13393 network=udp -41:10.197 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13398 [Follower]" leader= -41:10.198 [INFO] TestAgent.server: Adding LAN server: server="Node-c3557dc3-fdda-feb7-02dd-73b78800dc48 (Addr: tcp/127.0.0.1:13398) (DC: dc1)" -41:10.198 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c3557dc3-fdda-feb7-02dd-73b78800dc48.dc1 area=wan -41:10.198 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13393 network=tcp -41:10.199 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13394 network=tcp -41:10.199 [INFO] TestAgent: started state syncer -41:10.199 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13399 network=tcp -41:10.267 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:10.267 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13398 [Candidate]" term=2 -41:10.268 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:10.268 [DEBUG] TestAgent.server.raft: vote granted: from=c3557dc3-fdda-feb7-02dd-73b78800dc48 term=2 tally=1 -41:10.268 [INFO] TestAgent.server.raft: election won: tally=1 -41:10.268 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13398 [Leader]" -41:10.268 [INFO] TestAgent.server: cluster leadership acquired -41:10.268 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13398 -41:10.269 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:10.270 [INFO] TestAgent.server: New leader elected: payload=Node-c3557dc3-fdda-feb7-02dd-73b78800dc48 -41:10.272 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:10.272 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:10.272 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:10.272 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:10.273 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:10.273 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c3557dc3-fdda-feb7-02dd-73b78800dc48 -41:10.273 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c3557dc3-fdda-feb7-02dd-73b78800dc48 -41:10.274 [INFO] TestAgent.server: federation state anti-entropy synced -41:10.392 [INFO] TestAgent: Synced node info -41:10.393 [INFO] TestAgent: Synced service: service=web -41:10.393 [DEBUG] TestAgent: Check in sync: check=service:web -41:10.393 [INFO] TestAgent: Requesting shutdown -41:10.393 [INFO] TestAgent.server: shutting down server -41:10.393 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:10.394 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:10.394 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:10.394 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:10.394 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:10.394 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:10.394 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:10.394 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:10.394 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:10.394 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:10.394 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:10.394 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:10.394Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:10.394Z [INFO] agent.router.manager: shutting down -41:10.394 [INFO] TestAgent: consul server down -41:10.394 [INFO] TestAgent: shutdown complete -41:10.394 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13393 network=tcp -41:10.394 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13393 network=udp -41:10.394 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13394 network=tcp -41:10.895 [INFO] TestAgent: Waiting for endpoints to shut down -41:10.895 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:53.323Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:53.345Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:53.351 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:31e7ec63-a9a3-1063-83a7-26bb8e5af5e2 Address:127.0.0.1:24629}]" +32:53.352 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24629 [Follower]" leader= +32:53.352 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-31e7ec63-a9a3-1063-83a7-26bb8e5af5e2.dc1 127.0.0.1 +32:53.353 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-31e7ec63-a9a3-1063-83a7-26bb8e5af5e2 127.0.0.1 +2021-08-13T15:32:53.353Z [INFO] agent.router: Initializing LAN area manager +32:53.353 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24624 network=udp +32:53.353 [INFO] TestAgent.server: Adding LAN server: server="Node-31e7ec63-a9a3-1063-83a7-26bb8e5af5e2 (Addr: tcp/127.0.0.1:24629) (DC: dc1)" +32:53.353 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-31e7ec63-a9a3-1063-83a7-26bb8e5af5e2.dc1 area=wan +32:53.353 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24624 network=tcp +32:53.354 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24625 network=tcp +32:53.354 [INFO] TestAgent: started state syncer +32:53.354 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24630 network=tcp +32:53.396 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:53.397 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24629 [Candidate]" term=2 +32:53.397 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:53.397 [DEBUG] TestAgent.server.raft: vote granted: from=31e7ec63-a9a3-1063-83a7-26bb8e5af5e2 term=2 tally=1 +32:53.397 [INFO] TestAgent.server.raft: election won: tally=1 +32:53.397 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24629 [Leader]" +32:53.397 [INFO] TestAgent.server: cluster leadership acquired +32:53.397 [INFO] TestAgent.server: New leader elected: payload=Node-31e7ec63-a9a3-1063-83a7-26bb8e5af5e2 +32:53.397 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24629 +32:53.399 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:53.401 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:53.401 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:53.401 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:53.401 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:53.401 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:53.401 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-31e7ec63-a9a3-1063-83a7-26bb8e5af5e2 +32:53.401 [INFO] TestAgent.server: member joined, marking health alive: member=Node-31e7ec63-a9a3-1063-83a7-26bb8e5af5e2 +32:53.402 [INFO] TestAgent.server: federation state anti-entropy synced +32:53.457 [INFO] TestAgent: Synced node info +32:53.458 [INFO] TestAgent: Synced service: service=web +32:53.458 [DEBUG] TestAgent: Check in sync: check=service:web +32:53.458 [INFO] TestAgent: Requesting shutdown +32:53.458 [INFO] TestAgent.server: shutting down server +32:53.458 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:53.458 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:53.458 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:53.458 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:53.458 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:53.458 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:53.458 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +32:53.458 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:53.458 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:53.458 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:53.458 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:53.458Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:53.458Z [INFO] agent.router.manager: shutting down +32:53.458 [INFO] TestAgent: consul server down +32:53.458 [INFO] TestAgent: shutdown complete +32:53.458 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24624 network=tcp +32:53.458 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24624 network=udp +32:53.458 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24625 network=tcp +32:53.959 [INFO] TestAgent: Waiting for endpoints to shut down +32:53.959 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:53.323Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:53.353Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:53.356 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c78c9107-641e-2393-4c0c-4b33256bdd2d Address:127.0.0.1:24636}]" +32:53.356 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24636 [Follower]" leader= +32:53.357 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c78c9107-641e-2393-4c0c-4b33256bdd2d.dc1 127.0.0.1 +32:53.357 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c78c9107-641e-2393-4c0c-4b33256bdd2d 127.0.0.1 +2021-08-13T15:32:53.358Z [INFO] agent.router: Initializing LAN area manager +32:53.358 [INFO] TestAgent.server: Adding LAN server: server="Node-c78c9107-641e-2393-4c0c-4b33256bdd2d (Addr: tcp/127.0.0.1:24636) (DC: dc1)" +32:53.358 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c78c9107-641e-2393-4c0c-4b33256bdd2d.dc1 area=wan +32:53.358 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24631 network=udp +32:53.358 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24631 network=tcp +32:53.359 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24632 network=tcp +32:53.359 [INFO] TestAgent: started state syncer +32:53.359 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24637 network=tcp +32:53.406 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:53.406 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24636 [Candidate]" term=2 +32:53.406 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:53.406 [DEBUG] TestAgent.server.raft: vote granted: from=c78c9107-641e-2393-4c0c-4b33256bdd2d term=2 tally=1 +32:53.406 [INFO] TestAgent.server.raft: election won: tally=1 +32:53.406 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24636 [Leader]" +32:53.406 [INFO] TestAgent.server: cluster leadership acquired +32:53.407 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24636 +32:53.408 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:53.410 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:53.410 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:53.410 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:53.410 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:53.410 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:53.411 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c78c9107-641e-2393-4c0c-4b33256bdd2d +32:53.411 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c78c9107-641e-2393-4c0c-4b33256bdd2d +32:53.411 [INFO] TestAgent.server: New leader elected: payload=Node-c78c9107-641e-2393-4c0c-4b33256bdd2d +32:53.412 [INFO] TestAgent.server: federation state anti-entropy synced +32:53.601 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:53.602 [INFO] TestAgent: Synced node info +32:53.641 [DEBUG] TestAgent: Node info in sync +32:53.642 [INFO] TestAgent: Synced service: service=web +32:53.642 [DEBUG] TestAgent: Check in sync: check=service:web +32:53.642 [INFO] TestAgent: Requesting shutdown +32:53.642 [INFO] TestAgent.server: shutting down server +32:53.642 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:53.642 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:53.642 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:53.642 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:53.643 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:53.643 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:53.643 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:53.643 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:53.643 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:53.643 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:53.643 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:53.643Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:53.643Z [INFO] agent.router.manager: shutting down +32:53.643 [INFO] TestAgent: consul server down +32:53.643 [INFO] TestAgent: shutdown complete +32:53.643 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24631 network=tcp +32:53.643 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24631 network=udp +32:53.643 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24632 network=tcp +32:54.143 [INFO] TestAgent: Waiting for endpoints to shut down +32:54.143 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_RegisterService_ConnectNative (0.00s) - --- PASS: TestAgent_RegisterService_ConnectNative/normal (0.66s) - --- PASS: TestAgent_RegisterService_ConnectNative/service_manager (0.78s) + --- PASS: TestAgent_RegisterService_ConnectNative/normal (0.65s) + --- PASS: TestAgent_RegisterService_ConnectNative/service_manager (0.84s) === RUN TestAgent_RegisterService_ScriptCheck_ExecDisable === RUN TestAgent_RegisterService_ScriptCheck_ExecDisable/normal === PAUSE TestAgent_RegisterService_ScriptCheck_ExecDisable/normal @@ -9314,122 +9404,127 @@ === PAUSE TestAgent_RegisterService_ScriptCheck_ExecDisable/service_manager === CONT TestAgent_RegisterService_ScriptCheck_ExecDisable/normal === CONT TestAgent_RegisterService_ScriptCheck_ExecDisable/service_manager -2022-09-15T21:41:10.934Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:10.989Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:10.992 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:44a560d3-2d60-c6d8-8978-a4bbab561c79 Address:127.0.0.1:13419}]" -41:10.993 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-44a560d3-2d60-c6d8-8978-a4bbab561c79.dc1 127.0.0.1 -41:10.994 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13419 [Follower]" leader= -41:10.994 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-44a560d3-2d60-c6d8-8978-a4bbab561c79 127.0.0.1 -2022-09-15T21:41:10.994Z [INFO] agent.router: Initializing LAN area manager -41:10.995 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13414 network=udp -41:10.995 [INFO] TestAgent.server: Adding LAN server: server="Node-44a560d3-2d60-c6d8-8978-a4bbab561c79 (Addr: tcp/127.0.0.1:13419) (DC: dc1)" -41:10.995 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-44a560d3-2d60-c6d8-8978-a4bbab561c79.dc1 area=wan -41:10.995 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13414 network=tcp -41:10.996 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13415 network=tcp -41:10.996 [INFO] TestAgent: started state syncer -41:10.996 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13420 network=tcp -41:11.039 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:11.039 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13419 [Candidate]" term=2 -41:11.039 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:11.039 [DEBUG] TestAgent.server.raft: vote granted: from=44a560d3-2d60-c6d8-8978-a4bbab561c79 term=2 tally=1 -41:11.039 [INFO] TestAgent.server.raft: election won: tally=1 -41:11.039 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13419 [Leader]" -41:11.040 [INFO] TestAgent.server: cluster leadership acquired -41:11.040 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13419 -41:11.041 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:11.042 [INFO] TestAgent.server: New leader elected: payload=Node-44a560d3-2d60-c6d8-8978-a4bbab561c79 -41:11.044 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:11.044 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:11.044 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:11.044 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:11.044 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:11.044 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-44a560d3-2d60-c6d8-8978-a4bbab561c79 -41:11.044 [INFO] TestAgent.server: member joined, marking health alive: member=Node-44a560d3-2d60-c6d8-8978-a4bbab561c79 -41:11.045 [INFO] TestAgent.server: federation state anti-entropy synced -41:11.257 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:11.258 [INFO] TestAgent: Synced node info -41:11.260 [INFO] TestAgent: Requesting shutdown -41:11.260 [INFO] TestAgent.server: shutting down server -41:11.260 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:11.260 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:11.260 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:11.260 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:11.260 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:11.260 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:11.260 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:11.260 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:11.260 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:11.260 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:11.260 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:11.260Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:11.260Z [INFO] agent.router.manager: shutting down -41:11.260 [INFO] TestAgent: consul server down -41:11.260 [INFO] TestAgent: shutdown complete -41:11.260 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13414 network=tcp -41:11.260 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13414 network=udp -41:11.261 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13415 network=tcp -41:11.761 [INFO] TestAgent: Waiting for endpoints to shut down -41:11.761 [INFO] TestAgent: Endpoints down -2022-09-15T21:41:10.938Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:10.999Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:11.005 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:22ec6fe7-9e96-5945-9d49-46ad5833e990 Address:127.0.0.1:13412}]" -41:11.005 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13412 [Follower]" leader= -41:11.007 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-22ec6fe7-9e96-5945-9d49-46ad5833e990.dc1 127.0.0.1 -41:11.008 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-22ec6fe7-9e96-5945-9d49-46ad5833e990 127.0.0.1 -2022-09-15T21:41:11.008Z [INFO] agent.router: Initializing LAN area manager -41:11.008 [INFO] TestAgent.server: Adding LAN server: server="Node-22ec6fe7-9e96-5945-9d49-46ad5833e990 (Addr: tcp/127.0.0.1:13412) (DC: dc1)" -41:11.008 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-22ec6fe7-9e96-5945-9d49-46ad5833e990.dc1 area=wan -41:11.009 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13407 network=udp -41:11.009 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13407 network=tcp -41:11.010 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13408 network=tcp -41:11.010 [INFO] TestAgent: started state syncer -41:11.010 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13413 network=tcp -41:11.048 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:11.048 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13412 [Candidate]" term=2 -41:11.048 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:11.048 [DEBUG] TestAgent.server.raft: vote granted: from=22ec6fe7-9e96-5945-9d49-46ad5833e990 term=2 tally=1 -41:11.048 [INFO] TestAgent.server.raft: election won: tally=1 -41:11.048 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13412 [Leader]" -41:11.049 [INFO] TestAgent.server: cluster leadership acquired -41:11.049 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13412 -41:11.049 [INFO] TestAgent.server: New leader elected: payload=Node-22ec6fe7-9e96-5945-9d49-46ad5833e990 -41:11.050 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:11.053 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:11.053 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:11.053 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:11.053 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:11.053 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:11.053 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-22ec6fe7-9e96-5945-9d49-46ad5833e990 -41:11.053 [INFO] TestAgent.server: member joined, marking health alive: member=Node-22ec6fe7-9e96-5945-9d49-46ad5833e990 -41:11.054 [INFO] TestAgent.server: federation state anti-entropy synced -41:11.396 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:11.396 [INFO] TestAgent: Synced node info -41:11.396 [DEBUG] TestAgent: Node info in sync -41:11.435 [INFO] TestAgent: Requesting shutdown -41:11.435 [INFO] TestAgent.server: shutting down server -41:11.435 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:11.435 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:11.435 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:11.435 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:11.435 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:11.435 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:11.435 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:11.435 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:11.435 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:11.435 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:11.435 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:11.436Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:11.436Z [INFO] agent.router.manager: shutting down -41:11.436 [INFO] TestAgent: consul server down -41:11.436 [INFO] TestAgent: shutdown complete -41:11.436 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13407 network=tcp -41:11.436 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13407 network=udp -41:11.436 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13408 network=tcp -41:11.936 [INFO] TestAgent: Waiting for endpoints to shut down -41:11.936 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:54.160Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:54.181Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:54.186 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:81f7d18f-e7e6-42f5-ae8a-a2a6acf5aba1 Address:127.0.0.1:24650}]" +32:54.186 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24650 [Follower]" leader= +32:54.188 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-81f7d18f-e7e6-42f5-ae8a-a2a6acf5aba1.dc1 127.0.0.1 +32:54.189 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-81f7d18f-e7e6-42f5-ae8a-a2a6acf5aba1 127.0.0.1 +2021-08-13T15:32:54.189Z [INFO] agent.router: Initializing LAN area manager +32:54.189 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-81f7d18f-e7e6-42f5-ae8a-a2a6acf5aba1.dc1 area=wan +32:54.189 [INFO] TestAgent.server: Adding LAN server: server="Node-81f7d18f-e7e6-42f5-ae8a-a2a6acf5aba1 (Addr: tcp/127.0.0.1:24650) (DC: dc1)" +32:54.190 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24645 network=udp +32:54.190 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24645 network=tcp +32:54.191 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24646 network=tcp +32:54.191 [INFO] TestAgent: started state syncer +32:54.191 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24651 network=tcp +32:54.228 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:54.228 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24650 [Candidate]" term=2 +32:54.228 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:54.228 [DEBUG] TestAgent.server.raft: vote granted: from=81f7d18f-e7e6-42f5-ae8a-a2a6acf5aba1 term=2 tally=1 +32:54.228 [INFO] TestAgent.server.raft: election won: tally=1 +32:54.228 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24650 [Leader]" +32:54.228 [INFO] TestAgent.server: cluster leadership acquired +32:54.228 [INFO] TestAgent.server: New leader elected: payload=Node-81f7d18f-e7e6-42f5-ae8a-a2a6acf5aba1 +32:54.228 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24650 +32:54.230 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:54.232 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:54.232 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:54.232 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:54.232 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:54.232 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:54.232 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-81f7d18f-e7e6-42f5-ae8a-a2a6acf5aba1 +32:54.232 [INFO] TestAgent.server: member joined, marking health alive: member=Node-81f7d18f-e7e6-42f5-ae8a-a2a6acf5aba1 +32:54.233 [INFO] TestAgent.server: federation state anti-entropy synced +32:54.307 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:54.308 [INFO] TestAgent: Synced node info +32:54.308 [DEBUG] TestAgent: Node info in sync +32:54.470 [INFO] TestAgent: Requesting shutdown +32:54.470 [INFO] TestAgent.server: shutting down server +32:54.470 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:54.470 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:54.470 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:54.470 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:54.470 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:54.470 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:54.470 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:54.470 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:54.470 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:54.470 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:54.470 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:54.470Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:54.470Z [INFO] agent.router.manager: shutting down +32:54.470 [INFO] TestAgent: consul server down +32:54.470 [INFO] TestAgent: shutdown complete +32:54.470 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24645 network=tcp +32:54.470 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24645 network=udp +32:54.470 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24646 network=tcp +32:54.971 [INFO] TestAgent: Waiting for endpoints to shut down +32:54.971 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:54.160Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:54.191Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:54.193 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3ea24bbd-58f8-0fd6-bffa-20c594dc5e28 Address:127.0.0.1:24643}]" +32:54.193 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24643 [Follower]" leader= +32:54.194 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3ea24bbd-58f8-0fd6-bffa-20c594dc5e28.dc1 127.0.0.1 +32:54.195 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3ea24bbd-58f8-0fd6-bffa-20c594dc5e28 127.0.0.1 +2021-08-13T15:32:54.195Z [INFO] agent.router: Initializing LAN area manager +32:54.195 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3ea24bbd-58f8-0fd6-bffa-20c594dc5e28.dc1 area=wan +32:54.195 [INFO] TestAgent.server: Adding LAN server: server="Node-3ea24bbd-58f8-0fd6-bffa-20c594dc5e28 (Addr: tcp/127.0.0.1:24643) (DC: dc1)" +32:54.195 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24638 network=tcp +32:54.195 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24638 network=udp +32:54.196 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24639 network=tcp +32:54.196 [INFO] TestAgent: started state syncer +32:54.196 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24644 network=tcp +32:54.232 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:54.232 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24643 [Candidate]" term=2 +32:54.232 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:54.232 [DEBUG] TestAgent.server.raft: vote granted: from=3ea24bbd-58f8-0fd6-bffa-20c594dc5e28 term=2 tally=1 +32:54.232 [INFO] TestAgent.server.raft: election won: tally=1 +32:54.232 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24643 [Leader]" +32:54.232 [INFO] TestAgent.server: cluster leadership acquired +32:54.232 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24643 +32:54.233 [INFO] TestAgent.server: New leader elected: payload=Node-3ea24bbd-58f8-0fd6-bffa-20c594dc5e28 +32:54.233 [INFO] TestAgent: Synced node info +32:54.234 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:54.236 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:54.236 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:54.236 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:54.236 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:54.236 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:54.236 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3ea24bbd-58f8-0fd6-bffa-20c594dc5e28 +32:54.236 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3ea24bbd-58f8-0fd6-bffa-20c594dc5e28 +32:54.236 [INFO] TestAgent.server: federation state anti-entropy synced +32:54.629 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:54.629 [DEBUG] TestAgent: Node info in sync +32:54.629 [DEBUG] TestAgent: Node info in sync +32:54.644 [INFO] TestAgent: Requesting shutdown +32:54.644 [INFO] TestAgent.server: shutting down server +32:54.644 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:54.644 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:54.644 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:54.644 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:54.644 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:54.644 [DEBUG] TestAgent: Node info in sync +32:54.644 [INFO] TestAgent: Deregistered service: service=test +32:54.644 [DEBUG] TestAgent: Node info in sync +32:54.645 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:54.645 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:54.645 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:54.645 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:54.645 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:54.645 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:54.645Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:54.645Z [INFO] agent.router.manager: shutting down +32:54.645 [INFO] TestAgent: consul server down +32:54.645 [INFO] TestAgent: shutdown complete +32:54.645 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24638 network=tcp +32:54.645 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24638 network=udp +32:54.645 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24639 network=tcp +32:55.145 [INFO] TestAgent: Waiting for endpoints to shut down +32:55.145 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_RegisterService_ScriptCheck_ExecDisable (0.00s) - --- PASS: TestAgent_RegisterService_ScriptCheck_ExecDisable/service_manager (0.86s) - --- PASS: TestAgent_RegisterService_ScriptCheck_ExecDisable/normal (1.04s) + --- PASS: TestAgent_RegisterService_ScriptCheck_ExecDisable/service_manager (0.83s) + --- PASS: TestAgent_RegisterService_ScriptCheck_ExecDisable/normal (1.00s) === RUN TestAgent_RegisterService_ScriptCheck_ExecRemoteDisable === RUN TestAgent_RegisterService_ScriptCheck_ExecRemoteDisable/normal === PAUSE TestAgent_RegisterService_ScriptCheck_ExecRemoteDisable/normal @@ -9437,119 +9532,122 @@ === PAUSE TestAgent_RegisterService_ScriptCheck_ExecRemoteDisable/service_manager === CONT TestAgent_RegisterService_ScriptCheck_ExecRemoteDisable/normal === CONT TestAgent_RegisterService_ScriptCheck_ExecRemoteDisable/service_manager -2022-09-15T21:41:11.955Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:11.971Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:11.978 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b2971fb4-a356-141f-284c-183eee6b7ce3 Address:127.0.0.1:13426}]" -41:11.980 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b2971fb4-a356-141f-284c-183eee6b7ce3.dc1 127.0.0.1 -41:11.981 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b2971fb4-a356-141f-284c-183eee6b7ce3 127.0.0.1 -2022-09-15T21:41:11.981Z [INFO] agent.router: Initializing LAN area manager -41:11.982 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13426 [Follower]" leader= -41:11.982 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13421 network=udp -41:11.984 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b2971fb4-a356-141f-284c-183eee6b7ce3.dc1 area=wan -41:11.984 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13421 network=tcp -41:11.987 [INFO] TestAgent.server: Adding LAN server: server="Node-b2971fb4-a356-141f-284c-183eee6b7ce3 (Addr: tcp/127.0.0.1:13426) (DC: dc1)" -41:11.991 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13422 network=tcp -41:11.991 [INFO] TestAgent: started state syncer -41:11.994 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13427 network=tcp -41:12.037 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:12.037 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13426 [Candidate]" term=2 -41:12.037 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:12.037 [DEBUG] TestAgent.server.raft: vote granted: from=b2971fb4-a356-141f-284c-183eee6b7ce3 term=2 tally=1 -41:12.037 [INFO] TestAgent.server.raft: election won: tally=1 -41:12.037 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13426 [Leader]" -41:12.037 [INFO] TestAgent.server: cluster leadership acquired -41:12.038 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13426 -41:12.038 [INFO] TestAgent.server: New leader elected: payload=Node-b2971fb4-a356-141f-284c-183eee6b7ce3 -41:12.039 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:12.042 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:12.042 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:12.042 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:12.042 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:12.042 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:12.042 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b2971fb4-a356-141f-284c-183eee6b7ce3 -41:12.042 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b2971fb4-a356-141f-284c-183eee6b7ce3 -41:12.043 [INFO] TestAgent.server: federation state anti-entropy synced -41:12.139 [INFO] TestAgent: Requesting shutdown -41:12.139 [INFO] TestAgent.server: shutting down server -41:12.139 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:12.139 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:12.139 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:12.139 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:12.139 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:12.139 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:12.139 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:12.139 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:12.139 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:12.139 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:12.139 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:12.139 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:12.139Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:12.139Z [INFO] agent.router.manager: shutting down -41:12.139 [INFO] TestAgent: consul server down -41:12.140 [INFO] TestAgent: shutdown complete -41:12.140 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13421 network=tcp -41:12.140 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13421 network=udp -41:12.140 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13422 network=tcp -41:12.640 [INFO] TestAgent: Waiting for endpoints to shut down -41:12.640 [INFO] TestAgent: Endpoints down -2022-09-15T21:41:11.955Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:11.971Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:11.978 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:dbbfc325-ce80-dbff-1f8c-09db4c16ed47 Address:127.0.0.1:13433}]" -41:11.986 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-dbbfc325-ce80-dbff-1f8c-09db4c16ed47.dc1 127.0.0.1 -41:11.986 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13433 [Follower]" leader= -41:11.994 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-dbbfc325-ce80-dbff-1f8c-09db4c16ed47 127.0.0.1 -2022-09-15T21:41:11.994Z [INFO] agent.router: Initializing LAN area manager -41:11.994 [INFO] TestAgent.server: Adding LAN server: server="Node-dbbfc325-ce80-dbff-1f8c-09db4c16ed47 (Addr: tcp/127.0.0.1:13433) (DC: dc1)" -41:11.994 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13428 network=udp -41:11.997 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13428 network=tcp -41:11.998 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13429 network=tcp -41:11.998 [INFO] TestAgent: started state syncer -41:11.998 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13434 network=tcp -41:12.001 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-dbbfc325-ce80-dbff-1f8c-09db4c16ed47.dc1 area=wan -41:12.057 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:12.057 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13433 [Candidate]" term=2 -41:12.058 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:12.058 [DEBUG] TestAgent.server.raft: vote granted: from=dbbfc325-ce80-dbff-1f8c-09db4c16ed47 term=2 tally=1 -41:12.058 [INFO] TestAgent.server.raft: election won: tally=1 -41:12.058 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13433 [Leader]" -41:12.058 [INFO] TestAgent.server: cluster leadership acquired -41:12.058 [INFO] TestAgent.server: New leader elected: payload=Node-dbbfc325-ce80-dbff-1f8c-09db4c16ed47 -41:12.058 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13433 -41:12.060 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:12.062 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:12.062 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:12.062 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:12.062 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:12.062 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:12.062 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-dbbfc325-ce80-dbff-1f8c-09db4c16ed47 -41:12.062 [INFO] TestAgent.server: member joined, marking health alive: member=Node-dbbfc325-ce80-dbff-1f8c-09db4c16ed47 -41:12.063 [INFO] TestAgent.server: federation state anti-entropy synced -41:12.246 [INFO] TestAgent: Requesting shutdown -41:12.247 [INFO] TestAgent.server: shutting down server -41:12.247 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:12.247 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:12.247 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:12.247 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:12.247 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:12.247 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:12.247 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:12.247 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:12.247 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:12.247 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:12.247 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:12.247 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:12.247Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:12.247Z [INFO] agent.router.manager: shutting down -41:12.247 [INFO] TestAgent: consul server down -41:12.247 [INFO] TestAgent: shutdown complete -41:12.247 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13428 network=tcp -41:12.247 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13428 network=udp -41:12.247 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13429 network=tcp -41:12.748 [INFO] TestAgent: Waiting for endpoints to shut down -41:12.748 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:55.163Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:55.179Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:55.182 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e3486b09-e405-57c1-56c9-ba89a67fccc4 Address:127.0.0.1:24657}]" +32:55.182 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24657 [Follower]" leader= +32:55.183 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e3486b09-e405-57c1-56c9-ba89a67fccc4.dc1 127.0.0.1 +32:55.184 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e3486b09-e405-57c1-56c9-ba89a67fccc4 127.0.0.1 +2021-08-13T15:32:55.184Z [INFO] agent.router: Initializing LAN area manager +32:55.184 [INFO] TestAgent.server: Adding LAN server: server="Node-e3486b09-e405-57c1-56c9-ba89a67fccc4 (Addr: tcp/127.0.0.1:24657) (DC: dc1)" +32:55.184 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e3486b09-e405-57c1-56c9-ba89a67fccc4.dc1 area=wan +32:55.184 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24652 network=tcp +32:55.184 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24652 network=udp +32:55.197 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24653 network=tcp +32:55.197 [INFO] TestAgent: started state syncer +32:55.199 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24658 network=tcp +32:55.223 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:55.223 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24657 [Candidate]" term=2 +32:55.223 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:55.223 [DEBUG] TestAgent.server.raft: vote granted: from=e3486b09-e405-57c1-56c9-ba89a67fccc4 term=2 tally=1 +32:55.223 [INFO] TestAgent.server.raft: election won: tally=1 +32:55.223 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24657 [Leader]" +32:55.223 [INFO] TestAgent.server: cluster leadership acquired +32:55.224 [INFO] TestAgent.server: New leader elected: payload=Node-e3486b09-e405-57c1-56c9-ba89a67fccc4 +32:55.224 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24657 +32:55.225 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:55.228 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:55.228 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:55.228 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:55.228 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:55.228 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:55.228 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e3486b09-e405-57c1-56c9-ba89a67fccc4 +32:55.228 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e3486b09-e405-57c1-56c9-ba89a67fccc4 +32:55.229 [INFO] TestAgent: Synced node info +32:55.229 [INFO] TestAgent.server: federation state anti-entropy synced +32:55.229 [DEBUG] TestAgent: Node info in sync +32:55.281 [INFO] TestAgent: Requesting shutdown +32:55.281 [INFO] TestAgent.server: shutting down server +32:55.281 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:55.281 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:55.281 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:55.281 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:55.281 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:55.281 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:55.281 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:55.281 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:55.281 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:55.281 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:55.281 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:55.281Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:55.281Z [INFO] agent.router.manager: shutting down +32:55.282 [INFO] TestAgent: consul server down +32:55.282 [INFO] TestAgent: shutdown complete +32:55.282 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24652 network=tcp +32:55.282 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24652 network=udp +32:55.282 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24653 network=tcp +32:55.782 [INFO] TestAgent: Waiting for endpoints to shut down +32:55.782 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:55.180Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:55.213Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:55.217 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:23d3562b-e830-b48d-4a9f-6b2ea8631246 Address:127.0.0.1:24664}]" +32:55.217 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24664 [Follower]" leader= +32:55.217 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-23d3562b-e830-b48d-4a9f-6b2ea8631246.dc1 127.0.0.1 +32:55.218 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-23d3562b-e830-b48d-4a9f-6b2ea8631246 127.0.0.1 +2021-08-13T15:32:55.218Z [INFO] agent.router: Initializing LAN area manager +32:55.219 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24659 network=udp +32:55.219 [INFO] TestAgent.server: Adding LAN server: server="Node-23d3562b-e830-b48d-4a9f-6b2ea8631246 (Addr: tcp/127.0.0.1:24664) (DC: dc1)" +32:55.219 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-23d3562b-e830-b48d-4a9f-6b2ea8631246.dc1 area=wan +32:55.219 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24659 network=tcp +32:55.220 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24660 network=tcp +32:55.220 [INFO] TestAgent: started state syncer +32:55.220 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24665 network=tcp +32:55.254 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:55.254 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24664 [Candidate]" term=2 +32:55.254 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:55.254 [DEBUG] TestAgent.server.raft: vote granted: from=23d3562b-e830-b48d-4a9f-6b2ea8631246 term=2 tally=1 +32:55.254 [INFO] TestAgent.server.raft: election won: tally=1 +32:55.254 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24664 [Leader]" +32:55.254 [INFO] TestAgent.server: cluster leadership acquired +32:55.255 [INFO] TestAgent.server: New leader elected: payload=Node-23d3562b-e830-b48d-4a9f-6b2ea8631246 +32:55.255 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24664 +32:55.256 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:55.258 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:55.258 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:55.258 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:55.258 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:55.258 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:55.258 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-23d3562b-e830-b48d-4a9f-6b2ea8631246 +32:55.259 [INFO] TestAgent.server: member joined, marking health alive: member=Node-23d3562b-e830-b48d-4a9f-6b2ea8631246 +32:55.259 [INFO] TestAgent.server: federation state anti-entropy synced +32:55.593 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:55.594 [INFO] TestAgent: Synced node info +32:55.594 [DEBUG] TestAgent: Node info in sync +32:55.623 [INFO] TestAgent: Requesting shutdown +32:55.623 [INFO] TestAgent.server: shutting down server +32:55.623 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:55.623 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:55.623 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:55.623 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:55.623 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:55.623 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:55.623 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:55.623 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:55.623 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:55.623 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:55.623 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:55.623Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:55.623Z [INFO] agent.router.manager: shutting down +32:55.623 [INFO] TestAgent: consul server down +32:55.623 [INFO] TestAgent: shutdown complete +32:55.623 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24659 network=tcp +32:55.623 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24659 network=udp +32:55.623 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24660 network=tcp +32:56.124 [INFO] TestAgent: Waiting for endpoints to shut down +32:56.124 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_RegisterService_ScriptCheck_ExecRemoteDisable (0.00s) - --- PASS: TestAgent_RegisterService_ScriptCheck_ExecRemoteDisable/normal (0.70s) - --- PASS: TestAgent_RegisterService_ScriptCheck_ExecRemoteDisable/service_manager (0.81s) + --- PASS: TestAgent_RegisterService_ScriptCheck_ExecRemoteDisable/normal (0.64s) + --- PASS: TestAgent_RegisterService_ScriptCheck_ExecRemoteDisable/service_manager (0.98s) === RUN TestAgent_DeregisterService === PAUSE TestAgent_DeregisterService === RUN TestAgent_DeregisterService_ACLDeny @@ -9647,713 +9745,715 @@ === PAUSE TestAgent_MultiStartStop/#08 === RUN TestAgent_MultiStartStop/#09 === PAUSE TestAgent_MultiStartStop/#09 -=== CONT TestAgent_MultiStartStop/#09 -=== CONT TestAgent_MultiStartStop/#02 -=== CONT TestAgent_MultiStartStop/#03 -=== CONT TestAgent_MultiStartStop/#01 === CONT TestAgent_MultiStartStop/#00 +=== CONT TestAgent_MultiStartStop/#05 === CONT TestAgent_MultiStartStop/#08 -=== CONT TestAgent_MultiStartStop/#07 === CONT TestAgent_MultiStartStop/#06 -2022-09-15T21:41:12.838Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:12.888Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:12.902 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1410237a-6907-b5c6-2f4e-0250e75180b8 Address:127.0.0.1:13461}]" -41:12.903 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1410237a-6907-b5c6-2f4e-0250e75180b8.dc1 127.0.0.1 -41:12.903 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13461 [Follower]" leader= -41:12.975 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1410237a-6907-b5c6-2f4e-0250e75180b8 127.0.0.1 -2022-09-15T21:41:12.975Z [INFO] agent.router: Initializing LAN area manager -41:12.975 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13456 network=udp -41:12.977 [INFO] TestAgent.server: Adding LAN server: server="Node-1410237a-6907-b5c6-2f4e-0250e75180b8 (Addr: tcp/127.0.0.1:13461) (DC: dc1)" -41:12.977 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1410237a-6907-b5c6-2f4e-0250e75180b8.dc1 area=wan -41:12.980 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:12.980 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13461 [Candidate]" term=2 -41:12.981 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:12.981 [DEBUG] TestAgent.server.raft: vote granted: from=1410237a-6907-b5c6-2f4e-0250e75180b8 term=2 tally=1 -41:12.981 [INFO] TestAgent.server.raft: election won: tally=1 -41:12.981 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13461 [Leader]" -41:12.981 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13456 network=tcp -41:12.981 [INFO] TestAgent.server: cluster leadership acquired -41:12.981 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13461 -41:12.983 [INFO] TestAgent.server: New leader elected: payload=Node-1410237a-6907-b5c6-2f4e-0250e75180b8 -41:12.984 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:12.987 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13457 network=tcp -41:12.987 [INFO] TestAgent: started state syncer -41:12.987 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:12.987 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:12.987 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13462 network=tcp -41:12.987 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:12.987 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:12.987 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:12.987 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1410237a-6907-b5c6-2f4e-0250e75180b8 -41:12.987 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1410237a-6907-b5c6-2f4e-0250e75180b8 -41:12.989 [INFO] TestAgent: Synced node info -41:12.989 [DEBUG] TestAgent: Node info in sync -41:12.989 [INFO] TestAgent.server: federation state anti-entropy synced -41:13.238 [INFO] TestAgent: Requesting shutdown -41:13.238 [INFO] TestAgent.server: shutting down server -41:13.238 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:13.238 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:13.238 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:13.238 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:13.238 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:13.238 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:13.238 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:13.238 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:13.238 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:13.238 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:13.238 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:13.238Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:13.239Z [INFO] agent.router.manager: shutting down -41:13.239 [INFO] TestAgent: consul server down -41:13.239 [INFO] TestAgent: shutdown complete -41:13.239 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13456 network=tcp -41:13.239 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13456 network=udp -41:13.239 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13457 network=tcp -41:13.739 [INFO] TestAgent: Waiting for endpoints to shut down -41:13.739 [INFO] TestAgent: Endpoints down -=== CONT TestAgent_MultiStartStop/#05 -2022-09-15T21:41:12.768Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:12.812Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:12.814 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7d7956f3-ce44-11e4-280b-e3524599d7fe Address:127.0.0.1:13440}]" -41:12.815 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7d7956f3-ce44-11e4-280b-e3524599d7fe.dc1 127.0.0.1 -41:12.816 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7d7956f3-ce44-11e4-280b-e3524599d7fe 127.0.0.1 -2022-09-15T21:41:12.816Z [INFO] agent.router: Initializing LAN area manager -41:12.817 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13435 network=udp -41:12.818 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13440 [Follower]" leader= -41:12.819 [INFO] TestAgent.server: Adding LAN server: server="Node-7d7956f3-ce44-11e4-280b-e3524599d7fe (Addr: tcp/127.0.0.1:13440) (DC: dc1)" -41:12.819 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7d7956f3-ce44-11e4-280b-e3524599d7fe.dc1 area=wan -41:12.822 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13435 network=tcp -41:12.823 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13436 network=tcp -41:12.823 [INFO] TestAgent: started state syncer -41:12.823 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13441 network=tcp -41:12.877 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:12.877 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13440 [Candidate]" term=2 -41:12.877 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:12.877 [DEBUG] TestAgent.server.raft: vote granted: from=7d7956f3-ce44-11e4-280b-e3524599d7fe term=2 tally=1 -41:12.877 [INFO] TestAgent.server.raft: election won: tally=1 -41:12.877 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13440 [Leader]" -41:12.878 [INFO] TestAgent.server: cluster leadership acquired -41:12.878 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13440 -41:12.879 [INFO] TestAgent.server: New leader elected: payload=Node-7d7956f3-ce44-11e4-280b-e3524599d7fe -41:12.881 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:12.905 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:12.905 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:12.905 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:12.905 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:12.905 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:12.905 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7d7956f3-ce44-11e4-280b-e3524599d7fe -41:12.905 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7d7956f3-ce44-11e4-280b-e3524599d7fe -41:12.907 [INFO] TestAgent.server: federation state anti-entropy synced -41:13.254 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:13.255 [INFO] TestAgent: Synced node info -41:13.255 [DEBUG] TestAgent: Node info in sync -41:13.259 [INFO] TestAgent: Requesting shutdown -41:13.259 [INFO] TestAgent.server: shutting down server -41:13.259 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:13.259 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:13.259 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:13.259 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:13.259 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:13.259 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:13.259 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:13.260 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:13.260 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:13.260 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:13.260 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:13.260Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:13.260Z [INFO] agent.router.manager: shutting down -41:13.260 [INFO] TestAgent: consul server down -41:13.260 [INFO] TestAgent: shutdown complete -41:13.260 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13435 network=tcp -41:13.260 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13435 network=udp -41:13.260 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13436 network=tcp -41:13.760 [INFO] TestAgent: Waiting for endpoints to shut down -41:13.760 [INFO] TestAgent: Endpoints down +=== CONT TestAgent_MultiStartStop/#03 +=== CONT TestAgent_MultiStartStop/#07 +=== CONT TestAgent_MultiStartStop/#09 === CONT TestAgent_MultiStartStop/#04 -2022-09-15T21:41:12.819Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:12.837Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:12.849 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:50af30e1-a81e-0f69-9983-78e8d9651f5d Address:127.0.0.1:13489}]" -41:12.855 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13489 [Follower]" leader= -41:12.856 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-50af30e1-a81e-0f69-9983-78e8d9651f5d.dc1 127.0.0.1 -41:12.858 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-50af30e1-a81e-0f69-9983-78e8d9651f5d 127.0.0.1 -2022-09-15T21:41:12.858Z [INFO] agent.router: Initializing LAN area manager -41:12.860 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13484 network=udp -41:12.871 [INFO] TestAgent.server: Adding LAN server: server="Node-50af30e1-a81e-0f69-9983-78e8d9651f5d (Addr: tcp/127.0.0.1:13489) (DC: dc1)" -41:12.879 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-50af30e1-a81e-0f69-9983-78e8d9651f5d.dc1 area=wan -41:12.879 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13484 network=tcp -41:12.900 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13485 network=tcp -41:12.900 [INFO] TestAgent: started state syncer -41:12.900 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13490 network=tcp -41:12.907 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:12.907 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13489 [Candidate]" term=2 -41:12.907 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:12.907 [DEBUG] TestAgent.server.raft: vote granted: from=50af30e1-a81e-0f69-9983-78e8d9651f5d term=2 tally=1 -41:12.907 [INFO] TestAgent.server.raft: election won: tally=1 -41:12.907 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13489 [Leader]" -41:12.908 [INFO] TestAgent.server: cluster leadership acquired -41:12.908 [INFO] TestAgent.server: New leader elected: payload=Node-50af30e1-a81e-0f69-9983-78e8d9651f5d -41:12.908 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13489 -41:12.909 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:12.911 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:12.911 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:12.911 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:12.911 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:12.911 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:12.911 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-50af30e1-a81e-0f69-9983-78e8d9651f5d -41:12.911 [INFO] TestAgent.server: member joined, marking health alive: member=Node-50af30e1-a81e-0f69-9983-78e8d9651f5d -41:12.912 [INFO] TestAgent.server: federation state anti-entropy synced -41:13.275 [INFO] TestAgent: Requesting shutdown -41:13.275 [INFO] TestAgent.server: shutting down server -41:13.275 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:13.275 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:13.275 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:13.275 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:13.275 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:13.275 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:13.275 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:13.275 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:13.276 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:13.276 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:13.276 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:13.276 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:13.276Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:13.276Z [INFO] agent.router.manager: shutting down -41:13.276 [INFO] TestAgent: consul server down -41:13.276 [INFO] TestAgent: shutdown complete -41:13.276 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13484 network=tcp -41:13.276 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13484 network=udp -41:13.276 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13485 network=tcp -41:13.777 [INFO] TestAgent: Waiting for endpoints to shut down -41:13.777 [INFO] TestAgent: Endpoints down -2022-09-15T21:41:12.769Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:12.813Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:12.816 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8dc37c9c-15a7-28e0-bd50-d047d46557e3 Address:127.0.0.1:13447}]" -41:12.816 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8dc37c9c-15a7-28e0-bd50-d047d46557e3.dc1 127.0.0.1 -41:12.817 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8dc37c9c-15a7-28e0-bd50-d047d46557e3 127.0.0.1 -2022-09-15T21:41:12.817Z [INFO] agent.router: Initializing LAN area manager -41:12.818 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13442 network=udp -41:12.818 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13447 [Follower]" leader= -41:12.818 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8dc37c9c-15a7-28e0-bd50-d047d46557e3.dc1 area=wan -41:12.818 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13442 network=tcp -41:12.818 [INFO] TestAgent.server: Adding LAN server: server="Node-8dc37c9c-15a7-28e0-bd50-d047d46557e3 (Addr: tcp/127.0.0.1:13447) (DC: dc1)" -41:12.819 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13443 network=tcp -41:12.819 [INFO] TestAgent: started state syncer -41:12.819 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13448 network=tcp -41:12.886 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:12.886 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13447 [Candidate]" term=2 -41:12.886 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:12.886 [DEBUG] TestAgent.server.raft: vote granted: from=8dc37c9c-15a7-28e0-bd50-d047d46557e3 term=2 tally=1 -41:12.886 [INFO] TestAgent.server.raft: election won: tally=1 -41:12.886 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13447 [Leader]" -41:12.886 [INFO] TestAgent.server: cluster leadership acquired -41:12.886 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13447 -41:12.890 [INFO] TestAgent.server: New leader elected: payload=Node-8dc37c9c-15a7-28e0-bd50-d047d46557e3 -41:12.900 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:12.906 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:12.906 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:12.906 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:12.906 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:12.906 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:12.906 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8dc37c9c-15a7-28e0-bd50-d047d46557e3 -41:12.906 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8dc37c9c-15a7-28e0-bd50-d047d46557e3 -41:12.907 [INFO] TestAgent.server: federation state anti-entropy synced -41:13.027 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:13.028 [INFO] TestAgent: Synced node info -41:13.028 [DEBUG] TestAgent: Node info in sync -41:13.387 [INFO] TestAgent: Requesting shutdown -41:13.387 [INFO] TestAgent.server: shutting down server -41:13.387 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:13.387 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:13.387 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:13.387 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:13.387 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:13.387 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:13.387 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:13.387 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:13.387 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:13.387 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:13.388 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:13.388Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:13.388Z [INFO] agent.router.manager: shutting down -41:13.388 [INFO] TestAgent: consul server down -41:13.388 [INFO] TestAgent: shutdown complete -41:13.388 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13442 network=tcp -41:13.388 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13442 network=udp -41:13.388 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13443 network=tcp -41:13.888 [INFO] TestAgent: Waiting for endpoints to shut down -41:13.888 [INFO] TestAgent: Endpoints down -2022-09-15T21:41:12.883Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:12.967Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:12.975 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0774a8c9-221d-9d6c-8474-a7d779efd435 Address:127.0.0.1:13468}]" -41:12.977 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0774a8c9-221d-9d6c-8474-a7d779efd435.dc1 127.0.0.1 -41:12.978 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13468 [Follower]" leader= -41:12.982 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0774a8c9-221d-9d6c-8474-a7d779efd435 127.0.0.1 -2022-09-15T21:41:12.982Z [INFO] agent.router: Initializing LAN area manager -41:12.983 [INFO] TestAgent.server: Adding LAN server: server="Node-0774a8c9-221d-9d6c-8474-a7d779efd435 (Addr: tcp/127.0.0.1:13468) (DC: dc1)" -41:12.983 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0774a8c9-221d-9d6c-8474-a7d779efd435.dc1 area=wan -41:12.984 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13463 network=tcp -41:12.984 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13463 network=udp -41:12.990 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13464 network=tcp -41:12.990 [INFO] TestAgent: started state syncer -41:12.991 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13469 network=tcp -41:13.017 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:13.017 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13468 [Candidate]" term=2 -41:13.017 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:13.017 [DEBUG] TestAgent.server.raft: vote granted: from=0774a8c9-221d-9d6c-8474-a7d779efd435 term=2 tally=1 -41:13.017 [INFO] TestAgent.server.raft: election won: tally=1 -41:13.017 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13468 [Leader]" -41:13.017 [INFO] TestAgent.server: cluster leadership acquired -41:13.017 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13468 -41:13.018 [INFO] TestAgent.server: New leader elected: payload=Node-0774a8c9-221d-9d6c-8474-a7d779efd435 -41:13.019 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:13.021 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:13.021 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:13.021 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:13.021 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:13.021 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:13.021 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0774a8c9-221d-9d6c-8474-a7d779efd435 -41:13.021 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0774a8c9-221d-9d6c-8474-a7d779efd435 -41:13.022 [INFO] TestAgent.server: federation state anti-entropy synced -41:13.095 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:13.096 [INFO] TestAgent: Synced node info -41:13.398 [INFO] TestAgent: Requesting shutdown -41:13.398 [INFO] TestAgent.server: shutting down server -41:13.398 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:13.398 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:13.398 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:13.398 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:13.398 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:13.399 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:13.399 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:13.399 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:13.399 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:13.399 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:13.399 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:13.399Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:13.399Z [INFO] agent.router.manager: shutting down -41:13.399 [INFO] TestAgent: consul server down -41:13.399 [INFO] TestAgent: shutdown complete -41:13.399 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13463 network=tcp -41:13.399 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13463 network=udp -41:13.399 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13464 network=tcp -41:13.899 [INFO] TestAgent: Waiting for endpoints to shut down -41:13.899 [INFO] TestAgent: Endpoints down -2022-09-15T21:41:12.822Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:12.837Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:12.852 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3c8fc2d3-efb0-80e1-3b9e-b93e4e7f5e7b Address:127.0.0.1:13454}]" -41:12.854 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13454 [Follower]" leader= -41:12.859 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3c8fc2d3-efb0-80e1-3b9e-b93e4e7f5e7b.dc1 127.0.0.1 -41:12.878 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3c8fc2d3-efb0-80e1-3b9e-b93e4e7f5e7b 127.0.0.1 -2022-09-15T21:41:12.878Z [INFO] agent.router: Initializing LAN area manager -41:12.879 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13449 network=udp -41:12.879 [INFO] TestAgent.server: Adding LAN server: server="Node-3c8fc2d3-efb0-80e1-3b9e-b93e4e7f5e7b (Addr: tcp/127.0.0.1:13454) (DC: dc1)" -41:12.879 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3c8fc2d3-efb0-80e1-3b9e-b93e4e7f5e7b.dc1 area=wan -41:12.879 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13449 network=tcp -41:12.881 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13450 network=tcp -41:12.881 [INFO] TestAgent: started state syncer -41:12.881 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13455 network=tcp -41:12.901 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:12.901 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13454 [Candidate]" term=2 -41:12.901 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:12.901 [DEBUG] TestAgent.server.raft: vote granted: from=3c8fc2d3-efb0-80e1-3b9e-b93e4e7f5e7b term=2 tally=1 -41:12.901 [INFO] TestAgent.server.raft: election won: tally=1 -41:12.901 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13454 [Leader]" -41:12.902 [INFO] TestAgent.server: cluster leadership acquired -41:12.902 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13454 -41:12.903 [INFO] TestAgent.server: New leader elected: payload=Node-3c8fc2d3-efb0-80e1-3b9e-b93e4e7f5e7b -41:12.904 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:12.906 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:12.906 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:12.906 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:12.906 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:12.906 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:12.906 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3c8fc2d3-efb0-80e1-3b9e-b93e4e7f5e7b -41:12.906 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3c8fc2d3-efb0-80e1-3b9e-b93e4e7f5e7b -41:12.907 [INFO] TestAgent.server: federation state anti-entropy synced -41:13.187 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:13.188 [INFO] TestAgent: Synced node info -41:13.437 [INFO] TestAgent: Requesting shutdown -41:13.437 [INFO] TestAgent.server: shutting down server -41:13.437 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:13.437 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:13.437 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:13.437 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:13.437 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:13.437 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:13.437 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:13.437 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:13.437 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:13.437 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:13.437 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:13.438Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:13.438Z [INFO] agent.router.manager: shutting down -41:13.438 [INFO] TestAgent: consul server down -41:13.438 [INFO] TestAgent: shutdown complete -41:13.438 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13449 network=tcp -41:13.438 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13449 network=udp -41:13.438 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13450 network=tcp -41:13.938 [INFO] TestAgent: Waiting for endpoints to shut down -41:13.938 [INFO] TestAgent: Endpoints down -2022-09-15T21:41:12.877Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:12.970Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:12.993 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fb98ef70-a965-42df-23ce-a1d0135e48e2 Address:127.0.0.1:13482}]" -41:12.994 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13482 [Follower]" leader= -41:12.996 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fb98ef70-a965-42df-23ce-a1d0135e48e2.dc1 127.0.0.1 -41:12.997 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fb98ef70-a965-42df-23ce-a1d0135e48e2 127.0.0.1 -2022-09-15T21:41:12.997Z [INFO] agent.router: Initializing LAN area manager -41:12.997 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fb98ef70-a965-42df-23ce-a1d0135e48e2.dc1 area=wan -41:12.997 [INFO] TestAgent.server: Adding LAN server: server="Node-fb98ef70-a965-42df-23ce-a1d0135e48e2 (Addr: tcp/127.0.0.1:13482) (DC: dc1)" -41:12.997 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13477 network=udp -41:12.997 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13477 network=tcp -41:12.999 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13478 network=tcp -41:12.999 [INFO] TestAgent: started state syncer -41:12.999 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13483 network=tcp -41:13.033 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:13.033 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13482 [Candidate]" term=2 -41:13.033 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:13.033 [DEBUG] TestAgent.server.raft: vote granted: from=fb98ef70-a965-42df-23ce-a1d0135e48e2 term=2 tally=1 -41:13.033 [INFO] TestAgent.server.raft: election won: tally=1 -41:13.033 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13482 [Leader]" -41:13.033 [INFO] TestAgent.server: cluster leadership acquired -41:13.034 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13482 -41:13.035 [INFO] TestAgent.server: New leader elected: payload=Node-fb98ef70-a965-42df-23ce-a1d0135e48e2 -41:13.035 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:13.037 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:13.037 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:13.037 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:13.038 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:13.038 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:13.038 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fb98ef70-a965-42df-23ce-a1d0135e48e2 -41:13.038 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fb98ef70-a965-42df-23ce-a1d0135e48e2 -41:13.038 [INFO] TestAgent.server: federation state anti-entropy synced -41:13.293 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:13.293 [INFO] TestAgent: Synced node info -41:13.523 [INFO] TestAgent: Requesting shutdown -41:13.523 [INFO] TestAgent.server: shutting down server -41:13.523 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:13.523 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:13.523 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:13.523 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:13.523 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:13.523 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:13.523 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:13.523 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:13.523 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:13.523 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:13.523 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:13.523Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:13.523Z [INFO] agent.router.manager: shutting down -41:13.523 [INFO] TestAgent: consul server down -41:13.523 [INFO] TestAgent: shutdown complete -41:13.523 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13477 network=tcp -41:13.524 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13477 network=udp -41:13.524 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13478 network=tcp -41:14.024 [INFO] TestAgent: Waiting for endpoints to shut down -41:14.024 [INFO] TestAgent: Endpoints down -2022-09-15T21:41:12.833Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:12.891Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:12.967 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ce7bf8a7-2d42-32a9-0ee1-c226ef677a2e Address:127.0.0.1:13475}]" -41:12.968 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ce7bf8a7-2d42-32a9-0ee1-c226ef677a2e.dc1 127.0.0.1 -41:12.974 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13475 [Follower]" leader= -41:12.980 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ce7bf8a7-2d42-32a9-0ee1-c226ef677a2e 127.0.0.1 -2022-09-15T21:41:12.980Z [INFO] agent.router: Initializing LAN area manager -41:12.987 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ce7bf8a7-2d42-32a9-0ee1-c226ef677a2e.dc1 area=wan -41:12.990 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13470 network=udp -41:12.990 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13470 network=tcp -41:12.991 [INFO] TestAgent.server: Adding LAN server: server="Node-ce7bf8a7-2d42-32a9-0ee1-c226ef677a2e (Addr: tcp/127.0.0.1:13475) (DC: dc1)" -41:12.992 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13471 network=tcp -41:12.992 [INFO] TestAgent: started state syncer -41:12.993 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13476 network=tcp -41:13.014 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:13.014 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13475 [Candidate]" term=2 -41:13.014 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:13.014 [DEBUG] TestAgent.server.raft: vote granted: from=ce7bf8a7-2d42-32a9-0ee1-c226ef677a2e term=2 tally=1 -41:13.014 [INFO] TestAgent.server.raft: election won: tally=1 -41:13.014 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13475 [Leader]" -41:13.014 [INFO] TestAgent.server: cluster leadership acquired -41:13.014 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13475 -41:13.016 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:13.017 [INFO] TestAgent.server: New leader elected: payload=Node-ce7bf8a7-2d42-32a9-0ee1-c226ef677a2e -41:13.019 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:13.019 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:13.019 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:13.019 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:13.019 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:13.019 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ce7bf8a7-2d42-32a9-0ee1-c226ef677a2e -41:13.019 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ce7bf8a7-2d42-32a9-0ee1-c226ef677a2e -41:13.020 [INFO] TestAgent.server: federation state anti-entropy synced -41:13.356 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:13.356 [INFO] TestAgent: Synced node info -41:13.357 [DEBUG] TestAgent: Node info in sync -41:13.577 [INFO] TestAgent: Requesting shutdown -41:13.577 [INFO] TestAgent.server: shutting down server -41:13.577 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:13.577 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:13.577 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:13.577 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:13.577 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:13.578 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:13.578 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:13.578 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:13.578 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:13.578 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:13.578 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:13.578Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:13.578Z [INFO] agent.router.manager: shutting down -41:13.578 [INFO] TestAgent: consul server down -41:13.578 [INFO] TestAgent: shutdown complete -41:13.578 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13470 network=tcp -41:13.578 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13470 network=udp -41:13.578 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13471 network=tcp -41:14.078 [INFO] TestAgent: Waiting for endpoints to shut down -41:14.078 [INFO] TestAgent: Endpoints down -2022-09-15T21:41:13.778Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:13.795Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:13.798 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:65fabec1-6c9f-f94b-c553-6df675d47654 Address:127.0.0.1:13503}]" -41:13.798 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13503 [Follower]" leader= -41:13.799 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-65fabec1-6c9f-f94b-c553-6df675d47654.dc1 127.0.0.1 -41:13.800 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-65fabec1-6c9f-f94b-c553-6df675d47654 127.0.0.1 -2022-09-15T21:41:13.800Z [INFO] agent.router: Initializing LAN area manager -41:13.800 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13498 network=udp -41:13.800 [INFO] TestAgent.server: Adding LAN server: server="Node-65fabec1-6c9f-f94b-c553-6df675d47654 (Addr: tcp/127.0.0.1:13503) (DC: dc1)" -41:13.801 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-65fabec1-6c9f-f94b-c553-6df675d47654.dc1 area=wan -41:13.801 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13498 network=tcp -41:13.802 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13499 network=tcp -41:13.802 [INFO] TestAgent: started state syncer -41:13.802 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13504 network=tcp -41:13.844 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:13.844 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13503 [Candidate]" term=2 -41:13.845 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:13.845 [DEBUG] TestAgent.server.raft: vote granted: from=65fabec1-6c9f-f94b-c553-6df675d47654 term=2 tally=1 -41:13.845 [INFO] TestAgent.server.raft: election won: tally=1 -41:13.845 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13503 [Leader]" -41:13.845 [INFO] TestAgent.server: cluster leadership acquired -41:13.845 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13503 -41:13.846 [INFO] TestAgent.server: New leader elected: payload=Node-65fabec1-6c9f-f94b-c553-6df675d47654 -41:13.847 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:13.850 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:13.850 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:13.850 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:13.850 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:13.850 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:13.850 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-65fabec1-6c9f-f94b-c553-6df675d47654 -41:13.850 [INFO] TestAgent.server: member joined, marking health alive: member=Node-65fabec1-6c9f-f94b-c553-6df675d47654 -41:13.851 [INFO] TestAgent.server: federation state anti-entropy synced -41:14.142 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:14.143 [INFO] TestAgent: Synced node info -41:14.177 [INFO] TestAgent: Requesting shutdown -41:14.177 [INFO] TestAgent.server: shutting down server -41:14.177 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:14.177 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:14.177 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:14.177 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:14.177 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:14.177 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:14.177 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:14.177 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:14.177 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:14.177 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:14.177 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:14.178Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:14.178Z [INFO] agent.router.manager: shutting down -41:14.178 [INFO] TestAgent: consul server down -41:14.178 [INFO] TestAgent: shutdown complete -41:14.178 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13498 network=tcp -41:14.178 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13498 network=udp -41:14.178 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13499 network=tcp -41:14.678 [INFO] TestAgent: Waiting for endpoints to shut down -41:14.678 [INFO] TestAgent: Endpoints down -2022-09-15T21:41:13.757Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:13.773Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:13.776 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9808c436-ca4c-52fb-83a9-59165989548e Address:127.0.0.1:13496}]" -41:13.777 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13496 [Follower]" leader= -41:13.777 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9808c436-ca4c-52fb-83a9-59165989548e.dc1 127.0.0.1 -41:13.778 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9808c436-ca4c-52fb-83a9-59165989548e 127.0.0.1 -2022-09-15T21:41:13.778Z [INFO] agent.router: Initializing LAN area manager -41:13.779 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13491 network=udp -41:13.779 [INFO] TestAgent.server: Adding LAN server: server="Node-9808c436-ca4c-52fb-83a9-59165989548e (Addr: tcp/127.0.0.1:13496) (DC: dc1)" -41:13.779 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9808c436-ca4c-52fb-83a9-59165989548e.dc1 area=wan -41:13.779 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13491 network=tcp -41:13.780 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13492 network=tcp -41:13.780 [INFO] TestAgent: started state syncer -41:13.780 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13497 network=tcp -41:13.827 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:13.827 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13496 [Candidate]" term=2 -41:13.828 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:13.828 [DEBUG] TestAgent.server.raft: vote granted: from=9808c436-ca4c-52fb-83a9-59165989548e term=2 tally=1 -41:13.828 [INFO] TestAgent.server.raft: election won: tally=1 -41:13.828 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13496 [Leader]" -41:13.828 [INFO] TestAgent.server: cluster leadership acquired -41:13.828 [INFO] TestAgent.server: New leader elected: payload=Node-9808c436-ca4c-52fb-83a9-59165989548e -41:13.828 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13496 -41:13.829 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:13.832 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:13.832 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:13.832 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:13.832 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:13.832 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:13.832 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9808c436-ca4c-52fb-83a9-59165989548e -41:13.832 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9808c436-ca4c-52fb-83a9-59165989548e -41:13.833 [INFO] TestAgent.server: federation state anti-entropy synced -41:14.017 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:14.018 [INFO] TestAgent: Synced node info -41:14.196 [INFO] TestAgent: Requesting shutdown -41:14.196 [INFO] TestAgent.server: shutting down server -41:14.196 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:14.196 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:14.196 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:14.196 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:14.196 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:14.196 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:14.196 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:14.196 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:14.196 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:14.196 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:14.196 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:14.196Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:14.196Z [INFO] agent.router.manager: shutting down -41:14.197 [INFO] TestAgent: consul server down -41:14.197 [INFO] TestAgent: shutdown complete -41:14.197 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13491 network=tcp -41:14.197 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13491 network=udp -41:14.197 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13492 network=tcp -41:14.697 [INFO] TestAgent: Waiting for endpoints to shut down -41:14.697 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:56.195Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:56.259Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:56.262 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8c27bcd8-9ee2-9e7d-b0d3-aa99bec02dd7 Address:127.0.0.1:24692}]" +32:56.265 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24692 [Follower]" leader= +32:56.308 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8c27bcd8-9ee2-9e7d-b0d3-aa99bec02dd7.dc1 127.0.0.1 +32:56.309 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8c27bcd8-9ee2-9e7d-b0d3-aa99bec02dd7 127.0.0.1 +2021-08-13T15:32:56.309Z [INFO] agent.router: Initializing LAN area manager +32:56.309 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24687 network=udp +32:56.310 [INFO] TestAgent.server: Adding LAN server: server="Node-8c27bcd8-9ee2-9e7d-b0d3-aa99bec02dd7 (Addr: tcp/127.0.0.1:24692) (DC: dc1)" +32:56.310 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8c27bcd8-9ee2-9e7d-b0d3-aa99bec02dd7.dc1 area=wan +32:56.310 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24687 network=tcp +32:56.311 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24688 network=tcp +32:56.311 [INFO] TestAgent: started state syncer +32:56.311 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24693 network=tcp +32:56.329 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:56.329 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24692 [Candidate]" term=2 +32:56.329 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:56.329 [DEBUG] TestAgent.server.raft: vote granted: from=8c27bcd8-9ee2-9e7d-b0d3-aa99bec02dd7 term=2 tally=1 +32:56.329 [INFO] TestAgent.server.raft: election won: tally=1 +32:56.329 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24692 [Leader]" +32:56.329 [INFO] TestAgent.server: cluster leadership acquired +32:56.329 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24692 +32:56.331 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:56.332 [INFO] TestAgent.server: New leader elected: payload=Node-8c27bcd8-9ee2-9e7d-b0d3-aa99bec02dd7 +32:56.333 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:56.333 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:56.333 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:56.333 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:56.333 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:56.333 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8c27bcd8-9ee2-9e7d-b0d3-aa99bec02dd7 +32:56.333 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8c27bcd8-9ee2-9e7d-b0d3-aa99bec02dd7 +32:56.335 [INFO] TestAgent.server: federation state anti-entropy synced +32:56.592 [INFO] TestAgent: Requesting shutdown +32:56.592 [INFO] TestAgent.server: shutting down server +32:56.592 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:56.592 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:56.592 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:56.592 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:56.592 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:56.592 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +32:56.592 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:56.592 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:56.592 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:56.592 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:56.592 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:56.592 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:56.592Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:56.593Z [INFO] agent.router.manager: shutting down +32:56.593 [INFO] TestAgent: consul server down +32:56.593 [INFO] TestAgent: shutdown complete +32:56.593 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24687 network=tcp +32:56.593 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24687 network=udp +32:56.593 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24688 network=tcp +32:57.095 [INFO] TestAgent: Waiting for endpoints to shut down +32:57.095 [INFO] TestAgent: Endpoints down +=== CONT TestAgent_MultiStartStop/#02 +2021-08-13T15:32:56.147Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:56.225Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:56.228 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:206e08fc-1726-6f7e-9736-f9ebaf1895be Address:127.0.0.1:24671}]" +32:56.229 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-206e08fc-1726-6f7e-9736-f9ebaf1895be.dc1 127.0.0.1 +32:56.230 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-206e08fc-1726-6f7e-9736-f9ebaf1895be 127.0.0.1 +2021-08-13T15:32:56.230Z [INFO] agent.router: Initializing LAN area manager +32:56.230 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24666 network=udp +32:56.230 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24671 [Follower]" leader= +32:56.231 [INFO] TestAgent.server: Adding LAN server: server="Node-206e08fc-1726-6f7e-9736-f9ebaf1895be (Addr: tcp/127.0.0.1:24671) (DC: dc1)" +32:56.231 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-206e08fc-1726-6f7e-9736-f9ebaf1895be.dc1 area=wan +32:56.231 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24666 network=tcp +32:56.232 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24667 network=tcp +32:56.232 [INFO] TestAgent: started state syncer +32:56.232 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24672 network=tcp +32:56.277 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:56.277 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24671 [Candidate]" term=2 +32:56.277 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:56.278 [DEBUG] TestAgent.server.raft: vote granted: from=206e08fc-1726-6f7e-9736-f9ebaf1895be term=2 tally=1 +32:56.278 [INFO] TestAgent.server.raft: election won: tally=1 +32:56.278 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24671 [Leader]" +32:56.278 [INFO] TestAgent.server: cluster leadership acquired +32:56.278 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24671 +32:56.279 [INFO] TestAgent.server: New leader elected: payload=Node-206e08fc-1726-6f7e-9736-f9ebaf1895be +32:56.279 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:56.282 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:56.282 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:56.282 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:56.282 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:56.282 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:56.282 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-206e08fc-1726-6f7e-9736-f9ebaf1895be +32:56.282 [INFO] TestAgent.server: member joined, marking health alive: member=Node-206e08fc-1726-6f7e-9736-f9ebaf1895be +32:56.283 [INFO] TestAgent.server: federation state anti-entropy synced +32:56.593 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:56.594 [INFO] TestAgent: Synced node info +32:56.669 [INFO] TestAgent: Requesting shutdown +32:56.669 [INFO] TestAgent.server: shutting down server +32:56.669 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:56.669 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:56.669 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:56.669 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:56.669 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:56.669 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:56.669 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:56.669 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:56.669 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:56.669 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:56.670 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:56.670Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:56.670Z [INFO] agent.router.manager: shutting down +32:56.670 [INFO] TestAgent: consul server down +32:56.670 [INFO] TestAgent: shutdown complete +32:56.670 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24666 network=tcp +32:56.670 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24666 network=udp +32:56.670 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24667 network=tcp +32:57.170 [INFO] TestAgent: Waiting for endpoints to shut down +32:57.170 [INFO] TestAgent: Endpoints down +=== CONT TestAgent_MultiStartStop/#01 +2021-08-13T15:32:56.229Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:56.308Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:56.311 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5b18eec4-e91c-2c28-5289-b0f563633be6 Address:127.0.0.1:24713}]" +32:56.311 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24713 [Follower]" leader= +32:56.312 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5b18eec4-e91c-2c28-5289-b0f563633be6.dc1 127.0.0.1 +32:56.313 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5b18eec4-e91c-2c28-5289-b0f563633be6 127.0.0.1 +2021-08-13T15:32:56.313Z [INFO] agent.router: Initializing LAN area manager +32:56.313 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24708 network=udp +32:56.313 [INFO] TestAgent.server: Adding LAN server: server="Node-5b18eec4-e91c-2c28-5289-b0f563633be6 (Addr: tcp/127.0.0.1:24713) (DC: dc1)" +32:56.313 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5b18eec4-e91c-2c28-5289-b0f563633be6.dc1 area=wan +32:56.313 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24708 network=tcp +32:56.314 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24709 network=tcp +32:56.314 [INFO] TestAgent: started state syncer +32:56.314 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24714 network=tcp +32:56.352 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:56.352 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24713 [Candidate]" term=2 +32:56.352 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:56.352 [DEBUG] TestAgent.server.raft: vote granted: from=5b18eec4-e91c-2c28-5289-b0f563633be6 term=2 tally=1 +32:56.352 [INFO] TestAgent.server.raft: election won: tally=1 +32:56.352 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24713 [Leader]" +32:56.352 [INFO] TestAgent.server: cluster leadership acquired +32:56.352 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24713 +32:56.353 [INFO] TestAgent.server: New leader elected: payload=Node-5b18eec4-e91c-2c28-5289-b0f563633be6 +32:56.354 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:56.356 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:56.356 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:56.356 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:56.356 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:56.356 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:56.361 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5b18eec4-e91c-2c28-5289-b0f563633be6 +32:56.361 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5b18eec4-e91c-2c28-5289-b0f563633be6 +32:56.363 [INFO] TestAgent.server: federation state anti-entropy synced +32:56.599 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:56.600 [INFO] TestAgent: Synced node info +32:56.600 [DEBUG] TestAgent: Node info in sync +32:56.775 [INFO] TestAgent: Requesting shutdown +32:56.775 [INFO] TestAgent.server: shutting down server +32:56.775 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:56.775 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:56.775 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:56.775 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:56.775 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:56.775 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:56.776 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:56.776 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:56.775 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:56.775 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:56.776 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:56.776Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:56.776Z [INFO] agent.router.manager: shutting down +32:56.776 [INFO] TestAgent: consul server down +32:56.776 [INFO] TestAgent: shutdown complete +32:56.776 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24708 network=tcp +32:56.776 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24708 network=udp +32:56.776 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24709 network=tcp +32:57.277 [INFO] TestAgent: Waiting for endpoints to shut down +32:57.277 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:56.151Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:56.185Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:56.188 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a2ad225d-0b46-5c17-0d2b-dde5b26bca47 Address:127.0.0.1:24678}]" +32:56.189 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a2ad225d-0b46-5c17-0d2b-dde5b26bca47.dc1 127.0.0.1 +32:56.190 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a2ad225d-0b46-5c17-0d2b-dde5b26bca47 127.0.0.1 +2021-08-13T15:32:56.190Z [INFO] agent.router: Initializing LAN area manager +32:56.190 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24673 network=udp +32:56.191 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24678 [Follower]" leader= +32:56.191 [INFO] TestAgent.server: Adding LAN server: server="Node-a2ad225d-0b46-5c17-0d2b-dde5b26bca47 (Addr: tcp/127.0.0.1:24678) (DC: dc1)" +32:56.191 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a2ad225d-0b46-5c17-0d2b-dde5b26bca47.dc1 area=wan +32:56.191 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24673 network=tcp +32:56.192 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24674 network=tcp +32:56.192 [INFO] TestAgent: started state syncer +32:56.192 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24679 network=tcp +32:56.252 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:56.252 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24678 [Candidate]" term=2 +32:56.254 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:56.254 [DEBUG] TestAgent.server.raft: vote granted: from=a2ad225d-0b46-5c17-0d2b-dde5b26bca47 term=2 tally=1 +32:56.254 [INFO] TestAgent.server.raft: election won: tally=1 +32:56.254 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24678 [Leader]" +32:56.259 [INFO] TestAgent.server: cluster leadership acquired +32:56.260 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24678 +32:56.261 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:56.262 [INFO] TestAgent.server: New leader elected: payload=Node-a2ad225d-0b46-5c17-0d2b-dde5b26bca47 +32:56.263 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:56.263 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:56.263 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:56.263 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:56.263 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:56.263 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a2ad225d-0b46-5c17-0d2b-dde5b26bca47 +32:56.263 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a2ad225d-0b46-5c17-0d2b-dde5b26bca47 +32:56.265 [INFO] TestAgent.server: federation state anti-entropy synced +32:56.551 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:56.552 [INFO] TestAgent: Synced node info +32:56.552 [DEBUG] TestAgent: Node info in sync +32:56.687 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:56.687 [DEBUG] TestAgent: Node info in sync +32:56.805 [INFO] TestAgent: Requesting shutdown +32:56.805 [INFO] TestAgent.server: shutting down server +32:56.805 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:56.805 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:56.805 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:56.805 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:56.805 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:56.805 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:56.805 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:56.805 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:56.805 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:56.805 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:56.805 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:56.805Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:56.805Z [INFO] agent.router.manager: shutting down +32:56.805 [INFO] TestAgent: consul server down +32:56.805 [INFO] TestAgent: shutdown complete +32:56.805 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24673 network=tcp +32:56.806 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24673 network=udp +32:56.806 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24674 network=tcp +32:57.306 [INFO] TestAgent: Waiting for endpoints to shut down +32:57.306 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:56.222Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:56.262Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:56.265 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f5b10b72-d48b-af9d-c1fd-9d1d40d5e344 Address:127.0.0.1:24699}]" +32:56.265 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24699 [Follower]" leader= +32:56.266 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f5b10b72-d48b-af9d-c1fd-9d1d40d5e344.dc1 127.0.0.1 +32:56.308 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f5b10b72-d48b-af9d-c1fd-9d1d40d5e344 127.0.0.1 +2021-08-13T15:32:56.308Z [INFO] agent.router: Initializing LAN area manager +32:56.308 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24694 network=udp +32:56.308 [INFO] TestAgent.server: Adding LAN server: server="Node-f5b10b72-d48b-af9d-c1fd-9d1d40d5e344 (Addr: tcp/127.0.0.1:24699) (DC: dc1)" +32:56.309 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f5b10b72-d48b-af9d-c1fd-9d1d40d5e344.dc1 area=wan +32:56.309 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24694 network=tcp +32:56.310 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24695 network=tcp +32:56.310 [INFO] TestAgent: started state syncer +32:56.310 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24700 network=tcp +32:56.311 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:56.311 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24699 [Candidate]" term=2 +32:56.312 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:56.312 [DEBUG] TestAgent.server.raft: vote granted: from=f5b10b72-d48b-af9d-c1fd-9d1d40d5e344 term=2 tally=1 +32:56.312 [INFO] TestAgent.server.raft: election won: tally=1 +32:56.312 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24699 [Leader]" +32:56.312 [INFO] TestAgent.server: cluster leadership acquired +32:56.312 [INFO] TestAgent.server: New leader elected: payload=Node-f5b10b72-d48b-af9d-c1fd-9d1d40d5e344 +32:56.312 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24699 +32:56.313 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:56.316 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:56.316 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:56.316 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:56.316 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:56.316 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:56.316 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f5b10b72-d48b-af9d-c1fd-9d1d40d5e344 +32:56.316 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f5b10b72-d48b-af9d-c1fd-9d1d40d5e344 +32:56.317 [INFO] TestAgent.server: federation state anti-entropy synced +32:56.655 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:56.655 [INFO] TestAgent: Synced node info +32:56.869 [INFO] TestAgent: Requesting shutdown +32:56.869 [INFO] TestAgent.server: shutting down server +32:56.869 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:56.869 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:56.869 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:56.869 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:56.869 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:56.869 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:56.869 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:56.869 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:56.869 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:56.869 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:56.869 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:56.870Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:56.870Z [INFO] agent.router.manager: shutting down +32:56.870 [INFO] TestAgent: consul server down +32:56.870 [INFO] TestAgent: shutdown complete +32:56.870 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24694 network=tcp +32:56.870 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24694 network=udp +32:56.870 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24695 network=tcp +32:57.370 [INFO] TestAgent: Waiting for endpoints to shut down +32:57.370 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:56.179Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:56.195Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:56.219 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:44202ea4-ca6d-4ef0-36c2-e968a77c9e44 Address:127.0.0.1:24685}]" +32:56.220 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24685 [Follower]" leader= +32:56.231 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-44202ea4-ca6d-4ef0-36c2-e968a77c9e44.dc1 127.0.0.1 +32:56.232 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-44202ea4-ca6d-4ef0-36c2-e968a77c9e44 127.0.0.1 +2021-08-13T15:32:56.232Z [INFO] agent.router: Initializing LAN area manager +32:56.232 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-44202ea4-ca6d-4ef0-36c2-e968a77c9e44.dc1 area=wan +32:56.232 [INFO] TestAgent.server: Adding LAN server: server="Node-44202ea4-ca6d-4ef0-36c2-e968a77c9e44 (Addr: tcp/127.0.0.1:24685) (DC: dc1)" +32:56.232 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24680 network=udp +32:56.243 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24680 network=tcp +32:56.245 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24681 network=tcp +32:56.246 [INFO] TestAgent: started state syncer +32:56.246 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24686 network=tcp +32:56.282 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:56.282 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24685 [Candidate]" term=2 +32:56.282 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:56.282 [DEBUG] TestAgent.server.raft: vote granted: from=44202ea4-ca6d-4ef0-36c2-e968a77c9e44 term=2 tally=1 +32:56.282 [INFO] TestAgent.server.raft: election won: tally=1 +32:56.282 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24685 [Leader]" +32:56.282 [INFO] TestAgent.server: cluster leadership acquired +32:56.283 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24685 +32:56.283 [INFO] TestAgent.server: New leader elected: payload=Node-44202ea4-ca6d-4ef0-36c2-e968a77c9e44 +32:56.284 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:56.286 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:56.286 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:56.286 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:56.286 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:56.286 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:56.286 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-44202ea4-ca6d-4ef0-36c2-e968a77c9e44 +32:56.286 [INFO] TestAgent.server: member joined, marking health alive: member=Node-44202ea4-ca6d-4ef0-36c2-e968a77c9e44 +32:56.288 [INFO] TestAgent.server: federation state anti-entropy synced +32:56.523 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:56.524 [INFO] TestAgent: Synced node info +32:56.524 [DEBUG] TestAgent: Node info in sync +32:56.874 [INFO] TestAgent: Requesting shutdown +32:56.874 [INFO] TestAgent.server: shutting down server +32:56.874 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:56.874 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:56.874 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:56.874 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:56.874 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:56.874 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:56.874 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:56.874 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:56.874 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:56.874 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:56.874 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:56.874Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:56.874Z [INFO] agent.router.manager: shutting down +32:56.874 [INFO] TestAgent: consul server down +32:56.874 [INFO] TestAgent: shutdown complete +32:56.874 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24680 network=tcp +32:56.874 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24680 network=udp +32:56.874 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24681 network=tcp +32:57.375 [INFO] TestAgent: Waiting for endpoints to shut down +32:57.375 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:56.192Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:56.225Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:56.228 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:870061f3-f18d-cd36-d6e5-e3fb3be370bb Address:127.0.0.1:24706}]" +32:56.229 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-870061f3-f18d-cd36-d6e5-e3fb3be370bb.dc1 127.0.0.1 +32:56.230 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-870061f3-f18d-cd36-d6e5-e3fb3be370bb 127.0.0.1 +2021-08-13T15:32:56.230Z [INFO] agent.router: Initializing LAN area manager +32:56.230 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24706 [Follower]" leader= +32:56.231 [INFO] TestAgent.server: Adding LAN server: server="Node-870061f3-f18d-cd36-d6e5-e3fb3be370bb (Addr: tcp/127.0.0.1:24706) (DC: dc1)" +32:56.231 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-870061f3-f18d-cd36-d6e5-e3fb3be370bb.dc1 area=wan +32:56.231 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24701 network=udp +32:56.231 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24701 network=tcp +32:56.232 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24702 network=tcp +32:56.232 [INFO] TestAgent: started state syncer +32:56.232 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24707 network=tcp +32:56.285 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:56.285 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24706 [Candidate]" term=2 +32:56.286 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:56.286 [DEBUG] TestAgent.server.raft: vote granted: from=870061f3-f18d-cd36-d6e5-e3fb3be370bb term=2 tally=1 +32:56.286 [INFO] TestAgent.server.raft: election won: tally=1 +32:56.286 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24706 [Leader]" +32:56.286 [INFO] TestAgent.server: cluster leadership acquired +32:56.286 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24706 +32:56.287 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:56.288 [INFO] TestAgent.server: New leader elected: payload=Node-870061f3-f18d-cd36-d6e5-e3fb3be370bb +32:56.290 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:56.290 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:56.290 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:56.290 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:56.290 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:56.290 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-870061f3-f18d-cd36-d6e5-e3fb3be370bb +32:56.290 [INFO] TestAgent.server: member joined, marking health alive: member=Node-870061f3-f18d-cd36-d6e5-e3fb3be370bb +32:56.291 [INFO] TestAgent.server: federation state anti-entropy synced +32:56.430 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:56.431 [INFO] TestAgent: Synced node info +32:56.431 [DEBUG] TestAgent: Node info in sync +32:56.911 [INFO] TestAgent: Requesting shutdown +32:56.911 [INFO] TestAgent.server: shutting down server +32:56.911 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:56.911 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:56.911 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:56.911 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:56.911 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:56.912 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:56.911 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:56.912 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:56.912 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:56.912 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:56.912 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:56.912Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:56.912Z [INFO] agent.router.manager: shutting down +32:56.912 [INFO] TestAgent: consul server down +32:56.912 [INFO] TestAgent: shutdown complete +32:56.912 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24701 network=tcp +32:56.912 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24701 network=udp +32:56.912 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24702 network=tcp +32:57.412 [INFO] TestAgent: Waiting for endpoints to shut down +32:57.412 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:56.257Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:56.308Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:56.311 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f47b18c7-a513-f596-b144-75acb6231371 Address:127.0.0.1:24720}]" +32:56.311 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24720 [Follower]" leader= +32:56.312 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f47b18c7-a513-f596-b144-75acb6231371.dc1 127.0.0.1 +32:56.313 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f47b18c7-a513-f596-b144-75acb6231371 127.0.0.1 +2021-08-13T15:32:56.313Z [INFO] agent.router: Initializing LAN area manager +32:56.313 [INFO] TestAgent.server: Adding LAN server: server="Node-f47b18c7-a513-f596-b144-75acb6231371 (Addr: tcp/127.0.0.1:24720) (DC: dc1)" +32:56.313 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f47b18c7-a513-f596-b144-75acb6231371.dc1 area=wan +32:56.313 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24715 network=udp +32:56.313 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24715 network=tcp +32:56.314 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24716 network=tcp +32:56.314 [INFO] TestAgent: started state syncer +32:56.314 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24721 network=tcp +32:56.371 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:56.371 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24720 [Candidate]" term=2 +32:56.371 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:56.371 [DEBUG] TestAgent.server.raft: vote granted: from=f47b18c7-a513-f596-b144-75acb6231371 term=2 tally=1 +32:56.371 [INFO] TestAgent.server.raft: election won: tally=1 +32:56.371 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24720 [Leader]" +32:56.372 [INFO] TestAgent.server: cluster leadership acquired +32:56.373 [INFO] TestAgent.server: New leader elected: payload=Node-f47b18c7-a513-f596-b144-75acb6231371 +32:56.373 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24720 +32:56.376 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:56.378 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:56.378 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:56.378 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:56.378 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:56.378 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:56.378 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f47b18c7-a513-f596-b144-75acb6231371 +32:56.378 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f47b18c7-a513-f596-b144-75acb6231371 +32:56.379 [INFO] TestAgent.server: federation state anti-entropy synced +32:56.656 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:56.657 [INFO] TestAgent: Synced node info +32:56.945 [INFO] TestAgent: Requesting shutdown +32:56.945 [INFO] TestAgent.server: shutting down server +32:56.945 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:56.945 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:56.945 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:56.945 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:56.945 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:56.945 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:56.946 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:56.945 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:56.946 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:56.946 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:56.946 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:56.946Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:56.946Z [INFO] agent.router.manager: shutting down +32:56.946 [INFO] TestAgent: consul server down +32:56.946 [INFO] TestAgent: shutdown complete +32:56.946 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24715 network=tcp +32:56.946 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24715 network=udp +32:56.946 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24716 network=tcp +32:57.446 [INFO] TestAgent: Waiting for endpoints to shut down +32:57.446 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:57.119Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:57.135Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:57.138 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:787eb911-516f-1bfc-081a-c64ba98988cf Address:127.0.0.1:24727}]" +32:57.138 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24727 [Follower]" leader= +32:57.139 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-787eb911-516f-1bfc-081a-c64ba98988cf.dc1 127.0.0.1 +32:57.139 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-787eb911-516f-1bfc-081a-c64ba98988cf 127.0.0.1 +2021-08-13T15:32:57.140Z [INFO] agent.router: Initializing LAN area manager +32:57.140 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24722 network=udp +32:57.140 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-787eb911-516f-1bfc-081a-c64ba98988cf.dc1 area=wan +32:57.140 [INFO] TestAgent.server: Adding LAN server: server="Node-787eb911-516f-1bfc-081a-c64ba98988cf (Addr: tcp/127.0.0.1:24727) (DC: dc1)" +32:57.140 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24722 network=tcp +32:57.141 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24723 network=tcp +32:57.141 [INFO] TestAgent: started state syncer +32:57.141 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24728 network=tcp +32:57.201 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:57.201 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24727 [Candidate]" term=2 +32:57.202 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:57.202 [DEBUG] TestAgent.server.raft: vote granted: from=787eb911-516f-1bfc-081a-c64ba98988cf term=2 tally=1 +32:57.202 [INFO] TestAgent.server.raft: election won: tally=1 +32:57.202 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24727 [Leader]" +32:57.202 [INFO] TestAgent.server: cluster leadership acquired +32:57.202 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24727 +32:57.203 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:57.205 [INFO] TestAgent.server: New leader elected: payload=Node-787eb911-516f-1bfc-081a-c64ba98988cf +32:57.206 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:57.206 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:57.206 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:57.206 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:57.206 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:57.206 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-787eb911-516f-1bfc-081a-c64ba98988cf +32:57.206 [INFO] TestAgent.server: member joined, marking health alive: member=Node-787eb911-516f-1bfc-081a-c64ba98988cf +32:57.207 [INFO] TestAgent.server: federation state anti-entropy synced +32:57.207 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:57.208 [INFO] TestAgent: Synced node info +32:57.785 [INFO] TestAgent: Requesting shutdown +32:57.785 [INFO] TestAgent.server: shutting down server +32:57.785 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:57.785 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:57.785 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:57.785 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:57.785 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:57.785 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:57.785 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:57.785 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:57.785 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:57.785 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:57.785 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:57.785Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:57.785Z [INFO] agent.router.manager: shutting down +32:57.785 [INFO] TestAgent: consul server down +32:57.785 [INFO] TestAgent: shutdown complete +32:57.785 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24722 network=tcp +32:57.785 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24722 network=udp +32:57.786 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24723 network=tcp +32:58.286 [INFO] TestAgent: Waiting for endpoints to shut down +32:58.286 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:57.188Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:57.203Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:57.206 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:242221cf-86f6-eb97-e1aa-54d9d99e75b6 Address:127.0.0.1:24734}]" +32:57.206 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24734 [Follower]" leader= +32:57.207 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-242221cf-86f6-eb97-e1aa-54d9d99e75b6.dc1 127.0.0.1 +32:57.208 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-242221cf-86f6-eb97-e1aa-54d9d99e75b6 127.0.0.1 +2021-08-13T15:32:57.208Z [INFO] agent.router: Initializing LAN area manager +32:57.208 [INFO] TestAgent.server: Adding LAN server: server="Node-242221cf-86f6-eb97-e1aa-54d9d99e75b6 (Addr: tcp/127.0.0.1:24734) (DC: dc1)" +32:57.208 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-242221cf-86f6-eb97-e1aa-54d9d99e75b6.dc1 area=wan +32:57.208 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24729 network=udp +32:57.208 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24729 network=tcp +32:57.209 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24730 network=tcp +32:57.209 [INFO] TestAgent: started state syncer +32:57.209 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24735 network=tcp +32:57.250 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:57.250 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24734 [Candidate]" term=2 +32:57.251 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:57.251 [DEBUG] TestAgent.server.raft: vote granted: from=242221cf-86f6-eb97-e1aa-54d9d99e75b6 term=2 tally=1 +32:57.251 [INFO] TestAgent.server.raft: election won: tally=1 +32:57.251 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24734 [Leader]" +32:57.251 [INFO] TestAgent.server: cluster leadership acquired +32:57.251 [INFO] TestAgent.server: New leader elected: payload=Node-242221cf-86f6-eb97-e1aa-54d9d99e75b6 +32:57.251 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24734 +32:57.252 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:57.255 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:57.255 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:57.255 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:57.255 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:57.255 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:57.255 [INFO] TestAgent.server: federation state anti-entropy synced +32:57.255 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-242221cf-86f6-eb97-e1aa-54d9d99e75b6 +32:57.255 [INFO] TestAgent.server: member joined, marking health alive: member=Node-242221cf-86f6-eb97-e1aa-54d9d99e75b6 +32:57.268 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:57.269 [INFO] TestAgent: Synced node info +32:57.798 [INFO] TestAgent: Requesting shutdown +32:57.798 [INFO] TestAgent.server: shutting down server +32:57.798 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:57.798 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:57.798 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:57.798 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:57.798 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:57.798 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:57.798 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:57.798 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:57.798 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:57.798 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:57.798 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:57.798Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:57.798Z [INFO] agent.router.manager: shutting down +32:57.798 [INFO] TestAgent: consul server down +32:57.798 [INFO] TestAgent: shutdown complete +32:57.798 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24729 network=tcp +32:57.798 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24729 network=udp +32:57.798 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24730 network=tcp +32:58.299 [INFO] TestAgent: Waiting for endpoints to shut down +32:58.299 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_MultiStartStop (0.00s) - --- PASS: TestAgent_MultiStartStop/#03 (0.99s) - --- PASS: TestAgent_MultiStartStop/#09 (1.01s) - --- PASS: TestAgent_MultiStartStop/#06 (1.03s) - --- PASS: TestAgent_MultiStartStop/#02 (1.14s) - --- PASS: TestAgent_MultiStartStop/#00 (1.15s) - --- PASS: TestAgent_MultiStartStop/#01 (1.19s) - --- PASS: TestAgent_MultiStartStop/#08 (1.27s) - --- PASS: TestAgent_MultiStartStop/#07 (1.33s) - --- PASS: TestAgent_MultiStartStop/#04 (0.92s) - --- PASS: TestAgent_MultiStartStop/#05 (0.96s) + --- PASS: TestAgent_MultiStartStop/#08 (0.96s) + --- PASS: TestAgent_MultiStartStop/#05 (1.04s) + --- PASS: TestAgent_MultiStartStop/#04 (1.14s) + --- PASS: TestAgent_MultiStartStop/#03 (1.17s) + --- PASS: TestAgent_MultiStartStop/#07 (1.24s) + --- PASS: TestAgent_MultiStartStop/#06 (1.24s) + --- PASS: TestAgent_MultiStartStop/#09 (1.28s) + --- PASS: TestAgent_MultiStartStop/#00 (1.32s) + --- PASS: TestAgent_MultiStartStop/#02 (1.19s) + --- PASS: TestAgent_MultiStartStop/#01 (1.13s) === RUN TestAgent_ConnectClusterIDConfig === RUN TestAgent_ConnectClusterIDConfig/default_TestAgent_has_fixed_cluster_id -2022-09-15T21:41:14.742Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:14.769Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:14.772 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:069f54ac-292c-7b47-ee13-fb1f9414f39c Address:127.0.0.1:13510}]" -41:14.773 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-069f54ac-292c-7b47-ee13-fb1f9414f39c.dc1 127.0.0.1 -41:14.774 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13510 [Follower]" leader= -41:14.774 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-069f54ac-292c-7b47-ee13-fb1f9414f39c 127.0.0.1 -2022-09-15T21:41:14.774Z [INFO] agent.router: Initializing LAN area manager -41:14.775 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13505 network=udp -41:14.775 [INFO] TestAgent.server: Adding LAN server: server="Node-069f54ac-292c-7b47-ee13-fb1f9414f39c (Addr: tcp/127.0.0.1:13510) (DC: dc1)" -41:14.775 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-069f54ac-292c-7b47-ee13-fb1f9414f39c.dc1 area=wan -41:14.775 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13505 network=tcp -41:14.776 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13506 network=tcp -41:14.776 [INFO] TestAgent: started state syncer -41:14.776 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13511 network=tcp -41:14.839 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:14.839 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13510 [Candidate]" term=2 -41:14.839 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:14.839 [DEBUG] TestAgent.server.raft: vote granted: from=069f54ac-292c-7b47-ee13-fb1f9414f39c term=2 tally=1 -41:14.839 [INFO] TestAgent.server.raft: election won: tally=1 -41:14.839 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13510 [Leader]" -41:14.839 [INFO] TestAgent.server: cluster leadership acquired -41:14.839 [INFO] TestAgent.server: New leader elected: payload=Node-069f54ac-292c-7b47-ee13-fb1f9414f39c -41:14.840 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13510 -41:14.841 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:14.844 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:14.844 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:14.844 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:14.844 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:14.844 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:14.844 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-069f54ac-292c-7b47-ee13-fb1f9414f39c -41:14.844 [INFO] TestAgent.server: member joined, marking health alive: member=Node-069f54ac-292c-7b47-ee13-fb1f9414f39c -41:14.845 [INFO] TestAgent.server: federation state anti-entropy synced -41:15.030 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:15.031 [INFO] TestAgent: Synced node info -41:15.031 [DEBUG] TestAgent: Node info in sync -41:15.186 [INFO] TestAgent: Requesting shutdown -41:15.186 [INFO] TestAgent.server: shutting down server -41:15.186 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:15.186 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:15.186 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:15.186 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:15.186 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:15.186 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:15.186 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:15.186 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:15.186 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:15.186 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:15.186 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:15.187Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:15.187Z [INFO] agent.router.manager: shutting down -41:15.187 [INFO] TestAgent: consul server down -41:15.187 [INFO] TestAgent: shutdown complete -41:15.187 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13505 network=tcp -41:15.187 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13505 network=udp -41:15.187 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13506 network=tcp -41:15.687 [INFO] TestAgent: Waiting for endpoints to shut down -41:15.687 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:58.315Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:58.362Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:58.365 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6c11bc15-fede-ff46-31e9-9bd9902d19a9 Address:127.0.0.1:24741}]" +32:58.365 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24741 [Follower]" leader= +32:58.366 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6c11bc15-fede-ff46-31e9-9bd9902d19a9.dc1 127.0.0.1 +32:58.366 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6c11bc15-fede-ff46-31e9-9bd9902d19a9 127.0.0.1 +2021-08-13T15:32:58.366Z [INFO] agent.router: Initializing LAN area manager +32:58.367 [INFO] TestAgent.server: Adding LAN server: server="Node-6c11bc15-fede-ff46-31e9-9bd9902d19a9 (Addr: tcp/127.0.0.1:24741) (DC: dc1)" +32:58.367 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6c11bc15-fede-ff46-31e9-9bd9902d19a9.dc1 area=wan +32:58.367 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24736 network=tcp +32:58.367 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24736 network=udp +32:58.368 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24737 network=tcp +32:58.368 [INFO] TestAgent: started state syncer +32:58.368 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24742 network=tcp +32:58.422 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:58.422 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24741 [Candidate]" term=2 +32:58.422 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:58.422 [DEBUG] TestAgent.server.raft: vote granted: from=6c11bc15-fede-ff46-31e9-9bd9902d19a9 term=2 tally=1 +32:58.422 [INFO] TestAgent.server.raft: election won: tally=1 +32:58.422 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24741 [Leader]" +32:58.423 [INFO] TestAgent.server: cluster leadership acquired +32:58.423 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24741 +32:58.424 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:58.426 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:58.426 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:58.426 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:58.426 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:58.426 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:58.426 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6c11bc15-fede-ff46-31e9-9bd9902d19a9 +32:58.426 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6c11bc15-fede-ff46-31e9-9bd9902d19a9 +32:58.427 [INFO] TestAgent.server: New leader elected: payload=Node-6c11bc15-fede-ff46-31e9-9bd9902d19a9 +32:58.427 [INFO] TestAgent.server: federation state anti-entropy synced +32:58.479 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:58.479 [INFO] TestAgent: Synced node info +32:58.750 [INFO] TestAgent: Requesting shutdown +32:58.750 [INFO] TestAgent.server: shutting down server +32:58.750 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:58.750 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:58.750 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:58.750 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:58.750 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:58.750 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:58.750 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:58.750 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:58.750 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:58.750 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:58.750 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:58.750Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:58.750Z [INFO] agent.router.manager: shutting down +32:58.751 [INFO] TestAgent: consul server down +32:58.751 [INFO] TestAgent: shutdown complete +32:58.751 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24736 network=tcp +32:58.751 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24736 network=udp +32:58.751 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24737 network=tcp +32:59.251 [INFO] TestAgent: Waiting for endpoints to shut down +32:59.251 [INFO] TestAgent: Endpoints down === RUN TestAgent_ConnectClusterIDConfig/no_cluster_ID_specified_sets_to_test_ID -2022-09-15T21:41:15.716Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:15.735Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:15.741 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:52d39fab-25fe-eb8c-5768-233efa3c75ba Address:127.0.0.1:13517}]" -41:15.741 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13517 [Follower]" leader= -41:15.742 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-52d39fab-25fe-eb8c-5768-233efa3c75ba.dc1 127.0.0.1 -41:15.743 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-52d39fab-25fe-eb8c-5768-233efa3c75ba 127.0.0.1 -2022-09-15T21:41:15.743Z [INFO] agent.router: Initializing LAN area manager -41:15.743 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13512 network=udp -41:15.743 [INFO] TestAgent.server: Adding LAN server: server="Node-52d39fab-25fe-eb8c-5768-233efa3c75ba (Addr: tcp/127.0.0.1:13517) (DC: dc1)" -41:15.743 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-52d39fab-25fe-eb8c-5768-233efa3c75ba.dc1 area=wan -41:15.744 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13512 network=tcp -41:15.745 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13513 network=tcp -41:15.745 [INFO] TestAgent: started state syncer -41:15.745 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13518 network=tcp -41:15.810 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:15.810 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13517 [Candidate]" term=2 -41:15.810 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:15.810 [DEBUG] TestAgent.server.raft: vote granted: from=52d39fab-25fe-eb8c-5768-233efa3c75ba term=2 tally=1 -41:15.810 [INFO] TestAgent.server.raft: election won: tally=1 -41:15.810 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13517 [Leader]" -41:15.810 [INFO] TestAgent.server: cluster leadership acquired -41:15.811 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13517 -41:15.812 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:15.813 [INFO] TestAgent.server: New leader elected: payload=Node-52d39fab-25fe-eb8c-5768-233efa3c75ba -41:15.819 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:15.819 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:15.819 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:15.819 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:15.819 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:15.819 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-52d39fab-25fe-eb8c-5768-233efa3c75ba -41:15.819 [INFO] TestAgent.server: member joined, marking health alive: member=Node-52d39fab-25fe-eb8c-5768-233efa3c75ba -41:15.820 [INFO] TestAgent.server: federation state anti-entropy synced -41:15.924 [INFO] TestAgent: Requesting shutdown -41:15.924 [INFO] TestAgent.server: shutting down server -41:15.924 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:15.924 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:15.924 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:15.924 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:15.924 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:15.924 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:15.924 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:15.924 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:15.924 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:15.924 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:15.924 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:15.924 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:15.925Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:15.925Z [INFO] agent.router.manager: shutting down -41:15.925 [INFO] TestAgent: consul server down -41:15.925 [INFO] TestAgent: shutdown complete -41:15.925 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13512 network=tcp -41:15.925 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13512 network=udp -41:15.925 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13513 network=tcp -41:16.425 [INFO] TestAgent: Waiting for endpoints to shut down -41:16.425 [INFO] TestAgent: Endpoints down +2021-08-13T15:32:59.268Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:59.289Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:59.292 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4338f636-61d7-8455-13e2-b1e311f2728f Address:127.0.0.1:24748}]" +32:59.292 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4338f636-61d7-8455-13e2-b1e311f2728f.dc1 127.0.0.1 +32:59.293 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4338f636-61d7-8455-13e2-b1e311f2728f 127.0.0.1 +2021-08-13T15:32:59.293Z [INFO] agent.router: Initializing LAN area manager +32:59.293 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24743 network=udp +32:59.294 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24748 [Follower]" leader= +32:59.294 [INFO] TestAgent.server: Adding LAN server: server="Node-4338f636-61d7-8455-13e2-b1e311f2728f (Addr: tcp/127.0.0.1:24748) (DC: dc1)" +32:59.294 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4338f636-61d7-8455-13e2-b1e311f2728f.dc1 area=wan +32:59.294 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24743 network=tcp +32:59.295 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24744 network=tcp +32:59.295 [INFO] TestAgent: started state syncer +32:59.295 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24749 network=tcp +32:59.352 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:59.352 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24748 [Candidate]" term=2 +32:59.352 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:59.352 [DEBUG] TestAgent.server.raft: vote granted: from=4338f636-61d7-8455-13e2-b1e311f2728f term=2 tally=1 +32:59.352 [INFO] TestAgent.server.raft: election won: tally=1 +32:59.352 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24748 [Leader]" +32:59.352 [INFO] TestAgent.server: cluster leadership acquired +32:59.352 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24748 +32:59.354 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:59.355 [INFO] TestAgent.server: New leader elected: payload=Node-4338f636-61d7-8455-13e2-b1e311f2728f +32:59.356 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:59.356 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:59.356 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:59.356 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:59.356 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:59.356 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4338f636-61d7-8455-13e2-b1e311f2728f +32:59.356 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4338f636-61d7-8455-13e2-b1e311f2728f +32:59.357 [INFO] TestAgent.server: federation state anti-entropy synced +32:59.489 [INFO] TestAgent: Requesting shutdown +32:59.489 [INFO] TestAgent.server: shutting down server +32:59.489 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:59.489 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:59.489 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:59.489 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:59.489 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:59.489 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:59.489 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:59.489 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +32:59.489 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:59.489 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:59.489 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:59.489 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:59.489Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:59.490Z [INFO] agent.router.manager: shutting down +32:59.490 [INFO] TestAgent: consul server down +32:59.490 [INFO] TestAgent: shutdown complete +32:59.490 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24743 network=tcp +32:59.490 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24743 network=udp +32:59.490 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24744 network=tcp +32:59.990 [INFO] TestAgent: Waiting for endpoints to shut down +32:59.990 [INFO] TestAgent: Endpoints down === RUN TestAgent_ConnectClusterIDConfig/non-UUID_cluster_id_is_fatal -2022-09-15T21:41:16.445Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:16.463Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:16.463 [INFO] TestAgent: Requesting shutdown -41:16.463 [INFO] TestAgent: shutdown complete ---- PASS: TestAgent_ConnectClusterIDConfig (1.77s) - --- PASS: TestAgent_ConnectClusterIDConfig/default_TestAgent_has_fixed_cluster_id (0.99s) +2021-08-13T15:33:00.008Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:00.024Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:00.025 [INFO] TestAgent: Requesting shutdown +33:00.025 [INFO] TestAgent: shutdown complete +--- PASS: TestAgent_ConnectClusterIDConfig (1.73s) + --- PASS: TestAgent_ConnectClusterIDConfig/default_TestAgent_has_fixed_cluster_id (0.95s) --- PASS: TestAgent_ConnectClusterIDConfig/no_cluster_ID_specified_sets_to_test_ID (0.74s) - --- PASS: TestAgent_ConnectClusterIDConfig/non-UUID_cluster_id_is_fatal (0.04s) + --- PASS: TestAgent_ConnectClusterIDConfig/non-UUID_cluster_id_is_fatal (0.03s) === RUN TestAgent_StartStop === PAUSE TestAgent_StartStop === RUN TestAgent_RPCPing @@ -10377,170 +10477,170 @@ === RUN TestAgent_AddService/service_manager/one_check/check1_ttl === RUN TestAgent_AddService/service_manager/multiple_checks === RUN TestAgent_AddService/service_manager/multiple_checks/svcid2 +=== RUN TestAgent_AddService/service_manager/multiple_checks/check1 === RUN TestAgent_AddService/service_manager/multiple_checks/check-noname === RUN TestAgent_AddService/service_manager/multiple_checks/service:svcid2:3 === RUN TestAgent_AddService/service_manager/multiple_checks/service:svcid2:4 -=== RUN TestAgent_AddService/service_manager/multiple_checks/check1 -=== RUN TestAgent_AddService/service_manager/multiple_checks/service:svcid2:3_ttl -=== RUN TestAgent_AddService/service_manager/multiple_checks/service:svcid2:4_ttl === RUN TestAgent_AddService/service_manager/multiple_checks/check1_ttl === RUN TestAgent_AddService/service_manager/multiple_checks/check-noname_ttl +=== RUN TestAgent_AddService/service_manager/multiple_checks/service:svcid2:3_ttl +=== RUN TestAgent_AddService/service_manager/multiple_checks/service:svcid2:4_ttl === RUN TestAgent_AddService/normal/one_check === RUN TestAgent_AddService/normal/one_check/svcid1 === RUN TestAgent_AddService/normal/one_check/check1 === RUN TestAgent_AddService/normal/one_check/check1_ttl === RUN TestAgent_AddService/normal/multiple_checks === RUN TestAgent_AddService/normal/multiple_checks/svcid2 +=== RUN TestAgent_AddService/normal/multiple_checks/service:svcid2:4 === RUN TestAgent_AddService/normal/multiple_checks/check1 === RUN TestAgent_AddService/normal/multiple_checks/check-noname === RUN TestAgent_AddService/normal/multiple_checks/service:svcid2:3 -=== RUN TestAgent_AddService/normal/multiple_checks/service:svcid2:4 +=== RUN TestAgent_AddService/normal/multiple_checks/check1_ttl === RUN TestAgent_AddService/normal/multiple_checks/check-noname_ttl === RUN TestAgent_AddService/normal/multiple_checks/service:svcid2:3_ttl === RUN TestAgent_AddService/normal/multiple_checks/service:svcid2:4_ttl -=== RUN TestAgent_AddService/normal/multiple_checks/check1_ttl -2022-09-15T21:41:16.484Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:16.543Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:16.554 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2a993396-9fbe-9128-5987-e327e068d52d Address:127.0.0.1:13538}]" -41:16.555 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 -41:16.555 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13538 [Follower]" leader= -41:16.556 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 -2022-09-15T21:41:16.556Z [INFO] agent.router: Initializing LAN area manager -41:16.556 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13533 network=udp -41:16.557 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan -41:16.557 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:13538) (DC: dc1)" -41:16.557 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13533 network=tcp -41:16.558 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13534 network=tcp -41:16.558 [INFO] TestAgent: started state syncer -41:16.558 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13539 network=tcp -41:16.603 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:16.603 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13538 [Candidate]" term=2 -41:16.603 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:16.603 [DEBUG] TestAgent.server.raft: vote granted: from=2a993396-9fbe-9128-5987-e327e068d52d term=2 tally=1 -41:16.603 [INFO] TestAgent.server.raft: election won: tally=1 -41:16.603 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13538 [Leader]" -41:16.603 [INFO] TestAgent.server: cluster leadership acquired -41:16.604 [INFO] TestAgent.server: New leader elected: payload=node1 -41:16.604 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13538 -41:16.605 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:16.608 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:16.608 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:16.608 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:16.608 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:16.608 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:16.608 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 -41:16.608 [INFO] TestAgent.server: member joined, marking health alive: member=node1 -41:16.609 [INFO] TestAgent.server: federation state anti-entropy synced -41:16.611 [INFO] TestAgent: Requesting shutdown -41:16.611 [INFO] TestAgent.server: shutting down server -41:16.611 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:16.611 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:16.611 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:16.611 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:16.611 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:16.611 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:16.611 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:16.611 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:16.611 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:16.611 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:16.611 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:16.611 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:16.611Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:16.611Z [INFO] agent.router.manager: shutting down -41:16.611 [INFO] TestAgent: consul server down -41:16.611 [INFO] TestAgent: shutdown complete -41:16.611 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13533 network=tcp -41:16.611 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13533 network=udp -41:16.612 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13534 network=tcp -41:17.112 [INFO] TestAgent: Waiting for endpoints to shut down -41:17.112 [INFO] TestAgent: Endpoints down -2022-09-15T21:41:16.481Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:16.538Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:16.541 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8dc63462-9af4-bce7-b93d-8894893b343f Address:127.0.0.1:13531}]" -41:16.541 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13531 [Follower]" leader= -41:16.542 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 -41:16.543 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 -2022-09-15T21:41:16.543Z [INFO] agent.router: Initializing LAN area manager -41:16.543 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13526 network=udp -41:16.543 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:13531) (DC: dc1)" -41:16.543 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan -41:16.544 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13526 network=tcp -41:16.544 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13527 network=tcp -41:16.544 [INFO] TestAgent: started state syncer -41:16.545 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13532 network=tcp -41:16.604 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:16.604 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13531 [Candidate]" term=2 -41:16.604 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:16.604 [DEBUG] TestAgent.server.raft: vote granted: from=8dc63462-9af4-bce7-b93d-8894893b343f term=2 tally=1 -41:16.604 [INFO] TestAgent.server.raft: election won: tally=1 -41:16.604 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13531 [Leader]" -41:16.604 [INFO] TestAgent.server: cluster leadership acquired -41:16.604 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13531 -41:16.605 [INFO] TestAgent.server: New leader elected: payload=node1 -41:16.607 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:16.609 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:16.609 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:16.609 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:16.609 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:16.609 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:16.609 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 -41:16.609 [INFO] TestAgent.server: member joined, marking health alive: member=node1 -41:16.610 [INFO] TestAgent.server: federation state anti-entropy synced -41:16.617 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:16.617 [INFO] TestAgent: Synced node info -41:16.734 [INFO] TestAgent: Requesting shutdown -41:16.734 [INFO] TestAgent.server: shutting down server -41:16.734 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:16.734 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:16.734 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:16.734 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:16.734 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:16.735 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:16.735 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:16.735 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:16.735 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:16.735 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:16.735 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:16.735Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:16.735Z [INFO] agent.router.manager: shutting down -41:16.735 [INFO] TestAgent: consul server down -41:16.735 [INFO] TestAgent: shutdown complete -41:16.735 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13526 network=tcp -41:16.735 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13526 network=udp -41:16.735 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13527 network=tcp -41:17.236 [INFO] TestAgent: Waiting for endpoints to shut down -41:17.236 [INFO] TestAgent: Endpoints down +2021-08-13T15:33:00.056Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:00.126Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:00.129 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9665ba72-528f-e287-9479-14afe768a7d6 Address:127.0.0.1:24769}]" +33:00.129 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 +33:00.130 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 +2021-08-13T15:33:00.130Z [INFO] agent.router: Initializing LAN area manager +33:00.130 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24764 network=udp +33:00.131 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24769 [Follower]" leader= +33:00.131 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan +33:00.131 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:24769) (DC: dc1)" +33:00.131 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24764 network=tcp +33:00.132 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24765 network=tcp +33:00.132 [INFO] TestAgent: started state syncer +33:00.132 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24770 network=tcp +33:00.179 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:00.179 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24769 [Candidate]" term=2 +33:00.179 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:00.179 [DEBUG] TestAgent.server.raft: vote granted: from=9665ba72-528f-e287-9479-14afe768a7d6 term=2 tally=1 +33:00.179 [INFO] TestAgent.server.raft: election won: tally=1 +33:00.179 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24769 [Leader]" +33:00.179 [INFO] TestAgent.server: cluster leadership acquired +33:00.180 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24769 +33:00.181 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:00.181 [INFO] TestAgent.server: New leader elected: payload=node1 +33:00.185 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:00.185 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:00.185 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:00.185 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:00.185 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:00.186 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 +33:00.186 [INFO] TestAgent.server: member joined, marking health alive: member=node1 +33:00.187 [INFO] TestAgent.server: federation state anti-entropy synced +33:00.372 [INFO] TestAgent: Requesting shutdown +33:00.372 [INFO] TestAgent.server: shutting down server +33:00.372 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:00.372 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:00.372 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:00.372 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:00.372 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:00.372 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:00.372 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:00.373 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:00.373 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:00.373 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +33:00.373 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:00.373 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +2021-08-13T15:33:00.373Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:00.373Z [INFO] agent.router.manager: shutting down +33:00.373 [INFO] TestAgent: consul server down +33:00.373 [INFO] TestAgent: shutdown complete +33:00.373 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24764 network=tcp +33:00.373 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24764 network=udp +33:00.373 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24765 network=tcp +33:00.873 [INFO] TestAgent: Waiting for endpoints to shut down +33:00.873 [INFO] TestAgent: Endpoints down +2021-08-13T15:33:00.077Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:00.117Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:00.126 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d53b9ff0-84e1-12b4-91a2-e0b9110f71be Address:127.0.0.1:24762}]" +33:00.127 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 +33:00.128 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 +2021-08-13T15:33:00.128Z [INFO] agent.router: Initializing LAN area manager +33:00.128 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24757 network=udp +33:00.128 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24762 [Follower]" leader= +33:00.129 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:24762) (DC: dc1)" +33:00.129 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan +33:00.129 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24757 network=tcp +33:00.130 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24758 network=tcp +33:00.130 [INFO] TestAgent: started state syncer +33:00.130 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24763 network=tcp +33:00.165 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:00.165 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24762 [Candidate]" term=2 +33:00.165 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:00.165 [DEBUG] TestAgent.server.raft: vote granted: from=d53b9ff0-84e1-12b4-91a2-e0b9110f71be term=2 tally=1 +33:00.165 [INFO] TestAgent.server.raft: election won: tally=1 +33:00.165 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24762 [Leader]" +33:00.165 [INFO] TestAgent.server: cluster leadership acquired +33:00.165 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24762 +33:00.166 [INFO] TestAgent.server: New leader elected: payload=node1 +33:00.167 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:00.169 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:00.169 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:00.169 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:00.169 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:00.169 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:00.169 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 +33:00.169 [INFO] TestAgent.server: member joined, marking health alive: member=node1 +33:00.170 [INFO] TestAgent.server: federation state anti-entropy synced +33:00.283 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:00.284 [INFO] TestAgent: Synced node info +33:00.430 [INFO] TestAgent: Requesting shutdown +33:00.430 [INFO] TestAgent.server: shutting down server +33:00.430 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:00.430 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:00.430 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:00.430 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:00.430 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:00.430 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:00.430 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:00.430 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:00.430 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:00.430 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +33:00.430 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +2021-08-13T15:33:00.430Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:00.430Z [INFO] agent.router.manager: shutting down +33:00.430 [INFO] TestAgent: consul server down +33:00.430 [INFO] TestAgent: shutdown complete +33:00.430 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24757 network=tcp +33:00.430 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24757 network=udp +33:00.430 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24758 network=tcp +33:00.931 [INFO] TestAgent: Waiting for endpoints to shut down +33:00.931 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_AddService (0.00s) - --- PASS: TestAgent_AddService/service_manager (0.65s) + --- PASS: TestAgent_AddService/service_manager (0.85s) --- PASS: TestAgent_AddService/service_manager/one_check (0.00s) --- PASS: TestAgent_AddService/service_manager/one_check/svcid1 (0.00s) --- PASS: TestAgent_AddService/service_manager/one_check/check1 (0.00s) --- PASS: TestAgent_AddService/service_manager/one_check/check1_ttl (0.00s) --- PASS: TestAgent_AddService/service_manager/multiple_checks (0.00s) --- PASS: TestAgent_AddService/service_manager/multiple_checks/svcid2 (0.00s) + --- PASS: TestAgent_AddService/service_manager/multiple_checks/check1 (0.00s) --- PASS: TestAgent_AddService/service_manager/multiple_checks/check-noname (0.00s) --- PASS: TestAgent_AddService/service_manager/multiple_checks/service:svcid2:3 (0.00s) --- PASS: TestAgent_AddService/service_manager/multiple_checks/service:svcid2:4 (0.00s) - --- PASS: TestAgent_AddService/service_manager/multiple_checks/check1 (0.00s) - --- PASS: TestAgent_AddService/service_manager/multiple_checks/service:svcid2:3_ttl (0.00s) - --- PASS: TestAgent_AddService/service_manager/multiple_checks/service:svcid2:4_ttl (0.00s) --- PASS: TestAgent_AddService/service_manager/multiple_checks/check1_ttl (0.00s) --- PASS: TestAgent_AddService/service_manager/multiple_checks/check-noname_ttl (0.00s) - --- PASS: TestAgent_AddService/normal (0.77s) + --- PASS: TestAgent_AddService/service_manager/multiple_checks/service:svcid2:3_ttl (0.00s) + --- PASS: TestAgent_AddService/service_manager/multiple_checks/service:svcid2:4_ttl (0.00s) + --- PASS: TestAgent_AddService/normal (0.91s) --- PASS: TestAgent_AddService/normal/one_check (0.00s) --- PASS: TestAgent_AddService/normal/one_check/svcid1 (0.00s) --- PASS: TestAgent_AddService/normal/one_check/check1 (0.00s) --- PASS: TestAgent_AddService/normal/one_check/check1_ttl (0.00s) --- PASS: TestAgent_AddService/normal/multiple_checks (0.00s) --- PASS: TestAgent_AddService/normal/multiple_checks/svcid2 (0.00s) + --- PASS: TestAgent_AddService/normal/multiple_checks/service:svcid2:4 (0.00s) --- PASS: TestAgent_AddService/normal/multiple_checks/check1 (0.00s) --- PASS: TestAgent_AddService/normal/multiple_checks/check-noname (0.00s) --- PASS: TestAgent_AddService/normal/multiple_checks/service:svcid2:3 (0.00s) - --- PASS: TestAgent_AddService/normal/multiple_checks/service:svcid2:4 (0.00s) + --- PASS: TestAgent_AddService/normal/multiple_checks/check1_ttl (0.00s) --- PASS: TestAgent_AddService/normal/multiple_checks/check-noname_ttl (0.00s) --- PASS: TestAgent_AddService/normal/multiple_checks/service:svcid2:3_ttl (0.00s) --- PASS: TestAgent_AddService/normal/multiple_checks/service:svcid2:4_ttl (0.00s) - --- PASS: TestAgent_AddService/normal/multiple_checks/check1_ttl (0.00s) === RUN TestAgent_AddServices_AliasUpdateCheckNotReverted === RUN TestAgent_AddServices_AliasUpdateCheckNotReverted/normal === PAUSE TestAgent_AddServices_AliasUpdateCheckNotReverted/normal @@ -10548,233 +10648,175 @@ === PAUSE TestAgent_AddServices_AliasUpdateCheckNotReverted/service_manager === CONT TestAgent_AddServices_AliasUpdateCheckNotReverted/normal === CONT TestAgent_AddServices_AliasUpdateCheckNotReverted/service_manager -2022-09-15T21:41:17.255Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:17.272Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:17.275 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:db6885bf-0539-2678-9047-4cd78d823fe8 Address:127.0.0.1:13545}]" -41:17.275 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13545 [Follower]" leader= -41:17.276 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 -41:17.277 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 -2022-09-15T21:41:17.277Z [INFO] agent.router: Initializing LAN area manager -41:17.277 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13540 network=udp -41:17.282 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan -41:17.284 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:13545) (DC: dc1)" -41:17.285 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13540 network=tcp -41:17.298 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13541 network=tcp -41:17.299 [INFO] TestAgent: started state syncer -41:17.299 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13546 network=tcp -41:17.328 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:17.328 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13545 [Candidate]" term=2 -41:17.328 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:17.328 [DEBUG] TestAgent.server.raft: vote granted: from=db6885bf-0539-2678-9047-4cd78d823fe8 term=2 tally=1 -41:17.328 [INFO] TestAgent.server.raft: election won: tally=1 -41:17.328 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13545 [Leader]" -41:17.328 [INFO] TestAgent.server: cluster leadership acquired -41:17.329 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13545 -41:17.330 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:17.333 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:17.333 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:17.333 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:17.333 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:17.333 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:17.333 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 -41:17.333 [INFO] TestAgent.server: member joined, marking health alive: member=node1 -41:17.333 [INFO] TestAgent.server: New leader elected: payload=node1 -41:17.334 [INFO] TestAgent.server: federation state anti-entropy synced -41:17.656 [INFO] TestAgent: Requesting shutdown -41:17.656 [INFO] TestAgent.server: shutting down server -41:17.656 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:17.656 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:17.656 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:17.656 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:17.656 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:17.656 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:17.656 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:17.656 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:17.657 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -41:17.657 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:17.657 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:17.657 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -2022-09-15T21:41:17.657Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:17.657Z [INFO] agent.router.manager: shutting down -41:17.657 [INFO] TestAgent: consul server down -41:17.657 [INFO] TestAgent: shutdown complete -41:17.657 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13540 network=tcp -41:17.657 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13540 network=udp -41:17.657 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13541 network=tcp -41:18.158 [INFO] TestAgent: Waiting for endpoints to shut down -41:18.158 [INFO] TestAgent: Endpoints down -2022-09-15T21:41:17.270Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:17.316Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:17.319 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:21488225-e9ed-267e-5688-4e0ac8de27ba Address:127.0.0.1:13552}]" -41:17.319 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13552 [Follower]" leader= -41:17.320 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 -41:17.321 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 -2022-09-15T21:41:17.321Z [INFO] agent.router: Initializing LAN area manager -41:17.321 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13547 network=udp -41:17.321 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:13552) (DC: dc1)" -41:17.321 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan -41:17.321 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13547 network=tcp -41:17.322 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13548 network=tcp -41:17.322 [INFO] TestAgent: started state syncer -41:17.322 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13553 network=tcp -41:17.373 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:17.373 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13552 [Candidate]" term=2 -41:17.373 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:17.373 [DEBUG] TestAgent.server.raft: vote granted: from=21488225-e9ed-267e-5688-4e0ac8de27ba term=2 tally=1 -41:17.373 [INFO] TestAgent.server.raft: election won: tally=1 -41:17.373 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13552 [Leader]" -41:17.374 [INFO] TestAgent.server: cluster leadership acquired -41:17.374 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13552 -41:17.375 [INFO] TestAgent.server: New leader elected: payload=node1 -41:17.375 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:17.378 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:17.378 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:17.378 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:17.378 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:17.378 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:17.378 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 -41:17.378 [INFO] TestAgent.server: member joined, marking health alive: member=node1 -41:17.379 [INFO] TestAgent.server: federation state anti-entropy synced -41:17.441 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:17.441 [INFO] TestAgent: Synced node info -41:17.660 [INFO] TestAgent: Requesting shutdown -41:17.660 [INFO] TestAgent.server: shutting down server -41:17.660 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:17.660 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:17.660 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:17.660 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:17.660 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:17.660 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:17.660 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:17.661 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:17.661 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:17.661 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:17.661 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:17.661Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:17.661Z [INFO] agent.router.manager: shutting down -41:17.661 [INFO] TestAgent: consul server down -41:17.661 [INFO] TestAgent: shutdown complete -41:17.661 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13547 network=tcp -41:17.661 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13547 network=udp -41:17.661 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13548 network=tcp -41:18.161 [INFO] TestAgent: Waiting for endpoints to shut down -41:18.161 [INFO] TestAgent: Endpoints down +2021-08-13T15:33:00.985Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:01.009Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:01.012 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:859eea84-dfe1-5c1e-e441-3b4adf8fa4eb Address:127.0.0.1:24783}]" +33:01.012 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24783 [Follower]" leader= +33:01.012 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 +33:01.013 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 +2021-08-13T15:33:01.013Z [INFO] agent.router: Initializing LAN area manager +33:01.014 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24778 network=udp +33:01.014 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:24783) (DC: dc1)" +33:01.014 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan +33:01.014 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24778 network=tcp +33:01.015 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24779 network=tcp +33:01.015 [INFO] TestAgent: started state syncer +33:01.015 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24784 network=tcp +33:01.061 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:01.061 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24783 [Candidate]" term=2 +33:01.061 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:01.061 [DEBUG] TestAgent.server.raft: vote granted: from=859eea84-dfe1-5c1e-e441-3b4adf8fa4eb term=2 tally=1 +33:01.061 [INFO] TestAgent.server.raft: election won: tally=1 +33:01.061 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24783 [Leader]" +33:01.061 [INFO] TestAgent.server: cluster leadership acquired +33:01.061 [INFO] TestAgent.server: New leader elected: payload=node1 +33:01.061 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24783 +33:01.063 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:01.065 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:01.065 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:01.065 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:01.065 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:01.065 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:01.065 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 +33:01.065 [INFO] TestAgent.server: member joined, marking health alive: member=node1 +33:01.066 [INFO] TestAgent.server: federation state anti-entropy synced +33:01.272 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:01.272 [INFO] TestAgent: Synced node info +33:01.272 [DEBUG] TestAgent: Node info in sync +33:01.391 [INFO] TestAgent: Requesting shutdown +33:01.391 [INFO] TestAgent.server: shutting down server +33:01.391 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:01.391 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:01.391 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:01.391 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:01.391 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:01.391 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:01.391 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:01.391 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:01.391 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:01.391 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:01.391 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:01.392Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:01.392Z [INFO] agent.router.manager: shutting down +33:01.392 [INFO] TestAgent: consul server down +33:01.392 [INFO] TestAgent: shutdown complete +33:01.392 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24778 network=tcp +33:01.392 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24778 network=udp +33:01.392 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24779 network=tcp +33:01.892 [INFO] TestAgent: Waiting for endpoints to shut down +33:01.892 [INFO] TestAgent: Endpoints down +2021-08-13T15:33:00.959Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:01.000Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:01.003 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4aa50478-549e-265b-412f-9a0b8fbd5ed7 Address:127.0.0.1:24776}]" +33:01.004 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 +33:01.005 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 +2021-08-13T15:33:01.005Z [INFO] agent.router: Initializing LAN area manager +33:01.005 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24771 network=udp +33:01.005 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24776 [Follower]" leader= +33:01.006 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:24776) (DC: dc1)" +33:01.006 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan +33:01.006 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24771 network=tcp +33:01.007 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24772 network=tcp +33:01.007 [INFO] TestAgent: started state syncer +33:01.007 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24777 network=tcp +33:01.074 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:01.074 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24776 [Candidate]" term=2 +33:01.075 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:01.075 [DEBUG] TestAgent.server.raft: vote granted: from=4aa50478-549e-265b-412f-9a0b8fbd5ed7 term=2 tally=1 +33:01.075 [INFO] TestAgent.server.raft: election won: tally=1 +33:01.075 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24776 [Leader]" +33:01.075 [INFO] TestAgent.server: cluster leadership acquired +33:01.075 [INFO] TestAgent.server: New leader elected: payload=node1 +33:01.075 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24776 +33:01.076 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:01.079 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:01.079 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:01.079 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:01.079 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:01.079 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:01.079 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 +33:01.079 [INFO] TestAgent.server: member joined, marking health alive: member=node1 +33:01.079 [INFO] TestAgent.server: federation state anti-entropy synced +33:01.250 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:01.251 [INFO] TestAgent: Synced node info +33:01.427 [INFO] TestAgent: Requesting shutdown +33:01.428 [INFO] TestAgent.server: shutting down server +33:01.428 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:01.428 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:01.428 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:01.428 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:01.428 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:01.428 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:01.428 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:01.428 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:01.428 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:01.428 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:01.428 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:01.428Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:01.428Z [INFO] agent.router.manager: shutting down +33:01.428 [INFO] TestAgent: consul server down +33:01.428 [INFO] TestAgent: shutdown complete +33:01.428 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24771 network=tcp +33:01.428 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24771 network=udp +33:01.428 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24772 network=tcp +33:01.928 [INFO] TestAgent: Waiting for endpoints to shut down +33:01.928 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_AddServices_AliasUpdateCheckNotReverted (0.00s) - --- PASS: TestAgent_AddServices_AliasUpdateCheckNotReverted/normal (0.92s) - --- PASS: TestAgent_AddServices_AliasUpdateCheckNotReverted/service_manager (0.92s) + --- PASS: TestAgent_AddServices_AliasUpdateCheckNotReverted/service_manager (0.96s) + --- PASS: TestAgent_AddServices_AliasUpdateCheckNotReverted/normal (1.00s) === RUN TestAgent_CheckAliasRPC -[DEBUG] Service Removed {serviceAlias-9021491549310976979 {}} , err:= -[DEBUG] Service Removed {serviceAlias-5064500450404135988 {}} , err:= -[DEBUG] Service Removed {serviceAlias-4873557783575023733 {}} , err:= -[DEBUG] Service Removed {serviceAlias-5579994848648371008 {}} , err:= -[DEBUG] STARTING shutdown for TestAgent_CheckAliasRPC 2022-09-15 21:41:20.24584508 +0000 UTC m=+33.662351121 -[DEBUG] DONE shutdown for TestAgent_CheckAliasRPC 2022-09-15 21:41:20.25023426 +0000 UTC m=+33.666740301 -2022-09-15T21:41:18.184Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:18.206Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:18.209 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7ed36c67-db84-d8c4-0d4c-fe72d74f41c1 Address:127.0.0.1:13559}]" -41:18.209 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13559 [Follower]" leader= -41:18.210 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 -41:18.211 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 -2022-09-15T21:41:18.211Z [INFO] agent.router: Initializing LAN area manager -41:18.211 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13554 network=udp -41:18.212 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:13559) (DC: dc1)" -41:18.212 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan -41:18.212 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13554 network=tcp -41:18.213 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13555 network=tcp -41:18.213 [INFO] TestAgent: started state syncer -41:18.213 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13560 network=tcp -41:18.253 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:18.253 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13559 [Candidate]" term=2 -41:18.254 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:18.254 [DEBUG] TestAgent.server.raft: vote granted: from=7ed36c67-db84-d8c4-0d4c-fe72d74f41c1 term=2 tally=1 -41:18.254 [INFO] TestAgent.server.raft: election won: tally=1 -41:18.254 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13559 [Leader]" -41:18.254 [INFO] TestAgent.server: cluster leadership acquired -41:18.254 [INFO] TestAgent.server: New leader elected: payload=node1 -41:18.255 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13559 -41:18.256 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:18.259 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:18.259 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:18.259 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:18.259 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:18.259 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:18.259 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 -41:18.259 [INFO] TestAgent.server: member joined, marking health alive: member=node1 -41:18.260 [INFO] TestAgent.server: federation state anti-entropy synced -41:18.397 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:18.398 [INFO] TestAgent: Synced node info -41:20.123 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:20.123 [DEBUG] TestAgent: Node info in sync -41:20.123 [INFO] TestAgent: Synced service: service=svcid1 -41:20.123 [DEBUG] TestAgent: Node info in sync -41:20.123 [DEBUG] TestAgent: Service in sync: service=svcid1 -41:20.144 [DEBUG] TestAgent: Service tag will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: tag="tag-2022-09-15 21:41:20.14439092 +0000 UTC m=+33.560897041" -41:20.144 [DEBUG] TestAgent: Node info in sync -41:20.145 [INFO] TestAgent: Synced service: service=serviceAlias-9021491549310976979 -41:20.146 [INFO] TestAgent: Synced service: service=serviceAlias-5064500450404135988 -41:20.147 [INFO] TestAgent: Synced service: service=serviceAlias-4873557783575023733 -41:20.148 [INFO] TestAgent: Synced service: service=serviceAlias-5579994848648371008 -41:20.149 [INFO] TestAgent: Synced service: service=svcid1 -41:20.149 [DEBUG] TestAgent: Check in sync: check=check-4873557783575023733 -41:20.149 [DEBUG] TestAgent: Check in sync: check=check-5579994848648371008 -41:20.149 [DEBUG] TestAgent: Check in sync: check=check-9021491549310976979 -41:20.149 [DEBUG] TestAgent: Check in sync: check=check-5064500450404135988 -41:20.149 [DEBUG] TestAgent: Node info in sync -41:20.149 [DEBUG] TestAgent: Service in sync: service=serviceAlias-5579994848648371008 -41:20.149 [DEBUG] TestAgent: Service in sync: service=svcid1 -41:20.149 [DEBUG] TestAgent: Service in sync: service=serviceAlias-9021491549310976979 -41:20.149 [DEBUG] TestAgent: Service in sync: service=serviceAlias-5064500450404135988 -41:20.149 [DEBUG] TestAgent: Service in sync: service=serviceAlias-4873557783575023733 -41:20.149 [DEBUG] TestAgent: Check in sync: check=check-5064500450404135988 -41:20.150 [INFO] TestAgent: Synced check: check=check-4873557783575023733 -41:20.150 [DEBUG] TestAgent: Check in sync: check=check-5579994848648371008 -41:20.150 [DEBUG] TestAgent: Check in sync: check=check-9021491549310976979 -41:20.150 [DEBUG] TestAgent: Node info in sync -41:20.150 [DEBUG] TestAgent: Service in sync: service=svcid1 -41:20.150 [DEBUG] TestAgent: Service in sync: service=serviceAlias-9021491549310976979 -41:20.150 [DEBUG] TestAgent: Service in sync: service=serviceAlias-5064500450404135988 -41:20.150 [DEBUG] TestAgent: Service in sync: service=serviceAlias-4873557783575023733 -41:20.150 [DEBUG] TestAgent: Service in sync: service=serviceAlias-5579994848648371008 -41:20.150 [DEBUG] TestAgent: Check in sync: check=check-9021491549310976979 -41:20.150 [DEBUG] TestAgent: Check in sync: check=check-5064500450404135988 -41:20.150 [DEBUG] TestAgent: Check in sync: check=check-4873557783575023733 -41:20.151 [INFO] TestAgent: Synced check: check=check-5579994848648371008 -41:20.151 [DEBUG] TestAgent: Node info in sync -41:20.151 [DEBUG] TestAgent: Service in sync: service=svcid1 -41:20.151 [DEBUG] TestAgent: Service in sync: service=serviceAlias-9021491549310976979 -41:20.151 [DEBUG] TestAgent: Service in sync: service=serviceAlias-5064500450404135988 -41:20.151 [DEBUG] TestAgent: Service in sync: service=serviceAlias-4873557783575023733 -41:20.151 [DEBUG] TestAgent: Service in sync: service=serviceAlias-5579994848648371008 -41:20.151 [DEBUG] TestAgent: Check in sync: check=check-5579994848648371008 -41:20.151 [DEBUG] TestAgent: Check in sync: check=check-9021491549310976979 -41:20.152 [INFO] TestAgent: Synced check: check=check-5064500450404135988 -41:20.152 [DEBUG] TestAgent: Check in sync: check=check-4873557783575023733 -41:20.244 [DEBUG] TestAgent: Service tag will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: tag="tag-2022-09-15 21:41:20.24472178 +0000 UTC m=+33.661228041" -41:20.245 [DEBUG] TestAgent: Service tag will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: tag="tag-2022-09-15 21:41:20.24503546 +0000 UTC m=+33.661541561" -41:20.245 [DEBUG] TestAgent: Service tag will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: tag="tag-2022-09-15 21:41:20.245281 +0000 UTC m=+33.661787041" -41:20.245 [DEBUG] TestAgent: Service tag will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: tag="tag-2022-09-15 21:41:20.245439 +0000 UTC m=+33.661945041" -41:20.245 [DEBUG] TestAgent: Service tag will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: tag="tag-2022-09-15 21:41:20.24560954 +0000 UTC m=+33.662115581" -41:20.245 [DEBUG] TestAgent: Service tag will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: tag="tag-2022-09-15 21:41:20.24576604 +0000 UTC m=+33.662272081" -41:20.245 [DEBUG] TestAgent: Node info in sync -41:20.245 [INFO] TestAgent: Requesting shutdown -41:20.246 [INFO] TestAgent: Synced service: service=svcid1 -41:20.247 [INFO] TestAgent: Deregistered service: service=serviceAlias-9021491549310976979 -41:20.248 [INFO] TestAgent: Deregistered service: service=serviceAlias-5064500450404135988 -41:20.249 [INFO] TestAgent: Deregistered service: service=serviceAlias-4873557783575023733 -41:20.249 [INFO] TestAgent: Deregistered service: service=serviceAlias-5579994848648371008 -41:20.249 [DEBUG] TestAgent: Check in sync: check=check-9021491549310976979 -41:20.249 [DEBUG] TestAgent: Check in sync: check=check-5064500450404135988 -41:20.249 [DEBUG] TestAgent: Check in sync: check=check-4873557783575023733 -41:20.249 [DEBUG] TestAgent: Check in sync: check=check-5579994848648371008 -41:20.249 [DEBUG] TestAgent: Service tag will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: tag="tag-2022-09-15 21:41:20.24587068 +0000 UTC m=+33.662376781" -41:20.250 [DEBUG] TestAgent: Node info in sync -41:20.250 [DEBUG] TestAgent: Service in sync: service=svcid1 -41:20.250 [DEBUG] TestAgent: Check in sync: check=check-5579994848648371008 -41:20.250 [DEBUG] TestAgent: Check in sync: check=check-9021491549310976979 -41:20.250 [DEBUG] TestAgent: Check in sync: check=check-5064500450404135988 -41:20.250 [DEBUG] TestAgent: Check in sync: check=check-4873557783575023733 -41:20.250 [DEBUG] TestAgent: Node info in sync ---- PASS: TestAgent_CheckAliasRPC (2.09s) +[DEBUG] Service Removed {serviceAlias-248877271729650034 {}} , err:= +[DEBUG] Service Removed {serviceAlias-7433418641229794478 {}} , err:= +[DEBUG] Service Removed {serviceAlias-5324506607960678668 {}} , err:= +[DEBUG] Service Removed {serviceAlias-6853124940970460888 {}} , err:= +[DEBUG] STARTING shutdown for TestAgent_CheckAliasRPC 2021-08-13 15:33:02.499034286 +0000 UTC m=+30.838357081 +[DEBUG] DONE shutdown for TestAgent_CheckAliasRPC 2021-08-13 15:33:02.499105566 +0000 UTC m=+30.838428381 +2021-08-13T15:33:01.948Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:01.965Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:01.968 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f39f28c0-12a9-9fc4-e56a-e730ddac719b Address:127.0.0.1:24790}]" +33:01.968 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24790 [Follower]" leader= +33:01.969 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 +33:01.970 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 +2021-08-13T15:33:01.970Z [INFO] agent.router: Initializing LAN area manager +33:01.970 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24785 network=udp +33:01.971 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:24790) (DC: dc1)" +33:01.971 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan +33:01.971 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24785 network=tcp +33:01.972 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24786 network=tcp +33:01.972 [INFO] TestAgent: started state syncer +33:01.972 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24791 network=tcp +33:02.005 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:02.005 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24790 [Candidate]" term=2 +33:02.005 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:02.005 [DEBUG] TestAgent.server.raft: vote granted: from=f39f28c0-12a9-9fc4-e56a-e730ddac719b term=2 tally=1 +33:02.005 [INFO] TestAgent.server.raft: election won: tally=1 +33:02.005 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24790 [Leader]" +33:02.005 [INFO] TestAgent.server: cluster leadership acquired +33:02.006 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24790 +33:02.006 [INFO] TestAgent.server: New leader elected: payload=node1 +33:02.012 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:02.015 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:02.015 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:02.015 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:02.015 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:02.015 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:02.015 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 +33:02.015 [INFO] TestAgent.server: member joined, marking health alive: member=node1 +33:02.017 [INFO] TestAgent.server: federation state anti-entropy synced +33:02.383 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:02.383 [INFO] TestAgent: Synced node info +33:02.384 [INFO] TestAgent: Synced service: service=svcid1 +33:02.384 [DEBUG] TestAgent: Node info in sync +33:02.384 [DEBUG] TestAgent: Service in sync: service=svcid1 +33:02.397 [DEBUG] TestAgent: Service tag will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: tag="tag-2021-08-13 15:33:02.397675692 +0000 UTC m=+30.736998547" +33:02.498 [DEBUG] TestAgent: Service tag will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: tag="tag-2021-08-13 15:33:02.498070924 +0000 UTC m=+30.837393859" +33:02.498 [DEBUG] TestAgent: Service tag will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: tag="tag-2021-08-13 15:33:02.498345064 +0000 UTC m=+30.837667879" +33:02.498 [DEBUG] TestAgent: Service tag will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: tag="tag-2021-08-13 15:33:02.498583245 +0000 UTC m=+30.837906060" +33:02.498 [DEBUG] TestAgent: Service tag will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: tag="tag-2021-08-13 15:33:02.498736405 +0000 UTC m=+30.838059240" +33:02.498 [DEBUG] TestAgent: Service tag will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: tag="tag-2021-08-13 15:33:02.498856466 +0000 UTC m=+30.838179281" +33:02.499 [DEBUG] TestAgent: Service tag will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: tag="tag-2021-08-13 15:33:02.498968266 +0000 UTC m=+30.838291101" +33:02.499 [DEBUG] TestAgent: Service tag will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: tag="tag-2021-08-13 15:33:02.499052686 +0000 UTC m=+30.838375521" +--- PASS: TestAgent_CheckAliasRPC (0.57s) === RUN TestAgent_AddServiceNoExec === RUN TestAgent_AddServiceNoExec/normal === PAUSE TestAgent_AddServiceNoExec/normal @@ -10785,121 +10827,123 @@ [WARN] freeport: 6 out of 7 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:20.293Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:20.311Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:20.314 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:48fc3438-c426-9b5e-7a5f-76be516bdb7c Address:127.0.0.1:13573}]" -41:20.315 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 -41:20.316 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 -2022-09-15T21:41:20.316Z [INFO] agent.router: Initializing LAN area manager -41:20.316 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13568 network=udp -41:20.316 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13573 [Follower]" leader= -41:20.317 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:13573) (DC: dc1)" -41:20.317 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan -41:20.317 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13568 network=tcp -41:20.318 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13569 network=tcp -41:20.318 [INFO] TestAgent: started state syncer -41:20.319 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13574 network=tcp -41:20.374 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:20.374 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13573 [Candidate]" term=2 -41:20.375 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:20.375 [DEBUG] TestAgent.server.raft: vote granted: from=48fc3438-c426-9b5e-7a5f-76be516bdb7c term=2 tally=1 -41:20.375 [INFO] TestAgent.server.raft: election won: tally=1 -41:20.375 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13573 [Leader]" -41:20.375 [INFO] TestAgent.server: cluster leadership acquired -41:20.375 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13573 -41:20.376 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:20.378 [INFO] TestAgent.server: New leader elected: payload=node1 -41:20.379 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:20.379 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:20.379 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:20.379 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:20.379 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:20.379 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 -41:20.379 [INFO] TestAgent.server: member joined, marking health alive: member=node1 -41:20.380 [INFO] TestAgent.server: federation state anti-entropy synced -41:20.385 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:20.385 [INFO] TestAgent: Synced node info -41:20.427 [INFO] TestAgent: Requesting shutdown -41:20.428 [INFO] TestAgent.server: shutting down server -41:20.428 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:20.428 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:20.428 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:20.428 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:20.428 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:20.428 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:20.428 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:20.428 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:20.428 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:20.428 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:20.428 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:20.428Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:20.428Z [INFO] agent.router.manager: shutting down -41:20.428 [INFO] TestAgent: consul server down -41:20.428 [INFO] TestAgent: shutdown complete -41:20.428 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13568 network=tcp -41:20.428 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13568 network=udp -41:20.428 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13569 network=tcp -41:20.929 [INFO] TestAgent: Waiting for endpoints to shut down -41:20.929 [INFO] TestAgent: Endpoints down -[WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:20.296Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:20.313Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:20.316 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8409ae0e-16ea-a55a-a952-ebeffa9bfe97 Address:127.0.0.1:13566}]" -41:20.317 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 -41:20.317 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13566 [Follower]" leader= -41:20.318 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 -2022-09-15T21:41:20.318Z [INFO] agent.router: Initializing LAN area manager -41:20.318 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:13566) (DC: dc1)" -41:20.318 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan -41:20.318 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13561 network=udp -41:20.318 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13561 network=tcp -41:20.319 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13562 network=tcp -41:20.319 [INFO] TestAgent: started state syncer -41:20.319 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13567 network=tcp -41:20.376 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:20.376 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13566 [Candidate]" term=2 -41:20.376 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:20.376 [DEBUG] TestAgent.server.raft: vote granted: from=8409ae0e-16ea-a55a-a952-ebeffa9bfe97 term=2 tally=1 -41:20.376 [INFO] TestAgent.server.raft: election won: tally=1 -41:20.376 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13566 [Leader]" -41:20.376 [INFO] TestAgent.server: cluster leadership acquired -41:20.377 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13566 -41:20.378 [INFO] TestAgent.server: New leader elected: payload=node1 -41:20.378 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:20.380 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:20.380 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:20.380 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:20.380 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:20.380 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:20.380 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 -41:20.380 [INFO] TestAgent.server: member joined, marking health alive: member=node1 -41:20.381 [INFO] TestAgent.server: federation state anti-entropy synced -41:20.537 [INFO] TestAgent: Requesting shutdown -41:20.537 [INFO] TestAgent.server: shutting down server -41:20.537 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:20.537 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:20.537 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:20.537 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:20.537 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:20.537 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:20.537 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:20.537 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:20.537 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:20.538 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:20.538 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:20.538 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:20.538Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:20.538Z [INFO] agent.router.manager: shutting down -41:20.538 [INFO] TestAgent: consul server down -41:20.538 [INFO] TestAgent: shutdown complete -41:20.538 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13561 network=tcp -41:20.538 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13561 network=udp -41:20.538 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13562 network=tcp -41:21.038 [INFO] TestAgent: Waiting for endpoints to shut down -41:21.038 [INFO] TestAgent: Endpoints down +[WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:33:02.517Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:02.541Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:02.548 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d57bca3b-188d-b8cd-3138-d06835e1d604 Address:127.0.0.1:24797}]" +33:02.548 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24797 [Follower]" leader= +33:02.549 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 +33:02.550 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 +2021-08-13T15:33:02.550Z [INFO] agent.router: Initializing LAN area manager +33:02.550 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:24797) (DC: dc1)" +33:02.550 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan +33:02.550 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24792 network=udp +33:02.550 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24792 network=tcp +33:02.551 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24793 network=tcp +33:02.551 [INFO] TestAgent: started state syncer +33:02.551 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24798 network=tcp +33:02.601 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:02.601 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24797 [Candidate]" term=2 +33:02.601 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:02.601 [DEBUG] TestAgent.server.raft: vote granted: from=d57bca3b-188d-b8cd-3138-d06835e1d604 term=2 tally=1 +33:02.601 [INFO] TestAgent.server.raft: election won: tally=1 +33:02.601 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24797 [Leader]" +33:02.601 [INFO] TestAgent.server: cluster leadership acquired +33:02.602 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24797 +33:02.603 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:02.605 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:02.605 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:02.605 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:02.605 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:02.605 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:02.605 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 +33:02.605 [INFO] TestAgent.server: member joined, marking health alive: member=node1 +33:02.606 [INFO] TestAgent.server: New leader elected: payload=node1 +33:02.606 [INFO] TestAgent.server: federation state anti-entropy synced +33:02.816 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:02.817 [INFO] TestAgent: Synced node info +33:02.830 [INFO] TestAgent: Requesting shutdown +33:02.830 [INFO] TestAgent.server: shutting down server +33:02.830 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:02.830 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:02.830 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:02.830 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:02.830 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:02.830 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:02.830 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:02.831 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:02.831 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:02.831 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:02.831 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:02.831Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:02.831Z [INFO] agent.router.manager: shutting down +33:02.831 [INFO] TestAgent: consul server down +33:02.831 [INFO] TestAgent: shutdown complete +33:02.831 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24792 network=tcp +33:02.831 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24792 network=udp +33:02.831 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24793 network=tcp +33:03.331 [INFO] TestAgent: Waiting for endpoints to shut down +33:03.331 [INFO] TestAgent: Endpoints down +2021-08-13T15:33:02.527Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:02.554Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:02.557 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:05c9d4af-a63d-27a6-6e1d-bb8c947c734b Address:127.0.0.1:24804}]" +33:02.557 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24804 [Follower]" leader= +33:02.558 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 +33:02.559 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 +2021-08-13T15:33:02.559Z [INFO] agent.router: Initializing LAN area manager +33:02.559 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24799 network=udp +33:02.559 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:24804) (DC: dc1)" +33:02.559 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan +33:02.560 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24799 network=tcp +33:02.573 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24800 network=tcp +33:02.573 [INFO] TestAgent: started state syncer +33:02.573 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24805 network=tcp +33:02.609 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:02.610 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24804 [Candidate]" term=2 +33:02.610 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:02.610 [DEBUG] TestAgent.server.raft: vote granted: from=05c9d4af-a63d-27a6-6e1d-bb8c947c734b term=2 tally=1 +33:02.610 [INFO] TestAgent.server.raft: election won: tally=1 +33:02.610 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24804 [Leader]" +33:02.610 [INFO] TestAgent.server: cluster leadership acquired +33:02.610 [INFO] TestAgent.server: New leader elected: payload=node1 +33:02.610 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24804 +33:02.611 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:02.614 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:02.614 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:02.614 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:02.614 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:02.614 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:02.614 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 +33:02.614 [INFO] TestAgent.server: member joined, marking health alive: member=node1 +33:02.615 [INFO] TestAgent.server: federation state anti-entropy synced +33:02.715 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:02.716 [INFO] TestAgent: Synced node info +33:02.716 [DEBUG] TestAgent: Node info in sync +33:02.921 [INFO] TestAgent: Requesting shutdown +33:02.921 [INFO] TestAgent.server: shutting down server +33:02.921 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:02.921 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:02.921 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:02.921 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:02.921 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:02.921 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:02.921 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:02.921 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:02.921 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:02.921 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:02.921 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:02.922Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:02.922Z [INFO] agent.router.manager: shutting down +33:02.922 [INFO] TestAgent: consul server down +33:02.922 [INFO] TestAgent: shutdown complete +33:02.922 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24799 network=tcp +33:02.922 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24799 network=udp +33:02.922 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24800 network=tcp +33:03.422 [INFO] TestAgent: Waiting for endpoints to shut down +33:03.422 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_AddServiceNoExec (0.00s) - --- PASS: TestAgent_AddServiceNoExec/service_manager (0.68s) - --- PASS: TestAgent_AddServiceNoExec/normal (0.79s) + --- PASS: TestAgent_AddServiceNoExec/normal (0.83s) + --- PASS: TestAgent_AddServiceNoExec/service_manager (0.92s) === RUN TestAgent_AddServiceNoRemoteExec === RUN TestAgent_AddServiceNoRemoteExec/normal === PAUSE TestAgent_AddServiceNoRemoteExec/normal @@ -10907,122 +10951,123 @@ === PAUSE TestAgent_AddServiceNoRemoteExec/service_manager === CONT TestAgent_AddServiceNoRemoteExec/normal === CONT TestAgent_AddServiceNoRemoteExec/service_manager -[WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 6 out of 20 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:21.068Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:21.111Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:21.113 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:494eb4e2-f9ea-6e3e-a7c0-c5a778744010 Address:127.0.0.1:13587}]" -41:21.114 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13587 [Follower]" leader= -41:21.114 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 -41:21.115 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 -2022-09-15T21:41:21.115Z [INFO] agent.router: Initializing LAN area manager -41:21.116 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan -41:21.116 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13582 network=udp -41:21.116 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:13587) (DC: dc1)" -41:21.116 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13582 network=tcp -41:21.117 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13583 network=tcp -41:21.117 [INFO] TestAgent: started state syncer -41:21.117 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13588 network=tcp -41:21.162 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:21.162 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13587 [Candidate]" term=2 -41:21.162 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:21.162 [DEBUG] TestAgent.server.raft: vote granted: from=494eb4e2-f9ea-6e3e-a7c0-c5a778744010 term=2 tally=1 -41:21.162 [INFO] TestAgent.server.raft: election won: tally=1 -41:21.162 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13587 [Leader]" -41:21.162 [INFO] TestAgent.server: cluster leadership acquired -41:21.162 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13587 -41:21.164 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:21.164 [INFO] TestAgent.server: New leader elected: payload=node1 -41:21.166 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:21.166 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:21.166 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:21.166 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:21.166 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:21.166 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 -41:21.166 [INFO] TestAgent.server: member joined, marking health alive: member=node1 -41:21.167 [INFO] TestAgent.server: federation state anti-entropy synced -41:21.171 [INFO] TestAgent: Requesting shutdown -41:21.171 [INFO] TestAgent.server: shutting down server -41:21.171 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:21.171 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:21.171 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:21.171 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:21.171 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:21.172 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:21.172 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:21.172 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:21.172 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:21.172 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:21.172 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:21.172 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:21.172Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:21.172Z [INFO] agent.router.manager: shutting down -41:21.172 [INFO] TestAgent: consul server down -41:21.172 [INFO] TestAgent: shutdown complete -41:21.172 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13582 network=tcp -41:21.172 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13582 network=udp -41:21.172 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13583 network=tcp -41:21.674 [INFO] TestAgent: Waiting for endpoints to shut down -41:21.674 [INFO] TestAgent: Endpoints down +2021-08-13T15:33:03.452Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:03.488Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:03.490 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c7d45b7c-d164-fcd6-d8b8-3117790d7c39 Address:127.0.0.1:24818}]" +33:03.491 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24818 [Follower]" leader= +33:03.493 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 +33:03.494 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 +2021-08-13T15:33:03.494Z [INFO] agent.router: Initializing LAN area manager +33:03.494 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:24818) (DC: dc1)" +33:03.495 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan +33:03.495 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24813 network=udp +33:03.495 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24813 network=tcp +33:03.496 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24814 network=tcp +33:03.496 [INFO] TestAgent: started state syncer +33:03.496 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24819 network=tcp +33:03.556 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:03.556 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24818 [Candidate]" term=2 +33:03.556 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:03.556 [DEBUG] TestAgent.server.raft: vote granted: from=c7d45b7c-d164-fcd6-d8b8-3117790d7c39 term=2 tally=1 +33:03.556 [INFO] TestAgent.server.raft: election won: tally=1 +33:03.556 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24818 [Leader]" +33:03.556 [INFO] TestAgent.server: cluster leadership acquired +33:03.557 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24818 +33:03.558 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:03.558 [INFO] TestAgent.server: New leader elected: payload=node1 +33:03.560 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:03.560 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:03.560 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:03.560 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:03.560 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:03.560 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 +33:03.560 [INFO] TestAgent.server: member joined, marking health alive: member=node1 +33:03.561 [INFO] TestAgent.server: federation state anti-entropy synced +33:03.596 [INFO] TestAgent: Requesting shutdown +33:03.596 [INFO] TestAgent.server: shutting down server +33:03.596 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:03.596 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:03.596 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:03.596 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:03.596 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:03.596 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:03.596 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:03.596 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:03.596 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:03.596 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:03.596 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:03.596 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:03.596Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:03.596Z [INFO] agent.router.manager: shutting down +33:03.597 [INFO] TestAgent: consul server down +33:03.597 [INFO] TestAgent: shutdown complete +33:03.597 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24813 network=tcp +33:03.597 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24813 network=udp +33:03.597 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24814 network=tcp +33:04.097 [INFO] TestAgent: Waiting for endpoints to shut down +33:04.097 [INFO] TestAgent: Endpoints down [WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:21.065Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:21.107Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:21.112 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:652735e5-f228-bb89-84e6-868b44e2f728 Address:127.0.0.1:13580}]" -41:21.113 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 -41:21.113 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 -2022-09-15T21:41:21.113Z [INFO] agent.router: Initializing LAN area manager -41:21.114 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13580 [Follower]" leader= -41:21.114 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:13580) (DC: dc1)" -41:21.114 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13575 network=udp -41:21.114 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan -41:21.114 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13575 network=tcp -41:21.115 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13576 network=tcp -41:21.115 [INFO] TestAgent: started state syncer -41:21.115 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13581 network=tcp -41:21.160 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:21.161 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13580 [Candidate]" term=2 -41:21.161 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:21.161 [DEBUG] TestAgent.server.raft: vote granted: from=652735e5-f228-bb89-84e6-868b44e2f728 term=2 tally=1 -41:21.161 [INFO] TestAgent.server.raft: election won: tally=1 -41:21.161 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13580 [Leader]" -41:21.161 [INFO] TestAgent.server: cluster leadership acquired -41:21.161 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13580 -41:21.162 [INFO] TestAgent.server: New leader elected: payload=node1 -41:21.163 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:21.165 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:21.165 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:21.165 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:21.165 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:21.165 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:21.165 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 -41:21.166 [INFO] TestAgent.server: member joined, marking health alive: member=node1 -41:21.166 [INFO] TestAgent.server: federation state anti-entropy synced -41:21.445 [INFO] TestAgent: Requesting shutdown -41:21.445 [INFO] TestAgent.server: shutting down server -41:21.445 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:21.445 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:21.445 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:21.445 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:21.445 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:21.445 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:21.445 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:21.445 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:21.445 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:21.446 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:21.446 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:21.446 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:21.446Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:21.446Z [INFO] agent.router.manager: shutting down -41:21.446 [INFO] TestAgent: consul server down -41:21.446 [INFO] TestAgent: shutdown complete -41:21.446 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13575 network=tcp -41:21.446 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13575 network=udp -41:21.446 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13576 network=tcp -41:21.946 [INFO] TestAgent: Waiting for endpoints to shut down -41:21.946 [INFO] TestAgent: Endpoints down +2021-08-13T15:33:03.458Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:03.486Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:03.493 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b1ce11fa-1a24-dba9-381f-9eff55d5cdf9 Address:127.0.0.1:24811}]" +33:03.493 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24811 [Follower]" leader= +33:03.495 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 +33:03.496 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 +2021-08-13T15:33:03.496Z [INFO] agent.router: Initializing LAN area manager +33:03.496 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:24811) (DC: dc1)" +33:03.496 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24806 network=udp +33:03.496 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan +33:03.496 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24806 network=tcp +33:03.497 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24807 network=tcp +33:03.497 [INFO] TestAgent: started state syncer +33:03.497 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24812 network=tcp +33:03.536 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:03.536 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24811 [Candidate]" term=2 +33:03.539 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:03.539 [DEBUG] TestAgent.server.raft: vote granted: from=b1ce11fa-1a24-dba9-381f-9eff55d5cdf9 term=2 tally=1 +33:03.539 [INFO] TestAgent.server.raft: election won: tally=1 +33:03.539 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24811 [Leader]" +33:03.539 [INFO] TestAgent.server: cluster leadership acquired +33:03.540 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24811 +33:03.541 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:03.543 [INFO] TestAgent.server: New leader elected: payload=node1 +33:03.545 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:03.545 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:03.545 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:03.545 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:03.545 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:03.545 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 +33:03.545 [INFO] TestAgent.server: member joined, marking health alive: member=node1 +33:03.546 [INFO] TestAgent.server: federation state anti-entropy synced +33:03.796 [INFO] TestAgent: Requesting shutdown +33:03.796 [INFO] TestAgent.server: shutting down server +33:03.796 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:03.796 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:03.796 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:03.796 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:03.796 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:03.796 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:03.796 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:03.796 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:03.796 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:03.796 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:03.796 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:03.796 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:03.797Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:03.797Z [INFO] agent.router.manager: shutting down +33:03.797 [INFO] TestAgent: consul server down +33:03.797 [INFO] TestAgent: shutdown complete +33:03.797 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24806 network=tcp +33:03.797 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24806 network=udp +33:03.797 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24807 network=tcp +33:04.297 [INFO] TestAgent: Waiting for endpoints to shut down +33:04.297 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_AddServiceNoRemoteExec (0.00s) - --- PASS: TestAgent_AddServiceNoRemoteExec/service_manager (0.63s) - --- PASS: TestAgent_AddServiceNoRemoteExec/normal (0.91s) + --- PASS: TestAgent_AddServiceNoRemoteExec/service_manager (0.67s) + --- PASS: TestAgent_AddServiceNoRemoteExec/normal (0.87s) === RUN TestCacheRateLimit agent_test.go:749: DM-skipped --- SKIP: TestCacheRateLimit (0.00s) @@ -11030,250 +11075,247 @@ [WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:21.966Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:21.984Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:21.988 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6f1c8121-d1ae-3323-a420-31ac6d64d104 Address:127.0.0.1:13594}]" -41:21.988 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13594 [Follower]" leader= -41:21.989 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6f1c8121-d1ae-3323-a420-31ac6d64d104.dc1 127.0.0.1 -41:21.989 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6f1c8121-d1ae-3323-a420-31ac6d64d104 127.0.0.1 -2022-09-15T21:41:21.990Z [INFO] agent.router: Initializing LAN area manager -41:21.990 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13589 network=udp -41:21.990 [INFO] TestAgent.server: Adding LAN server: server="Node-6f1c8121-d1ae-3323-a420-31ac6d64d104 (Addr: tcp/127.0.0.1:13594) (DC: dc1)" -41:21.990 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6f1c8121-d1ae-3323-a420-31ac6d64d104.dc1 area=wan -41:21.990 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13589 network=tcp -41:21.991 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13590 network=tcp -41:21.991 [INFO] TestAgent: started state syncer -41:21.991 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13595 network=tcp -41:22.041 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:22.041 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13594 [Candidate]" term=2 -41:22.041 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:22.041 [DEBUG] TestAgent.server.raft: vote granted: from=6f1c8121-d1ae-3323-a420-31ac6d64d104 term=2 tally=1 -41:22.041 [INFO] TestAgent.server.raft: election won: tally=1 -41:22.041 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13594 [Leader]" -41:22.042 [INFO] TestAgent.server: cluster leadership acquired -41:22.042 [INFO] TestAgent.server: New leader elected: payload=Node-6f1c8121-d1ae-3323-a420-31ac6d64d104 -41:22.042 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13594 -41:22.043 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:22.046 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:22.046 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:22.046 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:22.046 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:22.046 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:22.046 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6f1c8121-d1ae-3323-a420-31ac6d64d104 -41:22.046 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6f1c8121-d1ae-3323-a420-31ac6d64d104 -41:22.047 [INFO] TestAgent.server: federation state anti-entropy synced -41:22.236 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:22.237 [INFO] TestAgent: Synced node info -41:22.268 [WARN] TestAgent: Service name will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: service=my_service -41:22.269 [INFO] TestAgent: Requesting shutdown -41:22.269 [INFO] TestAgent.server: shutting down server -41:22.269 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:22.269 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:22.269 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:22.269 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:22.269 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:22.269 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:22.269 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:22.269 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:22.269 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:22.269 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -41:22.269 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2022-09-15T21:41:22.269Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:22.269Z [INFO] agent.router.manager: shutting down -41:22.269 [INFO] TestAgent: consul server down -41:22.269 [INFO] TestAgent: shutdown complete -41:22.269 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13589 network=tcp -41:22.269 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13589 network=udp -41:22.270 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13590 network=tcp -41:22.770 [INFO] TestAgent: Waiting for endpoints to shut down -41:22.770 [INFO] TestAgent: Endpoints down ---- PASS: TestAddServiceIPv4TaggedDefault (0.82s) +2021-08-13T15:33:04.315Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:04.332Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:04.335 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f50a4412-a1a1-d708-0cf1-014bb542e4d1 Address:127.0.0.1:24825}]" +33:04.335 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24825 [Follower]" leader= +33:04.336 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f50a4412-a1a1-d708-0cf1-014bb542e4d1.dc1 127.0.0.1 +33:04.337 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f50a4412-a1a1-d708-0cf1-014bb542e4d1 127.0.0.1 +2021-08-13T15:33:04.337Z [INFO] agent.router: Initializing LAN area manager +33:04.337 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f50a4412-a1a1-d708-0cf1-014bb542e4d1.dc1 area=wan +33:04.337 [INFO] TestAgent.server: Adding LAN server: server="Node-f50a4412-a1a1-d708-0cf1-014bb542e4d1 (Addr: tcp/127.0.0.1:24825) (DC: dc1)" +33:04.337 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24820 network=tcp +33:04.337 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24820 network=udp +33:04.338 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24821 network=tcp +33:04.338 [INFO] TestAgent: started state syncer +33:04.338 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24826 network=tcp +33:04.396 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:04.396 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24825 [Candidate]" term=2 +33:04.396 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:04.396 [DEBUG] TestAgent.server.raft: vote granted: from=f50a4412-a1a1-d708-0cf1-014bb542e4d1 term=2 tally=1 +33:04.396 [INFO] TestAgent.server.raft: election won: tally=1 +33:04.396 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24825 [Leader]" +33:04.397 [INFO] TestAgent.server: cluster leadership acquired +33:04.397 [INFO] TestAgent.server: New leader elected: payload=Node-f50a4412-a1a1-d708-0cf1-014bb542e4d1 +33:04.397 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24825 +33:04.398 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:04.401 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:04.401 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:04.401 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:04.401 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:04.401 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:04.401 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f50a4412-a1a1-d708-0cf1-014bb542e4d1 +33:04.401 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f50a4412-a1a1-d708-0cf1-014bb542e4d1 +33:04.402 [INFO] TestAgent.server: federation state anti-entropy synced +33:04.691 [WARN] TestAgent: Service name will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: service=my_service +33:04.691 [INFO] TestAgent: Requesting shutdown +33:04.691 [INFO] TestAgent.server: shutting down server +33:04.691 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:04.691 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:04.691 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:04.691 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:04.691 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:04.691 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:04.691 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:04.691 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:04.691 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:04.691 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:04.691 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:04.691 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:04.692Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:04.692Z [INFO] agent.router.manager: shutting down +33:04.692 [INFO] TestAgent: consul server down +33:04.692 [INFO] TestAgent: shutdown complete +33:04.692 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24820 network=tcp +33:04.692 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24820 network=udp +33:04.692 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24821 network=tcp +33:05.192 [INFO] TestAgent: Waiting for endpoints to shut down +33:05.192 [INFO] TestAgent: Endpoints down +--- PASS: TestAddServiceIPv4TaggedDefault (0.89s) === RUN TestAddServiceIPv6TaggedDefault [WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:22.797Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:22.815Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:22.818 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:12d3e89d-d4d5-094d-b1e2-a131bfca5d6c Address:127.0.0.1:13601}]" -41:22.819 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-12d3e89d-d4d5-094d-b1e2-a131bfca5d6c.dc1 127.0.0.1 -41:22.820 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-12d3e89d-d4d5-094d-b1e2-a131bfca5d6c 127.0.0.1 -2022-09-15T21:41:22.820Z [INFO] agent.router: Initializing LAN area manager -41:22.821 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13596 network=udp -41:22.821 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13601 [Follower]" leader= -41:22.821 [INFO] TestAgent.server: Adding LAN server: server="Node-12d3e89d-d4d5-094d-b1e2-a131bfca5d6c (Addr: tcp/127.0.0.1:13601) (DC: dc1)" -41:22.821 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-12d3e89d-d4d5-094d-b1e2-a131bfca5d6c.dc1 area=wan -41:22.823 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13596 network=tcp -41:22.824 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13597 network=tcp -41:22.824 [INFO] TestAgent: started state syncer -41:22.824 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13602 network=tcp -41:22.878 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:22.878 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13601 [Candidate]" term=2 -41:22.878 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:22.878 [DEBUG] TestAgent.server.raft: vote granted: from=12d3e89d-d4d5-094d-b1e2-a131bfca5d6c term=2 tally=1 -41:22.878 [INFO] TestAgent.server.raft: election won: tally=1 -41:22.878 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13601 [Leader]" -41:22.879 [INFO] TestAgent.server: cluster leadership acquired -41:22.879 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13601 -41:22.880 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:22.882 [INFO] TestAgent.server: New leader elected: payload=Node-12d3e89d-d4d5-094d-b1e2-a131bfca5d6c -41:22.882 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:22.882 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:22.882 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:22.882 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:22.882 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:22.882 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-12d3e89d-d4d5-094d-b1e2-a131bfca5d6c -41:22.882 [INFO] TestAgent.server: member joined, marking health alive: member=Node-12d3e89d-d4d5-094d-b1e2-a131bfca5d6c -41:22.884 [INFO] TestAgent.server: federation state anti-entropy synced -41:22.999 [WARN] TestAgent: Service name will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: service=my_service -41:22.999 [INFO] TestAgent: Requesting shutdown -41:22.999 [INFO] TestAgent.server: shutting down server -41:22.999 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:22.999 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:22.999 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:22.999 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:22.999 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:23.000 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:23.000 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:23.000 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:23.000 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:23.000 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:23.000 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:23.000 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:23.000Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:23.000Z [INFO] agent.router.manager: shutting down -41:23.000 [INFO] TestAgent: consul server down -41:23.000 [INFO] TestAgent: shutdown complete -41:23.000 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13596 network=tcp -41:23.000 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13596 network=udp -41:23.000 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13597 network=tcp -41:23.500 [INFO] TestAgent: Waiting for endpoints to shut down -41:23.500 [INFO] TestAgent: Endpoints down ---- PASS: TestAddServiceIPv6TaggedDefault (0.73s) +[WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:33:05.208Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:05.224Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:05.226 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7eb99e8a-2416-d5aa-ebc0-b14175f9adc9 Address:127.0.0.1:24832}]" +33:05.227 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24832 [Follower]" leader= +33:05.227 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7eb99e8a-2416-d5aa-ebc0-b14175f9adc9.dc1 127.0.0.1 +33:05.228 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7eb99e8a-2416-d5aa-ebc0-b14175f9adc9 127.0.0.1 +2021-08-13T15:33:05.228Z [INFO] agent.router: Initializing LAN area manager +33:05.229 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24827 network=udp +33:05.229 [INFO] TestAgent.server: Adding LAN server: server="Node-7eb99e8a-2416-d5aa-ebc0-b14175f9adc9 (Addr: tcp/127.0.0.1:24832) (DC: dc1)" +33:05.229 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7eb99e8a-2416-d5aa-ebc0-b14175f9adc9.dc1 area=wan +33:05.229 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24827 network=tcp +33:05.230 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24828 network=tcp +33:05.230 [INFO] TestAgent: started state syncer +33:05.230 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24833 network=tcp +33:05.288 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:05.288 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24832 [Candidate]" term=2 +33:05.288 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:05.288 [DEBUG] TestAgent.server.raft: vote granted: from=7eb99e8a-2416-d5aa-ebc0-b14175f9adc9 term=2 tally=1 +33:05.288 [INFO] TestAgent.server.raft: election won: tally=1 +33:05.288 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24832 [Leader]" +33:05.288 [INFO] TestAgent.server: cluster leadership acquired +33:05.289 [INFO] TestAgent.server: New leader elected: payload=Node-7eb99e8a-2416-d5aa-ebc0-b14175f9adc9 +33:05.289 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24832 +33:05.290 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:05.293 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:05.293 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:05.293 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:05.293 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:05.293 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:05.293 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7eb99e8a-2416-d5aa-ebc0-b14175f9adc9 +33:05.293 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7eb99e8a-2416-d5aa-ebc0-b14175f9adc9 +33:05.294 [INFO] TestAgent.server: federation state anti-entropy synced +33:05.564 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:05.564 [INFO] TestAgent: Synced node info +33:05.581 [WARN] TestAgent: Service name will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: service=my_service +33:05.581 [INFO] TestAgent: Requesting shutdown +33:05.581 [INFO] TestAgent.server: shutting down server +33:05.581 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:05.581 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:05.581 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:05.581 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:05.581 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:05.581 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:05.581 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:05.581 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:05.581 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:05.581 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:05.581 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:05.581Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:05.581Z [INFO] agent.router.manager: shutting down +33:05.581 [INFO] TestAgent: consul server down +33:05.581 [INFO] TestAgent: shutdown complete +33:05.581 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24827 network=tcp +33:05.581 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24827 network=udp +33:05.582 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24828 network=tcp +33:06.082 [INFO] TestAgent: Waiting for endpoints to shut down +33:06.082 [INFO] TestAgent: Endpoints down +--- PASS: TestAddServiceIPv6TaggedDefault (0.89s) === RUN TestAddServiceIPv4TaggedSet [WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:23.544Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:23.564Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:23.567 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9f97937b-cbc1-09d4-9c5a-aaea4a5fc2e0 Address:127.0.0.1:13608}]" -41:23.568 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13608 [Follower]" leader= -41:23.568 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9f97937b-cbc1-09d4-9c5a-aaea4a5fc2e0.dc1 127.0.0.1 -41:23.569 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9f97937b-cbc1-09d4-9c5a-aaea4a5fc2e0 127.0.0.1 -2022-09-15T21:41:23.569Z [INFO] agent.router: Initializing LAN area manager -41:23.570 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9f97937b-cbc1-09d4-9c5a-aaea4a5fc2e0.dc1 area=wan -41:23.570 [INFO] TestAgent.server: Adding LAN server: server="Node-9f97937b-cbc1-09d4-9c5a-aaea4a5fc2e0 (Addr: tcp/127.0.0.1:13608) (DC: dc1)" -41:23.570 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13603 network=udp -41:23.570 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13603 network=tcp -41:23.571 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13604 network=tcp -41:23.571 [INFO] TestAgent: started state syncer -41:23.571 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13609 network=tcp -41:23.622 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:23.622 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13608 [Candidate]" term=2 -41:23.622 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:23.622 [DEBUG] TestAgent.server.raft: vote granted: from=9f97937b-cbc1-09d4-9c5a-aaea4a5fc2e0 term=2 tally=1 -41:23.622 [INFO] TestAgent.server.raft: election won: tally=1 -41:23.622 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13608 [Leader]" -41:23.622 [INFO] TestAgent.server: cluster leadership acquired -41:23.623 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13608 -41:23.623 [INFO] TestAgent.server: New leader elected: payload=Node-9f97937b-cbc1-09d4-9c5a-aaea4a5fc2e0 -41:23.624 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:23.631 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:23.631 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:23.631 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:23.631 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:23.631 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:23.631 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9f97937b-cbc1-09d4-9c5a-aaea4a5fc2e0 -41:23.631 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9f97937b-cbc1-09d4-9c5a-aaea4a5fc2e0 -41:23.632 [INFO] TestAgent.server: federation state anti-entropy synced -41:23.846 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:23.847 [INFO] TestAgent: Synced node info -41:23.847 [DEBUG] TestAgent: Node info in sync -41:23.852 [WARN] TestAgent: Service name will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: service=my_service -41:23.852 [INFO] TestAgent: Requesting shutdown -41:23.852 [INFO] TestAgent.server: shutting down server -41:23.852 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:23.852 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:23.852 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:23.852 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:23.852 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:23.852 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:23.852 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:23.852 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:23.852 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:23.852 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -41:23.853 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2022-09-15T21:41:23.853Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:23.853Z [INFO] agent.router.manager: shutting down -41:23.853 [INFO] TestAgent: consul server down -41:23.853 [INFO] TestAgent: shutdown complete -41:23.853 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13603 network=tcp -41:23.853 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13603 network=udp -41:23.853 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13604 network=tcp -41:24.353 [INFO] TestAgent: Waiting for endpoints to shut down -41:24.353 [INFO] TestAgent: Endpoints down ---- PASS: TestAddServiceIPv4TaggedSet (0.85s) +2021-08-13T15:33:06.118Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:06.137Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:06.140 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e6111e3b-6d1f-cf6c-d811-4c8ad33c378a Address:127.0.0.1:24839}]" +33:06.140 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e6111e3b-6d1f-cf6c-d811-4c8ad33c378a.dc1 127.0.0.1 +33:06.141 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e6111e3b-6d1f-cf6c-d811-4c8ad33c378a 127.0.0.1 +2021-08-13T15:33:06.141Z [INFO] agent.router: Initializing LAN area manager +33:06.142 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24834 network=udp +33:06.142 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24839 [Follower]" leader= +33:06.142 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e6111e3b-6d1f-cf6c-d811-4c8ad33c378a.dc1 area=wan +33:06.142 [INFO] TestAgent.server: Adding LAN server: server="Node-e6111e3b-6d1f-cf6c-d811-4c8ad33c378a (Addr: tcp/127.0.0.1:24839) (DC: dc1)" +33:06.142 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24834 network=tcp +33:06.143 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24835 network=tcp +33:06.143 [INFO] TestAgent: started state syncer +33:06.143 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24840 network=tcp +33:06.203 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:06.203 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24839 [Candidate]" term=2 +33:06.203 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:06.203 [DEBUG] TestAgent.server.raft: vote granted: from=e6111e3b-6d1f-cf6c-d811-4c8ad33c378a term=2 tally=1 +33:06.203 [INFO] TestAgent.server.raft: election won: tally=1 +33:06.203 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24839 [Leader]" +33:06.203 [INFO] TestAgent.server: cluster leadership acquired +33:06.204 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24839 +33:06.205 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:06.207 [INFO] TestAgent.server: New leader elected: payload=Node-e6111e3b-6d1f-cf6c-d811-4c8ad33c378a +33:06.208 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:06.208 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:06.208 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:06.208 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:06.208 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:06.208 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e6111e3b-6d1f-cf6c-d811-4c8ad33c378a +33:06.208 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e6111e3b-6d1f-cf6c-d811-4c8ad33c378a +33:06.209 [INFO] TestAgent.server: federation state anti-entropy synced +33:06.231 [WARN] TestAgent: Service name will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: service=my_service +33:06.231 [INFO] TestAgent: Requesting shutdown +33:06.231 [INFO] TestAgent.server: shutting down server +33:06.231 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:06.231 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:06.231 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:06.231 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:06.232 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:06.232 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:06.232 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:06.232 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:06.232 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:06.232 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:06.232 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:06.232 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:06.232Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:06.232Z [INFO] agent.router.manager: shutting down +33:06.232 [INFO] TestAgent: consul server down +33:06.232 [INFO] TestAgent: shutdown complete +33:06.232 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24834 network=tcp +33:06.232 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24834 network=udp +33:06.232 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24835 network=tcp +33:06.733 [INFO] TestAgent: Waiting for endpoints to shut down +33:06.733 [INFO] TestAgent: Endpoints down +--- PASS: TestAddServiceIPv4TaggedSet (0.65s) === RUN TestAddServiceIPv6TaggedSet [WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:24.372Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:24.389Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:24.391 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a5f02add-8dc0-6e36-71ea-c2507004d9dc Address:127.0.0.1:13615}]" -41:24.392 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13615 [Follower]" leader= -41:24.392 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a5f02add-8dc0-6e36-71ea-c2507004d9dc.dc1 127.0.0.1 -41:24.393 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a5f02add-8dc0-6e36-71ea-c2507004d9dc 127.0.0.1 -2022-09-15T21:41:24.393Z [INFO] agent.router: Initializing LAN area manager -41:24.394 [INFO] TestAgent.server: Adding LAN server: server="Node-a5f02add-8dc0-6e36-71ea-c2507004d9dc (Addr: tcp/127.0.0.1:13615) (DC: dc1)" -41:24.394 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a5f02add-8dc0-6e36-71ea-c2507004d9dc.dc1 area=wan -41:24.394 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13610 network=tcp -41:24.394 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13610 network=udp -41:24.395 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13611 network=tcp -41:24.395 [INFO] TestAgent: started state syncer -41:24.395 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13616 network=tcp -41:24.455 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:24.455 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13615 [Candidate]" term=2 -41:24.455 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:24.455 [DEBUG] TestAgent.server.raft: vote granted: from=a5f02add-8dc0-6e36-71ea-c2507004d9dc term=2 tally=1 -41:24.455 [INFO] TestAgent.server.raft: election won: tally=1 -41:24.455 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13615 [Leader]" -41:24.455 [INFO] TestAgent.server: cluster leadership acquired -41:24.456 [INFO] TestAgent.server: New leader elected: payload=Node-a5f02add-8dc0-6e36-71ea-c2507004d9dc -41:24.456 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13615 -41:24.457 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:24.460 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:24.460 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:24.460 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:24.460 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:24.460 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:24.460 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a5f02add-8dc0-6e36-71ea-c2507004d9dc -41:24.460 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a5f02add-8dc0-6e36-71ea-c2507004d9dc -41:24.461 [INFO] TestAgent.server: federation state anti-entropy synced -41:24.685 [WARN] TestAgent: Service name will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: service=my_service -41:24.685 [INFO] TestAgent: Requesting shutdown -41:24.685 [INFO] TestAgent.server: shutting down server -41:24.686 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:24.686 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:24.686 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:24.686 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:24.686 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:24.686 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:24.686 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:24.686 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:24.686 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:24.686 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:24.686 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:24.686 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:24.686Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:24.686Z [INFO] agent.router.manager: shutting down -41:24.686 [INFO] TestAgent: consul server down -41:24.686 [INFO] TestAgent: shutdown complete -41:24.686 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13610 network=tcp -41:24.686 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13610 network=udp -41:24.686 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13611 network=tcp -41:25.187 [INFO] TestAgent: Waiting for endpoints to shut down -41:25.187 [INFO] TestAgent: Endpoints down ---- PASS: TestAddServiceIPv6TaggedSet (0.83s) +2021-08-13T15:33:06.749Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:06.765Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:06.767 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:39300dff-ddaa-6bd0-daa6-9d59345c9c94 Address:127.0.0.1:24846}]" +33:06.767 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24846 [Follower]" leader= +33:06.768 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-39300dff-ddaa-6bd0-daa6-9d59345c9c94.dc1 127.0.0.1 +33:06.769 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-39300dff-ddaa-6bd0-daa6-9d59345c9c94 127.0.0.1 +2021-08-13T15:33:06.769Z [INFO] agent.router: Initializing LAN area manager +33:06.769 [INFO] TestAgent.server: Adding LAN server: server="Node-39300dff-ddaa-6bd0-daa6-9d59345c9c94 (Addr: tcp/127.0.0.1:24846) (DC: dc1)" +33:06.769 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-39300dff-ddaa-6bd0-daa6-9d59345c9c94.dc1 area=wan +33:06.770 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24841 network=udp +33:06.770 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24841 network=tcp +33:06.770 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24842 network=tcp +33:06.771 [INFO] TestAgent: started state syncer +33:06.771 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24847 network=tcp +33:06.828 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:06.828 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24846 [Candidate]" term=2 +33:06.828 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:06.828 [DEBUG] TestAgent.server.raft: vote granted: from=39300dff-ddaa-6bd0-daa6-9d59345c9c94 term=2 tally=1 +33:06.828 [INFO] TestAgent.server.raft: election won: tally=1 +33:06.828 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24846 [Leader]" +33:06.828 [INFO] TestAgent.server: cluster leadership acquired +33:06.828 [INFO] TestAgent.server: New leader elected: payload=Node-39300dff-ddaa-6bd0-daa6-9d59345c9c94 +33:06.829 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24846 +33:06.830 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:06.832 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:06.832 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:06.832 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:06.832 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:06.832 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:06.833 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-39300dff-ddaa-6bd0-daa6-9d59345c9c94 +33:06.833 [INFO] TestAgent.server: member joined, marking health alive: member=Node-39300dff-ddaa-6bd0-daa6-9d59345c9c94 +33:06.833 [INFO] TestAgent.server: federation state anti-entropy synced +33:06.940 [WARN] TestAgent: Service name will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: service=my_service +33:06.940 [INFO] TestAgent: Requesting shutdown +33:06.941 [INFO] TestAgent.server: shutting down server +33:06.941 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:06.941 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:06.941 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:06.941 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:06.941 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:06.941 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:06.941 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:06.941 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:06.941 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:06.941 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:06.941 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:06.941 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:06.941Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:06.941Z [INFO] agent.router.manager: shutting down +33:06.941 [INFO] TestAgent: consul server down +33:06.941 [INFO] TestAgent: shutdown complete +33:06.941 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24841 network=tcp +33:06.941 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24841 network=udp +33:06.941 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24842 network=tcp +33:07.442 [INFO] TestAgent: Waiting for endpoints to shut down +33:07.442 [INFO] TestAgent: Endpoints down +--- PASS: TestAddServiceIPv6TaggedSet (0.71s) === RUN TestAgent_RemoveService === RUN TestAgent_RemoveService/normal === PAUSE TestAgent_RemoveService/normal @@ -11284,136 +11326,135 @@ [WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:25.232Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:25.250Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:25.253 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:70f57084-b54c-fccc-79fb-1c2510ab581e Address:127.0.0.1:13629}]" -41:25.253 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13629 [Follower]" leader= -41:25.254 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-70f57084-b54c-fccc-79fb-1c2510ab581e.dc1 127.0.0.1 -41:25.255 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-70f57084-b54c-fccc-79fb-1c2510ab581e 127.0.0.1 -2022-09-15T21:41:25.255Z [INFO] agent.router: Initializing LAN area manager -41:25.255 [INFO] TestAgent.server: Adding LAN server: server="Node-70f57084-b54c-fccc-79fb-1c2510ab581e (Addr: tcp/127.0.0.1:13629) (DC: dc1)" -41:25.255 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-70f57084-b54c-fccc-79fb-1c2510ab581e.dc1 area=wan -41:25.255 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13624 network=udp -41:25.255 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13624 network=tcp -41:25.256 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13625 network=tcp -41:25.256 [INFO] TestAgent: started state syncer -41:25.256 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13630 network=tcp -41:25.298 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:25.298 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13629 [Candidate]" term=2 -41:25.298 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:25.298 [DEBUG] TestAgent.server.raft: vote granted: from=70f57084-b54c-fccc-79fb-1c2510ab581e term=2 tally=1 -41:25.298 [INFO] TestAgent.server.raft: election won: tally=1 -41:25.298 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13629 [Leader]" -41:25.298 [INFO] TestAgent.server: cluster leadership acquired -41:25.299 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13629 -41:25.300 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:25.301 [INFO] TestAgent.server: New leader elected: payload=Node-70f57084-b54c-fccc-79fb-1c2510ab581e -41:25.303 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:25.303 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:25.303 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:25.303 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:25.303 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:25.303 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-70f57084-b54c-fccc-79fb-1c2510ab581e -41:25.303 [INFO] TestAgent.server: member joined, marking health alive: member=Node-70f57084-b54c-fccc-79fb-1c2510ab581e -41:25.304 [INFO] TestAgent.server: federation state anti-entropy synced -41:25.318 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:25.318 [INFO] TestAgent: Synced node info -41:25.318 [DEBUG] TestAgent: Node info in sync -41:25.335 [WARN] TestAgent: Failed to deregister service: service=redis error="Service {"redis" {}} does not exist" -41:25.335 [DEBUG] TestAgent: removed check: check=service:memcache -41:25.335 [DEBUG] TestAgent: removed check: check=check2 -41:25.335 [DEBUG] TestAgent: removed service: service=memcache -41:25.336 [DEBUG] TestAgent: removed check: check=service:redis:1 -41:25.336 [DEBUG] TestAgent: removed check: check=service:redis:2 -41:25.336 [DEBUG] TestAgent: removed service: service=redis -41:25.336 [INFO] TestAgent: Requesting shutdown -41:25.336 [INFO] TestAgent.server: shutting down server -41:25.336 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:25.336 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:25.336 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:25.336 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:25.336 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:25.336 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:25.336 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:25.336 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:25.336 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:25.336 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:25.336 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:25.336Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:25.337Z [INFO] agent.router.manager: shutting down -41:25.337 [INFO] TestAgent: consul server down -41:25.337 [INFO] TestAgent: shutdown complete -41:25.337 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13624 network=tcp -41:25.337 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13624 network=udp -41:25.337 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13625 network=tcp -41:25.837 [INFO] TestAgent: Waiting for endpoints to shut down -41:25.837 [INFO] TestAgent: Endpoints down -2022-09-15T21:41:25.210Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:25.240Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:25.242 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8ae515ad-e25a-f90b-c515-b5e76c0ba2aa Address:127.0.0.1:13622}]" -41:25.243 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13622 [Follower]" leader= -41:25.243 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8ae515ad-e25a-f90b-c515-b5e76c0ba2aa.dc1 127.0.0.1 -41:25.244 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8ae515ad-e25a-f90b-c515-b5e76c0ba2aa 127.0.0.1 -2022-09-15T21:41:25.244Z [INFO] agent.router: Initializing LAN area manager -41:25.245 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13617 network=udp -41:25.245 [INFO] TestAgent.server: Adding LAN server: server="Node-8ae515ad-e25a-f90b-c515-b5e76c0ba2aa (Addr: tcp/127.0.0.1:13622) (DC: dc1)" -41:25.245 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8ae515ad-e25a-f90b-c515-b5e76c0ba2aa.dc1 area=wan -41:25.245 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13617 network=tcp -41:25.246 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13618 network=tcp -41:25.246 [INFO] TestAgent: started state syncer -41:25.247 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13623 network=tcp -41:25.311 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:25.311 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13622 [Candidate]" term=2 -41:25.311 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:25.311 [DEBUG] TestAgent.server.raft: vote granted: from=8ae515ad-e25a-f90b-c515-b5e76c0ba2aa term=2 tally=1 -41:25.311 [INFO] TestAgent.server.raft: election won: tally=1 -41:25.311 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13622 [Leader]" -41:25.311 [INFO] TestAgent.server: cluster leadership acquired -41:25.312 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13622 -41:25.312 [INFO] TestAgent.server: New leader elected: payload=Node-8ae515ad-e25a-f90b-c515-b5e76c0ba2aa -41:25.313 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:25.316 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:25.316 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:25.316 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:25.316 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:25.316 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:25.316 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8ae515ad-e25a-f90b-c515-b5e76c0ba2aa -41:25.316 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8ae515ad-e25a-f90b-c515-b5e76c0ba2aa -41:25.317 [INFO] TestAgent.server: federation state anti-entropy synced -41:25.342 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:25.343 [INFO] TestAgent: Synced node info -41:25.381 [WARN] TestAgent: Failed to deregister service: service=redis error="Service {"redis" {}} does not exist" -41:25.382 [DEBUG] TestAgent: removed check: check=service:memcache -41:25.382 [DEBUG] TestAgent: removed check: check=check2 -41:25.382 [DEBUG] TestAgent: removed service: service=memcache -41:25.382 [DEBUG] TestAgent: removed check: check=service:redis:2 -41:25.382 [DEBUG] TestAgent: removed check: check=service:redis:1 -41:25.382 [DEBUG] TestAgent: removed service: service=redis -41:25.382 [INFO] TestAgent: Requesting shutdown -41:25.382 [INFO] TestAgent.server: shutting down server -41:25.382 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:25.382 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:25.382 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:25.382 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:25.382 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:25.383 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:25.383 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:25.383 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:25.383 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:25.383 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:25.383 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:25.383Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:25.383Z [INFO] agent.router.manager: shutting down -41:25.383 [INFO] TestAgent: consul server down -41:25.383 [INFO] TestAgent: shutdown complete -41:25.383 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13617 network=tcp -41:25.383 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13617 network=udp -41:25.383 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13618 network=tcp -41:25.883 [INFO] TestAgent: Waiting for endpoints to shut down -41:25.884 [INFO] TestAgent: Endpoints down +2021-08-13T15:33:07.475Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:07.509Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:07.512 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1246a309-95b0-c61a-273c-a9c6ee89fc58 Address:127.0.0.1:24853}]" +33:07.512 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24853 [Follower]" leader= +33:07.512 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1246a309-95b0-c61a-273c-a9c6ee89fc58.dc1 127.0.0.1 +33:07.514 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1246a309-95b0-c61a-273c-a9c6ee89fc58 127.0.0.1 +2021-08-13T15:33:07.514Z [INFO] agent.router: Initializing LAN area manager +33:07.514 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24848 network=udp +33:07.514 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1246a309-95b0-c61a-273c-a9c6ee89fc58.dc1 area=wan +33:07.514 [INFO] TestAgent.server: Adding LAN server: server="Node-1246a309-95b0-c61a-273c-a9c6ee89fc58 (Addr: tcp/127.0.0.1:24853) (DC: dc1)" +33:07.514 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24848 network=tcp +33:07.515 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24849 network=tcp +33:07.515 [INFO] TestAgent: started state syncer +33:07.515 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24854 network=tcp +33:07.576 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:07.576 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24853 [Candidate]" term=2 +33:07.577 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:07.577 [DEBUG] TestAgent.server.raft: vote granted: from=1246a309-95b0-c61a-273c-a9c6ee89fc58 term=2 tally=1 +33:07.577 [INFO] TestAgent.server.raft: election won: tally=1 +33:07.577 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24853 [Leader]" +33:07.577 [INFO] TestAgent.server: cluster leadership acquired +33:07.577 [INFO] TestAgent.server: New leader elected: payload=Node-1246a309-95b0-c61a-273c-a9c6ee89fc58 +33:07.577 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24853 +33:07.579 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:07.581 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:07.581 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:07.581 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:07.581 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:07.581 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:07.581 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1246a309-95b0-c61a-273c-a9c6ee89fc58 +33:07.581 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1246a309-95b0-c61a-273c-a9c6ee89fc58 +33:07.582 [INFO] TestAgent.server: federation state anti-entropy synced +33:07.586 [WARN] TestAgent: Failed to deregister service: service=redis error="Service {"redis" {}} does not exist" +33:07.586 [DEBUG] TestAgent: removed check: check=service:memcache +33:07.586 [DEBUG] TestAgent: removed check: check=check2 +33:07.586 [DEBUG] TestAgent: removed service: service=memcache +33:07.586 [DEBUG] TestAgent: removed check: check=service:redis:2 +33:07.586 [DEBUG] TestAgent: removed check: check=service:redis:1 +33:07.586 [DEBUG] TestAgent: removed service: service=redis +33:07.586 [INFO] TestAgent: Requesting shutdown +33:07.587 [INFO] TestAgent.server: shutting down server +33:07.587 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:07.587 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:07.587 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:07.587 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:07.587 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:07.587 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:07.587 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:07.587 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:07.587 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:07.587 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:07.587 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:07.587 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:07.587Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:07.587Z [INFO] agent.router.manager: shutting down +33:07.588 [INFO] TestAgent: consul server down +33:07.588 [INFO] TestAgent: shutdown complete +33:07.588 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24848 network=tcp +33:07.588 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24848 network=udp +33:07.588 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24849 network=tcp +33:08.088 [INFO] TestAgent: Waiting for endpoints to shut down +33:08.088 [INFO] TestAgent: Endpoints down +[WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:33:07.475Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:07.507Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:07.511 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5de1ff10-bde4-3a22-df0b-cf2c6d9b4453 Address:127.0.0.1:24860}]" +33:07.511 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24860 [Follower]" leader= +33:07.512 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5de1ff10-bde4-3a22-df0b-cf2c6d9b4453.dc1 127.0.0.1 +33:07.512 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5de1ff10-bde4-3a22-df0b-cf2c6d9b4453 127.0.0.1 +2021-08-13T15:33:07.512Z [INFO] agent.router: Initializing LAN area manager +33:07.513 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5de1ff10-bde4-3a22-df0b-cf2c6d9b4453.dc1 area=wan +33:07.513 [INFO] TestAgent.server: Adding LAN server: server="Node-5de1ff10-bde4-3a22-df0b-cf2c6d9b4453 (Addr: tcp/127.0.0.1:24860) (DC: dc1)" +33:07.513 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24855 network=tcp +33:07.513 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24855 network=udp +33:07.514 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24856 network=tcp +33:07.514 [INFO] TestAgent: started state syncer +33:07.514 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24861 network=tcp +33:07.552 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:07.552 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24860 [Candidate]" term=2 +33:07.552 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:07.552 [DEBUG] TestAgent.server.raft: vote granted: from=5de1ff10-bde4-3a22-df0b-cf2c6d9b4453 term=2 tally=1 +33:07.552 [INFO] TestAgent.server.raft: election won: tally=1 +33:07.552 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24860 [Leader]" +33:07.553 [INFO] TestAgent.server: cluster leadership acquired +33:07.553 [INFO] TestAgent.server: New leader elected: payload=Node-5de1ff10-bde4-3a22-df0b-cf2c6d9b4453 +33:07.553 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24860 +33:07.554 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:07.557 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:07.557 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:07.557 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:07.557 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:07.557 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:07.557 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5de1ff10-bde4-3a22-df0b-cf2c6d9b4453 +33:07.557 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5de1ff10-bde4-3a22-df0b-cf2c6d9b4453 +33:07.558 [INFO] TestAgent.server: federation state anti-entropy synced +33:07.619 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:07.619 [INFO] TestAgent: Synced node info +33:07.897 [WARN] TestAgent: Failed to deregister service: service=redis error="Service {"redis" {}} does not exist" +33:07.897 [DEBUG] TestAgent: removed check: check=check2 +33:07.897 [DEBUG] TestAgent: removed check: check=service:memcache +33:07.897 [DEBUG] TestAgent: removed service: service=memcache +33:07.897 [DEBUG] TestAgent: removed check: check=service:redis:1 +33:07.897 [DEBUG] TestAgent: removed check: check=service:redis:2 +33:07.897 [DEBUG] TestAgent: removed service: service=redis +33:07.898 [INFO] TestAgent: Requesting shutdown +33:07.898 [INFO] TestAgent.server: shutting down server +33:07.898 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:07.898 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:07.898 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:07.898 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:07.898 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:07.898 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:07.898 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:07.898 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:07.898 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:07.898 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:07.898 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:07.898Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:07.898Z [INFO] agent.router.manager: shutting down +33:07.898 [INFO] TestAgent: consul server down +33:07.898 [INFO] TestAgent: shutdown complete +33:07.898 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24855 network=tcp +33:07.898 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24855 network=udp +33:07.899 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24856 network=tcp +33:08.399 [INFO] TestAgent: Waiting for endpoints to shut down +33:08.399 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_RemoveService (0.00s) - --- PASS: TestAgent_RemoveService/service_manager (0.65s) - --- PASS: TestAgent_RemoveService/normal (0.70s) + --- PASS: TestAgent_RemoveService/normal (0.65s) + --- PASS: TestAgent_RemoveService/service_manager (0.95s) === RUN TestAgent_RemoveServiceRemovesAllChecks === RUN TestAgent_RemoveServiceRemovesAllChecks/normal === PAUSE TestAgent_RemoveServiceRemovesAllChecks/normal @@ -11421,132 +11462,128 @@ === PAUSE TestAgent_RemoveServiceRemovesAllChecks/service_manager === CONT TestAgent_RemoveServiceRemovesAllChecks/normal === CONT TestAgent_RemoveServiceRemovesAllChecks/service_manager -[WARN] freeport: 6 out of 20 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:25.902Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:25.945Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:25.952 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d688cea4-4a8d-9ec8-86d1-d93cd048cd6a Address:127.0.0.1:13643}]" -41:25.953 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 -41:25.954 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13643 [Follower]" leader= -41:25.955 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 -2022-09-15T21:41:25.955Z [INFO] agent.router: Initializing LAN area manager -41:25.956 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13638 network=udp -41:25.956 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:13643) (DC: dc1)" -41:25.956 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan -41:25.956 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13638 network=tcp -41:25.957 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13639 network=tcp -41:25.957 [INFO] TestAgent: started state syncer -41:25.957 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13644 network=tcp -41:26.017 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:26.017 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13643 [Candidate]" term=2 -41:26.018 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:26.018 [DEBUG] TestAgent.server.raft: vote granted: from=d688cea4-4a8d-9ec8-86d1-d93cd048cd6a term=2 tally=1 -41:26.018 [INFO] TestAgent.server.raft: election won: tally=1 -41:26.018 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13643 [Leader]" -41:26.018 [INFO] TestAgent.server: cluster leadership acquired -41:26.018 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13643 -41:26.020 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:26.021 [INFO] TestAgent.server: New leader elected: payload=node1 -41:26.039 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:26.039 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:26.039 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:26.039 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:26.039 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:26.041 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 -41:26.041 [INFO] TestAgent.server: member joined, marking health alive: member=node1 -41:26.041 [INFO] TestAgent.server: federation state anti-entropy synced -41:26.078 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:26.078 [INFO] TestAgent: Synced node info -41:26.078 [DEBUG] TestAgent: Node info in sync -41:26.314 [DEBUG] TestAgent: removed check: check=chk2 -41:26.314 [DEBUG] TestAgent: removed check: check=chk1 -41:26.314 [DEBUG] TestAgent: removed service: service=redis -41:26.314 [INFO] TestAgent: Requesting shutdown -41:26.314 [INFO] TestAgent.server: shutting down server -41:26.314 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:26.314 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:26.314 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:26.314 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:26.314 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:26.314 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:26.314 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:26.314 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:26.314 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:26.314 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:26.314 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:26.314Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:26.314Z [INFO] agent.router.manager: shutting down -41:26.314 [INFO] TestAgent: consul server down -41:26.314 [INFO] TestAgent: shutdown complete -41:26.314 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13638 network=tcp -41:26.314 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13638 network=udp -41:26.315 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13639 network=tcp -41:26.815 [INFO] TestAgent: Waiting for endpoints to shut down -41:26.815 [INFO] TestAgent: Endpoints down -2022-09-15T21:41:25.904Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:25.950Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:25.953 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9a444fea-497e-5d15-8d29-7c62ed20b445 Address:127.0.0.1:13636}]" -41:25.954 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 -41:25.954 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13636 [Follower]" leader= -41:25.954 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 -2022-09-15T21:41:25.954Z [INFO] agent.router: Initializing LAN area manager -41:25.955 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13631 network=udp -41:25.955 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:13636) (DC: dc1)" -41:25.955 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan -41:25.955 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13631 network=tcp -41:25.956 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13632 network=tcp -41:25.956 [INFO] TestAgent: started state syncer -41:25.956 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13637 network=tcp -41:26.022 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:26.022 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13636 [Candidate]" term=2 -41:26.031 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:26.033 [DEBUG] TestAgent.server.raft: vote granted: from=9a444fea-497e-5d15-8d29-7c62ed20b445 term=2 tally=1 -41:26.033 [INFO] TestAgent.server.raft: election won: tally=1 -41:26.033 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13636 [Leader]" -41:26.033 [INFO] TestAgent.server: cluster leadership acquired -41:26.033 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13636 -41:26.035 [INFO] TestAgent.server: New leader elected: payload=node1 -41:26.035 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:26.038 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:26.041 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:26.041 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:26.041 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:26.041 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:26.042 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 -41:26.042 [INFO] TestAgent.server: federation state anti-entropy synced -41:26.042 [INFO] TestAgent.server: member joined, marking health alive: member=node1 -41:26.310 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:26.311 [INFO] TestAgent: Synced node info -41:26.358 [DEBUG] TestAgent: removed check: check=chk1 -41:26.358 [DEBUG] TestAgent: removed check: check=chk2 -41:26.358 [DEBUG] TestAgent: removed service: service=redis -41:26.358 [INFO] TestAgent: Requesting shutdown -41:26.358 [INFO] TestAgent.server: shutting down server -41:26.358 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:26.358 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:26.358 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:26.358 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:26.358 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:26.358 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:26.359 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:26.359 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:26.359 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:26.359 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:26.359 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:26.359Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:26.359Z [INFO] agent.router.manager: shutting down -41:26.359 [INFO] TestAgent: consul server down -41:26.359 [INFO] TestAgent: shutdown complete -41:26.359 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13631 network=tcp -41:26.359 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13631 network=udp -41:26.359 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13632 network=tcp -41:26.859 [INFO] TestAgent: Waiting for endpoints to shut down -41:26.859 [INFO] TestAgent: Endpoints down +2021-08-13T15:33:08.419Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:08.473Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:08.476 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b6567815-1e5b-bff2-2679-0c3501673c3d Address:127.0.0.1:24867}]" +33:08.477 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24867 [Follower]" leader= +33:08.477 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 +33:08.478 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 +2021-08-13T15:33:08.478Z [INFO] agent.router: Initializing LAN area manager +33:08.478 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:24867) (DC: dc1)" +33:08.478 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan +33:08.478 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24862 network=udp +33:08.478 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24862 network=tcp +33:08.479 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24863 network=tcp +33:08.479 [INFO] TestAgent: started state syncer +33:08.479 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24868 network=tcp +33:08.521 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:08.521 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24867 [Candidate]" term=2 +33:08.521 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:08.521 [DEBUG] TestAgent.server.raft: vote granted: from=b6567815-1e5b-bff2-2679-0c3501673c3d term=2 tally=1 +33:08.521 [INFO] TestAgent.server.raft: election won: tally=1 +33:08.521 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24867 [Leader]" +33:08.521 [INFO] TestAgent.server: cluster leadership acquired +33:08.522 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24867 +33:08.523 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:08.525 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:08.525 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:08.525 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:08.525 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:08.525 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:08.526 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 +33:08.526 [INFO] TestAgent.server: member joined, marking health alive: member=node1 +33:08.526 [INFO] TestAgent.server: New leader elected: payload=node1 +33:08.532 [INFO] TestAgent.server: federation state anti-entropy synced +33:08.635 [DEBUG] TestAgent: removed check: check=chk1 +33:08.636 [DEBUG] TestAgent: removed check: check=chk2 +33:08.636 [DEBUG] TestAgent: removed service: service=redis +33:08.636 [INFO] TestAgent: Requesting shutdown +33:08.636 [INFO] TestAgent.server: shutting down server +33:08.636 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:08.636 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:08.636 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:08.636 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:08.636 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:08.636 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:08.636 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:08.636 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:08.636 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:08.636 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:08.636 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:08.636 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:08.636Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:08.636Z [INFO] agent.router.manager: shutting down +33:08.636 [INFO] TestAgent: consul server down +33:08.636 [INFO] TestAgent: shutdown complete +33:08.636 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24862 network=tcp +33:08.637 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24862 network=udp +33:08.637 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24863 network=tcp +33:09.137 [INFO] TestAgent: Waiting for endpoints to shut down +33:09.137 [INFO] TestAgent: Endpoints down +2021-08-13T15:33:08.416Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:08.466Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:08.469 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4daa03ff-df4a-f460-c3c4-dd5bee941dbf Address:127.0.0.1:24874}]" +33:08.470 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24874 [Follower]" leader= +33:08.470 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 +33:08.475 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 +2021-08-13T15:33:08.475Z [INFO] agent.router: Initializing LAN area manager +33:08.475 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:24874) (DC: dc1)" +33:08.475 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan +33:08.476 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24869 network=udp +33:08.476 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24869 network=tcp +33:08.476 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24870 network=tcp +33:08.476 [INFO] TestAgent: started state syncer +33:08.477 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24875 network=tcp +33:08.527 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:08.527 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24874 [Candidate]" term=2 +33:08.527 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:08.527 [DEBUG] TestAgent.server.raft: vote granted: from=4daa03ff-df4a-f460-c3c4-dd5bee941dbf term=2 tally=1 +33:08.527 [INFO] TestAgent.server.raft: election won: tally=1 +33:08.527 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24874 [Leader]" +33:08.527 [INFO] TestAgent.server: cluster leadership acquired +33:08.527 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24874 +33:08.529 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:08.531 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:08.531 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:08.531 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:08.531 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:08.531 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:08.531 [INFO] TestAgent.server: New leader elected: payload=node1 +33:08.531 [INFO] TestAgent.server: federation state anti-entropy synced +33:08.531 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 +33:08.531 [INFO] TestAgent.server: member joined, marking health alive: member=node1 +33:08.707 [DEBUG] TestAgent: removed check: check=chk1 +33:08.707 [DEBUG] TestAgent: removed check: check=chk2 +33:08.707 [DEBUG] TestAgent: removed service: service=redis +33:08.707 [INFO] TestAgent: Requesting shutdown +33:08.707 [INFO] TestAgent.server: shutting down server +33:08.707 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:08.707 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:08.707 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:08.707 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:08.707 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:08.707 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:08.707 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:08.707 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:08.707 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:08.707 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +33:08.707 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:08.707 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2021-08-13T15:33:08.708Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:08.708Z [INFO] agent.router.manager: shutting down +33:08.708 [INFO] TestAgent: consul server down +33:08.708 [INFO] TestAgent: shutdown complete +33:08.708 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24869 network=tcp +33:08.708 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24869 network=udp +33:08.708 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24870 network=tcp +33:09.208 [INFO] TestAgent: Waiting for endpoints to shut down +33:09.208 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_RemoveServiceRemovesAllChecks (0.00s) - --- PASS: TestAgent_RemoveServiceRemovesAllChecks/service_manager (0.93s) - --- PASS: TestAgent_RemoveServiceRemovesAllChecks/normal (0.98s) + --- PASS: TestAgent_RemoveServiceRemovesAllChecks/normal (0.74s) + --- PASS: TestAgent_RemoveServiceRemovesAllChecks/service_manager (0.81s) === RUN TestAgent_IndexChurn === PAUSE TestAgent_IndexChurn === RUN TestAgent_AddCheck @@ -11597,96 +11634,96 @@ [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -41:27.521 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-e193aba4-d2d2-fcfe-f07a-20a5b058e4d2 127.0.0.1 -2022-09-15T21:41:27.521Z [INFO] agent.router: Initializing LAN area manager -41:27.522 [INFO] TestAgent.client.serf.lan: serf: Attempting re-join to previously known node: Node-e1b42f8f-df60-790f-9018-42dd9f34c9bd: 127.0.0.1:13655 -41:27.522 [DEBUG] TestAgent.client.memberlist.lan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:13655: connect: connection refused -41:27.522 [WARN] TestAgent.client.serf.lan: serf: Failed to re-join any previously known node -41:27.526 [DEBUG] TestAgent: restored service definition from file: service=redis file=/tmp/consul-test/TestAgent_PersistService_service_manager-TestAgent-agent585262787/services/86a1b907d54bf7010394bf316e183e67 -41:27.527 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13666 network=udp -41:27.527 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13666 network=tcp -41:27.528 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13667 network=tcp -41:27.528 [INFO] TestAgent: started state syncer -41:27.528 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13672 network=tcp -2022-09-15T21:41:27.528Z [WARN] agent.router.manager: No servers available -41:27.528 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" -41:27.529 [INFO] TestAgent: Requesting shutdown -41:27.529 [INFO] TestAgent.client: shutting down client -41:27.529 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave -41:27.529 [INFO] TestAgent: consul client down -41:27.529 [INFO] TestAgent: shutdown complete -41:27.529 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13666 network=tcp -41:27.529 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13666 network=udp -41:27.529 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13667 network=tcp -41:28.030 [INFO] TestAgent: Waiting for endpoints to shut down -41:28.030 [INFO] TestAgent: Endpoints down -41:26.928 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-e1b42f8f-df60-790f-9018-42dd9f34c9bd 127.0.0.1 -2022-09-15T21:41:26.928Z [INFO] agent.router: Initializing LAN area manager -41:26.929 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13652 network=udp -41:26.929 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13652 network=tcp -41:26.930 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13653 network=tcp -41:26.930 [INFO] TestAgent: started state syncer -41:26.932 [INFO] TestAgent: Requesting shutdown -41:26.932 [INFO] TestAgent.client: shutting down client -41:26.932 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave -41:26.932 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13658 network=tcp -41:26.932 [ERROR] TestAgent: gRPC server failed: error="grpc: the server has been stopped" -2022-09-15T21:41:26.932Z [WARN] agent.router.manager: No servers available -41:26.932 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" -41:26.932 [INFO] TestAgent: consul client down -41:26.932 [INFO] TestAgent: shutdown complete -41:26.932 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13652 network=tcp -41:26.932 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13652 network=udp -41:26.932 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13653 network=tcp -41:27.432 [INFO] TestAgent: Waiting for endpoints to shut down -41:27.432 [INFO] TestAgent: Endpoints down -41:27.537 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-fecf9bca-f9ef-2132-3c15-4c48a0ed3069 127.0.0.1 -2022-09-15T21:41:27.537Z [INFO] agent.router: Initializing LAN area manager -41:27.537 [DEBUG] TestAgent: restored service definition from file: service=redis file=/tmp/consul-test/TestAgent_PersistService_normal-TestAgent-agent160407028/services/86a1b907d54bf7010394bf316e183e67 -41:27.538 [INFO] TestAgent.client.serf.lan: serf: Attempting re-join to previously known node: Node-c0e4ea91-9355-513f-f6ae-ea1822b2051a: 127.0.0.1:13648 -41:27.538 [DEBUG] TestAgent.client.memberlist.lan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:13648: connect: connection refused -41:27.538 [WARN] TestAgent.client.serf.lan: serf: Failed to re-join any previously known node -41:27.538 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13659 network=tcp -41:27.538 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13659 network=udp -41:27.539 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13660 network=tcp -41:27.539 [INFO] TestAgent: started state syncer -41:27.540 [INFO] TestAgent: Requesting shutdown -41:27.540 [INFO] TestAgent.client: shutting down client -41:27.540 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave -41:27.540 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13665 network=tcp -41:27.540 [ERROR] TestAgent: gRPC server failed: error="grpc: the server has been stopped" -2022-09-15T21:41:27.540Z [WARN] agent.router.manager: No servers available -41:27.540 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" -41:27.540 [INFO] TestAgent: consul client down -41:27.540 [INFO] TestAgent: shutdown complete -41:27.540 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13659 network=tcp -41:27.540 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13659 network=udp -41:27.540 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13660 network=tcp -41:28.041 [INFO] TestAgent: Waiting for endpoints to shut down -41:28.041 [INFO] TestAgent: Endpoints down -41:26.936 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-c0e4ea91-9355-513f-f6ae-ea1822b2051a 127.0.0.1 -2022-09-15T21:41:26.936Z [INFO] agent.router: Initializing LAN area manager -41:26.936 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13645 network=udp -41:26.936 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13645 network=tcp -41:26.937 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13646 network=tcp -41:26.937 [INFO] TestAgent: started state syncer -41:26.939 [INFO] TestAgent: Requesting shutdown -41:26.939 [INFO] TestAgent.client: shutting down client -41:26.939 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave -41:26.939 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13651 network=tcp -41:26.939 [ERROR] TestAgent: gRPC server failed: error="grpc: the server has been stopped" -2022-09-15T21:41:26.940Z [WARN] agent.router.manager: No servers available -41:26.940 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" -41:26.940 [INFO] TestAgent: consul client down -41:26.940 [INFO] TestAgent: shutdown complete -41:26.940 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13645 network=tcp -41:26.940 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13645 network=udp -41:26.940 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13646 network=tcp -41:27.441 [INFO] TestAgent: Waiting for endpoints to shut down -41:27.441 [INFO] TestAgent: Endpoints down +33:09.825 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-16aa8e45-c044-1081-c490-df321c504a27 127.0.0.1 +2021-08-13T15:33:09.825Z [INFO] agent.router: Initializing LAN area manager +33:09.826 [DEBUG] TestAgent: restored service definition from file: service=redis file=/tmp/consul-test/TestAgent_PersistService_service_manager-TestAgent-agent709150222/services/86a1b907d54bf7010394bf316e183e67 +33:09.826 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24890 network=udp +33:09.826 [INFO] TestAgent.client.serf.lan: serf: Attempting re-join to previously known node: Node-24b92765-8bfe-99f3-5391-b23a804c17e7: 127.0.0.1:24886 +33:09.827 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24890 network=tcp +33:09.827 [DEBUG] TestAgent.client.memberlist.lan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:24886: connect: connection refused +33:09.827 [WARN] TestAgent.client.serf.lan: serf: Failed to re-join any previously known node +33:09.831 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24891 network=tcp +33:09.832 [INFO] TestAgent: started state syncer +33:09.832 [INFO] TestAgent: Requesting shutdown +33:09.832 [INFO] TestAgent.client: shutting down client +33:09.832 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave +33:09.832 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24896 network=tcp +33:09.832 [ERROR] TestAgent: gRPC server failed: error="grpc: the server has been stopped" +2021-08-13T15:33:09.832Z [WARN] agent.router.manager: No servers available +33:09.832 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" +33:09.833 [INFO] TestAgent: consul client down +33:09.833 [INFO] TestAgent: shutdown complete +33:09.833 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24890 network=tcp +33:09.833 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24890 network=udp +33:09.833 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24891 network=tcp +33:10.333 [INFO] TestAgent: Waiting for endpoints to shut down +33:10.333 [INFO] TestAgent: Endpoints down +33:09.278 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-24b92765-8bfe-99f3-5391-b23a804c17e7 127.0.0.1 +2021-08-13T15:33:09.278Z [INFO] agent.router: Initializing LAN area manager +33:09.279 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24883 network=udp +33:09.283 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24883 network=tcp +33:09.289 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24884 network=tcp +33:09.289 [INFO] TestAgent: started state syncer +33:09.290 [INFO] TestAgent: Requesting shutdown +33:09.290 [INFO] TestAgent.client: shutting down client +33:09.290 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave +33:09.290 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24889 network=tcp +33:09.290 [ERROR] TestAgent: gRPC server failed: error="grpc: the server has been stopped" +2021-08-13T15:33:09.290Z [WARN] agent.router.manager: No servers available +33:09.290 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" +33:09.290 [INFO] TestAgent: consul client down +33:09.290 [INFO] TestAgent: shutdown complete +33:09.290 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24883 network=tcp +33:09.290 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24883 network=udp +33:09.291 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24884 network=tcp +33:09.791 [INFO] TestAgent: Waiting for endpoints to shut down +33:09.791 [INFO] TestAgent: Endpoints down +33:09.846 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-eee1bf53-530b-09a8-54d5-9c1cc8091cb6 127.0.0.1 +2021-08-13T15:33:09.846Z [INFO] agent.router: Initializing LAN area manager +33:09.847 [DEBUG] TestAgent: restored service definition from file: service=redis file=/tmp/consul-test/TestAgent_PersistService_normal-TestAgent-agent420081835/services/86a1b907d54bf7010394bf316e183e67 +33:09.847 [INFO] TestAgent.client.serf.lan: serf: Attempting re-join to previously known node: Node-13b2ac66-aa7c-a4cd-6ec0-014ad63f4aaf: 127.0.0.1:24879 +33:09.847 [DEBUG] TestAgent.client.memberlist.lan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:24879: connect: connection refused +33:09.847 [WARN] TestAgent.client.serf.lan: serf: Failed to re-join any previously known node +33:09.851 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24897 network=udp +33:09.851 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24897 network=tcp +33:09.852 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24898 network=tcp +33:09.852 [INFO] TestAgent: started state syncer +33:09.853 [INFO] TestAgent: Requesting shutdown +33:09.853 [INFO] TestAgent.client: shutting down client +33:09.853 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave +33:09.853 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24903 network=tcp +33:09.853 [ERROR] TestAgent: gRPC server failed: error="grpc: the server has been stopped" +2021-08-13T15:33:09.853Z [WARN] agent.router.manager: No servers available +33:09.853 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" +33:09.853 [INFO] TestAgent: consul client down +33:09.853 [INFO] TestAgent: shutdown complete +33:09.853 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24897 network=tcp +33:09.853 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24897 network=udp +33:09.853 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24898 network=tcp +33:10.353 [INFO] TestAgent: Waiting for endpoints to shut down +33:10.353 [INFO] TestAgent: Endpoints down +33:09.289 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-13b2ac66-aa7c-a4cd-6ec0-014ad63f4aaf 127.0.0.1 +2021-08-13T15:33:09.289Z [INFO] agent.router: Initializing LAN area manager +33:09.290 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24876 network=udp +33:09.290 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24876 network=tcp +33:09.291 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24877 network=tcp +33:09.291 [INFO] TestAgent: started state syncer +33:09.291 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24882 network=tcp +2021-08-13T15:33:09.291Z [WARN] agent.router.manager: No servers available +33:09.291 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" +33:09.292 [INFO] TestAgent: Requesting shutdown +33:09.292 [INFO] TestAgent.client: shutting down client +33:09.292 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave +33:09.292 [INFO] TestAgent: consul client down +33:09.292 [INFO] TestAgent: shutdown complete +33:09.292 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24876 network=tcp +33:09.292 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24876 network=udp +33:09.292 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24877 network=tcp +33:09.795 [INFO] TestAgent: Waiting for endpoints to shut down +33:09.795 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_PersistService (0.00s) - --- PASS: TestAgent_PersistService/service_manager (1.16s) - --- PASS: TestAgent_PersistService/normal (1.18s) + --- PASS: TestAgent_PersistService/service_manager (1.12s) + --- PASS: TestAgent_PersistService/normal (1.14s) === RUN TestAgent_persistedService_compat === RUN TestAgent_persistedService_compat/normal === PAUSE TestAgent_persistedService_compat/normal @@ -11697,125 +11734,122 @@ [WARN] freeport: 6 out of 34 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:28.062Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:28.098Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:28.106 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:548722cf-4827-0007-765d-ef075ca88ec2 Address:127.0.0.1:13678}]" -41:28.106 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13678 [Follower]" leader= -41:28.107 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-548722cf-4827-0007-765d-ef075ca88ec2.dc1 127.0.0.1 -41:28.109 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-548722cf-4827-0007-765d-ef075ca88ec2 127.0.0.1 -2022-09-15T21:41:28.109Z [INFO] agent.router: Initializing LAN area manager -41:28.109 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13673 network=udp -41:28.109 [INFO] TestAgent.server: Adding LAN server: server="Node-548722cf-4827-0007-765d-ef075ca88ec2 (Addr: tcp/127.0.0.1:13678) (DC: dc1)" -41:28.110 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-548722cf-4827-0007-765d-ef075ca88ec2.dc1 area=wan -41:28.110 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13673 network=tcp -41:28.112 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13674 network=tcp -41:28.112 [INFO] TestAgent: started state syncer -41:28.112 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13679 network=tcp -41:28.176 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:28.176 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13678 [Candidate]" term=2 -41:28.177 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:28.177 [DEBUG] TestAgent.server.raft: vote granted: from=548722cf-4827-0007-765d-ef075ca88ec2 term=2 tally=1 -41:28.177 [INFO] TestAgent.server.raft: election won: tally=1 -41:28.177 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13678 [Leader]" -41:28.177 [INFO] TestAgent.server: cluster leadership acquired -41:28.177 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13678 -41:28.177 [INFO] TestAgent.server: New leader elected: payload=Node-548722cf-4827-0007-765d-ef075ca88ec2 -41:28.179 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:28.181 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:28.181 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:28.182 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:28.182 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:28.182 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:28.182 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-548722cf-4827-0007-765d-ef075ca88ec2 -41:28.182 [INFO] TestAgent.server: member joined, marking health alive: member=Node-548722cf-4827-0007-765d-ef075ca88ec2 -41:28.210 [INFO] TestAgent.server: federation state anti-entropy synced -41:28.290 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:28.291 [INFO] TestAgent: Synced node info -41:28.361 [DEBUG] TestAgent: restored service definition from file: service=redis file=/tmp/consul-test/TestAgent_persistedService_compat_normal-TestAgent-agent906857286/services/86a1b907d54bf7010394bf316e183e67 -41:28.361 [INFO] TestAgent: Requesting shutdown -41:28.361 [INFO] TestAgent.server: shutting down server -41:28.361 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:28.361 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:28.361 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:28.361 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:28.361 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:28.361 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:28.361 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:28.361 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:28.361 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:28.361 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:28.362 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:28.362Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:28.362Z [INFO] agent.router.manager: shutting down -41:28.362 [INFO] TestAgent: consul server down -41:28.362 [INFO] TestAgent: shutdown complete -41:28.362 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13673 network=tcp -41:28.362 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13673 network=udp -41:28.362 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13674 network=tcp -41:28.863 [INFO] TestAgent: Waiting for endpoints to shut down -41:28.863 [INFO] TestAgent: Endpoints down -[WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:28.060Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:28.078Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:28.081 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:072d1122-ed6c-c505-d092-912cb8e210d2 Address:127.0.0.1:13685}]" -41:28.081 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13685 [Follower]" leader= -41:28.083 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-072d1122-ed6c-c505-d092-912cb8e210d2.dc1 127.0.0.1 -41:28.102 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-072d1122-ed6c-c505-d092-912cb8e210d2 127.0.0.1 -2022-09-15T21:41:28.102Z [INFO] agent.router: Initializing LAN area manager -41:28.102 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13680 network=udp -41:28.103 [INFO] TestAgent.server: Adding LAN server: server="Node-072d1122-ed6c-c505-d092-912cb8e210d2 (Addr: tcp/127.0.0.1:13685) (DC: dc1)" -41:28.103 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-072d1122-ed6c-c505-d092-912cb8e210d2.dc1 area=wan -41:28.103 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13680 network=tcp -41:28.105 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13681 network=tcp -41:28.105 [INFO] TestAgent: started state syncer -41:28.105 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13686 network=tcp -41:28.144 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:28.144 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13685 [Candidate]" term=2 -41:28.145 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:28.145 [DEBUG] TestAgent.server.raft: vote granted: from=072d1122-ed6c-c505-d092-912cb8e210d2 term=2 tally=1 -41:28.145 [INFO] TestAgent.server.raft: election won: tally=1 -41:28.145 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13685 [Leader]" -41:28.145 [INFO] TestAgent.server: cluster leadership acquired -41:28.145 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13685 -41:28.146 [INFO] TestAgent.server: New leader elected: payload=Node-072d1122-ed6c-c505-d092-912cb8e210d2 -41:28.147 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:28.151 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:28.151 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:28.151 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:28.151 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:28.151 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:28.151 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-072d1122-ed6c-c505-d092-912cb8e210d2 -41:28.151 [INFO] TestAgent.server: member joined, marking health alive: member=Node-072d1122-ed6c-c505-d092-912cb8e210d2 -41:28.152 [INFO] TestAgent.server: federation state anti-entropy synced -41:28.242 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:28.242 [INFO] TestAgent: Synced node info -41:28.242 [DEBUG] TestAgent: Node info in sync -41:28.522 [DEBUG] TestAgent: restored service definition from file: service=redis file=/tmp/consul-test/TestAgent_persistedService_compat_service_manager-TestAgent-agent575868909/services/86a1b907d54bf7010394bf316e183e67 -41:28.522 [INFO] TestAgent: Requesting shutdown -41:28.522 [INFO] TestAgent.server: shutting down server -41:28.522 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:28.522 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:28.522 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:28.522 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:28.522 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:28.522 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:28.522 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:28.522 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:28.522 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:28.522 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:28.522 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:28.523Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:28.523Z [INFO] agent.router.manager: shutting down -41:28.523 [INFO] TestAgent: consul server down -41:28.523 [INFO] TestAgent: shutdown complete -41:28.523 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13680 network=tcp -41:28.523 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13680 network=udp -41:28.523 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13681 network=tcp -41:29.024 [INFO] TestAgent: Waiting for endpoints to shut down -41:29.025 [INFO] TestAgent: Endpoints down +2021-08-13T15:33:10.404Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:10.420Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:10.426 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fd48e466-8971-8578-2c62-3d0df55b1ee0 Address:127.0.0.1:24909}]" +33:10.427 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fd48e466-8971-8578-2c62-3d0df55b1ee0.dc1 127.0.0.1 +33:10.427 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24909 [Follower]" leader= +33:10.428 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fd48e466-8971-8578-2c62-3d0df55b1ee0 127.0.0.1 +2021-08-13T15:33:10.428Z [INFO] agent.router: Initializing LAN area manager +33:10.428 [INFO] TestAgent.server: Adding LAN server: server="Node-fd48e466-8971-8578-2c62-3d0df55b1ee0 (Addr: tcp/127.0.0.1:24909) (DC: dc1)" +33:10.428 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24904 network=udp +33:10.428 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fd48e466-8971-8578-2c62-3d0df55b1ee0.dc1 area=wan +33:10.428 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24904 network=tcp +33:10.429 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24905 network=tcp +33:10.429 [INFO] TestAgent: started state syncer +33:10.429 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24910 network=tcp +33:10.483 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:10.483 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24909 [Candidate]" term=2 +33:10.483 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:10.483 [DEBUG] TestAgent.server.raft: vote granted: from=fd48e466-8971-8578-2c62-3d0df55b1ee0 term=2 tally=1 +33:10.483 [INFO] TestAgent.server.raft: election won: tally=1 +33:10.483 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24909 [Leader]" +33:10.483 [INFO] TestAgent.server: cluster leadership acquired +33:10.484 [INFO] TestAgent.server: New leader elected: payload=Node-fd48e466-8971-8578-2c62-3d0df55b1ee0 +33:10.484 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24909 +33:10.485 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:10.487 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:10.487 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:10.487 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:10.487 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:10.487 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:10.487 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fd48e466-8971-8578-2c62-3d0df55b1ee0 +33:10.487 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fd48e466-8971-8578-2c62-3d0df55b1ee0 +33:10.488 [INFO] TestAgent.server: federation state anti-entropy synced +33:10.528 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:10.529 [INFO] TestAgent: Synced node info +33:10.667 [DEBUG] TestAgent: restored service definition from file: service=redis file=/tmp/consul-test/TestAgent_persistedService_compat_normal-TestAgent-agent374817813/services/86a1b907d54bf7010394bf316e183e67 +33:10.668 [INFO] TestAgent: Requesting shutdown +33:10.668 [INFO] TestAgent.server: shutting down server +33:10.668 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:10.668 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:10.668 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:10.668 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:10.668 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:10.668 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:10.668 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:10.668 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:10.668 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:10.668 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:10.668 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:10.668Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:10.668Z [INFO] agent.router.manager: shutting down +33:10.668 [INFO] TestAgent: consul server down +33:10.668 [INFO] TestAgent: shutdown complete +33:10.668 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24904 network=tcp +33:10.668 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24904 network=udp +33:10.668 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24905 network=tcp +33:11.168 [INFO] TestAgent: Waiting for endpoints to shut down +33:11.168 [INFO] TestAgent: Endpoints down +2021-08-13T15:33:10.397Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:10.413Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:10.423 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:41356a43-6ff0-322d-6caf-e0cce1f665c6 Address:127.0.0.1:24916}]" +33:10.424 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24916 [Follower]" leader= +33:10.424 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-41356a43-6ff0-322d-6caf-e0cce1f665c6.dc1 127.0.0.1 +33:10.425 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-41356a43-6ff0-322d-6caf-e0cce1f665c6 127.0.0.1 +2021-08-13T15:33:10.425Z [INFO] agent.router: Initializing LAN area manager +33:10.425 [INFO] TestAgent.server: Adding LAN server: server="Node-41356a43-6ff0-322d-6caf-e0cce1f665c6 (Addr: tcp/127.0.0.1:24916) (DC: dc1)" +33:10.426 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-41356a43-6ff0-322d-6caf-e0cce1f665c6.dc1 area=wan +33:10.426 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24911 network=udp +33:10.426 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24911 network=tcp +33:10.427 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24912 network=tcp +33:10.427 [INFO] TestAgent: started state syncer +33:10.427 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24917 network=tcp +33:10.471 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:10.471 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24916 [Candidate]" term=2 +33:10.472 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:10.472 [DEBUG] TestAgent.server.raft: vote granted: from=41356a43-6ff0-322d-6caf-e0cce1f665c6 term=2 tally=1 +33:10.472 [INFO] TestAgent.server.raft: election won: tally=1 +33:10.472 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24916 [Leader]" +33:10.472 [INFO] TestAgent.server: cluster leadership acquired +33:10.472 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24916 +33:10.473 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:10.475 [INFO] TestAgent.server: New leader elected: payload=Node-41356a43-6ff0-322d-6caf-e0cce1f665c6 +33:10.476 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:10.476 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:10.476 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:10.476 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:10.476 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:10.476 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-41356a43-6ff0-322d-6caf-e0cce1f665c6 +33:10.476 [INFO] TestAgent.server: member joined, marking health alive: member=Node-41356a43-6ff0-322d-6caf-e0cce1f665c6 +33:10.477 [INFO] TestAgent.server: federation state anti-entropy synced +33:10.766 [DEBUG] TestAgent: restored service definition from file: service=redis file=/tmp/consul-test/TestAgent_persistedService_compat_service_manager-TestAgent-agent763737712/services/86a1b907d54bf7010394bf316e183e67 +33:10.766 [INFO] TestAgent: Requesting shutdown +33:10.766 [INFO] TestAgent.server: shutting down server +33:10.766 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:10.766 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:10.766 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:10.766 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:10.766 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:10.766 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:10.766 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:10.766 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:10.766 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:10.766 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:10.766 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:10.766 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:10.766Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:10.766Z [INFO] agent.router.manager: shutting down +33:10.767 [INFO] TestAgent: consul server down +33:10.767 [INFO] TestAgent: shutdown complete +33:10.767 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24911 network=tcp +33:10.767 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24911 network=udp +33:10.767 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24912 network=tcp +33:11.267 [INFO] TestAgent: Waiting for endpoints to shut down +33:11.267 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_persistedService_compat (0.00s) - --- PASS: TestAgent_persistedService_compat/normal (0.82s) - --- PASS: TestAgent_persistedService_compat/service_manager (0.98s) + --- PASS: TestAgent_persistedService_compat/normal (0.81s) + --- PASS: TestAgent_persistedService_compat/service_manager (0.91s) === RUN TestAgent_PurgeService === RUN TestAgent_PurgeService/normal === PAUSE TestAgent_PurgeService/normal @@ -11823,126 +11857,129 @@ === PAUSE TestAgent_PurgeService/service_manager === CONT TestAgent_PurgeService/normal === CONT TestAgent_PurgeService/service_manager -[WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 6 out of 20 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:29.055Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:29.072Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:29.075 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:479a5501-666f-1fe0-da5d-56effdfba6c4 Address:127.0.0.1:13699}]" -41:29.075 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13699 [Follower]" leader= -41:29.076 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-479a5501-666f-1fe0-da5d-56effdfba6c4.dc1 127.0.0.1 -41:29.077 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-479a5501-666f-1fe0-da5d-56effdfba6c4 127.0.0.1 -2022-09-15T21:41:29.077Z [INFO] agent.router: Initializing LAN area manager -41:29.077 [INFO] TestAgent.server: Adding LAN server: server="Node-479a5501-666f-1fe0-da5d-56effdfba6c4 (Addr: tcp/127.0.0.1:13699) (DC: dc1)" -41:29.077 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-479a5501-666f-1fe0-da5d-56effdfba6c4.dc1 area=wan -41:29.077 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13694 network=udp -41:29.077 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13694 network=tcp -41:29.078 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13695 network=tcp -41:29.078 [INFO] TestAgent: started state syncer -41:29.079 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13700 network=tcp -41:29.133 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:29.133 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13699 [Candidate]" term=2 -41:29.134 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:29.134 [DEBUG] TestAgent.server.raft: vote granted: from=479a5501-666f-1fe0-da5d-56effdfba6c4 term=2 tally=1 -41:29.134 [INFO] TestAgent.server.raft: election won: tally=1 -41:29.134 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13699 [Leader]" -41:29.134 [INFO] TestAgent.server: cluster leadership acquired -41:29.134 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13699 -41:29.135 [INFO] TestAgent.server: New leader elected: payload=Node-479a5501-666f-1fe0-da5d-56effdfba6c4 -41:29.135 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:29.144 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:29.144 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:29.144 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:29.144 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:29.144 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:29.144 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-479a5501-666f-1fe0-da5d-56effdfba6c4 -41:29.144 [INFO] TestAgent.server: member joined, marking health alive: member=Node-479a5501-666f-1fe0-da5d-56effdfba6c4 -41:29.146 [INFO] TestAgent.server: federation state anti-entropy synced -41:29.158 [DEBUG] TestAgent: removed service: service=redis -41:29.158 [DEBUG] TestAgent: removed service: service=redis -41:29.158 [INFO] TestAgent: Requesting shutdown -41:29.158 [INFO] TestAgent.server: shutting down server -41:29.159 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:29.159 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:29.159 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:29.159 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:29.159 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:29.159 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:29.159 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:29.159 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:29.159 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:29.159 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:29.159 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:29.159 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:29.159Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:29.159Z [INFO] agent.router.manager: shutting down -41:29.159 [INFO] TestAgent: consul server down -41:29.159 [INFO] TestAgent: shutdown complete -41:29.160 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13694 network=tcp -41:29.160 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13694 network=udp -41:29.160 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13695 network=tcp -41:29.660 [INFO] TestAgent: Waiting for endpoints to shut down -41:29.660 [INFO] TestAgent: Endpoints down +[WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:33:11.311Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:11.338Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:11.345 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0d09f81b-a35a-78e3-9cc1-df13d1a97794 Address:127.0.0.1:24923}]" +33:11.345 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24923 [Follower]" leader= +33:11.345 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0d09f81b-a35a-78e3-9cc1-df13d1a97794.dc1 127.0.0.1 +33:11.346 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0d09f81b-a35a-78e3-9cc1-df13d1a97794 127.0.0.1 +2021-08-13T15:33:11.346Z [INFO] agent.router: Initializing LAN area manager +33:11.346 [INFO] TestAgent.server: Adding LAN server: server="Node-0d09f81b-a35a-78e3-9cc1-df13d1a97794 (Addr: tcp/127.0.0.1:24923) (DC: dc1)" +33:11.347 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0d09f81b-a35a-78e3-9cc1-df13d1a97794.dc1 area=wan +33:11.347 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24918 network=udp +33:11.347 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24918 network=tcp +33:11.348 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24919 network=tcp +33:11.348 [INFO] TestAgent: started state syncer +33:11.348 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24924 network=tcp +33:11.396 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:11.396 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24923 [Candidate]" term=2 +33:11.397 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:11.397 [DEBUG] TestAgent.server.raft: vote granted: from=0d09f81b-a35a-78e3-9cc1-df13d1a97794 term=2 tally=1 +33:11.397 [INFO] TestAgent.server.raft: election won: tally=1 +33:11.397 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24923 [Leader]" +33:11.397 [INFO] TestAgent.server: cluster leadership acquired +33:11.397 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24923 +33:11.398 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:11.398 [INFO] TestAgent.server: New leader elected: payload=Node-0d09f81b-a35a-78e3-9cc1-df13d1a97794 +33:11.400 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:11.400 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:11.400 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:11.400 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:11.400 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:11.400 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0d09f81b-a35a-78e3-9cc1-df13d1a97794 +33:11.400 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0d09f81b-a35a-78e3-9cc1-df13d1a97794 +33:11.401 [INFO] TestAgent.server: federation state anti-entropy synced +33:11.421 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:11.421 [INFO] TestAgent: Synced node info +33:11.509 [DEBUG] TestAgent: removed service: service=redis +33:11.510 [DEBUG] TestAgent: removed service: service=redis +33:11.510 [INFO] TestAgent: Requesting shutdown +33:11.510 [INFO] TestAgent.server: shutting down server +33:11.510 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:11.510 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:11.510 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:11.510 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:11.510 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:11.510 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:11.510 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:11.510 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:11.510 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:11.510 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:11.510 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:11.510Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:11.510Z [INFO] agent.router.manager: shutting down +33:11.510 [INFO] TestAgent: consul server down +33:11.510 [INFO] TestAgent: shutdown complete +33:11.510 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24918 network=tcp +33:11.510 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24918 network=udp +33:11.510 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24919 network=tcp +33:12.010 [INFO] TestAgent: Waiting for endpoints to shut down +33:12.011 [INFO] TestAgent: Endpoints down [WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:29.055Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:29.073Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:29.076 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9f199aa1-dc60-c325-7ecc-148436776af7 Address:127.0.0.1:13692}]" -41:29.076 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13692 [Follower]" leader= -41:29.077 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9f199aa1-dc60-c325-7ecc-148436776af7.dc1 127.0.0.1 -41:29.077 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9f199aa1-dc60-c325-7ecc-148436776af7 127.0.0.1 -2022-09-15T21:41:29.077Z [INFO] agent.router: Initializing LAN area manager -41:29.078 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9f199aa1-dc60-c325-7ecc-148436776af7.dc1 area=wan -41:29.078 [INFO] TestAgent.server: Adding LAN server: server="Node-9f199aa1-dc60-c325-7ecc-148436776af7 (Addr: tcp/127.0.0.1:13692) (DC: dc1)" -41:29.078 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13687 network=udp -41:29.078 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13687 network=tcp -41:29.079 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13688 network=tcp -41:29.079 [INFO] TestAgent: started state syncer -41:29.079 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13693 network=tcp -41:29.138 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:29.138 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13692 [Candidate]" term=2 -41:29.140 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:29.140 [DEBUG] TestAgent.server.raft: vote granted: from=9f199aa1-dc60-c325-7ecc-148436776af7 term=2 tally=1 -41:29.140 [INFO] TestAgent.server.raft: election won: tally=1 -41:29.140 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13692 [Leader]" -41:29.146 [INFO] TestAgent.server: cluster leadership acquired -41:29.147 [INFO] TestAgent.server: New leader elected: payload=Node-9f199aa1-dc60-c325-7ecc-148436776af7 -41:29.147 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13692 -41:29.149 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:29.152 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:29.152 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:29.152 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:29.152 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:29.152 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:29.152 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9f199aa1-dc60-c325-7ecc-148436776af7 -41:29.152 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9f199aa1-dc60-c325-7ecc-148436776af7 -41:29.162 [INFO] TestAgent.server: federation state anti-entropy synced -41:29.334 [DEBUG] TestAgent: removed service: service=redis -41:29.334 [DEBUG] TestAgent: removed service: service=redis -41:29.334 [INFO] TestAgent: Requesting shutdown -41:29.334 [INFO] TestAgent.server: shutting down server -41:29.334 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:29.334 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:29.334 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:29.334 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:29.334 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:29.335 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:29.335 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:29.335 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:29.335 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:29.335 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:29.335 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:29.335 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:29.335Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:29.335Z [INFO] agent.router.manager: shutting down -41:29.335 [INFO] TestAgent: consul server down -41:29.335 [INFO] TestAgent: shutdown complete -41:29.335 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13687 network=tcp -41:29.335 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13687 network=udp -41:29.335 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13688 network=tcp -41:29.838 [INFO] TestAgent: Waiting for endpoints to shut down -41:29.838 [INFO] TestAgent: Endpoints down +2021-08-13T15:33:11.304Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:11.349Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:11.351 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:08c1c8cc-484e-fb64-4faa-779f6e59ed4e Address:127.0.0.1:24930}]" +33:11.351 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24930 [Follower]" leader= +33:11.352 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-08c1c8cc-484e-fb64-4faa-779f6e59ed4e.dc1 127.0.0.1 +33:11.353 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-08c1c8cc-484e-fb64-4faa-779f6e59ed4e 127.0.0.1 +2021-08-13T15:33:11.353Z [INFO] agent.router: Initializing LAN area manager +33:11.353 [INFO] TestAgent.server: Adding LAN server: server="Node-08c1c8cc-484e-fb64-4faa-779f6e59ed4e (Addr: tcp/127.0.0.1:24930) (DC: dc1)" +33:11.353 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-08c1c8cc-484e-fb64-4faa-779f6e59ed4e.dc1 area=wan +33:11.353 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24925 network=udp +33:11.353 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24925 network=tcp +33:11.354 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24926 network=tcp +33:11.354 [INFO] TestAgent: started state syncer +33:11.354 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24931 network=tcp +33:11.396 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:11.396 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24930 [Candidate]" term=2 +33:11.396 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:11.396 [DEBUG] TestAgent.server.raft: vote granted: from=08c1c8cc-484e-fb64-4faa-779f6e59ed4e term=2 tally=1 +33:11.396 [INFO] TestAgent.server.raft: election won: tally=1 +33:11.396 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24930 [Leader]" +33:11.396 [INFO] TestAgent.server: cluster leadership acquired +33:11.397 [INFO] TestAgent.server: New leader elected: payload=Node-08c1c8cc-484e-fb64-4faa-779f6e59ed4e +33:11.397 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24930 +33:11.398 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:11.400 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:11.400 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:11.400 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:11.400 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:11.400 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:11.400 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-08c1c8cc-484e-fb64-4faa-779f6e59ed4e +33:11.400 [INFO] TestAgent.server: member joined, marking health alive: member=Node-08c1c8cc-484e-fb64-4faa-779f6e59ed4e +33:11.401 [INFO] TestAgent.server: federation state anti-entropy synced +33:11.557 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:11.557 [INFO] TestAgent: Synced node info +33:11.638 [DEBUG] TestAgent: removed service: service=redis +33:11.639 [DEBUG] TestAgent: removed service: service=redis +33:11.639 [INFO] TestAgent: Requesting shutdown +33:11.639 [INFO] TestAgent.server: shutting down server +33:11.639 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:11.639 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:11.639 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:11.639 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:11.639 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:11.639 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:11.639 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:11.639 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:11.639 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:11.639 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:11.639 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:11.639Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:11.639Z [INFO] agent.router.manager: shutting down +33:11.639 [INFO] TestAgent: consul server down +33:11.639 [INFO] TestAgent: shutdown complete +33:11.639 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24925 network=tcp +33:11.640 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24925 network=udp +33:11.640 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24926 network=tcp +33:12.140 [INFO] TestAgent: Waiting for endpoints to shut down +33:12.140 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_PurgeService (0.00s) - --- PASS: TestAgent_PurgeService/service_manager (0.63s) - --- PASS: TestAgent_PurgeService/normal (0.81s) + --- PASS: TestAgent_PurgeService/normal (0.74s) + --- PASS: TestAgent_PurgeService/service_manager (0.87s) === RUN TestAgent_PurgeServiceOnDuplicate === RUN TestAgent_PurgeServiceOnDuplicate/normal === PAUSE TestAgent_PurgeServiceOnDuplicate/normal @@ -11954,92 +11991,93 @@ [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -41:30.453 [INFO] Agent2.client.serf.lan: serf: EventMemberJoin: Node-15b283a8-9e86-b476-7eaf-f82277a31764 127.0.0.1 -2022-09-15T21:41:30.453Z [INFO] agent.router: Initializing LAN area manager -41:30.453 [DEBUG] Agent2: service exists, not restoring from file: service=redis file=/tmp/consul-test/TestAgent_PurgeServiceOnDuplicate_normal-TestAgent-agent639016922/services/86a1b907d54bf7010394bf316e183e67 -41:30.454 [INFO] Agent2: Started DNS server: address=127.0.0.1:13715 network=udp -41:30.454 [INFO] Agent2.client.serf.lan: serf: Attempting re-join to previously known node: Node-bce398f6-55d4-9272-fe47-f899a7f83e83: 127.0.0.1:13704 -41:30.454 [INFO] Agent2: Started DNS server: address=127.0.0.1:13715 network=tcp -41:30.456 [INFO] Agent2: Started HTTP server: address=127.0.0.1:13716 network=tcp -41:30.456 [INFO] Agent2: started state syncer -41:30.457 [INFO] Agent2: Started gRPC server: address=127.0.0.1:13721 network=tcp -2022-09-15T21:41:30.457Z [WARN] agent.router.manager: No servers available -41:30.457 [INFO] Agent2: Requesting shutdown -41:30.457 [ERROR] Agent2.anti_entropy: failed to sync remote state: error="No known Consul servers" -41:30.457 [INFO] Agent2.client: shutting down client -41:30.457 [WARN] Agent2.client.serf.lan: serf: Shutdown without a Leave -41:30.457 [INFO] Agent2: consul client down -41:30.457 [INFO] Agent2: shutdown complete -41:30.457 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:13715 network=tcp -41:30.457 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:13715 network=udp -41:30.458 [INFO] Agent2: Stopping server: protocol=HTTP address=127.0.0.1:13716 network=tcp -41:30.462 [DEBUG] Agent2.client.memberlist.lan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:13704: connect: connection refused -41:30.462 [WARN] Agent2.client.serf.lan: serf: Failed to re-join any previously known node -41:30.958 [INFO] Agent2: Waiting for endpoints to shut down -41:30.958 [INFO] Agent2: Endpoints down -41:29.891 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-bce398f6-55d4-9272-fe47-f899a7f83e83 127.0.0.1 -2022-09-15T21:41:29.891Z [INFO] agent.router: Initializing LAN area manager -41:29.891 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13701 network=udp -41:29.892 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13701 network=tcp -41:29.892 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13702 network=tcp -41:29.893 [INFO] TestAgent: started state syncer -41:29.894 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13707 network=tcp -41:29.894 [INFO] TestAgent: Requesting shutdown -41:29.894 [INFO] TestAgent.client: shutting down client -41:29.894 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave -41:29.894 [INFO] TestAgent: consul client down -41:29.894 [INFO] TestAgent: shutdown complete -41:29.894 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13701 network=tcp -41:29.894 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13701 network=udp -41:29.894 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13702 network=tcp -41:30.395 [INFO] TestAgent: Waiting for endpoints to shut down -41:30.395 [INFO] TestAgent: Endpoints down -41:30.492 [INFO] Agent2.client.serf.lan: serf: EventMemberJoin: Node-dc917028-9649-6637-7588-c0db459488c8 127.0.0.1 -2022-09-15T21:41:30.492Z [INFO] agent.router: Initializing LAN area manager -41:30.492 [DEBUG] Agent2: service exists, not restoring from file: service=redis file=/tmp/consul-test/TestAgent_PurgeServiceOnDuplicate_service_manager-TestAgent-agent789241201/services/86a1b907d54bf7010394bf316e183e67 -41:30.493 [INFO] Agent2: Started DNS server: address=127.0.0.1:13722 network=udp -41:30.493 [INFO] Agent2.client.serf.lan: serf: Attempting re-join to previously known node: Node-a9cbe103-58ca-b3f3-d6c7-5b4888a4160d: 127.0.0.1:13711 -41:30.493 [DEBUG] Agent2.client.memberlist.lan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:13711: connect: connection refused -41:30.493 [WARN] Agent2.client.serf.lan: serf: Failed to re-join any previously known node -41:30.493 [INFO] Agent2: Started DNS server: address=127.0.0.1:13722 network=tcp -41:30.494 [INFO] Agent2: Started HTTP server: address=127.0.0.1:13723 network=tcp -41:30.494 [INFO] Agent2: started state syncer -41:30.495 [INFO] Agent2: Requesting shutdown -41:30.495 [INFO] Agent2.client: shutting down client -41:30.495 [WARN] Agent2.client.serf.lan: serf: Shutdown without a Leave -41:30.495 [INFO] Agent2: Started gRPC server: address=127.0.0.1:13728 network=tcp -41:30.495 [ERROR] Agent2: gRPC server failed: error="grpc: the server has been stopped" -2022-09-15T21:41:30.496Z [WARN] agent.router.manager: No servers available -41:30.496 [ERROR] Agent2.anti_entropy: failed to sync remote state: error="No known Consul servers" -41:30.496 [INFO] Agent2: consul client down -41:30.496 [INFO] Agent2: shutdown complete -41:30.496 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:13722 network=tcp -41:30.496 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:13722 network=udp -41:30.496 [INFO] Agent2: Stopping server: protocol=HTTP address=127.0.0.1:13723 network=tcp -41:30.996 [INFO] Agent2: Waiting for endpoints to shut down -41:30.996 [INFO] Agent2: Endpoints down -41:29.937 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-a9cbe103-58ca-b3f3-d6c7-5b4888a4160d 127.0.0.1 -2022-09-15T21:41:29.937Z [INFO] agent.router: Initializing LAN area manager -41:29.937 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13708 network=tcp -41:29.937 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13708 network=udp -41:29.938 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13709 network=tcp -41:29.939 [INFO] TestAgent: started state syncer -41:29.939 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13714 network=tcp -2022-09-15T21:41:29.939Z [WARN] agent.router.manager: No servers available -41:29.939 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" -41:29.940 [INFO] TestAgent: Requesting shutdown -41:29.940 [INFO] TestAgent.client: shutting down client -41:29.940 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave -41:29.940 [INFO] TestAgent: consul client down -41:29.940 [INFO] TestAgent: shutdown complete -41:29.940 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13708 network=tcp -41:29.940 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13708 network=udp -41:29.940 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13709 network=tcp -41:30.442 [INFO] TestAgent: Waiting for endpoints to shut down -41:30.442 [INFO] TestAgent: Endpoints down +33:12.732 [INFO] Agent2.client.serf.lan: serf: EventMemberJoin: Node-3d3a01e6-06c3-f66a-3498-74bb5c5dc751 127.0.0.1 +2021-08-13T15:33:12.732Z [INFO] agent.router: Initializing LAN area manager +33:12.733 [INFO] Agent2.client.serf.lan: serf: Attempting re-join to previously known node: Node-e21c09e6-b090-1bdd-d789-a0e12bd22fcc: 127.0.0.1:24935 +33:12.733 [DEBUG] Agent2: service exists, not restoring from file: service=redis file=/tmp/consul-test/TestAgent_PurgeServiceOnDuplicate_normal-TestAgent-agent450317337/services/86a1b907d54bf7010394bf316e183e67 +33:12.733 [DEBUG] Agent2.client.memberlist.lan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:24935: connect: connection refused +33:12.733 [WARN] Agent2.client.serf.lan: serf: Failed to re-join any previously known node +33:12.733 [INFO] Agent2: Started DNS server: address=127.0.0.1:24946 network=udp +33:12.733 [INFO] Agent2: Started DNS server: address=127.0.0.1:24946 network=tcp +33:12.734 [INFO] Agent2: Started HTTP server: address=127.0.0.1:24947 network=tcp +33:12.734 [INFO] Agent2: started state syncer +33:12.735 [INFO] Agent2: Requesting shutdown +33:12.735 [INFO] Agent2.client: shutting down client +33:12.735 [WARN] Agent2.client.serf.lan: serf: Shutdown without a Leave +33:12.735 [INFO] Agent2: Started gRPC server: address=127.0.0.1:24952 network=tcp +33:12.735 [INFO] Agent2: consul client down +33:12.735 [INFO] Agent2: shutdown complete +33:12.735 [ERROR] Agent2: gRPC server failed: error="grpc: the server has been stopped" +33:12.735 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:24946 network=tcp +33:12.735 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:24946 network=udp +2021-08-13T15:33:12.735Z [WARN] agent.router.manager: No servers available +33:12.735 [ERROR] Agent2.anti_entropy: failed to sync remote state: error="No known Consul servers" +33:12.735 [INFO] Agent2: Stopping server: protocol=HTTP address=127.0.0.1:24947 network=tcp +33:13.235 [INFO] Agent2: Waiting for endpoints to shut down +33:13.235 [INFO] Agent2: Endpoints down +33:12.191 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-e21c09e6-b090-1bdd-d789-a0e12bd22fcc 127.0.0.1 +2021-08-13T15:33:12.191Z [INFO] agent.router: Initializing LAN area manager +33:12.192 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24932 network=udp +33:12.192 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24932 network=tcp +33:12.193 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24933 network=tcp +33:12.193 [INFO] TestAgent: started state syncer +33:12.195 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24938 network=tcp +33:12.198 [INFO] TestAgent: Requesting shutdown +33:12.198 [INFO] TestAgent.client: shutting down client +33:12.198 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave +33:12.198 [INFO] TestAgent: consul client down +33:12.198 [INFO] TestAgent: shutdown complete +33:12.198 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24932 network=tcp +33:12.198 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24932 network=udp +33:12.198 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24933 network=tcp +33:12.698 [INFO] TestAgent: Waiting for endpoints to shut down +33:12.698 [INFO] TestAgent: Endpoints down +33:12.736 [INFO] Agent2.client.serf.lan: serf: EventMemberJoin: Node-bb60a692-0a21-4868-fa9e-f50d47842bd9 127.0.0.1 +2021-08-13T15:33:12.736Z [INFO] agent.router: Initializing LAN area manager +33:12.736 [INFO] Agent2.client.serf.lan: serf: Attempting re-join to previously known node: Node-0009d3ab-0be4-a01f-2ef2-89007a91827d: 127.0.0.1:24942 +33:12.736 [DEBUG] Agent2: service exists, not restoring from file: service=redis file=/tmp/consul-test/TestAgent_PurgeServiceOnDuplicate_service_manager-TestAgent-agent405376164/services/86a1b907d54bf7010394bf316e183e67 +33:12.736 [DEBUG] Agent2.client.memberlist.lan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:24942: connect: connection refused +33:12.736 [WARN] Agent2.client.serf.lan: serf: Failed to re-join any previously known node +33:12.736 [INFO] Agent2: Started DNS server: address=127.0.0.1:24953 network=udp +33:12.736 [INFO] Agent2: Started DNS server: address=127.0.0.1:24953 network=tcp +33:12.737 [INFO] Agent2: Started HTTP server: address=127.0.0.1:24954 network=tcp +33:12.737 [INFO] Agent2: started state syncer +33:12.737 [INFO] Agent2: Started gRPC server: address=127.0.0.1:24959 network=tcp +2021-08-13T15:33:12.737Z [WARN] agent.router.manager: No servers available +33:12.737 [ERROR] Agent2.anti_entropy: failed to sync remote state: error="No known Consul servers" +33:12.738 [INFO] Agent2: Requesting shutdown +33:12.738 [INFO] Agent2.client: shutting down client +33:12.738 [WARN] Agent2.client.serf.lan: serf: Shutdown without a Leave +33:12.738 [INFO] Agent2: consul client down +33:12.738 [INFO] Agent2: shutdown complete +33:12.738 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:24953 network=tcp +33:12.738 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:24953 network=udp +33:12.738 [INFO] Agent2: Stopping server: protocol=HTTP address=127.0.0.1:24954 network=tcp +33:13.238 [INFO] Agent2: Waiting for endpoints to shut down +33:13.238 [INFO] Agent2: Endpoints down +33:12.198 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-0009d3ab-0be4-a01f-2ef2-89007a91827d 127.0.0.1 +2021-08-13T15:33:12.198Z [INFO] agent.router: Initializing LAN area manager +33:12.199 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24939 network=udp +33:12.199 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24939 network=tcp +33:12.200 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24940 network=tcp +33:12.200 [INFO] TestAgent: started state syncer +33:12.201 [INFO] TestAgent: Requesting shutdown +33:12.201 [INFO] TestAgent.client: shutting down client +33:12.201 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave +33:12.201 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24945 network=tcp +33:12.201 [ERROR] TestAgent: gRPC server failed: error="grpc: the server has been stopped" +2021-08-13T15:33:12.201Z [WARN] agent.router.manager: No servers available +33:12.201 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" +33:12.201 [INFO] TestAgent: consul client down +33:12.201 [INFO] TestAgent: shutdown complete +33:12.201 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24939 network=tcp +33:12.202 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24939 network=udp +33:12.202 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24940 network=tcp +33:12.702 [INFO] TestAgent: Waiting for endpoints to shut down +33:12.702 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_PurgeServiceOnDuplicate (0.00s) - --- PASS: TestAgent_PurgeServiceOnDuplicate/normal (1.12s) - --- PASS: TestAgent_PurgeServiceOnDuplicate/service_manager (1.15s) + --- PASS: TestAgent_PurgeServiceOnDuplicate/normal (1.10s) + --- PASS: TestAgent_PurgeServiceOnDuplicate/service_manager (1.10s) === RUN TestAgent_PersistCheck === PAUSE TestAgent_PersistCheck === RUN TestAgent_PurgeCheck @@ -12062,126 +12100,121 @@ [WARN] freeport: 6 out of 34 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:31.019Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:31.055Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:31.062 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5244448e-b1e8-821e-cafa-84e18ecaf714 Address:127.0.0.1:13741}]" -41:31.062 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13741 [Follower]" leader= -41:31.063 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5244448e-b1e8-821e-cafa-84e18ecaf714.dc1 127.0.0.1 -41:31.073 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5244448e-b1e8-821e-cafa-84e18ecaf714 127.0.0.1 -2022-09-15T21:41:31.073Z [INFO] agent.router: Initializing LAN area manager -41:31.079 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5244448e-b1e8-821e-cafa-84e18ecaf714.dc1 area=wan -41:31.079 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13736 network=tcp -41:31.079 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13736 network=udp -41:31.082 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13737 network=tcp -41:31.082 [INFO] TestAgent: started state syncer -41:31.083 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13742 network=tcp -41:31.086 [INFO] TestAgent.server: Adding LAN server: server="Node-5244448e-b1e8-821e-cafa-84e18ecaf714 (Addr: tcp/127.0.0.1:13741) (DC: dc1)" -41:31.121 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:31.121 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13741 [Candidate]" term=2 -41:31.121 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:31.121 [DEBUG] TestAgent.server.raft: vote granted: from=5244448e-b1e8-821e-cafa-84e18ecaf714 term=2 tally=1 -41:31.121 [INFO] TestAgent.server.raft: election won: tally=1 -41:31.121 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13741 [Leader]" -41:31.122 [INFO] TestAgent.server: cluster leadership acquired -41:31.122 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13741 -41:31.124 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:31.126 [INFO] TestAgent.server: New leader elected: payload=Node-5244448e-b1e8-821e-cafa-84e18ecaf714 -41:31.127 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:31.127 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:31.127 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:31.127 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:31.127 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:31.127 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5244448e-b1e8-821e-cafa-84e18ecaf714 -41:31.127 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5244448e-b1e8-821e-cafa-84e18ecaf714 -41:31.128 [INFO] TestAgent.server: federation state anti-entropy synced -41:31.217 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:31.217 [INFO] TestAgent: Synced node info -41:31.218 [INFO] TestAgent: Synced service: service=rabbitmq -41:31.221 [INFO] TestAgent: Requesting shutdown -41:31.222 [INFO] TestAgent.server: shutting down server -41:31.222 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:31.222 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:31.222 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:31.222 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:31.222 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:31.222 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:31.222 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:31.222 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:31.222 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:31.222 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:31.222 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:31.222Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:31.222Z [INFO] agent.router.manager: shutting down -41:31.222 [INFO] TestAgent: consul server down -41:31.222 [INFO] TestAgent: shutdown complete -41:31.222 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13736 network=tcp -41:31.222 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13736 network=udp -41:31.222 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13737 network=tcp -41:31.723 [INFO] TestAgent: Waiting for endpoints to shut down -41:31.723 [INFO] TestAgent: Endpoints down -[WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:31.023Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:31.060Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:31.070 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:90d28c7d-6d31-d1fe-effc-82752341d36d Address:127.0.0.1:13734}]" -41:31.072 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-90d28c7d-6d31-d1fe-effc-82752341d36d.dc1 127.0.0.1 -41:31.073 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-90d28c7d-6d31-d1fe-effc-82752341d36d 127.0.0.1 -2022-09-15T21:41:31.073Z [INFO] agent.router: Initializing LAN area manager -41:31.073 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13734 [Follower]" leader= -41:31.080 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-90d28c7d-6d31-d1fe-effc-82752341d36d.dc1 area=wan -41:31.081 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13729 network=tcp -41:31.082 [INFO] TestAgent.server: Adding LAN server: server="Node-90d28c7d-6d31-d1fe-effc-82752341d36d (Addr: tcp/127.0.0.1:13734) (DC: dc1)" -41:31.083 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13729 network=udp -41:31.085 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13730 network=tcp -41:31.085 [INFO] TestAgent: started state syncer -41:31.085 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13735 network=tcp -41:31.120 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:31.120 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13734 [Candidate]" term=2 -41:31.121 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:31.121 [DEBUG] TestAgent.server.raft: vote granted: from=90d28c7d-6d31-d1fe-effc-82752341d36d term=2 tally=1 -41:31.121 [INFO] TestAgent.server.raft: election won: tally=1 -41:31.121 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13734 [Leader]" -41:31.121 [INFO] TestAgent.server: cluster leadership acquired -41:31.121 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13734 -41:31.123 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:31.126 [INFO] TestAgent.server: New leader elected: payload=Node-90d28c7d-6d31-d1fe-effc-82752341d36d -41:31.128 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:31.128 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:31.128 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:31.128 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:31.128 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:31.128 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-90d28c7d-6d31-d1fe-effc-82752341d36d -41:31.128 [INFO] TestAgent.server: member joined, marking health alive: member=Node-90d28c7d-6d31-d1fe-effc-82752341d36d -41:31.129 [INFO] TestAgent.server: federation state anti-entropy synced -41:31.334 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:31.335 [INFO] TestAgent: Synced node info -41:31.335 [INFO] TestAgent: Synced service: service=rabbitmq -41:31.335 [DEBUG] TestAgent: Node info in sync -41:31.335 [DEBUG] TestAgent: Service in sync: service=rabbitmq -41:31.423 [INFO] TestAgent: Requesting shutdown -41:31.423 [INFO] TestAgent.server: shutting down server -41:31.423 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:31.423 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:31.423 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:31.423 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:31.423 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:31.423 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:31.423 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:31.423 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:31.423 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:31.423 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:31.423 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:31.424Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:31.424Z [INFO] agent.router.manager: shutting down -41:31.424 [INFO] TestAgent: consul server down -41:31.424 [INFO] TestAgent: shutdown complete -41:31.424 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13729 network=tcp -41:31.424 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13729 network=udp -41:31.424 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13730 network=tcp -41:31.926 [INFO] TestAgent: Waiting for endpoints to shut down -41:31.926 [INFO] TestAgent: Endpoints down +2021-08-13T15:33:13.278Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:13.305Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:13.312 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:466f8628-f5a3-9f7f-f754-175d83ae135a Address:127.0.0.1:24972}]" +33:13.312 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24972 [Follower]" leader= +33:13.313 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-466f8628-f5a3-9f7f-f754-175d83ae135a.dc1 127.0.0.1 +33:13.313 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-466f8628-f5a3-9f7f-f754-175d83ae135a 127.0.0.1 +2021-08-13T15:33:13.313Z [INFO] agent.router: Initializing LAN area manager +33:13.314 [INFO] TestAgent.server: Adding LAN server: server="Node-466f8628-f5a3-9f7f-f754-175d83ae135a (Addr: tcp/127.0.0.1:24972) (DC: dc1)" +33:13.314 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-466f8628-f5a3-9f7f-f754-175d83ae135a.dc1 area=wan +33:13.314 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24967 network=tcp +33:13.314 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24967 network=udp +33:13.315 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24968 network=tcp +33:13.315 [INFO] TestAgent: started state syncer +33:13.315 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24973 network=tcp +33:13.364 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:13.364 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24972 [Candidate]" term=2 +33:13.365 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:13.365 [DEBUG] TestAgent.server.raft: vote granted: from=466f8628-f5a3-9f7f-f754-175d83ae135a term=2 tally=1 +33:13.365 [INFO] TestAgent.server.raft: election won: tally=1 +33:13.365 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24972 [Leader]" +33:13.365 [INFO] TestAgent.server: cluster leadership acquired +33:13.365 [INFO] TestAgent.server: New leader elected: payload=Node-466f8628-f5a3-9f7f-f754-175d83ae135a +33:13.365 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24972 +33:13.366 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:13.369 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:13.369 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:13.369 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:13.369 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:13.369 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:13.369 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-466f8628-f5a3-9f7f-f754-175d83ae135a +33:13.369 [INFO] TestAgent.server: member joined, marking health alive: member=Node-466f8628-f5a3-9f7f-f754-175d83ae135a +33:13.370 [INFO] TestAgent.server: federation state anti-entropy synced +33:13.402 [INFO] TestAgent: Requesting shutdown +33:13.402 [INFO] TestAgent.server: shutting down server +33:13.402 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:13.402 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:13.402 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:13.402 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:13.402 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:13.402 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:13.402 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:13.402 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:13.402 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:13.402 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:13.402 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:13.402 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:13.403Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:13.403Z [INFO] agent.router.manager: shutting down +33:13.403 [INFO] TestAgent: consul server down +33:13.403 [INFO] TestAgent: shutdown complete +33:13.403 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24967 network=tcp +33:13.403 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24967 network=udp +33:13.403 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24968 network=tcp +33:13.903 [INFO] TestAgent: Waiting for endpoints to shut down +33:13.903 [INFO] TestAgent: Endpoints down +2021-08-13T15:33:13.274Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:13.315Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:13.317 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9afb1050-c9fc-a5fd-59b1-c80317e8935b Address:127.0.0.1:24965}]" +33:13.317 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24965 [Follower]" leader= +33:13.318 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9afb1050-c9fc-a5fd-59b1-c80317e8935b.dc1 127.0.0.1 +33:13.319 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9afb1050-c9fc-a5fd-59b1-c80317e8935b 127.0.0.1 +2021-08-13T15:33:13.319Z [INFO] agent.router: Initializing LAN area manager +33:13.319 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9afb1050-c9fc-a5fd-59b1-c80317e8935b.dc1 area=wan +33:13.319 [INFO] TestAgent.server: Adding LAN server: server="Node-9afb1050-c9fc-a5fd-59b1-c80317e8935b (Addr: tcp/127.0.0.1:24965) (DC: dc1)" +33:13.319 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24960 network=tcp +33:13.319 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24960 network=udp +33:13.320 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24961 network=tcp +33:13.320 [INFO] TestAgent: started state syncer +33:13.320 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24966 network=tcp +33:13.374 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:13.374 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24965 [Candidate]" term=2 +33:13.374 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:13.374 [DEBUG] TestAgent.server.raft: vote granted: from=9afb1050-c9fc-a5fd-59b1-c80317e8935b term=2 tally=1 +33:13.374 [INFO] TestAgent.server.raft: election won: tally=1 +33:13.374 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24965 [Leader]" +33:13.374 [INFO] TestAgent.server: cluster leadership acquired +33:13.375 [INFO] TestAgent.server: New leader elected: payload=Node-9afb1050-c9fc-a5fd-59b1-c80317e8935b +33:13.375 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24965 +33:13.376 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:13.378 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:13.378 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:13.378 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:13.378 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:13.378 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:13.378 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9afb1050-c9fc-a5fd-59b1-c80317e8935b +33:13.378 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9afb1050-c9fc-a5fd-59b1-c80317e8935b +33:13.379 [INFO] TestAgent.server: federation state anti-entropy synced +33:13.472 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:13.473 [INFO] TestAgent: Synced node info +33:13.474 [INFO] TestAgent: Synced service: service=rabbitmq +33:13.502 [INFO] TestAgent: Requesting shutdown +33:13.502 [INFO] TestAgent.server: shutting down server +33:13.502 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:13.502 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:13.502 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:13.502 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:13.502 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:13.502 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:13.502 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:13.502 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:13.502 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:13.502 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:13.502 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:13.502Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:13.502Z [INFO] agent.router.manager: shutting down +33:13.502 [INFO] TestAgent: consul server down +33:13.502 [INFO] TestAgent: shutdown complete +33:13.502 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24960 network=tcp +33:13.502 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24960 network=udp +33:13.503 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24961 network=tcp +33:14.003 [INFO] TestAgent: Waiting for endpoints to shut down +33:14.003 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_loadServices_token (0.00s) - --- PASS: TestAgent_loadServices_token/service_manager (0.72s) - --- PASS: TestAgent_loadServices_token/normal (0.93s) + --- PASS: TestAgent_loadServices_token/service_manager (0.66s) + --- PASS: TestAgent_loadServices_token/normal (0.76s) === RUN TestAgent_loadServices_sidecar === RUN TestAgent_loadServices_sidecar/normal === PAUSE TestAgent_loadServices_sidecar/normal @@ -12189,124 +12222,138 @@ === PAUSE TestAgent_loadServices_sidecar/service_manager === CONT TestAgent_loadServices_sidecar/normal === CONT TestAgent_loadServices_sidecar/service_manager -[WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 6 out of 20 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:31.971Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:31.999Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:32.011 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:407872b2-31a4-b518-c7b5-f71314e1164b Address:127.0.0.1:13755}]" -41:32.011 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13755 [Follower]" leader= -41:32.012 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-407872b2-31a4-b518-c7b5-f71314e1164b.dc1 127.0.0.1 -41:32.013 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-407872b2-31a4-b518-c7b5-f71314e1164b 127.0.0.1 -2022-09-15T21:41:32.013Z [INFO] agent.router: Initializing LAN area manager -41:32.013 [INFO] TestAgent.server: Adding LAN server: server="Node-407872b2-31a4-b518-c7b5-f71314e1164b (Addr: tcp/127.0.0.1:13755) (DC: dc1)" -41:32.013 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-407872b2-31a4-b518-c7b5-f71314e1164b.dc1 area=wan -41:32.013 [DEBUG] TestAgent: added local registration for service: service=rabbitmq-sidecar-proxy -41:32.013 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13750 network=udp -41:32.013 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13750 network=tcp -41:32.014 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13751 network=tcp -41:32.015 [INFO] TestAgent: started state syncer -41:32.015 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13756 network=tcp -41:32.074 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:32.074 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13755 [Candidate]" term=2 -41:32.074 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:32.074 [DEBUG] TestAgent.server.raft: vote granted: from=407872b2-31a4-b518-c7b5-f71314e1164b term=2 tally=1 -41:32.074 [INFO] TestAgent.server.raft: election won: tally=1 -41:32.074 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13755 [Leader]" -41:32.075 [INFO] TestAgent.server: cluster leadership acquired -41:32.082 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13755 -41:32.084 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:32.085 [INFO] TestAgent.server: New leader elected: payload=Node-407872b2-31a4-b518-c7b5-f71314e1164b -41:32.102 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:32.102 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:32.102 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:32.102 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:32.102 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:32.102 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-407872b2-31a4-b518-c7b5-f71314e1164b -41:32.102 [INFO] TestAgent.server: member joined, marking health alive: member=Node-407872b2-31a4-b518-c7b5-f71314e1164b -41:32.104 [INFO] TestAgent.server: federation state anti-entropy synced -41:32.246 [INFO] TestAgent: Requesting shutdown -41:32.247 [INFO] TestAgent.server: shutting down server -41:32.247 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:32.247 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:32.247 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:32.247 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:32.247 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:32.247 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:32.247 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:32.247 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:32.247 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:32.247 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:32.247 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:32.247 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:32.247Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:32.247Z [INFO] agent.router.manager: shutting down -41:32.247 [INFO] TestAgent: consul server down -41:32.247 [INFO] TestAgent: shutdown complete -41:32.247 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13750 network=tcp -41:32.247 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13750 network=udp -41:32.247 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13751 network=tcp -41:32.748 [INFO] TestAgent: Waiting for endpoints to shut down -41:32.748 [INFO] TestAgent: Endpoints down -[WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:31.968Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:32.005Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:32.007 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:41461790-43de-1521-0a60-24dda85a8130 Address:127.0.0.1:13748}]" -41:32.008 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13748 [Follower]" leader= -41:32.008 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-41461790-43de-1521-0a60-24dda85a8130.dc1 127.0.0.1 -41:32.009 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-41461790-43de-1521-0a60-24dda85a8130 127.0.0.1 -2022-09-15T21:41:32.009Z [INFO] agent.router: Initializing LAN area manager -41:32.010 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-41461790-43de-1521-0a60-24dda85a8130.dc1 area=wan -41:32.010 [INFO] TestAgent.server: Adding LAN server: server="Node-41461790-43de-1521-0a60-24dda85a8130 (Addr: tcp/127.0.0.1:13748) (DC: dc1)" -41:32.015 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13743 network=tcp -41:32.015 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13743 network=udp -41:32.015 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13744 network=tcp -41:32.015 [INFO] TestAgent: started state syncer -41:32.015 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13749 network=tcp -41:32.082 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:32.082 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13748 [Candidate]" term=2 -41:32.082 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:32.082 [DEBUG] TestAgent.server.raft: vote granted: from=41461790-43de-1521-0a60-24dda85a8130 term=2 tally=1 -41:32.082 [INFO] TestAgent.server.raft: election won: tally=1 -41:32.082 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13748 [Leader]" -41:32.082 [INFO] TestAgent.server: cluster leadership acquired -41:32.083 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13748 -41:32.086 [INFO] TestAgent.server: New leader elected: payload=Node-41461790-43de-1521-0a60-24dda85a8130 -41:32.095 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:32.100 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:32.100 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:32.100 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:32.100 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:32.100 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:32.100 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-41461790-43de-1521-0a60-24dda85a8130 -41:32.100 [INFO] TestAgent.server: member joined, marking health alive: member=Node-41461790-43de-1521-0a60-24dda85a8130 -41:32.102 [INFO] TestAgent.server: federation state anti-entropy synced -41:32.304 [INFO] TestAgent: Requesting shutdown -41:32.304 [INFO] TestAgent.server: shutting down server -41:32.304 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:32.304 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:32.304 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:32.304 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:32.304 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:32.305 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:32.305 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:32.305 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:32.305 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:32.305 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:32.305 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:32.305 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:32.305Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:32.305Z [INFO] agent.router.manager: shutting down -41:32.305 [INFO] TestAgent: consul server down -41:32.305 [INFO] TestAgent: shutdown complete -41:32.305 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13743 network=tcp -41:32.305 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13743 network=udp -41:32.305 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13744 network=tcp -41:32.806 [INFO] TestAgent: Waiting for endpoints to shut down -41:32.806 [INFO] TestAgent: Endpoints down +[WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:33:14.029Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:14.062Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:14.064 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:65831589-6ac0-c363-4b90-4e693cb05e37 Address:127.0.0.1:24979}]" +33:14.065 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24979 [Follower]" leader= +33:14.065 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-65831589-6ac0-c363-4b90-4e693cb05e37.dc1 127.0.0.1 +33:14.066 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-65831589-6ac0-c363-4b90-4e693cb05e37 127.0.0.1 +2021-08-13T15:33:14.066Z [INFO] agent.router: Initializing LAN area manager +33:14.066 [INFO] TestAgent.server: Adding LAN server: server="Node-65831589-6ac0-c363-4b90-4e693cb05e37 (Addr: tcp/127.0.0.1:24979) (DC: dc1)" +33:14.066 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-65831589-6ac0-c363-4b90-4e693cb05e37.dc1 area=wan +33:14.067 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24974 network=udp +33:14.067 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24974 network=tcp +33:14.067 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24975 network=tcp +33:14.068 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24980 network=tcp +33:14.068 [INFO] TestAgent: started state syncer +33:14.117 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:14.117 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24979 [Candidate]" term=2 +33:14.117 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:14.117 [DEBUG] TestAgent.server.raft: vote granted: from=65831589-6ac0-c363-4b90-4e693cb05e37 term=2 tally=1 +33:14.117 [INFO] TestAgent.server.raft: election won: tally=1 +33:14.117 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24979 [Leader]" +33:14.118 [INFO] TestAgent.server: cluster leadership acquired +33:14.118 [INFO] TestAgent.server: New leader elected: payload=Node-65831589-6ac0-c363-4b90-4e693cb05e37 +33:14.118 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24979 +33:14.119 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:14.121 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:14.121 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:14.121 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:14.121 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:14.121 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:14.122 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-65831589-6ac0-c363-4b90-4e693cb05e37 +33:14.122 [INFO] TestAgent.server: member joined, marking health alive: member=Node-65831589-6ac0-c363-4b90-4e693cb05e37 +33:14.123 [INFO] TestAgent.server: federation state anti-entropy synced +33:14.288 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:14.289 [INFO] TestAgent: Synced node info +33:14.289 [INFO] TestAgent: Synced service: service=rabbitmq +33:14.290 [INFO] TestAgent: Synced service: service=rabbitmq-sidecar-proxy +33:14.290 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:1 +33:14.290 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:2 +33:14.334 [INFO] TestAgent: Requesting shutdown +33:14.334 [INFO] TestAgent.server: shutting down server +33:14.335 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:14.335 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:14.335 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:14.335 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:14.335 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:14.335 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:14.335 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:14.335 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:14.335 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:14.335 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:14.335Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:14.335Z [INFO] agent.router.manager: shutting down +33:14.335 [INFO] TestAgent: consul server down +33:14.335 [INFO] TestAgent: shutdown complete +33:14.335 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24974 network=tcp +33:14.335 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24974 network=udp +33:14.335 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24975 network=tcp +33:14.837 [INFO] TestAgent: Waiting for endpoints to shut down +33:14.837 [INFO] TestAgent: Endpoints down +2021-08-13T15:33:14.024Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:14.063Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:14.065 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:44f26dc1-c316-0c28-4eb1-098ac217b5bd Address:127.0.0.1:24986}]" +33:14.065 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24986 [Follower]" leader= +33:14.066 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-44f26dc1-c316-0c28-4eb1-098ac217b5bd.dc1 127.0.0.1 +33:14.067 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-44f26dc1-c316-0c28-4eb1-098ac217b5bd 127.0.0.1 +2021-08-13T15:33:14.067Z [INFO] agent.router: Initializing LAN area manager +33:14.067 [INFO] TestAgent.server: Adding LAN server: server="Node-44f26dc1-c316-0c28-4eb1-098ac217b5bd (Addr: tcp/127.0.0.1:24986) (DC: dc1)" +33:14.067 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-44f26dc1-c316-0c28-4eb1-098ac217b5bd.dc1 area=wan +33:14.067 [DEBUG] TestAgent: added local registration for service: service=rabbitmq-sidecar-proxy +33:14.067 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24981 network=udp +33:14.067 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24981 network=tcp +33:14.068 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24982 network=tcp +33:14.068 [INFO] TestAgent: started state syncer +33:14.068 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24987 network=tcp +33:14.123 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:14.123 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24986 [Candidate]" term=2 +33:14.123 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:14.123 [DEBUG] TestAgent.server.raft: vote granted: from=44f26dc1-c316-0c28-4eb1-098ac217b5bd term=2 tally=1 +33:14.123 [INFO] TestAgent.server.raft: election won: tally=1 +33:14.123 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24986 [Leader]" +33:14.123 [INFO] TestAgent.server: cluster leadership acquired +33:14.124 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24986 +33:14.125 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:14.127 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:14.127 [INFO] TestAgent.server: New leader elected: payload=Node-44f26dc1-c316-0c28-4eb1-098ac217b5bd +33:14.127 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:14.127 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:14.127 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:14.127 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:14.127 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-44f26dc1-c316-0c28-4eb1-098ac217b5bd +33:14.127 [INFO] TestAgent.server: member joined, marking health alive: member=Node-44f26dc1-c316-0c28-4eb1-098ac217b5bd +33:14.128 [INFO] TestAgent.server: federation state anti-entropy synced +33:14.354 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:14.355 [INFO] TestAgent: Synced node info +33:14.356 [INFO] TestAgent: Synced service: service=rabbitmq +33:14.356 [INFO] TestAgent: Synced service: service=rabbitmq-sidecar-proxy +33:14.356 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:1 +33:14.356 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:2 +33:14.356 [DEBUG] TestAgent: Node info in sync +33:14.356 [DEBUG] TestAgent: Service in sync: service=rabbitmq +33:14.356 [DEBUG] TestAgent: Service in sync: service=rabbitmq-sidecar-proxy +33:14.356 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:1 +33:14.356 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:2 +33:14.365 [INFO] TestAgent: Requesting shutdown +33:14.365 [INFO] TestAgent.server: shutting down server +33:14.365 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:14.365 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:14.365 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:14.365 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:14.365 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:14.365 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:14.365 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:14.365 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:14.365 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:14.365 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:14.365 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:14.366Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:14.366Z [INFO] agent.router.manager: shutting down +33:14.366 [INFO] TestAgent: consul server down +33:14.366 [INFO] TestAgent: shutdown complete +33:14.366 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24981 network=tcp +33:14.366 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24981 network=udp +33:14.366 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24982 network=tcp +33:14.866 [INFO] TestAgent: Waiting for endpoints to shut down +33:14.866 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_loadServices_sidecar (0.00s) - --- PASS: TestAgent_loadServices_sidecar/service_manager (0.82s) - --- PASS: TestAgent_loadServices_sidecar/normal (0.88s) + --- PASS: TestAgent_loadServices_sidecar/normal (0.83s) + --- PASS: TestAgent_loadServices_sidecar/service_manager (0.86s) === RUN TestAgent_loadServices_sidecarSeparateToken === RUN TestAgent_loadServices_sidecarSeparateToken/normal === PAUSE TestAgent_loadServices_sidecarSeparateToken/normal @@ -12314,119 +12361,123 @@ === PAUSE TestAgent_loadServices_sidecarSeparateToken/service_manager === CONT TestAgent_loadServices_sidecarSeparateToken/normal === CONT TestAgent_loadServices_sidecarSeparateToken/service_manager -[WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 6 out of 20 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:32.830Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:32.847Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:32.850 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5eddfa85-ed2a-9164-70a9-4c21860b1e73 Address:127.0.0.1:13769}]" -41:32.851 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5eddfa85-ed2a-9164-70a9-4c21860b1e73.dc1 127.0.0.1 -41:32.851 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13769 [Follower]" leader= -41:32.854 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5eddfa85-ed2a-9164-70a9-4c21860b1e73 127.0.0.1 -2022-09-15T21:41:32.854Z [INFO] agent.router: Initializing LAN area manager -41:32.854 [INFO] TestAgent.server: Adding LAN server: server="Node-5eddfa85-ed2a-9164-70a9-4c21860b1e73 (Addr: tcp/127.0.0.1:13769) (DC: dc1)" -41:32.854 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5eddfa85-ed2a-9164-70a9-4c21860b1e73.dc1 area=wan -41:32.854 [DEBUG] TestAgent: added local registration for service: service=rabbitmq-sidecar-proxy -41:32.854 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13764 network=tcp -41:32.854 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13764 network=udp -41:32.855 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13765 network=tcp -41:32.855 [INFO] TestAgent: started state syncer -41:32.855 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13770 network=tcp -41:32.898 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:32.898 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13769 [Candidate]" term=2 -41:32.898 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:32.898 [DEBUG] TestAgent.server.raft: vote granted: from=5eddfa85-ed2a-9164-70a9-4c21860b1e73 term=2 tally=1 -41:32.898 [INFO] TestAgent.server.raft: election won: tally=1 -41:32.898 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13769 [Leader]" -41:32.898 [INFO] TestAgent.server: cluster leadership acquired -41:32.899 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13769 -41:32.900 [INFO] TestAgent.server: New leader elected: payload=Node-5eddfa85-ed2a-9164-70a9-4c21860b1e73 -41:32.900 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:32.903 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:32.903 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:32.903 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:32.903 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:32.903 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:32.903 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5eddfa85-ed2a-9164-70a9-4c21860b1e73 -41:32.903 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5eddfa85-ed2a-9164-70a9-4c21860b1e73 -41:32.904 [INFO] TestAgent.server: federation state anti-entropy synced -41:32.922 [INFO] TestAgent: Requesting shutdown -41:32.922 [INFO] TestAgent.server: shutting down server -41:32.922 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:32.922 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:32.922 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:32.922 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:32.922 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:32.922 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:32.922 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:32.922 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:32.922 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:32.922 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:32.922 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:32.922 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:32.923Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:32.923Z [INFO] agent.router.manager: shutting down -41:32.923 [INFO] TestAgent: consul server down -41:32.923 [INFO] TestAgent: shutdown complete -41:32.923 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13764 network=tcp -41:32.923 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13764 network=udp -41:32.923 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13765 network=tcp -41:33.423 [INFO] TestAgent: Waiting for endpoints to shut down -41:33.423 [INFO] TestAgent: Endpoints down -2022-09-15T21:41:32.830Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:32.847Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:32.850 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:98f11149-f5b6-2c84-cb47-464b4acd0278 Address:127.0.0.1:13762}]" -41:32.851 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-98f11149-f5b6-2c84-cb47-464b4acd0278.dc1 127.0.0.1 -41:32.851 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13762 [Follower]" leader= -41:32.851 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-98f11149-f5b6-2c84-cb47-464b4acd0278 127.0.0.1 -2022-09-15T21:41:32.851Z [INFO] agent.router: Initializing LAN area manager -41:32.852 [INFO] TestAgent.server: Adding LAN server: server="Node-98f11149-f5b6-2c84-cb47-464b4acd0278 (Addr: tcp/127.0.0.1:13762) (DC: dc1)" -41:32.852 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-98f11149-f5b6-2c84-cb47-464b4acd0278.dc1 area=wan -41:32.852 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13757 network=udp -41:32.852 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13757 network=tcp -41:32.853 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13758 network=tcp -41:32.853 [INFO] TestAgent: started state syncer -41:32.853 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13763 network=tcp -41:32.911 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:32.911 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13762 [Candidate]" term=2 -41:32.912 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:32.912 [DEBUG] TestAgent.server.raft: vote granted: from=98f11149-f5b6-2c84-cb47-464b4acd0278 term=2 tally=1 -41:32.912 [INFO] TestAgent.server.raft: election won: tally=1 -41:32.912 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13762 [Leader]" -41:32.912 [INFO] TestAgent: Requesting shutdown -41:32.912 [INFO] TestAgent.server: shutting down server -41:32.912 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:32.912 [INFO] TestAgent.server: cluster leadership acquired -41:32.913 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13762 -41:32.914 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:32.916 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:32.917 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:32.917 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:32.917 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:32.917 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:32.917 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-98f11149-f5b6-2c84-cb47-464b4acd0278 -41:32.917 [INFO] TestAgent.server: member joined, marking health alive: member=Node-98f11149-f5b6-2c84-cb47-464b4acd0278 -41:32.917 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:32.917 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:32.917 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:32.917 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:32.918 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:32.918 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -41:32.918 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:32.918 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:32.918 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:32.918 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2022-09-15T21:41:32.925Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:32.925Z [INFO] agent.router.manager: shutting down -41:32.925 [INFO] TestAgent: consul server down -41:32.925 [INFO] TestAgent: shutdown complete -41:32.925 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13757 network=tcp -41:32.926 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13757 network=udp -41:32.926 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13758 network=tcp -41:33.426 [INFO] TestAgent: Waiting for endpoints to shut down -41:33.426 [INFO] TestAgent: Endpoints down +[WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:33:14.885Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:14.925Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:14.928 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e0abc03b-5d73-48f4-0a28-bdf9dc138c1a Address:127.0.0.1:24993}]" +33:14.929 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e0abc03b-5d73-48f4-0a28-bdf9dc138c1a.dc1 127.0.0.1 +33:14.930 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e0abc03b-5d73-48f4-0a28-bdf9dc138c1a 127.0.0.1 +2021-08-13T15:33:14.930Z [INFO] agent.router: Initializing LAN area manager +33:14.930 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24988 network=udp +33:14.930 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24993 [Follower]" leader= +33:14.931 [INFO] TestAgent.server: Adding LAN server: server="Node-e0abc03b-5d73-48f4-0a28-bdf9dc138c1a (Addr: tcp/127.0.0.1:24993) (DC: dc1)" +33:14.931 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e0abc03b-5d73-48f4-0a28-bdf9dc138c1a.dc1 area=wan +33:14.931 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24988 network=tcp +33:14.932 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24989 network=tcp +33:14.932 [INFO] TestAgent: started state syncer +33:14.932 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24994 network=tcp +33:14.997 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:14.997 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24993 [Candidate]" term=2 +33:14.997 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:14.997 [DEBUG] TestAgent.server.raft: vote granted: from=e0abc03b-5d73-48f4-0a28-bdf9dc138c1a term=2 tally=1 +33:14.997 [INFO] TestAgent.server.raft: election won: tally=1 +33:14.997 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24993 [Leader]" +33:14.997 [INFO] TestAgent.server: cluster leadership acquired +33:14.997 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24993 +33:14.999 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:15.001 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:15.001 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:15.001 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:15.001 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:15.001 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:15.001 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e0abc03b-5d73-48f4-0a28-bdf9dc138c1a +33:15.001 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e0abc03b-5d73-48f4-0a28-bdf9dc138c1a +33:15.002 [INFO] TestAgent.server: New leader elected: payload=Node-e0abc03b-5d73-48f4-0a28-bdf9dc138c1a +33:15.003 [INFO] TestAgent.server: federation state anti-entropy synced +33:15.021 [INFO] TestAgent: Requesting shutdown +33:15.022 [INFO] TestAgent.server: shutting down server +33:15.022 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:15.022 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:15.022 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:15.022 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:15.022 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:15.022 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:15.022 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:15.022 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:15.022 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:15.022 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:15.022 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:15.022 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:15.022Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:15.022Z [INFO] agent.router.manager: shutting down +33:15.023 [INFO] TestAgent: consul server down +33:15.023 [INFO] TestAgent: shutdown complete +33:15.023 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24988 network=tcp +33:15.023 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24988 network=udp +33:15.023 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24989 network=tcp +33:15.523 [INFO] TestAgent: Waiting for endpoints to shut down +33:15.523 [INFO] TestAgent: Endpoints down +2021-08-13T15:33:14.885Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:14.903Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:14.906 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:03ed23c8-96b5-5f36-7622-7057b54013bd Address:127.0.0.1:25000}]" +33:14.906 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25000 [Follower]" leader= +33:14.909 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-03ed23c8-96b5-5f36-7622-7057b54013bd.dc1 127.0.0.1 +33:14.911 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-03ed23c8-96b5-5f36-7622-7057b54013bd 127.0.0.1 +2021-08-13T15:33:14.911Z [INFO] agent.router: Initializing LAN area manager +33:14.911 [DEBUG] TestAgent: added local registration for service: service=rabbitmq-sidecar-proxy +33:14.911 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24995 network=udp +33:14.912 [INFO] TestAgent.server: Adding LAN server: server="Node-03ed23c8-96b5-5f36-7622-7057b54013bd (Addr: tcp/127.0.0.1:25000) (DC: dc1)" +33:14.912 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-03ed23c8-96b5-5f36-7622-7057b54013bd.dc1 area=wan +33:14.913 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24995 network=tcp +33:14.915 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24996 network=tcp +33:14.915 [INFO] TestAgent: started state syncer +33:14.915 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25001 network=tcp +33:14.941 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:14.941 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25000 [Candidate]" term=2 +33:14.942 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:14.942 [DEBUG] TestAgent.server.raft: vote granted: from=03ed23c8-96b5-5f36-7622-7057b54013bd term=2 tally=1 +33:14.942 [INFO] TestAgent.server.raft: election won: tally=1 +33:14.942 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25000 [Leader]" +33:14.942 [INFO] TestAgent.server: cluster leadership acquired +33:14.942 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25000 +33:14.943 [INFO] TestAgent.server: New leader elected: payload=Node-03ed23c8-96b5-5f36-7622-7057b54013bd +33:14.943 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:14.946 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:14.946 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:14.946 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:14.946 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:14.946 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:14.946 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-03ed23c8-96b5-5f36-7622-7057b54013bd +33:14.946 [INFO] TestAgent.server: member joined, marking health alive: member=Node-03ed23c8-96b5-5f36-7622-7057b54013bd +33:14.947 [INFO] TestAgent.server: federation state anti-entropy synced +33:15.083 [INFO] TestAgent: Requesting shutdown +33:15.083 [INFO] TestAgent.server: shutting down server +33:15.084 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:15.084 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:15.084 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:15.084 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:15.084 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:15.084 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:15.084 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:15.084 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:15.084 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:15.084 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:15.084 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:15.084 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:15.084Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:15.084Z [INFO] agent.router.manager: shutting down +33:15.084 [INFO] TestAgent: consul server down +33:15.084 [INFO] TestAgent: shutdown complete +33:15.084 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24995 network=tcp +33:15.085 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24995 network=udp +33:15.085 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24996 network=tcp +33:15.585 [INFO] TestAgent: Waiting for endpoints to shut down +33:15.585 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_loadServices_sidecarSeparateToken (0.00s) - --- PASS: TestAgent_loadServices_sidecarSeparateToken/service_manager (0.61s) - --- PASS: TestAgent_loadServices_sidecarSeparateToken/normal (0.62s) + --- PASS: TestAgent_loadServices_sidecarSeparateToken/normal (0.66s) + --- PASS: TestAgent_loadServices_sidecarSeparateToken/service_manager (0.72s) === RUN TestAgent_loadServices_sidecarInheritMeta === RUN TestAgent_loadServices_sidecarInheritMeta/normal === PAUSE TestAgent_loadServices_sidecarInheritMeta/normal @@ -12437,121 +12488,125 @@ [WARN] freeport: 6 out of 20 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:33.487Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:33.533Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:33.540 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6ffba3f4-ef85-c28f-daf7-2987c5158964 Address:127.0.0.1:13783}]" -41:33.541 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6ffba3f4-ef85-c28f-daf7-2987c5158964.dc1 127.0.0.1 -41:33.541 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6ffba3f4-ef85-c28f-daf7-2987c5158964 127.0.0.1 -2022-09-15T21:41:33.541Z [INFO] agent.router: Initializing LAN area manager -41:33.542 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13783 [Follower]" leader= -41:33.542 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6ffba3f4-ef85-c28f-daf7-2987c5158964.dc1 area=wan -41:33.542 [INFO] TestAgent.server: Adding LAN server: server="Node-6ffba3f4-ef85-c28f-daf7-2987c5158964 (Addr: tcp/127.0.0.1:13783) (DC: dc1)" -41:33.542 [DEBUG] TestAgent: added local registration for service: service=rabbitmq-sidecar-proxy -41:33.542 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13778 network=udp -41:33.543 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13778 network=tcp -41:33.544 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13779 network=tcp -41:33.544 [INFO] TestAgent: started state syncer -41:33.544 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13784 network=tcp -41:33.591 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:33.592 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13783 [Candidate]" term=2 -41:33.592 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:33.592 [DEBUG] TestAgent.server.raft: vote granted: from=6ffba3f4-ef85-c28f-daf7-2987c5158964 term=2 tally=1 -41:33.592 [INFO] TestAgent.server.raft: election won: tally=1 -41:33.592 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13783 [Leader]" -41:33.592 [INFO] TestAgent.server: cluster leadership acquired -41:33.592 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13783 -41:33.592 [INFO] TestAgent.server: New leader elected: payload=Node-6ffba3f4-ef85-c28f-daf7-2987c5158964 -41:33.594 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:33.596 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:33.596 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:33.596 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:33.596 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:33.596 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:33.596 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6ffba3f4-ef85-c28f-daf7-2987c5158964 -41:33.596 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6ffba3f4-ef85-c28f-daf7-2987c5158964 -41:33.597 [INFO] TestAgent.server: federation state anti-entropy synced -41:33.745 [INFO] TestAgent: Requesting shutdown -41:33.745 [INFO] TestAgent.server: shutting down server -41:33.745 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:33.745 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:33.745 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:33.745 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:33.745 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:33.745 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:33.745 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:33.745 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:33.745 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -41:33.745 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:33.745 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:33.745 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -2022-09-15T21:41:33.746Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:33.746Z [INFO] agent.router.manager: shutting down -41:33.746 [INFO] TestAgent: consul server down -41:33.746 [INFO] TestAgent: shutdown complete -41:33.746 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13778 network=tcp -41:33.746 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13778 network=udp -41:33.746 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13779 network=tcp -41:34.246 [INFO] TestAgent: Waiting for endpoints to shut down -41:34.246 [INFO] TestAgent: Endpoints down -[WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:33.495Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:33.547Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:33.550 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4c84efa7-7fd6-da70-00e3-52a0d3804ac2 Address:127.0.0.1:13776}]" -41:33.551 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4c84efa7-7fd6-da70-00e3-52a0d3804ac2.dc1 127.0.0.1 -41:33.551 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4c84efa7-7fd6-da70-00e3-52a0d3804ac2 127.0.0.1 -2022-09-15T21:41:33.551Z [INFO] agent.router: Initializing LAN area manager -41:33.552 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13776 [Follower]" leader= -41:33.552 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13771 network=udp -41:33.552 [INFO] TestAgent.server: Adding LAN server: server="Node-4c84efa7-7fd6-da70-00e3-52a0d3804ac2 (Addr: tcp/127.0.0.1:13776) (DC: dc1)" -41:33.552 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4c84efa7-7fd6-da70-00e3-52a0d3804ac2.dc1 area=wan -41:33.552 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13771 network=tcp -41:33.553 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13772 network=tcp -41:33.553 [INFO] TestAgent: started state syncer -41:33.553 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13777 network=tcp -41:33.614 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:33.614 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13776 [Candidate]" term=2 -41:33.614 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:33.614 [DEBUG] TestAgent.server.raft: vote granted: from=4c84efa7-7fd6-da70-00e3-52a0d3804ac2 term=2 tally=1 -41:33.614 [INFO] TestAgent.server.raft: election won: tally=1 -41:33.614 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13776 [Leader]" -41:33.614 [INFO] TestAgent.server: cluster leadership acquired -41:33.615 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13776 -41:33.615 [INFO] TestAgent.server: New leader elected: payload=Node-4c84efa7-7fd6-da70-00e3-52a0d3804ac2 -41:33.617 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:33.620 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:33.620 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:33.620 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:33.620 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:33.620 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:33.620 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4c84efa7-7fd6-da70-00e3-52a0d3804ac2 -41:33.620 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4c84efa7-7fd6-da70-00e3-52a0d3804ac2 -41:33.621 [INFO] TestAgent.server: federation state anti-entropy synced -41:33.850 [INFO] TestAgent: Requesting shutdown -41:33.851 [INFO] TestAgent.server: shutting down server -41:33.851 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:33.851 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:33.851 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:33.851 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:33.851 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:33.851 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:33.851 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:33.851 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:33.851 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:33.851 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:33.851 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:33.851 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:33.851Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:33.851Z [INFO] agent.router.manager: shutting down -41:33.852 [INFO] TestAgent: consul server down -41:33.852 [INFO] TestAgent: shutdown complete -41:33.852 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13771 network=tcp -41:33.852 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13771 network=udp -41:33.852 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13772 network=tcp -41:34.352 [INFO] TestAgent: Waiting for endpoints to shut down -41:34.352 [INFO] TestAgent: Endpoints down +2021-08-13T15:33:15.602Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:15.618Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:15.621 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:593b8fca-5870-3668-d1a7-962795fb60a1 Address:127.0.0.1:25014}]" +33:15.621 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25014 [Follower]" leader= +33:15.622 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-593b8fca-5870-3668-d1a7-962795fb60a1.dc1 127.0.0.1 +33:15.623 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-593b8fca-5870-3668-d1a7-962795fb60a1 127.0.0.1 +2021-08-13T15:33:15.623Z [INFO] agent.router: Initializing LAN area manager +33:15.623 [DEBUG] TestAgent: added local registration for service: service=rabbitmq-sidecar-proxy +33:15.623 [INFO] TestAgent.server: Adding LAN server: server="Node-593b8fca-5870-3668-d1a7-962795fb60a1 (Addr: tcp/127.0.0.1:25014) (DC: dc1)" +33:15.623 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-593b8fca-5870-3668-d1a7-962795fb60a1.dc1 area=wan +33:15.624 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25009 network=tcp +33:15.624 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25009 network=udp +33:15.632 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25010 network=tcp +33:15.632 [INFO] TestAgent: started state syncer +33:15.632 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25015 network=tcp +33:15.661 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:15.661 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25014 [Candidate]" term=2 +33:15.662 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:15.662 [DEBUG] TestAgent.server.raft: vote granted: from=593b8fca-5870-3668-d1a7-962795fb60a1 term=2 tally=1 +33:15.662 [INFO] TestAgent.server.raft: election won: tally=1 +33:15.662 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25014 [Leader]" +33:15.662 [INFO] TestAgent.server: cluster leadership acquired +33:15.662 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25014 +33:15.662 [INFO] TestAgent.server: New leader elected: payload=Node-593b8fca-5870-3668-d1a7-962795fb60a1 +33:15.663 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:15.666 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:15.666 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:15.666 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:15.666 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:15.666 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:15.666 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-593b8fca-5870-3668-d1a7-962795fb60a1 +33:15.666 [INFO] TestAgent.server: member joined, marking health alive: member=Node-593b8fca-5870-3668-d1a7-962795fb60a1 +33:15.667 [INFO] TestAgent.server: federation state anti-entropy synced +33:15.930 [INFO] TestAgent: Requesting shutdown +33:15.931 [INFO] TestAgent.server: shutting down server +33:15.931 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:15.931 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:15.931 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:15.931 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:15.931 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:15.931 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:15.931 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:15.931 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:15.931 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:15.931 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:15.931 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +33:15.931 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +2021-08-13T15:33:15.931Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:15.931Z [INFO] agent.router.manager: shutting down +33:15.931 [INFO] TestAgent: consul server down +33:15.931 [INFO] TestAgent: shutdown complete +33:15.931 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25009 network=tcp +33:15.931 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25009 network=udp +33:15.931 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25010 network=tcp +33:16.432 [INFO] TestAgent: Waiting for endpoints to shut down +33:16.432 [INFO] TestAgent: Endpoints down +2021-08-13T15:33:15.602Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:15.618Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:15.621 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6d4b5c02-84af-ab96-e00b-aced939524d4 Address:127.0.0.1:25007}]" +33:15.621 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25007 [Follower]" leader= +33:15.622 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6d4b5c02-84af-ab96-e00b-aced939524d4.dc1 127.0.0.1 +33:15.623 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6d4b5c02-84af-ab96-e00b-aced939524d4 127.0.0.1 +2021-08-13T15:33:15.623Z [INFO] agent.router: Initializing LAN area manager +33:15.631 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25002 network=udp +33:15.632 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25002 network=tcp +33:15.632 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6d4b5c02-84af-ab96-e00b-aced939524d4.dc1 area=wan +33:15.632 [INFO] TestAgent.server: Adding LAN server: server="Node-6d4b5c02-84af-ab96-e00b-aced939524d4 (Addr: tcp/127.0.0.1:25007) (DC: dc1)" +33:15.633 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25003 network=tcp +33:15.633 [INFO] TestAgent: started state syncer +33:15.634 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25008 network=tcp +33:15.657 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:15.657 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25007 [Candidate]" term=2 +33:15.657 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:15.657 [DEBUG] TestAgent.server.raft: vote granted: from=6d4b5c02-84af-ab96-e00b-aced939524d4 term=2 tally=1 +33:15.657 [INFO] TestAgent.server.raft: election won: tally=1 +33:15.657 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25007 [Leader]" +33:15.657 [INFO] TestAgent.server: cluster leadership acquired +33:15.657 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25007 +33:15.659 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:15.659 [INFO] TestAgent.server: New leader elected: payload=Node-6d4b5c02-84af-ab96-e00b-aced939524d4 +33:15.661 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:15.661 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:15.661 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:15.661 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:15.661 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:15.661 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6d4b5c02-84af-ab96-e00b-aced939524d4 +33:15.661 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6d4b5c02-84af-ab96-e00b-aced939524d4 +33:15.662 [INFO] TestAgent.server: federation state anti-entropy synced +33:15.699 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:15.700 [INFO] TestAgent: Synced node info +33:15.701 [INFO] TestAgent: Synced service: service=rabbitmq-sidecar-proxy +33:15.702 [INFO] TestAgent: Synced service: service=rabbitmq +33:15.702 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:1 +33:15.702 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:2 +33:15.956 [INFO] TestAgent: Requesting shutdown +33:15.956 [INFO] TestAgent.server: shutting down server +33:15.956 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:15.956 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:15.956 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:15.956 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:15.956 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:15.956 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:15.956 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:15.956 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:15.956 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:15.956 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:15.956 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:15.956Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:15.956Z [INFO] agent.router.manager: shutting down +33:15.957 [INFO] TestAgent: consul server down +33:15.957 [INFO] TestAgent: shutdown complete +33:15.957 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25002 network=tcp +33:15.957 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25002 network=udp +33:15.957 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25003 network=tcp +33:16.457 [INFO] TestAgent: Waiting for endpoints to shut down +33:16.457 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_loadServices_sidecarInheritMeta (0.00s) - --- PASS: TestAgent_loadServices_sidecarInheritMeta/service_manager (0.82s) - --- PASS: TestAgent_loadServices_sidecarInheritMeta/normal (0.93s) + --- PASS: TestAgent_loadServices_sidecarInheritMeta/service_manager (0.85s) + --- PASS: TestAgent_loadServices_sidecarInheritMeta/normal (0.87s) === RUN TestAgent_loadServices_sidecarOverrideMeta === RUN TestAgent_loadServices_sidecarOverrideMeta/normal === PAUSE TestAgent_loadServices_sidecarOverrideMeta/normal @@ -12559,129 +12614,133 @@ === PAUSE TestAgent_loadServices_sidecarOverrideMeta/service_manager === CONT TestAgent_loadServices_sidecarOverrideMeta/normal === CONT TestAgent_loadServices_sidecarOverrideMeta/service_manager -[WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 6 out of 20 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:34.372Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:34.394Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:34.397 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:01839481-91b0-aeb7-e5a1-05feef258961 Address:127.0.0.1:13790}]" -41:34.397 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13790 [Follower]" leader= -41:34.402 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-01839481-91b0-aeb7-e5a1-05feef258961.dc1 127.0.0.1 -41:34.403 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-01839481-91b0-aeb7-e5a1-05feef258961 127.0.0.1 -2022-09-15T21:41:34.403Z [INFO] agent.router: Initializing LAN area manager -41:34.410 [INFO] TestAgent.server: Adding LAN server: server="Node-01839481-91b0-aeb7-e5a1-05feef258961 (Addr: tcp/127.0.0.1:13790) (DC: dc1)" -41:34.410 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-01839481-91b0-aeb7-e5a1-05feef258961.dc1 area=wan -41:34.412 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13785 network=udp -41:34.412 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13785 network=tcp -41:34.413 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13786 network=tcp -41:34.418 [INFO] TestAgent: started state syncer -41:34.418 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13791 network=tcp -41:34.450 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:34.450 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13790 [Candidate]" term=2 -41:34.450 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:34.450 [DEBUG] TestAgent.server.raft: vote granted: from=01839481-91b0-aeb7-e5a1-05feef258961 term=2 tally=1 -41:34.450 [INFO] TestAgent.server.raft: election won: tally=1 -41:34.450 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13790 [Leader]" -41:34.450 [INFO] TestAgent.server: cluster leadership acquired -41:34.450 [INFO] TestAgent.server: New leader elected: payload=Node-01839481-91b0-aeb7-e5a1-05feef258961 -41:34.450 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13790 -41:34.452 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:34.454 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:34.454 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:34.454 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:34.454 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:34.454 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:34.455 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-01839481-91b0-aeb7-e5a1-05feef258961 -41:34.455 [INFO] TestAgent.server: member joined, marking health alive: member=Node-01839481-91b0-aeb7-e5a1-05feef258961 -41:34.456 [INFO] TestAgent.server: federation state anti-entropy synced -41:34.614 [INFO] TestAgent: Requesting shutdown -41:34.614 [INFO] TestAgent.server: shutting down server -41:34.614 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:34.614 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:34.614 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:34.614 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:34.614 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:34.615 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:34.615 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:34.615 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:34.615 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:34.615 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:34.615 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:34.615 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:34.615Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:34.615Z [INFO] agent.router.manager: shutting down -41:34.615 [INFO] TestAgent: consul server down -41:34.615 [INFO] TestAgent: shutdown complete -41:34.615 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13785 network=tcp -41:34.615 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13785 network=udp -41:34.615 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13786 network=tcp -41:35.116 [INFO] TestAgent: Waiting for endpoints to shut down -41:35.116 [INFO] TestAgent: Endpoints down -[WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:34.379Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:34.415Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:34.420 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2f93f43b-c123-251c-9b88-8354e05425cb Address:127.0.0.1:13797}]" -41:34.421 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13797 [Follower]" leader= -41:34.422 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2f93f43b-c123-251c-9b88-8354e05425cb.dc1 127.0.0.1 -41:34.423 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2f93f43b-c123-251c-9b88-8354e05425cb 127.0.0.1 -2022-09-15T21:41:34.423Z [INFO] agent.router: Initializing LAN area manager -41:34.423 [INFO] TestAgent.server: Adding LAN server: server="Node-2f93f43b-c123-251c-9b88-8354e05425cb (Addr: tcp/127.0.0.1:13797) (DC: dc1)" -41:34.423 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2f93f43b-c123-251c-9b88-8354e05425cb.dc1 area=wan -41:34.423 [DEBUG] TestAgent: added local registration for service: service=rabbitmq-sidecar-proxy -41:34.424 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13792 network=udp -41:34.424 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13792 network=tcp -41:34.426 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13793 network=tcp -41:34.426 [INFO] TestAgent: started state syncer -41:34.426 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13798 network=tcp -41:34.490 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:34.490 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13797 [Candidate]" term=2 -41:34.491 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:34.491 [DEBUG] TestAgent.server.raft: vote granted: from=2f93f43b-c123-251c-9b88-8354e05425cb term=2 tally=1 -41:34.491 [INFO] TestAgent.server.raft: election won: tally=1 -41:34.491 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13797 [Leader]" -41:34.491 [INFO] TestAgent.server: cluster leadership acquired -41:34.491 [INFO] TestAgent.server: New leader elected: payload=Node-2f93f43b-c123-251c-9b88-8354e05425cb -41:34.491 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13797 -41:34.493 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:34.495 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:34.495 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:34.495 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:34.495 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:34.495 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:34.495 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2f93f43b-c123-251c-9b88-8354e05425cb -41:34.495 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2f93f43b-c123-251c-9b88-8354e05425cb -41:34.496 [INFO] TestAgent.server: federation state anti-entropy synced -41:34.580 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:34.581 [INFO] TestAgent: Synced node info -41:34.582 [INFO] TestAgent: Synced service: service=rabbitmq -41:34.583 [INFO] TestAgent: Synced service: service=rabbitmq-sidecar-proxy -41:34.583 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:1 -41:34.583 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:2 -41:34.856 [INFO] TestAgent: Requesting shutdown -41:34.856 [INFO] TestAgent.server: shutting down server -41:34.856 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:34.856 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:34.857 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:34.857 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:34.857 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:34.857 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:34.857 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:34.857 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:34.857 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:34.857 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -41:34.857 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2022-09-15T21:41:34.857Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:34.857Z [INFO] agent.router.manager: shutting down -41:34.857 [INFO] TestAgent: consul server down -41:34.857 [INFO] TestAgent: shutdown complete -41:34.857 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13792 network=tcp -41:34.857 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13792 network=udp -41:34.857 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13793 network=tcp -41:35.358 [INFO] TestAgent: Waiting for endpoints to shut down -41:35.358 [INFO] TestAgent: Endpoints down +2021-08-13T15:33:16.494Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:16.532Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:16.540 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4d73e6e0-1814-5066-b742-903b7bfee051 Address:127.0.0.1:25028}]" +33:16.540 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25028 [Follower]" leader= +33:16.540 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4d73e6e0-1814-5066-b742-903b7bfee051.dc1 127.0.0.1 +33:16.541 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4d73e6e0-1814-5066-b742-903b7bfee051 127.0.0.1 +2021-08-13T15:33:16.541Z [INFO] agent.router: Initializing LAN area manager +33:16.542 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4d73e6e0-1814-5066-b742-903b7bfee051.dc1 area=wan +33:16.542 [INFO] TestAgent.server: Adding LAN server: server="Node-4d73e6e0-1814-5066-b742-903b7bfee051 (Addr: tcp/127.0.0.1:25028) (DC: dc1)" +33:16.542 [DEBUG] TestAgent: added local registration for service: service=rabbitmq-sidecar-proxy +33:16.542 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25023 network=udp +33:16.542 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25023 network=tcp +33:16.543 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25024 network=tcp +33:16.543 [INFO] TestAgent: started state syncer +33:16.543 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25029 network=tcp +33:16.603 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:16.603 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25028 [Candidate]" term=2 +33:16.603 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:16.603 [DEBUG] TestAgent.server.raft: vote granted: from=4d73e6e0-1814-5066-b742-903b7bfee051 term=2 tally=1 +33:16.603 [INFO] TestAgent.server.raft: election won: tally=1 +33:16.603 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25028 [Leader]" +33:16.603 [INFO] TestAgent.server: cluster leadership acquired +33:16.604 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25028 +33:16.605 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:16.607 [INFO] TestAgent.server: New leader elected: payload=Node-4d73e6e0-1814-5066-b742-903b7bfee051 +33:16.614 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:16.614 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:16.614 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:16.614 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:16.614 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:16.614 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4d73e6e0-1814-5066-b742-903b7bfee051 +33:16.614 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4d73e6e0-1814-5066-b742-903b7bfee051 +33:16.615 [INFO] TestAgent.server: federation state anti-entropy synced +33:16.725 [INFO] TestAgent: Requesting shutdown +33:16.725 [INFO] TestAgent.server: shutting down server +33:16.725 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:16.725 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:16.725 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:16.725 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:16.725 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:16.725 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:16.725 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:16.725 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:16.725 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:16.725 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:16.725 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +33:16.725 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2021-08-13T15:33:16.725Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:16.725Z [INFO] agent.router.manager: shutting down +33:16.726 [INFO] TestAgent: consul server down +33:16.726 [INFO] TestAgent: shutdown complete +33:16.726 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25023 network=tcp +33:16.726 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25023 network=udp +33:16.726 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25024 network=tcp +33:17.226 [INFO] TestAgent: Waiting for endpoints to shut down +33:17.226 [INFO] TestAgent: Endpoints down +2021-08-13T15:33:16.496Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:16.535Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:16.538 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ed5db6ca-7f95-65f9-7b1d-0991df018c73 Address:127.0.0.1:25021}]" +33:16.538 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25021 [Follower]" leader= +33:16.543 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ed5db6ca-7f95-65f9-7b1d-0991df018c73.dc1 127.0.0.1 +33:16.544 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ed5db6ca-7f95-65f9-7b1d-0991df018c73 127.0.0.1 +2021-08-13T15:33:16.544Z [INFO] agent.router: Initializing LAN area manager +33:16.544 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25016 network=udp +33:16.544 [INFO] TestAgent.server: Adding LAN server: server="Node-ed5db6ca-7f95-65f9-7b1d-0991df018c73 (Addr: tcp/127.0.0.1:25021) (DC: dc1)" +33:16.544 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ed5db6ca-7f95-65f9-7b1d-0991df018c73.dc1 area=wan +33:16.545 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25016 network=tcp +33:16.546 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25017 network=tcp +33:16.546 [INFO] TestAgent: started state syncer +33:16.546 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25022 network=tcp +33:16.579 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:16.579 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25021 [Candidate]" term=2 +33:16.579 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:16.579 [DEBUG] TestAgent.server.raft: vote granted: from=ed5db6ca-7f95-65f9-7b1d-0991df018c73 term=2 tally=1 +33:16.579 [INFO] TestAgent.server.raft: election won: tally=1 +33:16.579 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25021 [Leader]" +33:16.579 [INFO] TestAgent.server: cluster leadership acquired +33:16.580 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25021 +33:16.581 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:16.583 [INFO] TestAgent.server: New leader elected: payload=Node-ed5db6ca-7f95-65f9-7b1d-0991df018c73 +33:16.591 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:16.591 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:16.591 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:16.591 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:16.591 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:16.592 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ed5db6ca-7f95-65f9-7b1d-0991df018c73 +33:16.592 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ed5db6ca-7f95-65f9-7b1d-0991df018c73 +33:16.593 [INFO] TestAgent.server: federation state anti-entropy synced +33:16.716 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:16.716 [INFO] TestAgent: Synced node info +33:16.717 [INFO] TestAgent: Synced service: service=rabbitmq +33:16.718 [INFO] TestAgent: Synced service: service=rabbitmq-sidecar-proxy +33:16.718 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:1 +33:16.718 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:2 +33:16.718 [DEBUG] TestAgent: Node info in sync +33:16.718 [DEBUG] TestAgent: Service in sync: service=rabbitmq +33:16.718 [DEBUG] TestAgent: Service in sync: service=rabbitmq-sidecar-proxy +33:16.718 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:1 +33:16.718 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:2 +33:16.741 [INFO] TestAgent: Requesting shutdown +33:16.741 [INFO] TestAgent.server: shutting down server +33:16.741 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:16.741 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:16.741 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:16.741 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:16.741 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:16.741 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:16.741 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:16.741 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:16.741 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:16.741 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:16.741 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:16.742Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:16.742Z [INFO] agent.router.manager: shutting down +33:16.742 [INFO] TestAgent: consul server down +33:16.742 [INFO] TestAgent: shutdown complete +33:16.742 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25016 network=tcp +33:16.742 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25016 network=udp +33:16.742 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25017 network=tcp +33:17.242 [INFO] TestAgent: Waiting for endpoints to shut down +33:17.242 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_loadServices_sidecarOverrideMeta (0.00s) - --- PASS: TestAgent_loadServices_sidecarOverrideMeta/normal (0.76s) - --- PASS: TestAgent_loadServices_sidecarOverrideMeta/service_manager (1.00s) + --- PASS: TestAgent_loadServices_sidecarOverrideMeta/service_manager (0.76s) + --- PASS: TestAgent_loadServices_sidecarOverrideMeta/normal (0.78s) === RUN TestAgent_unloadServices === RUN TestAgent_unloadServices/normal === PAUSE TestAgent_unloadServices/normal @@ -12689,129 +12748,127 @@ === PAUSE TestAgent_unloadServices/service_manager === CONT TestAgent_unloadServices/normal === CONT TestAgent_unloadServices/service_manager -[WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 6 out of 20 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:35.376Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:35.394Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:35.397 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1aab3483-7c0e-682c-37f9-3c061036efa8 Address:127.0.0.1:13804}]" -41:35.397 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13804 [Follower]" leader= -41:35.398 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1aab3483-7c0e-682c-37f9-3c061036efa8.dc1 127.0.0.1 -41:35.399 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1aab3483-7c0e-682c-37f9-3c061036efa8 127.0.0.1 -2022-09-15T21:41:35.399Z [INFO] agent.router: Initializing LAN area manager -41:35.399 [INFO] TestAgent.server: Adding LAN server: server="Node-1aab3483-7c0e-682c-37f9-3c061036efa8 (Addr: tcp/127.0.0.1:13804) (DC: dc1)" -41:35.399 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1aab3483-7c0e-682c-37f9-3c061036efa8.dc1 area=wan -41:35.399 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13799 network=udp -41:35.399 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13799 network=tcp -41:35.400 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13800 network=tcp -41:35.400 [INFO] TestAgent: started state syncer -41:35.401 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13805 network=tcp -41:35.441 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:35.441 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13804 [Candidate]" term=2 -41:35.441 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:35.441 [DEBUG] TestAgent.server.raft: vote granted: from=1aab3483-7c0e-682c-37f9-3c061036efa8 term=2 tally=1 -41:35.441 [INFO] TestAgent.server.raft: election won: tally=1 -41:35.441 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13804 [Leader]" -41:35.442 [INFO] TestAgent.server: cluster leadership acquired -41:35.442 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13804 -41:35.442 [INFO] TestAgent.server: New leader elected: payload=Node-1aab3483-7c0e-682c-37f9-3c061036efa8 -41:35.443 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:35.450 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:35.450 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:35.450 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:35.450 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:35.450 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:35.450 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1aab3483-7c0e-682c-37f9-3c061036efa8 -41:35.450 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1aab3483-7c0e-682c-37f9-3c061036efa8 -41:35.451 [INFO] TestAgent.server: federation state anti-entropy synced -41:35.493 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:35.494 [INFO] TestAgent: Synced node info -41:35.494 [DEBUG] TestAgent: Node info in sync -41:35.737 [DEBUG] TestAgent: removed service: service=redis -41:35.737 [INFO] TestAgent: Requesting shutdown -41:35.737 [INFO] TestAgent.server: shutting down server -41:35.737 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:35.737 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:35.737 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:35.737 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:35.737 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:35.738 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:35.738 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:35.738 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:35.738 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:35.738 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:35.738 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:35.738Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:35.738Z [INFO] agent.router.manager: shutting down -41:35.738 [INFO] TestAgent: consul server down -41:35.738 [INFO] TestAgent: shutdown complete -41:35.738 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13799 network=tcp -41:35.738 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13799 network=udp -41:35.738 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13800 network=tcp -41:36.239 [INFO] TestAgent: Waiting for endpoints to shut down -41:36.239 [INFO] TestAgent: Endpoints down +2021-08-13T15:33:17.280Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:17.298Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:17.301 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0a63cc9c-94fd-3da3-a44b-6b438e43c216 Address:127.0.0.1:25042}]" +33:17.301 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25042 [Follower]" leader= +33:17.302 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0a63cc9c-94fd-3da3-a44b-6b438e43c216.dc1 127.0.0.1 +33:17.315 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0a63cc9c-94fd-3da3-a44b-6b438e43c216 127.0.0.1 +2021-08-13T15:33:17.315Z [INFO] agent.router: Initializing LAN area manager +33:17.316 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25037 network=udp +33:17.316 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0a63cc9c-94fd-3da3-a44b-6b438e43c216.dc1 area=wan +33:17.316 [INFO] TestAgent.server: Adding LAN server: server="Node-0a63cc9c-94fd-3da3-a44b-6b438e43c216 (Addr: tcp/127.0.0.1:25042) (DC: dc1)" +33:17.316 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25037 network=tcp +33:17.317 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25038 network=tcp +33:17.317 [INFO] TestAgent: started state syncer +33:17.317 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25043 network=tcp +33:17.370 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:17.371 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25042 [Candidate]" term=2 +33:17.371 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:17.371 [DEBUG] TestAgent.server.raft: vote granted: from=0a63cc9c-94fd-3da3-a44b-6b438e43c216 term=2 tally=1 +33:17.371 [INFO] TestAgent.server.raft: election won: tally=1 +33:17.371 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25042 [Leader]" +33:17.371 [INFO] TestAgent.server: cluster leadership acquired +33:17.371 [INFO] TestAgent.server: New leader elected: payload=Node-0a63cc9c-94fd-3da3-a44b-6b438e43c216 +33:17.371 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25042 +33:17.373 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:17.375 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:17.375 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:17.375 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:17.375 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:17.375 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:17.375 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0a63cc9c-94fd-3da3-a44b-6b438e43c216 +33:17.375 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0a63cc9c-94fd-3da3-a44b-6b438e43c216 +33:17.376 [INFO] TestAgent.server: federation state anti-entropy synced +33:17.453 [DEBUG] TestAgent: removed service: service=redis +33:17.453 [INFO] TestAgent: Requesting shutdown +33:17.454 [INFO] TestAgent.server: shutting down server +33:17.454 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:17.454 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:17.454 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:17.454 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:17.454 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:17.454 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:17.454 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:17.454 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:17.454 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:17.454 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:17.454 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:17.454 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:17.454Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:17.454Z [INFO] agent.router.manager: shutting down +33:17.454 [INFO] TestAgent: consul server down +33:17.454 [INFO] TestAgent: shutdown complete +33:17.454 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25037 network=tcp +33:17.454 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25037 network=udp +33:17.454 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25038 network=tcp +33:17.955 [INFO] TestAgent: Waiting for endpoints to shut down +33:17.955 [INFO] TestAgent: Endpoints down [WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:35.380Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:35.397Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:35.400 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6aaa45d0-f42b-a2b6-ee0b-d862caf87944 Address:127.0.0.1:13811}]" -41:35.400 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13811 [Follower]" leader= -41:35.400 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6aaa45d0-f42b-a2b6-ee0b-d862caf87944.dc1 127.0.0.1 -41:35.401 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6aaa45d0-f42b-a2b6-ee0b-d862caf87944 127.0.0.1 -2022-09-15T21:41:35.401Z [INFO] agent.router: Initializing LAN area manager -41:35.401 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6aaa45d0-f42b-a2b6-ee0b-d862caf87944.dc1 area=wan -41:35.401 [INFO] TestAgent.server: Adding LAN server: server="Node-6aaa45d0-f42b-a2b6-ee0b-d862caf87944 (Addr: tcp/127.0.0.1:13811) (DC: dc1)" -41:35.402 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13806 network=udp -41:35.402 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13806 network=tcp -41:35.402 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13807 network=tcp -41:35.403 [INFO] TestAgent: started state syncer -41:35.403 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13812 network=tcp -41:35.436 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:35.436 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13811 [Candidate]" term=2 -41:35.437 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:35.437 [DEBUG] TestAgent.server.raft: vote granted: from=6aaa45d0-f42b-a2b6-ee0b-d862caf87944 term=2 tally=1 -41:35.437 [INFO] TestAgent.server.raft: election won: tally=1 -41:35.437 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13811 [Leader]" -41:35.437 [INFO] TestAgent.server: cluster leadership acquired -41:35.437 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13811 -41:35.438 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:35.439 [INFO] TestAgent.server: New leader elected: payload=Node-6aaa45d0-f42b-a2b6-ee0b-d862caf87944 -41:35.441 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:35.441 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:35.441 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:35.441 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:35.441 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:35.441 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6aaa45d0-f42b-a2b6-ee0b-d862caf87944 -41:35.441 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6aaa45d0-f42b-a2b6-ee0b-d862caf87944 -41:35.442 [INFO] TestAgent.server: federation state anti-entropy synced -41:35.576 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:35.576 [INFO] TestAgent: Synced node info -41:35.576 [DEBUG] TestAgent: Node info in sync -41:35.798 [DEBUG] TestAgent: removed service: service=redis -41:35.799 [INFO] TestAgent: Requesting shutdown -41:35.799 [INFO] TestAgent.server: shutting down server -41:35.799 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:35.799 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:35.799 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:35.799 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:35.799 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:35.799 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:35.799 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:35.799 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:35.799 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:35.799 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:35.799 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:35.799Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:35.799Z [INFO] agent.router.manager: shutting down -41:35.799 [INFO] TestAgent: consul server down -41:35.799 [INFO] TestAgent: shutdown complete -41:35.800 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13806 network=tcp -41:35.800 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13806 network=udp -41:35.800 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13807 network=tcp -41:36.300 [INFO] TestAgent: Waiting for endpoints to shut down -41:36.300 [INFO] TestAgent: Endpoints down +2021-08-13T15:33:17.284Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:17.339Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:17.341 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8524e5b2-3024-4416-38bf-ef91ceca17e3 Address:127.0.0.1:25035}]" +33:17.342 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25035 [Follower]" leader= +33:17.342 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8524e5b2-3024-4416-38bf-ef91ceca17e3.dc1 127.0.0.1 +33:17.368 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8524e5b2-3024-4416-38bf-ef91ceca17e3 127.0.0.1 +2021-08-13T15:33:17.368Z [INFO] agent.router: Initializing LAN area manager +33:17.368 [INFO] TestAgent.server: Adding LAN server: server="Node-8524e5b2-3024-4416-38bf-ef91ceca17e3 (Addr: tcp/127.0.0.1:25035) (DC: dc1)" +33:17.368 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8524e5b2-3024-4416-38bf-ef91ceca17e3.dc1 area=wan +33:17.368 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25030 network=tcp +33:17.368 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25030 network=udp +33:17.369 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25031 network=tcp +33:17.370 [INFO] TestAgent: started state syncer +33:17.370 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25036 network=tcp +33:17.400 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:17.400 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25035 [Candidate]" term=2 +33:17.400 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:17.400 [DEBUG] TestAgent.server.raft: vote granted: from=8524e5b2-3024-4416-38bf-ef91ceca17e3 term=2 tally=1 +33:17.400 [INFO] TestAgent.server.raft: election won: tally=1 +33:17.400 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25035 [Leader]" +33:17.400 [INFO] TestAgent.server: cluster leadership acquired +33:17.400 [INFO] TestAgent.server: New leader elected: payload=Node-8524e5b2-3024-4416-38bf-ef91ceca17e3 +33:17.400 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25035 +33:17.402 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:17.405 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:17.405 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:17.405 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:17.405 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:17.405 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:17.405 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8524e5b2-3024-4416-38bf-ef91ceca17e3 +33:17.405 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8524e5b2-3024-4416-38bf-ef91ceca17e3 +33:17.406 [INFO] TestAgent.server: federation state anti-entropy synced +33:17.628 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:17.629 [INFO] TestAgent: Synced node info +33:17.629 [DEBUG] TestAgent: Node info in sync +33:17.666 [DEBUG] TestAgent: removed service: service=redis +33:17.666 [INFO] TestAgent: Requesting shutdown +33:17.666 [INFO] TestAgent.server: shutting down server +33:17.666 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:17.666 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:17.667 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:17.667 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:17.667 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:17.667 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:17.667 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:17.667 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:17.667 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:17.667 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:17.667 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:17.667Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:17.667Z [INFO] agent.router.manager: shutting down +33:17.667 [INFO] TestAgent: consul server down +33:17.667 [INFO] TestAgent: shutdown complete +33:17.667 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25030 network=tcp +33:17.667 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25030 network=udp +33:17.667 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25031 network=tcp +33:18.167 [INFO] TestAgent: Waiting for endpoints to shut down +33:18.167 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_unloadServices (0.00s) - --- PASS: TestAgent_unloadServices/normal (0.88s) - --- PASS: TestAgent_unloadServices/service_manager (0.94s) + --- PASS: TestAgent_unloadServices/service_manager (0.71s) + --- PASS: TestAgent_unloadServices/normal (0.93s) === RUN TestAgent_Service_MaintenanceMode === PAUSE TestAgent_Service_MaintenanceMode === RUN TestAgent_Service_Reap @@ -12820,137 +12877,136 @@ [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:36.340Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:36.356Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:36.359 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:252d09e9-8f2f-261b-6d78-7a70f330e9e0 Address:127.0.0.1:13818}]" -41:36.360 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-252d09e9-8f2f-261b-6d78-7a70f330e9e0.dc1 127.0.0.1 -41:36.361 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-252d09e9-8f2f-261b-6d78-7a70f330e9e0 127.0.0.1 -2022-09-15T21:41:36.361Z [INFO] agent.router: Initializing LAN area manager -41:36.361 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13818 [Follower]" leader= -41:36.361 [INFO] TestAgent.server: Adding LAN server: server="Node-252d09e9-8f2f-261b-6d78-7a70f330e9e0 (Addr: tcp/127.0.0.1:13818) (DC: dc1)" -41:36.361 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-252d09e9-8f2f-261b-6d78-7a70f330e9e0.dc1 area=wan -41:36.361 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13813 network=udp -41:36.361 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13813 network=tcp -41:36.362 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13814 network=tcp -41:36.362 [INFO] TestAgent: started state syncer -41:36.362 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13819 network=tcp -41:36.418 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:36.418 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13818 [Candidate]" term=2 -41:36.419 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:36.419 [DEBUG] TestAgent.server.raft: vote granted: from=252d09e9-8f2f-261b-6d78-7a70f330e9e0 term=2 tally=1 -41:36.419 [INFO] TestAgent.server.raft: election won: tally=1 -41:36.419 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13818 [Leader]" -41:36.419 [INFO] TestAgent.server: cluster leadership acquired -41:36.419 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13818 -41:36.420 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:36.422 [INFO] TestAgent.server: New leader elected: payload=Node-252d09e9-8f2f-261b-6d78-7a70f330e9e0 -41:36.423 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:36.423 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:36.423 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:36.423 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:36.423 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:36.423 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-252d09e9-8f2f-261b-6d78-7a70f330e9e0 -41:36.423 [INFO] TestAgent.server: member joined, marking health alive: member=Node-252d09e9-8f2f-261b-6d78-7a70f330e9e0 -41:36.424 [INFO] TestAgent.server: federation state anti-entropy synced -41:36.517 [WARN] TestAgent: Check missed TTL, is now critical: check=service:redis -41:36.592 [DEBUG] TestAgent: Check status updated: check=service:redis status=passing -41:36.618 [WARN] TestAgent: Check missed TTL, is now critical: check=service:redis -41:36.664 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:36.665 [INFO] TestAgent: Synced node info -41:36.666 [INFO] TestAgent: Synced service: service=redis -41:36.666 [DEBUG] TestAgent: Check in sync: check=service:redis -41:36.821 [DEBUG] TestAgent: removed check: check=service:redis -41:36.821 [DEBUG] TestAgent: removed service: service=redis -41:36.821 [INFO] TestAgent: Check for service has been critical for too long; deregistered service: service=redis check=service:redis -41:37.096 [INFO] TestAgent: Requesting shutdown -41:37.096 [INFO] TestAgent.server: shutting down server -41:37.096 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:37.096 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:37.096 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:37.096 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:37.096 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:37.096 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:37.096 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:37.096 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:37.096 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:37.096 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:37.097 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:37.097Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:37.097Z [INFO] agent.router.manager: shutting down -41:37.097 [INFO] TestAgent: consul server down -41:37.097 [INFO] TestAgent: shutdown complete -41:37.097 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13813 network=tcp -41:37.097 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13813 network=udp -41:37.097 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13814 network=tcp -41:37.597 [INFO] TestAgent: Waiting for endpoints to shut down -41:37.597 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Service_Reap (1.30s) +[WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:33:18.186Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:18.203Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:18.206 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7076c3c9-b197-8cda-fb02-250f4c60539a Address:127.0.0.1:25049}]" +33:18.206 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25049 [Follower]" leader= +33:18.207 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7076c3c9-b197-8cda-fb02-250f4c60539a.dc1 127.0.0.1 +33:18.208 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7076c3c9-b197-8cda-fb02-250f4c60539a 127.0.0.1 +2021-08-13T15:33:18.208Z [INFO] agent.router: Initializing LAN area manager +33:18.208 [INFO] TestAgent.server: Adding LAN server: server="Node-7076c3c9-b197-8cda-fb02-250f4c60539a (Addr: tcp/127.0.0.1:25049) (DC: dc1)" +33:18.208 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7076c3c9-b197-8cda-fb02-250f4c60539a.dc1 area=wan +33:18.208 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25044 network=udp +33:18.208 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25044 network=tcp +33:18.209 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25045 network=tcp +33:18.209 [INFO] TestAgent: started state syncer +33:18.209 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25050 network=tcp +33:18.272 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:18.272 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25049 [Candidate]" term=2 +33:18.272 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:18.273 [DEBUG] TestAgent.server.raft: vote granted: from=7076c3c9-b197-8cda-fb02-250f4c60539a term=2 tally=1 +33:18.273 [INFO] TestAgent.server.raft: election won: tally=1 +33:18.273 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25049 [Leader]" +33:18.273 [INFO] TestAgent.server: cluster leadership acquired +33:18.273 [INFO] TestAgent.server: New leader elected: payload=Node-7076c3c9-b197-8cda-fb02-250f4c60539a +33:18.273 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25049 +33:18.274 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:18.277 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:18.277 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:18.277 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:18.277 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:18.277 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:18.277 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7076c3c9-b197-8cda-fb02-250f4c60539a +33:18.277 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7076c3c9-b197-8cda-fb02-250f4c60539a +33:18.278 [INFO] TestAgent.server: federation state anti-entropy synced +33:18.293 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:18.294 [INFO] TestAgent: Synced node info +33:18.294 [DEBUG] TestAgent: Node info in sync +33:18.540 [WARN] TestAgent: Check missed TTL, is now critical: check=service:redis +33:18.615 [DEBUG] TestAgent: Check status updated: check=service:redis status=passing +33:18.640 [WARN] TestAgent: Check missed TTL, is now critical: check=service:redis +33:18.860 [DEBUG] TestAgent: removed check: check=service:redis +33:18.860 [DEBUG] TestAgent: removed service: service=redis +33:18.860 [INFO] TestAgent: Check for service has been critical for too long; deregistered service: service=redis check=service:redis +33:19.116 [INFO] TestAgent: Requesting shutdown +33:19.116 [INFO] TestAgent.server: shutting down server +33:19.116 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:19.116 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:19.116 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:19.116 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:19.116 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:19.116 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:19.116 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:19.116 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:19.116 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:19.116 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:19.116 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:19.117Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:19.117Z [INFO] agent.router.manager: shutting down +33:19.117 [INFO] TestAgent: consul server down +33:19.117 [INFO] TestAgent: shutdown complete +33:19.117 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25044 network=tcp +33:19.117 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25044 network=udp +33:19.117 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25045 network=tcp +33:19.617 [INFO] TestAgent: Waiting for endpoints to shut down +33:19.617 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Service_Reap (1.45s) === RUN TestAgent_Service_NoReap [WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:37.618Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:37.636Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:37.639 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ab1b8e78-0ca3-4f30-c03e-cda3e9319f45 Address:127.0.0.1:13825}]" -41:37.639 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13825 [Follower]" leader= -41:37.640 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ab1b8e78-0ca3-4f30-c03e-cda3e9319f45.dc1 127.0.0.1 -41:37.641 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ab1b8e78-0ca3-4f30-c03e-cda3e9319f45 127.0.0.1 -2022-09-15T21:41:37.641Z [INFO] agent.router: Initializing LAN area manager -41:37.641 [INFO] TestAgent.server: Adding LAN server: server="Node-ab1b8e78-0ca3-4f30-c03e-cda3e9319f45 (Addr: tcp/127.0.0.1:13825) (DC: dc1)" -41:37.641 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ab1b8e78-0ca3-4f30-c03e-cda3e9319f45.dc1 area=wan -41:37.641 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13820 network=udp -41:37.641 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13820 network=tcp -41:37.642 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13821 network=tcp -41:37.642 [INFO] TestAgent: started state syncer -41:37.642 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13826 network=tcp -41:37.682 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:37.682 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13825 [Candidate]" term=2 -41:37.683 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:37.683 [DEBUG] TestAgent.server.raft: vote granted: from=ab1b8e78-0ca3-4f30-c03e-cda3e9319f45 term=2 tally=1 -41:37.683 [INFO] TestAgent.server.raft: election won: tally=1 -41:37.683 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13825 [Leader]" -41:37.683 [INFO] TestAgent.server: cluster leadership acquired -41:37.683 [INFO] TestAgent.server: New leader elected: payload=Node-ab1b8e78-0ca3-4f30-c03e-cda3e9319f45 -41:37.683 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13825 -41:37.684 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:37.687 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:37.687 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:37.687 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:37.687 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:37.687 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:37.687 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ab1b8e78-0ca3-4f30-c03e-cda3e9319f45 -41:37.687 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ab1b8e78-0ca3-4f30-c03e-cda3e9319f45 -41:37.688 [INFO] TestAgent.server: federation state anti-entropy synced -41:37.978 [WARN] TestAgent: Check missed TTL, is now critical: check=service:redis -41:38.035 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:38.035 [INFO] TestAgent: Synced node info -41:38.036 [INFO] TestAgent: Synced service: service=redis -41:38.036 [DEBUG] TestAgent: Check in sync: check=service:redis -41:38.353 [INFO] TestAgent: Requesting shutdown -41:38.353 [INFO] TestAgent.server: shutting down server -41:38.353 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:38.353 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:38.353 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:38.353 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:38.353 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:38.353 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:38.353 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:38.354 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:38.354 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:38.354 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:38.354 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:38.354Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:38.354Z [INFO] agent.router.manager: shutting down -41:38.354 [INFO] TestAgent: consul server down -41:38.354 [INFO] TestAgent: shutdown complete -41:38.354 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13820 network=tcp -41:38.354 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13820 network=udp -41:38.354 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13821 network=tcp -41:38.854 [INFO] TestAgent: Waiting for endpoints to shut down -41:38.854 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Service_NoReap (1.26s) +2021-08-13T15:33:19.635Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:19.662Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:19.665 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:996d5ed8-3380-1906-5844-5d97ceb20778 Address:127.0.0.1:25056}]" +33:19.665 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25056 [Follower]" leader= +33:19.666 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-996d5ed8-3380-1906-5844-5d97ceb20778.dc1 127.0.0.1 +33:19.667 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-996d5ed8-3380-1906-5844-5d97ceb20778 127.0.0.1 +2021-08-13T15:33:19.667Z [INFO] agent.router: Initializing LAN area manager +33:19.667 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-996d5ed8-3380-1906-5844-5d97ceb20778.dc1 area=wan +33:19.667 [INFO] TestAgent.server: Adding LAN server: server="Node-996d5ed8-3380-1906-5844-5d97ceb20778 (Addr: tcp/127.0.0.1:25056) (DC: dc1)" +33:19.667 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25051 network=tcp +33:19.667 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25051 network=udp +33:19.668 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25052 network=tcp +33:19.668 [INFO] TestAgent: started state syncer +33:19.668 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25057 network=tcp +33:19.701 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:19.701 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25056 [Candidate]" term=2 +33:19.702 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:19.702 [DEBUG] TestAgent.server.raft: vote granted: from=996d5ed8-3380-1906-5844-5d97ceb20778 term=2 tally=1 +33:19.702 [INFO] TestAgent.server.raft: election won: tally=1 +33:19.702 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25056 [Leader]" +33:19.702 [INFO] TestAgent.server: cluster leadership acquired +33:19.702 [INFO] TestAgent.server: New leader elected: payload=Node-996d5ed8-3380-1906-5844-5d97ceb20778 +33:19.702 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25056 +33:19.703 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:19.706 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:19.706 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:19.706 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:19.706 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:19.706 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:19.706 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-996d5ed8-3380-1906-5844-5d97ceb20778 +33:19.706 [INFO] TestAgent.server: member joined, marking health alive: member=Node-996d5ed8-3380-1906-5844-5d97ceb20778 +33:19.707 [INFO] TestAgent.server: federation state anti-entropy synced +33:19.753 [WARN] TestAgent: Check missed TTL, is now critical: check=service:redis +33:19.990 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:19.990 [INFO] TestAgent: Synced node info +33:19.991 [INFO] TestAgent: Synced service: service=redis +33:19.991 [DEBUG] TestAgent: Check in sync: check=service:redis +33:20.128 [INFO] TestAgent: Requesting shutdown +33:20.128 [INFO] TestAgent.server: shutting down server +33:20.128 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:20.128 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:20.128 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:20.128 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:20.128 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:20.128 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:20.128 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:20.128 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:20.128 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:20.128 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:20.128 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:20.129Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:20.129Z [INFO] agent.router.manager: shutting down +33:20.129 [INFO] TestAgent: consul server down +33:20.129 [INFO] TestAgent: shutdown complete +33:20.129 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25051 network=tcp +33:20.129 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25051 network=udp +33:20.129 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25052 network=tcp +33:20.629 [INFO] TestAgent: Waiting for endpoints to shut down +33:20.629 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Service_NoReap (1.01s) === RUN TestAgent_AddService_restoresSnapshot === RUN TestAgent_AddService_restoresSnapshot/normal === PAUSE TestAgent_AddService_restoresSnapshot/normal @@ -12961,121 +13017,123 @@ [WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:38.876Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:38.917Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:38.920 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:93b8525b-2298-7684-7667-228f596feafa Address:127.0.0.1:13839}]" -41:38.920 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13839 [Follower]" leader= -41:38.921 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-93b8525b-2298-7684-7667-228f596feafa.dc1 127.0.0.1 -41:38.922 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-93b8525b-2298-7684-7667-228f596feafa 127.0.0.1 -2022-09-15T21:41:38.922Z [INFO] agent.router: Initializing LAN area manager -41:38.922 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-93b8525b-2298-7684-7667-228f596feafa.dc1 area=wan -41:38.922 [INFO] TestAgent.server: Adding LAN server: server="Node-93b8525b-2298-7684-7667-228f596feafa (Addr: tcp/127.0.0.1:13839) (DC: dc1)" -41:38.923 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13834 network=udp -41:38.923 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13834 network=tcp -41:38.923 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13835 network=tcp -41:38.924 [INFO] TestAgent: started state syncer -41:38.924 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13840 network=tcp -41:38.965 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:38.965 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13839 [Candidate]" term=2 -41:38.965 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:38.965 [DEBUG] TestAgent.server.raft: vote granted: from=93b8525b-2298-7684-7667-228f596feafa term=2 tally=1 -41:38.965 [INFO] TestAgent.server.raft: election won: tally=1 -41:38.965 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13839 [Leader]" -41:38.965 [INFO] TestAgent.server: cluster leadership acquired -41:38.965 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13839 -41:38.966 [INFO] TestAgent.server: New leader elected: payload=Node-93b8525b-2298-7684-7667-228f596feafa -41:38.967 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:38.969 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:38.969 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:38.969 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:38.969 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:38.969 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:38.969 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-93b8525b-2298-7684-7667-228f596feafa -41:38.969 [INFO] TestAgent.server: member joined, marking health alive: member=Node-93b8525b-2298-7684-7667-228f596feafa -41:38.970 [INFO] TestAgent.server: federation state anti-entropy synced -41:39.066 [INFO] TestAgent: Requesting shutdown -41:39.066 [INFO] TestAgent.server: shutting down server -41:39.066 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:39.066 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:39.066 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:39.066 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:39.066 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:39.066 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:39.066 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:39.066 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:39.066 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:39.066 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:39.066 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:39.066 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:39.066Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:39.066Z [INFO] agent.router.manager: shutting down -41:39.066 [INFO] TestAgent: consul server down -41:39.066 [INFO] TestAgent: shutdown complete -41:39.067 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13834 network=tcp -41:39.067 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13834 network=udp -41:39.067 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13835 network=tcp -41:39.567 [INFO] TestAgent: Waiting for endpoints to shut down -41:39.567 [INFO] TestAgent: Endpoints down -2022-09-15T21:41:38.877Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:38.910Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:38.914 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:498a9715-6e8f-bbf6-b727-4d5b6ce73144 Address:127.0.0.1:13832}]" -41:38.915 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13832 [Follower]" leader= -41:38.915 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-498a9715-6e8f-bbf6-b727-4d5b6ce73144.dc1 127.0.0.1 -41:38.916 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-498a9715-6e8f-bbf6-b727-4d5b6ce73144 127.0.0.1 -2022-09-15T21:41:38.916Z [INFO] agent.router: Initializing LAN area manager -41:38.917 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13827 network=udp -41:38.917 [INFO] TestAgent.server: Adding LAN server: server="Node-498a9715-6e8f-bbf6-b727-4d5b6ce73144 (Addr: tcp/127.0.0.1:13832) (DC: dc1)" -41:38.917 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-498a9715-6e8f-bbf6-b727-4d5b6ce73144.dc1 area=wan -41:38.917 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13827 network=tcp -41:38.919 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13828 network=tcp -41:38.919 [INFO] TestAgent: started state syncer -41:38.919 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13833 network=tcp -41:38.959 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:38.959 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13832 [Candidate]" term=2 -41:38.959 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:38.959 [DEBUG] TestAgent.server.raft: vote granted: from=498a9715-6e8f-bbf6-b727-4d5b6ce73144 term=2 tally=1 -41:38.959 [INFO] TestAgent.server.raft: election won: tally=1 -41:38.959 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13832 [Leader]" -41:38.959 [INFO] TestAgent.server: cluster leadership acquired -41:38.959 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13832 -41:38.960 [INFO] TestAgent.server: New leader elected: payload=Node-498a9715-6e8f-bbf6-b727-4d5b6ce73144 -41:38.961 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:38.963 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:38.963 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:38.963 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:38.963 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:38.963 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:38.963 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-498a9715-6e8f-bbf6-b727-4d5b6ce73144 -41:38.963 [INFO] TestAgent.server: member joined, marking health alive: member=Node-498a9715-6e8f-bbf6-b727-4d5b6ce73144 -41:38.965 [INFO] TestAgent.server: federation state anti-entropy synced -41:39.065 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:39.065 [INFO] TestAgent: Synced node info -41:39.065 [DEBUG] TestAgent: Node info in sync -41:39.098 [INFO] TestAgent: Requesting shutdown -41:39.098 [INFO] TestAgent.server: shutting down server -41:39.098 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:39.098 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:39.098 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:39.098 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:39.098 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:39.098 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:39.098 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:39.098 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:39.099 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:39.099 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:39.099 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:39.099Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:39.099Z [INFO] agent.router.manager: shutting down -41:39.099 [INFO] TestAgent: consul server down -41:39.099 [INFO] TestAgent: shutdown complete -41:39.099 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13827 network=tcp -41:39.099 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13827 network=udp -41:39.099 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13828 network=tcp -41:39.599 [INFO] TestAgent: Waiting for endpoints to shut down -41:39.599 [INFO] TestAgent: Endpoints down +2021-08-13T15:33:20.648Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:20.665Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:20.668 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1c7c1f1e-5552-5069-b103-ce3b05e7936b Address:127.0.0.1:25070}]" +33:20.668 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25070 [Follower]" leader= +33:20.669 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1c7c1f1e-5552-5069-b103-ce3b05e7936b.dc1 127.0.0.1 +33:20.670 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1c7c1f1e-5552-5069-b103-ce3b05e7936b 127.0.0.1 +2021-08-13T15:33:20.670Z [INFO] agent.router: Initializing LAN area manager +33:20.670 [INFO] TestAgent.server: Adding LAN server: server="Node-1c7c1f1e-5552-5069-b103-ce3b05e7936b (Addr: tcp/127.0.0.1:25070) (DC: dc1)" +33:20.670 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25065 network=udp +33:20.670 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1c7c1f1e-5552-5069-b103-ce3b05e7936b.dc1 area=wan +33:20.670 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25065 network=tcp +33:20.671 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25066 network=tcp +33:20.671 [INFO] TestAgent: started state syncer +33:20.671 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25071 network=tcp +33:20.710 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:20.710 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25070 [Candidate]" term=2 +33:20.710 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:20.710 [DEBUG] TestAgent.server.raft: vote granted: from=1c7c1f1e-5552-5069-b103-ce3b05e7936b term=2 tally=1 +33:20.710 [INFO] TestAgent.server.raft: election won: tally=1 +33:20.710 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25070 [Leader]" +33:20.710 [INFO] TestAgent.server: cluster leadership acquired +33:20.710 [INFO] TestAgent.server: New leader elected: payload=Node-1c7c1f1e-5552-5069-b103-ce3b05e7936b +33:20.710 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25070 +33:20.712 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:20.714 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:20.714 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:20.714 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:20.714 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:20.714 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:20.714 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1c7c1f1e-5552-5069-b103-ce3b05e7936b +33:20.714 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1c7c1f1e-5552-5069-b103-ce3b05e7936b +33:20.715 [INFO] TestAgent.server: federation state anti-entropy synced +33:20.753 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:20.754 [INFO] TestAgent: Synced node info +33:20.836 [INFO] TestAgent: Requesting shutdown +33:20.837 [INFO] TestAgent.server: shutting down server +33:20.837 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:20.837 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:20.837 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:20.837 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:20.837 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:20.837 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:20.837 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:20.837 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:20.837 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:20.837 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:20.837 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:20.837Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:20.837Z [INFO] agent.router.manager: shutting down +33:20.837 [INFO] TestAgent: consul server down +33:20.837 [INFO] TestAgent: shutdown complete +33:20.837 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25065 network=tcp +33:20.837 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25065 network=udp +33:20.837 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25066 network=tcp +33:21.338 [INFO] TestAgent: Waiting for endpoints to shut down +33:21.338 [INFO] TestAgent: Endpoints down +[WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:33:20.648Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:20.665Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:20.668 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7e3776e7-cedb-6efe-45e5-5e5d5695ed86 Address:127.0.0.1:25063}]" +33:20.668 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25063 [Follower]" leader= +33:20.669 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7e3776e7-cedb-6efe-45e5-5e5d5695ed86.dc1 127.0.0.1 +33:20.670 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7e3776e7-cedb-6efe-45e5-5e5d5695ed86 127.0.0.1 +2021-08-13T15:33:20.670Z [INFO] agent.router: Initializing LAN area manager +33:20.670 [INFO] TestAgent.server: Adding LAN server: server="Node-7e3776e7-cedb-6efe-45e5-5e5d5695ed86 (Addr: tcp/127.0.0.1:25063) (DC: dc1)" +33:20.670 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7e3776e7-cedb-6efe-45e5-5e5d5695ed86.dc1 area=wan +33:20.670 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25058 network=udp +33:20.670 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25058 network=tcp +33:20.671 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25059 network=tcp +33:20.671 [INFO] TestAgent: started state syncer +33:20.671 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25064 network=tcp +33:20.703 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:20.703 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25063 [Candidate]" term=2 +33:20.703 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:20.703 [DEBUG] TestAgent.server.raft: vote granted: from=7e3776e7-cedb-6efe-45e5-5e5d5695ed86 term=2 tally=1 +33:20.703 [INFO] TestAgent.server.raft: election won: tally=1 +33:20.703 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25063 [Leader]" +33:20.703 [INFO] TestAgent.server: cluster leadership acquired +33:20.704 [INFO] TestAgent.server: New leader elected: payload=Node-7e3776e7-cedb-6efe-45e5-5e5d5695ed86 +33:20.704 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25063 +33:20.705 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:20.708 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:20.708 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:20.708 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:20.708 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:20.708 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:20.708 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7e3776e7-cedb-6efe-45e5-5e5d5695ed86 +33:20.708 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7e3776e7-cedb-6efe-45e5-5e5d5695ed86 +33:20.709 [INFO] TestAgent.server: federation state anti-entropy synced +33:20.719 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:20.719 [INFO] TestAgent: Synced node info +33:20.719 [DEBUG] TestAgent: Node info in sync +33:21.027 [INFO] TestAgent: Requesting shutdown +33:21.027 [INFO] TestAgent.server: shutting down server +33:21.027 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:21.027 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:21.027 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:21.027 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:21.027 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:21.027 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:21.027 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:21.027 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:21.027 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:21.027 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:21.027 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:21.028Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:21.028Z [INFO] agent.router.manager: shutting down +33:21.028 [INFO] TestAgent: consul server down +33:21.028 [INFO] TestAgent: shutdown complete +33:21.028 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25058 network=tcp +33:21.028 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25058 network=udp +33:21.028 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25059 network=tcp +33:21.528 [INFO] TestAgent: Waiting for endpoints to shut down +33:21.528 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_AddService_restoresSnapshot (0.00s) --- PASS: TestAgent_AddService_restoresSnapshot/service_manager (0.71s) - --- PASS: TestAgent_AddService_restoresSnapshot/normal (0.74s) + --- PASS: TestAgent_AddService_restoresSnapshot/normal (0.90s) === RUN TestAgent_AddCheck_restoresSnapshot === PAUSE TestAgent_AddCheck_restoresSnapshot === RUN TestAgent_NodeMaintenanceMode @@ -13200,7 +13258,7 @@ === RUN TestAgent_JoinWAN_viaMeshGateway === PAUSE TestAgent_JoinWAN_viaMeshGateway === RUN TestAutoConfig_Integration -[WARN] freeport: 6 out of 20 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! agent_test.go:4731: Forcing roots rotation on the server [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! @@ -13220,403 +13278,364 @@ [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:39.891Z [INFO] agent.auto_config: retrieving initial agent auto configuration remotely -41:39.960 [INFO] test-client.client.serf.lan: serf: EventMemberJoin: test-client 127.0.0.1 -2022-09-15T21:41:39.960Z [INFO] agent.router: Initializing LAN area manager -2022-09-15T21:41:39.960Z [INFO] agent.auto_config: auto-config started -41:39.960 [INFO] test-client: Started DNS server: address=127.0.0.1:13848 network=udp -2022-09-15T21:41:39.960Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:39.960Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:39.960Z [WARN] agent.router.manager: No servers available -41:39.960 [INFO] test-client: Started DNS server: address=127.0.0.1:13848 network=tcp -2022-09-15T21:41:39.960Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:39.960Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:39.960Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:39.960Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:39.960Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:39.960Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:39.961Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:39.961Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:39.961Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:39.961Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:39.961Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:39.961Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:39.961Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:39.961Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:39.961Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:39.961Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:39.961Z [WARN] agent.router.manager: No servers available -41:39.961 [INFO] test-client: Started HTTP server: address=127.0.0.1:13849 network=tcp -2022-09-15T21:41:39.961Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:39.961Z [WARN] agent.router.manager: No servers available -41:39.961 [INFO] test-client: started state syncer -2022-09-15T21:41:39.961Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:39.961Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:39.961Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:39.961Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:39.961Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:39.961Z [WARN] agent.router.manager: No servers available -41:39.961 [INFO] test-client: Started gRPC server: address=127.0.0.1:13854 network=tcp -2022-09-15T21:41:39.961Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:39.961Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:39.962Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:39.962Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:39.962Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:39.962Z [WARN] agent.router.manager: No servers available -41:39.962 [ERROR] test-client.anti_entropy: failed to sync remote state: error="No known Consul servers" -41:39.962 [INFO] test-client: Retry join is supported for the following discovery methods: cluster=LAN discovery_methods="aliyun aws gce mdns os packet" -41:39.962 [INFO] test-client: Joining cluster...: cluster=LAN -41:39.962 [INFO] test-client: (LAN) joining: lan_addresses=[127.0.0.1:13844] -2022-09-15T21:41:39.962Z [WARN] agent.router.manager: No servers available -41:39.963 [DEBUG] test-client.client.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:13844 -41:39.964 [INFO] test-client.client.serf.lan: serf: EventMemberJoin: Node-4254daf1-d949-9139-89ec-1c41982efbc5 127.0.0.1 -41:39.964 [INFO] test-client: (LAN) joined: number_of_nodes=1 -41:39.964 [DEBUG] test-client: systemd notify failed: error="No socket" -41:39.964 [INFO] test-client: Join cluster completed. Synced with initial agents: cluster=LAN num_agents=1 -41:39.964 [INFO] test-client.client: adding server: server="Node-4254daf1-d949-9139-89ec-1c41982efbc5 (Addr: tcp/127.0.0.1:13846) (DC: dc1)" -41:40.010 [DEBUG] test-client.client: transitioned out of legacy ACL mode -41:40.011 [INFO] test-client.client.serf.lan: serf: EventMemberUpdate: test-client -41:40.063 [DEBUG] test-client.client.serf.lan: serf: messageUserEventType: consul:new-leader -41:40.261 [DEBUG] test-client.client.serf.lan: serf: messageUserEventType: consul:new-leader -41:40.261 [DEBUG] test-client.client.serf.lan: serf: messageJoinType: test-client -41:40.463 [DEBUG] test-client.client.serf.lan: serf: messageUserEventType: consul:new-leader -41:40.463 [DEBUG] test-client.client.serf.lan: serf: messageJoinType: test-client -41:40.666 [DEBUG] test-client.client.serf.lan: serf: messageJoinType: test-client -41:40.666 [DEBUG] test-client.client.serf.lan: serf: messageUserEventType: consul:new-leader -41:40.666 [DEBUG] test-client.client.serf.lan: serf: messageJoinType: test-client -41:42.888 [DEBUG] test-client: Skipping remote check since it is managed automatically: check=serfHealth -41:42.889 [INFO] test-client: Synced node info -41:42.889 [DEBUG] test-client: Node info in sync -41:43.799 [INFO] test-client: Requesting shutdown -41:43.799 [INFO] test-client.client: shutting down client -41:43.799 [WARN] test-client.client.serf.lan: serf: Shutdown without a Leave -41:43.800 [INFO] test-client: consul client down -41:43.800 [INFO] test-client: shutdown complete -41:43.800 [INFO] test-client: Stopping server: protocol=DNS address=127.0.0.1:13848 network=tcp -41:43.800 [INFO] test-client: Stopping server: protocol=DNS address=127.0.0.1:13848 network=udp -41:43.800 [INFO] test-client: Stopping server: protocol=HTTP address=127.0.0.1:13849 network=tcp -41:43.800 [ERROR] test-client.client: RPC failed to server: method=ConnectCA.Roots server=127.0.0.1:13846 error="rpc error making call: EOF" -41:44.300 [INFO] test-client: Waiting for endpoints to shut down -41:44.300 [INFO] test-client: Endpoints down -2022-09-15T21:41:39.629Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:39.653Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:39.658 [INFO] TestAgent-Server.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4254daf1-d949-9139-89ec-1c41982efbc5 Address:127.0.0.1:13846}]" -41:39.658 [INFO] TestAgent-Server.server.raft: entering follower state: follower="Node at 127.0.0.1:13846 [Follower]" leader= -41:39.659 [INFO] TestAgent-Server.server.serf.wan: serf: EventMemberJoin: Node-4254daf1-d949-9139-89ec-1c41982efbc5.dc1 127.0.0.1 -41:39.660 [INFO] TestAgent-Server.server.serf.lan: serf: EventMemberJoin: Node-4254daf1-d949-9139-89ec-1c41982efbc5 127.0.0.1 -2022-09-15T21:41:39.660Z [INFO] agent.router: Initializing LAN area manager -41:39.660 [INFO] TestAgent-Server.server: Adding LAN server: server="Node-4254daf1-d949-9139-89ec-1c41982efbc5 (Addr: tcp/127.0.0.1:13846) (DC: dc1)" -41:39.660 [INFO] TestAgent-Server.server: Handled event for server in area: event=member-join server=Node-4254daf1-d949-9139-89ec-1c41982efbc5.dc1 area=wan -41:39.660 [INFO] TestAgent-Server: Started DNS server: address=127.0.0.1:13841 network=udp -41:39.660 [INFO] TestAgent-Server: Started DNS server: address=127.0.0.1:13841 network=tcp -41:39.661 [INFO] TestAgent-Server: Started HTTP server: address=127.0.0.1:13842 network=tcp -41:39.661 [INFO] TestAgent-Server: started state syncer -41:39.661 [INFO] TestAgent-Server: Started gRPC server: address=127.0.0.1:13847 network=tcp -41:39.698 [WARN] TestAgent-Server.server.raft: heartbeat timeout reached, starting election: last-leader= -41:39.698 [INFO] TestAgent-Server.server.raft: entering candidate state: node="Node at 127.0.0.1:13846 [Candidate]" term=2 -41:39.698 [DEBUG] TestAgent-Server.server.raft: votes: needed=1 -41:39.698 [DEBUG] TestAgent-Server.server.raft: vote granted: from=4254daf1-d949-9139-89ec-1c41982efbc5 term=2 tally=1 -41:39.698 [INFO] TestAgent-Server.server.raft: election won: tally=1 -41:39.698 [INFO] TestAgent-Server.server.raft: entering leader state: leader="Node at 127.0.0.1:13846 [Leader]" -41:39.699 [INFO] TestAgent-Server.server: cluster leadership acquired -41:39.699 [INFO] TestAgent-Server.server: initializing acls -41:39.699 [INFO] TestAgent-Server.server: Created ACL 'global-management' policy -41:39.700 [INFO] TestAgent-Server.server: Bootstrapped ACL master token from configuration -41:39.701 [INFO] TestAgent-Server.server: Created ACL anonymous token from configuration -41:39.701 [INFO] TestAgent-Server.leader: started routine: routine="legacy ACL token upgrade" -41:39.701 [INFO] TestAgent-Server.leader: started routine: routine="acl token reaping" -41:39.701 [INFO] TestAgent-Server.server.serf.lan: serf: EventMemberUpdate: Node-4254daf1-d949-9139-89ec-1c41982efbc5 -41:39.701 [INFO] TestAgent-Server.server.serf.wan: serf: EventMemberUpdate: Node-4254daf1-d949-9139-89ec-1c41982efbc5.dc1 -41:39.702 [INFO] TestAgent-Server.server: New leader elected: payload=Node-4254daf1-d949-9139-89ec-1c41982efbc5 -41:39.702 [INFO] TestAgent-Server.server: Updating LAN server: server="Node-4254daf1-d949-9139-89ec-1c41982efbc5 (Addr: tcp/127.0.0.1:13846) (DC: dc1)" -41:39.702 [INFO] TestAgent-Server.server: Handled event for server in area: event=member-update server=Node-4254daf1-d949-9139-89ec-1c41982efbc5.dc1 area=wan -41:39.703 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:39.706 [INFO] TestAgent-Server.server.connect: initialized primary datacenter CA with provider: provider=consul -41:39.706 [INFO] TestAgent-Server.leader: started routine: routine="federation state anti-entropy" -41:39.706 [INFO] TestAgent-Server.leader: started routine: routine="federation state pruning" -41:39.706 [INFO] TestAgent-Server.leader: started routine: routine="intermediate cert renew watch" -41:39.706 [INFO] TestAgent-Server.leader: started routine: routine="CA root pruning" -41:39.706 [DEBUG] TestAgent-Server.server: Skipping self join check for node since the cluster is too small: node=Node-4254daf1-d949-9139-89ec-1c41982efbc5 -41:39.706 [INFO] TestAgent-Server.server: member joined, marking health alive: member=Node-4254daf1-d949-9139-89ec-1c41982efbc5 -41:39.707 [INFO] TestAgent-Server.server: federation state anti-entropy synced -41:39.707 [DEBUG] TestAgent-Server.server: Skipping self join check for node since the cluster is too small: node=Node-4254daf1-d949-9139-89ec-1c41982efbc5 -41:39.722 [DEBUG] TestAgent-Server: Skipping remote check since it is managed automatically: check=serfHealth -41:39.723 [INFO] TestAgent-Server: Synced node info -41:39.723 [DEBUG] TestAgent-Server: Node info in sync -41:39.809 [DEBUG] TestAgent-Server.acl: dropping node from result due to ACLs: node=Node-4254daf1-d949-9139-89ec-1c41982efbc5 -41:39.963 [DEBUG] TestAgent-Server.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:47802 -41:39.964 [INFO] TestAgent-Server.server.serf.lan: serf: EventMemberJoin: test-client 127.0.0.1 -41:39.964 [INFO] TestAgent-Server.server: member joined, marking health alive: member=test-client -41:39.999 [DEBUG] connect.ca.consul: consul CA provider configured: id=38:c1:d9:a8:8f:c5:4f:f6:dc:e1:5e:22:1b:0a:1b:44:9f:cd:0a:f3:b1:ca:7b:73:0d:ef:d8:c4:69:bd:e8:00 is_primary=true -41:40.002 [INFO] TestAgent-Server.server.connect: CA rotated to new root under provider: provider=consul -41:40.162 [INFO] TestAgent-Server.server.serf.lan: serf: EventMemberUpdate: test-client -41:40.162 [DEBUG] TestAgent-Server.server.serf.lan: serf: messageJoinType: test-client -41:40.361 [DEBUG] TestAgent-Server.server.serf.lan: serf: messageJoinType: test-client -41:40.562 [DEBUG] TestAgent-Server.server.serf.lan: serf: messageJoinType: test-client -41:40.666 [DEBUG] TestAgent-Server.server.serf.lan: serf: messageJoinType: test-client -41:41.585 [DEBUG] TestAgent-Server: Skipping remote check since it is managed automatically: check=serfHealth -41:41.585 [DEBUG] TestAgent-Server: Node info in sync -41:44.300 [INFO] TestAgent-Server: Requesting shutdown -41:44.300 [INFO] TestAgent-Server.server: shutting down server -41:44.300 [DEBUG] TestAgent-Server.leader: stopping routine: routine="federation state anti-entropy" -41:44.300 [DEBUG] TestAgent-Server.leader: stopping routine: routine="federation state pruning" -41:44.300 [DEBUG] TestAgent-Server.leader: stopping routine: routine="intermediate cert renew watch" -41:44.300 [DEBUG] TestAgent-Server.leader: stopping routine: routine="CA root pruning" -41:44.300 [DEBUG] TestAgent-Server.leader: stopping routine: routine="legacy ACL token upgrade" -41:44.300 [DEBUG] TestAgent-Server.leader: stopping routine: routine="acl token reaping" -41:44.300 [WARN] TestAgent-Server.server.serf.lan: serf: Shutdown without a Leave -41:44.301 [INFO] TestAgent-Server.server: shutting down Connect CA roots monitor -41:44.301 [ERROR] TestAgent-Server.server: error performing anti-entropy sync of federation state: error="context canceled" -41:44.301 [DEBUG] TestAgent-Server.leader: stopped routine: routine="federation state anti-entropy" -41:44.301 [DEBUG] TestAgent-Server.leader: stopped routine: routine="federation state pruning" -41:44.301 [DEBUG] TestAgent-Server.leader: stopped routine: routine="intermediate cert renew watch" -41:44.301 [DEBUG] TestAgent-Server.leader: stopped routine: routine="CA root pruning" -41:44.301 [DEBUG] TestAgent-Server.leader: stopped routine: routine="legacy ACL token upgrade" -41:44.301 [DEBUG] TestAgent-Server.leader: stopped routine: routine="acl token reaping" -41:44.301 [WARN] TestAgent-Server.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:44.301Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:44.301Z [INFO] agent.router.manager: shutting down -41:44.301 [INFO] TestAgent-Server: consul server down -41:44.301 [INFO] TestAgent-Server: shutdown complete -41:44.301 [INFO] TestAgent-Server: Stopping server: protocol=DNS address=127.0.0.1:13841 network=tcp -41:44.301 [INFO] TestAgent-Server: Stopping server: protocol=DNS address=127.0.0.1:13841 network=udp -41:44.301 [INFO] TestAgent-Server: Stopping server: protocol=HTTP address=127.0.0.1:13842 network=tcp -41:44.802 [INFO] TestAgent-Server: Waiting for endpoints to shut down -41:44.802 [INFO] TestAgent-Server: Endpoints down ---- PASS: TestAutoConfig_Integration (5.20s) +2021-08-13T15:33:21.701Z [INFO] agent.auto_config: retrieving initial agent auto configuration remotely +33:21.752 [INFO] test-client.client.serf.lan: serf: EventMemberJoin: test-client 127.0.0.1 +2021-08-13T15:33:21.753Z [INFO] agent.router: Initializing LAN area manager +2021-08-13T15:33:21.753Z [INFO] agent.auto_config: auto-config started +2021-08-13T15:33:21.753Z [WARN] agent.router.manager: No servers available +2021-08-13T15:33:21.753Z [WARN] agent.router.manager: No servers available +2021-08-13T15:33:21.753Z [WARN] agent.router.manager: No servers available +33:21.753 [INFO] test-client: Started DNS server: address=127.0.0.1:25079 network=tcp +33:21.753 [INFO] test-client: Started DNS server: address=127.0.0.1:25079 network=udp +2021-08-13T15:33:21.753Z [WARN] agent.router.manager: No servers available +2021-08-13T15:33:21.753Z [WARN] agent.router.manager: No servers available +2021-08-13T15:33:21.753Z [WARN] agent.router.manager: No servers available +2021-08-13T15:33:21.753Z [WARN] agent.router.manager: No servers available +2021-08-13T15:33:21.753Z [WARN] agent.router.manager: No servers available +2021-08-13T15:33:21.753Z [WARN] agent.router.manager: No servers available +2021-08-13T15:33:21.753Z [WARN] agent.router.manager: No servers available +2021-08-13T15:33:21.753Z [WARN] agent.router.manager: No servers available +2021-08-13T15:33:21.753Z [WARN] agent.router.manager: No servers available +33:21.754 [INFO] test-client: Started HTTP server: address=127.0.0.1:25080 network=tcp +33:21.754 [INFO] test-client: started state syncer +33:21.754 [INFO] test-client: Started gRPC server: address=127.0.0.1:25085 network=tcp +33:21.754 [INFO] test-client: Retry join is supported for the following discovery methods: cluster=LAN discovery_methods="aliyun aws gce mdns os packet" +33:21.754 [INFO] test-client: Joining cluster...: cluster=LAN +33:21.754 [INFO] test-client: (LAN) joining: lan_addresses=[127.0.0.1:25075] +2021-08-13T15:33:21.754Z [WARN] agent.router.manager: No servers available +2021-08-13T15:33:21.754Z [WARN] agent.router.manager: No servers available +33:21.754 [ERROR] test-client.anti_entropy: failed to sync remote state: error="No known Consul servers" +33:21.754 [DEBUG] test-client.client.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:25075 +2021-08-13T15:33:21.755Z [WARN] agent.router.manager: No servers available +33:21.756 [INFO] test-client.client.serf.lan: serf: EventMemberJoin: Node-5f6de3fb-a120-6a36-5500-ffd99e33ccac 127.0.0.1 +33:21.756 [INFO] test-client: (LAN) joined: number_of_nodes=1 +33:21.756 [DEBUG] test-client: systemd notify failed: error="No socket" +33:21.756 [INFO] test-client: Join cluster completed. Synced with initial agents: cluster=LAN num_agents=1 +33:21.756 [INFO] test-client.client: adding server: server="Node-5f6de3fb-a120-6a36-5500-ffd99e33ccac (Addr: tcp/127.0.0.1:25077) (DC: dc1)" +33:21.791 [DEBUG] test-client.client.serf.lan: serf: messageUserEventType: consul:new-leader +33:21.803 [DEBUG] test-client.client: transitioned out of legacy ACL mode +33:21.803 [INFO] test-client.client.serf.lan: serf: EventMemberUpdate: test-client +33:21.991 [DEBUG] test-client.client.serf.lan: serf: messageUserEventType: consul:new-leader +33:21.991 [DEBUG] test-client.client.serf.lan: serf: messageJoinType: test-client +33:22.192 [DEBUG] test-client.client.serf.lan: serf: messageUserEventType: consul:new-leader +33:22.192 [DEBUG] test-client.client.serf.lan: serf: messageJoinType: test-client +33:22.392 [DEBUG] test-client.client.serf.lan: serf: messageUserEventType: consul:new-leader +33:22.392 [DEBUG] test-client.client.serf.lan: serf: messageJoinType: test-client +33:22.592 [DEBUG] test-client.client.serf.lan: serf: messageJoinType: test-client +33:22.650 [DEBUG] test-client: Skipping remote check since it is managed automatically: check=serfHealth +33:22.651 [INFO] test-client: Synced node info +33:22.651 [DEBUG] test-client: Node info in sync +33:25.516 [INFO] test-client: Requesting shutdown +33:25.516 [INFO] test-client.client: shutting down client +33:25.516 [WARN] test-client.client.serf.lan: serf: Shutdown without a Leave +33:25.516 [INFO] test-client: consul client down +33:25.516 [INFO] test-client: shutdown complete +33:25.516 [INFO] test-client: Stopping server: protocol=DNS address=127.0.0.1:25079 network=tcp +33:25.516 [INFO] test-client: Stopping server: protocol=DNS address=127.0.0.1:25079 network=udp +33:25.516 [INFO] test-client: Stopping server: protocol=HTTP address=127.0.0.1:25080 network=tcp +33:25.516 [ERROR] test-client.client: RPC failed to server: method=ConnectCA.Roots server=127.0.0.1:25077 error="rpc error making call: EOF" +33:26.016 [INFO] test-client: Waiting for endpoints to shut down +33:26.016 [INFO] test-client: Endpoints down +2021-08-13T15:33:21.553Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:21.584Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:21.589 [INFO] TestAgent-Server.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5f6de3fb-a120-6a36-5500-ffd99e33ccac Address:127.0.0.1:25077}]" +33:21.589 [INFO] TestAgent-Server.server.raft: entering follower state: follower="Node at 127.0.0.1:25077 [Follower]" leader= +33:21.590 [INFO] TestAgent-Server.server.serf.wan: serf: EventMemberJoin: Node-5f6de3fb-a120-6a36-5500-ffd99e33ccac.dc1 127.0.0.1 +33:21.590 [INFO] TestAgent-Server.server.serf.lan: serf: EventMemberJoin: Node-5f6de3fb-a120-6a36-5500-ffd99e33ccac 127.0.0.1 +2021-08-13T15:33:21.590Z [INFO] agent.router: Initializing LAN area manager +33:21.591 [INFO] TestAgent-Server.server: Adding LAN server: server="Node-5f6de3fb-a120-6a36-5500-ffd99e33ccac (Addr: tcp/127.0.0.1:25077) (DC: dc1)" +33:21.591 [INFO] TestAgent-Server.server: Handled event for server in area: event=member-join server=Node-5f6de3fb-a120-6a36-5500-ffd99e33ccac.dc1 area=wan +33:21.591 [INFO] TestAgent-Server: Started DNS server: address=127.0.0.1:25072 network=udp +33:21.591 [INFO] TestAgent-Server: Started DNS server: address=127.0.0.1:25072 network=tcp +33:21.592 [INFO] TestAgent-Server: Started HTTP server: address=127.0.0.1:25073 network=tcp +33:21.592 [INFO] TestAgent-Server: started state syncer +33:21.592 [INFO] TestAgent-Server: Started gRPC server: address=127.0.0.1:25078 network=tcp +33:21.632 [WARN] TestAgent-Server.server.raft: heartbeat timeout reached, starting election: last-leader= +33:21.632 [INFO] TestAgent-Server.server.raft: entering candidate state: node="Node at 127.0.0.1:25077 [Candidate]" term=2 +33:21.632 [DEBUG] TestAgent-Server.server.raft: votes: needed=1 +33:21.632 [DEBUG] TestAgent-Server.server.raft: vote granted: from=5f6de3fb-a120-6a36-5500-ffd99e33ccac term=2 tally=1 +33:21.632 [INFO] TestAgent-Server.server.raft: election won: tally=1 +33:21.632 [INFO] TestAgent-Server.server.raft: entering leader state: leader="Node at 127.0.0.1:25077 [Leader]" +33:21.632 [INFO] TestAgent-Server.server: cluster leadership acquired +33:21.633 [INFO] TestAgent-Server.server: New leader elected: payload=Node-5f6de3fb-a120-6a36-5500-ffd99e33ccac +33:21.633 [INFO] TestAgent-Server.server: initializing acls +33:21.633 [INFO] TestAgent-Server.server: Created ACL 'global-management' policy +33:21.634 [INFO] TestAgent-Server.server: Bootstrapped ACL master token from configuration +33:21.634 [INFO] TestAgent-Server.server: Created ACL anonymous token from configuration +33:21.634 [INFO] TestAgent-Server.leader: started routine: routine="legacy ACL token upgrade" +33:21.634 [INFO] TestAgent-Server.leader: started routine: routine="acl token reaping" +33:21.635 [INFO] TestAgent-Server.server.serf.lan: serf: EventMemberUpdate: Node-5f6de3fb-a120-6a36-5500-ffd99e33ccac +33:21.635 [INFO] TestAgent-Server.server: Updating LAN server: server="Node-5f6de3fb-a120-6a36-5500-ffd99e33ccac (Addr: tcp/127.0.0.1:25077) (DC: dc1)" +33:21.635 [INFO] TestAgent-Server.server.serf.wan: serf: EventMemberUpdate: Node-5f6de3fb-a120-6a36-5500-ffd99e33ccac.dc1 +33:21.635 [INFO] TestAgent-Server.server: Handled event for server in area: event=member-update server=Node-5f6de3fb-a120-6a36-5500-ffd99e33ccac.dc1 area=wan +33:21.636 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:21.638 [INFO] TestAgent-Server.server.connect: initialized primary datacenter CA with provider: provider=consul +33:21.638 [INFO] TestAgent-Server.leader: started routine: routine="federation state anti-entropy" +33:21.638 [INFO] TestAgent-Server.leader: started routine: routine="federation state pruning" +33:21.638 [INFO] TestAgent-Server.leader: started routine: routine="intermediate cert renew watch" +33:21.638 [INFO] TestAgent-Server.leader: started routine: routine="CA root pruning" +33:21.638 [DEBUG] TestAgent-Server.server: Skipping self join check for node since the cluster is too small: node=Node-5f6de3fb-a120-6a36-5500-ffd99e33ccac +33:21.638 [INFO] TestAgent-Server.server: member joined, marking health alive: member=Node-5f6de3fb-a120-6a36-5500-ffd99e33ccac +33:21.639 [INFO] TestAgent-Server.server: federation state anti-entropy synced +33:21.639 [DEBUG] TestAgent-Server.server: Skipping self join check for node since the cluster is too small: node=Node-5f6de3fb-a120-6a36-5500-ffd99e33ccac +33:21.754 [DEBUG] TestAgent-Server.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:42928 +33:21.756 [INFO] TestAgent-Server.server.serf.lan: serf: EventMemberJoin: test-client 127.0.0.1 +33:21.756 [INFO] TestAgent-Server.server: member joined, marking health alive: member=test-client +33:21.789 [DEBUG] connect.ca.consul: consul CA provider configured: id=48:4f:78:5b:3d:1b:1f:f2:87:6e:71:fa:8d:3b:d5:34:d2:8f:ef:19:34:30:e9:15:1f:fe:04:26:1a:b7:0c:1c is_primary=true +33:21.791 [INFO] TestAgent-Server.server.connect: CA rotated to new root under provider: provider=consul +33:21.848 [DEBUG] TestAgent-Server: Skipping remote check since it is managed automatically: check=serfHealth +33:21.849 [INFO] TestAgent-Server: Synced node info +33:21.954 [INFO] TestAgent-Server.server.serf.lan: serf: EventMemberUpdate: test-client +33:21.954 [DEBUG] TestAgent-Server.server.serf.lan: serf: messageJoinType: test-client +33:22.156 [DEBUG] TestAgent-Server.server.serf.lan: serf: messageJoinType: test-client +33:22.355 [DEBUG] TestAgent-Server.server.serf.lan: serf: messageJoinType: test-client +33:22.554 [DEBUG] TestAgent-Server.server.serf.lan: serf: messageJoinType: test-client +33:23.227 [DEBUG] TestAgent-Server: Skipping remote check since it is managed automatically: check=serfHealth +33:23.227 [DEBUG] TestAgent-Server: Node info in sync +33:23.227 [DEBUG] TestAgent-Server: Node info in sync +33:26.016 [INFO] TestAgent-Server: Requesting shutdown +33:26.017 [INFO] TestAgent-Server.server: shutting down server +33:26.017 [DEBUG] TestAgent-Server.leader: stopping routine: routine="CA root pruning" +33:26.017 [DEBUG] TestAgent-Server.leader: stopping routine: routine="legacy ACL token upgrade" +33:26.017 [DEBUG] TestAgent-Server.leader: stopping routine: routine="acl token reaping" +33:26.017 [DEBUG] TestAgent-Server.leader: stopping routine: routine="federation state anti-entropy" +33:26.017 [DEBUG] TestAgent-Server.leader: stopping routine: routine="federation state pruning" +33:26.017 [DEBUG] TestAgent-Server.leader: stopping routine: routine="intermediate cert renew watch" +33:26.017 [WARN] TestAgent-Server.server.serf.lan: serf: Shutdown without a Leave +33:26.017 [INFO] TestAgent-Server.server: shutting down Connect CA roots monitor +33:26.017 [ERROR] TestAgent-Server.server: error performing anti-entropy sync of federation state: error="context canceled" +33:26.017 [DEBUG] TestAgent-Server.leader: stopped routine: routine="federation state anti-entropy" +33:26.017 [DEBUG] TestAgent-Server.leader: stopped routine: routine="CA root pruning" +33:26.017 [DEBUG] TestAgent-Server.leader: stopped routine: routine="legacy ACL token upgrade" +33:26.017 [DEBUG] TestAgent-Server.leader: stopped routine: routine="acl token reaping" +33:26.017 [DEBUG] TestAgent-Server.leader: stopped routine: routine="federation state pruning" +33:26.017 [DEBUG] TestAgent-Server.leader: stopped routine: routine="intermediate cert renew watch" +33:26.017 [WARN] TestAgent-Server.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:26.017Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:26.017Z [INFO] agent.router.manager: shutting down +33:26.017 [INFO] TestAgent-Server: consul server down +33:26.017 [INFO] TestAgent-Server: shutdown complete +33:26.017 [INFO] TestAgent-Server: Stopping server: protocol=DNS address=127.0.0.1:25072 network=tcp +33:26.017 [INFO] TestAgent-Server: Stopping server: protocol=DNS address=127.0.0.1:25072 network=udp +33:26.017 [INFO] TestAgent-Server: Stopping server: protocol=HTTP address=127.0.0.1:25073 network=tcp +33:26.091 [DEBUG] TestAgent-Server.server.memberlist.lan: memberlist: Failed ping: test-client (timeout reached) +33:26.518 [INFO] TestAgent-Server: Waiting for endpoints to shut down +33:26.518 [INFO] TestAgent-Server: Endpoints down +--- PASS: TestAutoConfig_Integration (4.99s) === RUN TestAgent_AutoEncrypt [WARN] freeport: 6 out of 20 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:45.144Z [INFO] agent.auto_config: automatically upgraded to TLS -41:45.152 [INFO] test-client.client.serf.lan: serf: EventMemberJoin: test-client 127.0.0.1 -2022-09-15T21:41:45.152Z [INFO] agent.router: Initializing LAN area manager -2022-09-15T21:41:45.152Z [INFO] agent.auto_config: auto-config started -41:45.153 [INFO] test-client: Started DNS server: address=127.0.0.1:13862 network=udp -2022-09-15T21:41:45.153Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:45.153Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:45.153Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:45.153Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:45.154Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:45.154Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:45.154Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:45.154Z [WARN] agent.router.manager: No servers available -41:45.154 [INFO] test-client: Started DNS server: address=127.0.0.1:13862 network=tcp -2022-09-15T21:41:45.154Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:45.154Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:45.154Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:45.154Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:45.155Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:45.155Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:45.155Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:45.155Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:45.155Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:45.155Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:45.155Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:45.155Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:45.155Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:45.155Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:45.155Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:45.155Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:45.156Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:45.156Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:45.156Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:45.156Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:45.156Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:45.156Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:45.156Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:45.156Z [WARN] agent.router.manager: No servers available -41:45.159 [INFO] test-client: Started HTTPS server: address=127.0.0.1:13864 network=tcp -41:45.160 [INFO] test-client: started state syncer -2022-09-15T21:41:45.160Z [WARN] agent.router.manager: No servers available -2022-09-15T21:41:45.160Z [WARN] agent.router.manager: No servers available -41:45.160 [INFO] test-client: Started gRPC server: address=127.0.0.1:13868 network=tcp -41:45.160 [INFO] test-client: Retry join is supported for the following discovery methods: cluster=LAN discovery_methods="aliyun aws gce mdns os packet" -41:45.160 [INFO] test-client: Joining cluster...: cluster=LAN -41:45.160 [INFO] test-client: (LAN) joining: lan_addresses=[127.0.0.1:13858] -2022-09-15T21:41:45.161Z [WARN] agent.router.manager: No servers available -41:45.161 [ERROR] test-client.anti_entropy: failed to sync remote state: error="No known Consul servers" -41:45.161 [DEBUG] test-client.client.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:13858 -41:45.167 [INFO] test-client.client.serf.lan: serf: EventMemberJoin: Node-5ab02b64-0a30-d2dd-2b92-757e512cff50 127.0.0.1 -41:45.167 [INFO] test-client: (LAN) joined: number_of_nodes=1 -41:45.167 [DEBUG] test-client: systemd notify failed: error="No socket" -41:45.167 [INFO] test-client: Join cluster completed. Synced with initial agents: cluster=LAN num_agents=1 -41:45.168 [INFO] test-client.client: adding server: server="Node-5ab02b64-0a30-d2dd-2b92-757e512cff50 (Addr: tcp/127.0.0.1:13860) (DC: dc1)" -41:45.196 [INFO] test-client: Requesting shutdown -41:45.196 [INFO] test-client.client: shutting down client -41:45.196 [WARN] test-client.client.serf.lan: serf: Shutdown without a Leave -41:45.196 [INFO] test-client: consul client down -41:45.196 [INFO] test-client: shutdown complete -41:45.196 [INFO] test-client: Stopping server: protocol=DNS address=127.0.0.1:13862 network=tcp -41:45.197 [INFO] test-client: Stopping server: protocol=DNS address=127.0.0.1:13862 network=udp -41:45.197 [INFO] test-client: Stopping server: protocol=HTTPS address=127.0.0.1:13864 network=tcp -41:45.697 [INFO] test-client: Waiting for endpoints to shut down -41:45.697 [INFO] test-client: Endpoints down -2022-09-15T21:41:44.824Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:44.843Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:44.846 [INFO] test-server.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5ab02b64-0a30-d2dd-2b92-757e512cff50 Address:127.0.0.1:13860}]" -41:44.847 [INFO] test-server.server.raft: entering follower state: follower="Node at 127.0.0.1:13860 [Follower]" leader= -41:44.847 [INFO] test-server.server.serf.wan: serf: EventMemberJoin: Node-5ab02b64-0a30-d2dd-2b92-757e512cff50.dc1 127.0.0.1 -41:44.848 [INFO] test-server.server.serf.lan: serf: EventMemberJoin: Node-5ab02b64-0a30-d2dd-2b92-757e512cff50 127.0.0.1 -2022-09-15T21:41:44.848Z [INFO] agent.router: Initializing LAN area manager -41:44.848 [INFO] test-server.server: Adding LAN server: server="Node-5ab02b64-0a30-d2dd-2b92-757e512cff50 (Addr: tcp/127.0.0.1:13860) (DC: dc1)" -41:44.849 [INFO] test-server.server: Handled event for server in area: event=member-join server=Node-5ab02b64-0a30-d2dd-2b92-757e512cff50.dc1 area=wan -41:44.849 [INFO] test-server: Started DNS server: address=127.0.0.1:13855 network=tcp -41:44.849 [INFO] test-server: Started DNS server: address=127.0.0.1:13855 network=udp -41:44.850 [INFO] test-server: Started HTTP server: address=127.0.0.1:13856 network=tcp -41:44.850 [INFO] test-server: started state syncer -41:44.850 [INFO] test-server: Started gRPC server: address=127.0.0.1:13861 network=tcp -41:44.892 [WARN] test-server.server.raft: heartbeat timeout reached, starting election: last-leader= -41:44.892 [INFO] test-server.server.raft: entering candidate state: node="Node at 127.0.0.1:13860 [Candidate]" term=2 -41:44.893 [DEBUG] test-server.server.raft: votes: needed=1 -41:44.893 [DEBUG] test-server.server.raft: vote granted: from=5ab02b64-0a30-d2dd-2b92-757e512cff50 term=2 tally=1 -41:44.893 [INFO] test-server.server.raft: election won: tally=1 -41:44.893 [INFO] test-server.server.raft: entering leader state: leader="Node at 127.0.0.1:13860 [Leader]" -41:44.893 [INFO] test-server.server: cluster leadership acquired -41:44.893 [INFO] test-server.server: initializing acls -41:44.894 [INFO] test-server.server: New leader elected: payload=Node-5ab02b64-0a30-d2dd-2b92-757e512cff50 -41:44.894 [INFO] test-server.server: Created ACL 'global-management' policy -41:44.894 [INFO] test-server.server: Bootstrapped ACL master token from configuration -41:44.895 [INFO] test-server.server: Created ACL anonymous token from configuration -41:44.895 [INFO] test-server.leader: started routine: routine="legacy ACL token upgrade" -41:44.895 [INFO] test-server.leader: started routine: routine="acl token reaping" -41:44.895 [INFO] test-server.server.serf.lan: serf: EventMemberUpdate: Node-5ab02b64-0a30-d2dd-2b92-757e512cff50 -41:44.895 [INFO] test-server.server.serf.wan: serf: EventMemberUpdate: Node-5ab02b64-0a30-d2dd-2b92-757e512cff50.dc1 -41:44.897 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:44.898 [INFO] test-server.server: Updating LAN server: server="Node-5ab02b64-0a30-d2dd-2b92-757e512cff50 (Addr: tcp/127.0.0.1:13860) (DC: dc1)" -41:44.898 [INFO] test-server.server: Handled event for server in area: event=member-update server=Node-5ab02b64-0a30-d2dd-2b92-757e512cff50.dc1 area=wan -41:44.899 [INFO] test-server.server.connect: initialized primary datacenter CA with provider: provider=consul -41:44.899 [INFO] test-server.leader: started routine: routine="federation state anti-entropy" -41:44.899 [INFO] test-server.leader: started routine: routine="federation state pruning" -41:44.899 [INFO] test-server.leader: started routine: routine="intermediate cert renew watch" -41:44.899 [INFO] test-server.leader: started routine: routine="CA root pruning" -41:44.899 [DEBUG] test-server.server: Skipping self join check for node since the cluster is too small: node=Node-5ab02b64-0a30-d2dd-2b92-757e512cff50 -41:44.900 [INFO] test-server.server: member joined, marking health alive: member=Node-5ab02b64-0a30-d2dd-2b92-757e512cff50 -41:44.900 [DEBUG] test-server.server: Skipping self join check for node since the cluster is too small: node=Node-5ab02b64-0a30-d2dd-2b92-757e512cff50 -41:44.901 [INFO] test-server.server: federation state anti-entropy synced -41:45.072 [DEBUG] test-server: Skipping remote check since it is managed automatically: check=serfHealth -41:45.073 [INFO] test-server: Synced node info -41:45.073 [DEBUG] test-server: Node info in sync -41:45.092 [DEBUG] test-server.acl: dropping node from result due to ACLs: node=Node-5ab02b64-0a30-d2dd-2b92-757e512cff50 -41:45.161 [DEBUG] test-server.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:52392 -41:45.163 [INFO] test-server.server.serf.lan: serf: EventMemberJoin: test-client 127.0.0.1 -41:45.164 [INFO] test-server.server: member joined, marking health alive: member=test-client -41:45.500 [DEBUG] test-server: Skipping remote check since it is managed automatically: check=serfHealth -41:45.500 [DEBUG] test-server: Node info in sync -41:45.697 [INFO] test-server: Requesting shutdown -41:45.697 [INFO] test-server.server: shutting down server -41:45.697 [DEBUG] test-server.leader: stopping routine: routine="CA root pruning" -41:45.697 [DEBUG] test-server.leader: stopping routine: routine="legacy ACL token upgrade" -41:45.697 [DEBUG] test-server.leader: stopping routine: routine="acl token reaping" -41:45.697 [DEBUG] test-server.leader: stopping routine: routine="federation state anti-entropy" -41:45.697 [DEBUG] test-server.leader: stopping routine: routine="federation state pruning" -41:45.697 [DEBUG] test-server.leader: stopping routine: routine="intermediate cert renew watch" -41:45.697 [WARN] test-server.server.serf.lan: serf: Shutdown without a Leave -41:45.697 [ERROR] test-server.server: error performing anti-entropy sync of federation state: error="context canceled" -41:45.697 [DEBUG] test-server.leader: stopped routine: routine="federation state anti-entropy" -41:45.697 [INFO] test-server.server: shutting down Connect CA roots monitor -41:45.697 [DEBUG] test-server.leader: stopped routine: routine="CA root pruning" -41:45.697 [DEBUG] test-server.leader: stopped routine: routine="legacy ACL token upgrade" -41:45.697 [DEBUG] test-server.leader: stopped routine: routine="acl token reaping" -41:45.697 [DEBUG] test-server.leader: stopped routine: routine="federation state pruning" -41:45.697 [DEBUG] test-server.leader: stopped routine: routine="intermediate cert renew watch" -41:45.697 [WARN] test-server.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:45.698Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:45.698Z [INFO] agent.router.manager: shutting down -41:45.698 [INFO] test-server: consul server down -41:45.698 [INFO] test-server: shutdown complete -41:45.698 [INFO] test-server: Stopping server: protocol=DNS address=127.0.0.1:13855 network=tcp -41:45.698 [INFO] test-server: Stopping server: protocol=DNS address=127.0.0.1:13855 network=udp -41:45.698 [INFO] test-server: Stopping server: protocol=HTTP address=127.0.0.1:13856 network=tcp -41:46.198 [INFO] test-server: Waiting for endpoints to shut down -41:46.198 [INFO] test-server: Endpoints down ---- PASS: TestAgent_AutoEncrypt (1.40s) +[WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:33:27.044Z [INFO] agent.auto_config: automatically upgraded to TLS +33:27.046 [INFO] test-client.client.serf.lan: serf: EventMemberJoin: test-client 127.0.0.1 +2021-08-13T15:33:27.046Z [INFO] agent.router: Initializing LAN area manager +2021-08-13T15:33:27.046Z [INFO] agent.auto_config: auto-config started +2021-08-13T15:33:27.046Z [WARN] agent.router.manager: No servers available +2021-08-13T15:33:27.046Z [WARN] agent.router.manager: No servers available +2021-08-13T15:33:27.046Z [WARN] agent.router.manager: No servers available +2021-08-13T15:33:27.046Z [WARN] agent.router.manager: No servers available +33:27.046 [INFO] test-client: Started DNS server: address=127.0.0.1:25093 network=udp +2021-08-13T15:33:27.046Z [WARN] agent.router.manager: No servers available +2021-08-13T15:33:27.046Z [WARN] agent.router.manager: No servers available +2021-08-13T15:33:27.046Z [WARN] agent.router.manager: No servers available +33:27.046 [INFO] test-client: Started DNS server: address=127.0.0.1:25093 network=tcp +2021-08-13T15:33:27.046Z [WARN] agent.router.manager: No servers available +2021-08-13T15:33:27.046Z [WARN] agent.router.manager: No servers available +2021-08-13T15:33:27.046Z [WARN] agent.router.manager: No servers available +2021-08-13T15:33:27.046Z [WARN] agent.router.manager: No servers available +2021-08-13T15:33:27.046Z [WARN] agent.router.manager: No servers available +33:27.048 [INFO] test-client: Started HTTPS server: address=127.0.0.1:25095 network=tcp +33:27.048 [INFO] test-client: started state syncer +2021-08-13T15:33:27.049Z [WARN] agent.router.manager: No servers available +2021-08-13T15:33:27.049Z [WARN] agent.router.manager: No servers available +33:27.049 [INFO] test-client: Started gRPC server: address=127.0.0.1:25099 network=tcp +33:27.049 [INFO] test-client: Retry join is supported for the following discovery methods: cluster=LAN discovery_methods="aliyun aws gce mdns os packet" +33:27.049 [INFO] test-client: Joining cluster...: cluster=LAN +33:27.049 [INFO] test-client: (LAN) joining: lan_addresses=[127.0.0.1:25089] +2021-08-13T15:33:27.049Z [WARN] agent.router.manager: No servers available +33:27.049 [ERROR] test-client.anti_entropy: failed to sync remote state: error="No known Consul servers" +33:27.049 [DEBUG] test-client.client.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:25089 +33:27.050 [INFO] test-client.client.serf.lan: serf: EventMemberJoin: Node-01a7cc13-9082-ae8b-15cf-8039d6f0f264 127.0.0.1 +33:27.050 [INFO] test-client: (LAN) joined: number_of_nodes=1 +33:27.050 [DEBUG] test-client: systemd notify failed: error="No socket" +33:27.050 [INFO] test-client: Join cluster completed. Synced with initial agents: cluster=LAN num_agents=1 +33:27.050 [INFO] test-client.client: adding server: server="Node-01a7cc13-9082-ae8b-15cf-8039d6f0f264 (Addr: tcp/127.0.0.1:25091) (DC: dc1)" +33:27.081 [INFO] test-client: Requesting shutdown +33:27.081 [INFO] test-client.client: shutting down client +33:27.081 [WARN] test-client.client.serf.lan: serf: Shutdown without a Leave +33:27.081 [INFO] test-client: consul client down +33:27.081 [INFO] test-client: shutdown complete +33:27.081 [INFO] test-client: Stopping server: protocol=DNS address=127.0.0.1:25093 network=tcp +33:27.082 [INFO] test-client: Stopping server: protocol=DNS address=127.0.0.1:25093 network=udp +33:27.082 [INFO] test-client: Stopping server: protocol=HTTPS address=127.0.0.1:25095 network=tcp +33:27.582 [INFO] test-client: Waiting for endpoints to shut down +33:27.582 [INFO] test-client: Endpoints down +2021-08-13T15:33:26.538Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:26.555Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:26.559 [INFO] test-server.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:01a7cc13-9082-ae8b-15cf-8039d6f0f264 Address:127.0.0.1:25091}]" +33:26.559 [INFO] test-server.server.raft: entering follower state: follower="Node at 127.0.0.1:25091 [Follower]" leader= +33:26.560 [INFO] test-server.server.serf.wan: serf: EventMemberJoin: Node-01a7cc13-9082-ae8b-15cf-8039d6f0f264.dc1 127.0.0.1 +33:26.561 [INFO] test-server.server.serf.lan: serf: EventMemberJoin: Node-01a7cc13-9082-ae8b-15cf-8039d6f0f264 127.0.0.1 +2021-08-13T15:33:26.561Z [INFO] agent.router: Initializing LAN area manager +33:26.561 [INFO] test-server.server: Handled event for server in area: event=member-join server=Node-01a7cc13-9082-ae8b-15cf-8039d6f0f264.dc1 area=wan +33:26.561 [INFO] test-server.server: Adding LAN server: server="Node-01a7cc13-9082-ae8b-15cf-8039d6f0f264 (Addr: tcp/127.0.0.1:25091) (DC: dc1)" +33:26.561 [INFO] test-server: Started DNS server: address=127.0.0.1:25086 network=tcp +33:26.561 [INFO] test-server: Started DNS server: address=127.0.0.1:25086 network=udp +33:26.562 [INFO] test-server: Started HTTP server: address=127.0.0.1:25087 network=tcp +33:26.562 [INFO] test-server: started state syncer +33:26.562 [INFO] test-server: Started gRPC server: address=127.0.0.1:25092 network=tcp +33:26.611 [DEBUG] test-server.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +33:26.613 [WARN] test-server.server.raft: heartbeat timeout reached, starting election: last-leader= +33:26.613 [INFO] test-server.server.raft: entering candidate state: node="Node at 127.0.0.1:25091 [Candidate]" term=2 +33:26.613 [DEBUG] test-server.server.raft: votes: needed=1 +33:26.613 [DEBUG] test-server.server.raft: vote granted: from=01a7cc13-9082-ae8b-15cf-8039d6f0f264 term=2 tally=1 +33:26.613 [INFO] test-server.server.raft: election won: tally=1 +33:26.613 [INFO] test-server.server.raft: entering leader state: leader="Node at 127.0.0.1:25091 [Leader]" +33:26.614 [INFO] test-server.server: cluster leadership acquired +33:26.614 [INFO] test-server.server: initializing acls +33:26.614 [INFO] test-server.server: Created ACL 'global-management' policy +33:26.615 [INFO] test-server.server: New leader elected: payload=Node-01a7cc13-9082-ae8b-15cf-8039d6f0f264 +33:26.615 [INFO] test-server.server: Bootstrapped ACL master token from configuration +33:26.616 [INFO] test-server.server: Created ACL anonymous token from configuration +33:26.616 [INFO] test-server.leader: started routine: routine="legacy ACL token upgrade" +33:26.616 [INFO] test-server.leader: started routine: routine="acl token reaping" +33:26.616 [INFO] test-server.server.serf.lan: serf: EventMemberUpdate: Node-01a7cc13-9082-ae8b-15cf-8039d6f0f264 +33:26.616 [INFO] test-server.server: Updating LAN server: server="Node-01a7cc13-9082-ae8b-15cf-8039d6f0f264 (Addr: tcp/127.0.0.1:25091) (DC: dc1)" +33:26.616 [INFO] test-server.server.serf.wan: serf: EventMemberUpdate: Node-01a7cc13-9082-ae8b-15cf-8039d6f0f264.dc1 +33:26.616 [INFO] test-server.server: Handled event for server in area: event=member-update server=Node-01a7cc13-9082-ae8b-15cf-8039d6f0f264.dc1 area=wan +33:26.617 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:26.620 [INFO] test-server.server.connect: initialized primary datacenter CA with provider: provider=consul +33:26.620 [INFO] test-server.leader: started routine: routine="federation state anti-entropy" +33:26.620 [INFO] test-server.leader: started routine: routine="federation state pruning" +33:26.620 [INFO] test-server.leader: started routine: routine="intermediate cert renew watch" +33:26.620 [INFO] test-server.leader: started routine: routine="CA root pruning" +33:26.620 [DEBUG] test-server.server: Skipping self join check for node since the cluster is too small: node=Node-01a7cc13-9082-ae8b-15cf-8039d6f0f264 +33:26.620 [INFO] test-server.server: member joined, marking health alive: member=Node-01a7cc13-9082-ae8b-15cf-8039d6f0f264 +33:26.620 [INFO] test-server.server: federation state anti-entropy synced +33:26.621 [DEBUG] test-server.server: Skipping self join check for node since the cluster is too small: node=Node-01a7cc13-9082-ae8b-15cf-8039d6f0f264 +33:26.938 [DEBUG] test-server: Skipping remote check since it is managed automatically: check=serfHealth +33:26.939 [INFO] test-server: Synced node info +33:27.001 [DEBUG] test-server.acl: dropping node from result due to ACLs: node=Node-01a7cc13-9082-ae8b-15cf-8039d6f0f264 +33:27.049 [DEBUG] test-server.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:52044 +33:27.050 [INFO] test-server.server.serf.lan: serf: EventMemberJoin: test-client 127.0.0.1 +33:27.050 [INFO] test-server.server: member joined, marking health alive: member=test-client +33:27.582 [INFO] test-server: Requesting shutdown +33:27.582 [INFO] test-server.server: shutting down server +33:27.582 [DEBUG] test-server.leader: stopping routine: routine="CA root pruning" +33:27.582 [DEBUG] test-server.leader: stopping routine: routine="legacy ACL token upgrade" +33:27.582 [DEBUG] test-server.leader: stopping routine: routine="acl token reaping" +33:27.582 [DEBUG] test-server.leader: stopping routine: routine="federation state anti-entropy" +33:27.582 [DEBUG] test-server.leader: stopping routine: routine="federation state pruning" +33:27.582 [DEBUG] test-server.leader: stopping routine: routine="intermediate cert renew watch" +33:27.582 [WARN] test-server.server.serf.lan: serf: Shutdown without a Leave +33:27.582 [DEBUG] test-server.leader: stopped routine: routine="CA root pruning" +33:27.582 [DEBUG] test-server.leader: stopped routine: routine="federation state pruning" +33:27.582 [DEBUG] test-server.leader: stopped routine: routine="legacy ACL token upgrade" +33:27.582 [DEBUG] test-server.leader: stopped routine: routine="acl token reaping" +33:27.582 [INFO] test-server.server: shutting down Connect CA roots monitor +33:27.582 [ERROR] test-server.server: error performing anti-entropy sync of federation state: error="context canceled" +33:27.582 [DEBUG] test-server.leader: stopped routine: routine="federation state anti-entropy" +33:27.582 [DEBUG] test-server.leader: stopped routine: routine="intermediate cert renew watch" +33:27.582 [WARN] test-server.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:27.582Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:27.582Z [INFO] agent.router.manager: shutting down +33:27.583 [INFO] test-server: consul server down +33:27.583 [INFO] test-server: shutdown complete +33:27.583 [INFO] test-server: Stopping server: protocol=DNS address=127.0.0.1:25086 network=tcp +33:27.583 [INFO] test-server: Stopping server: protocol=DNS address=127.0.0.1:25086 network=udp +33:27.583 [INFO] test-server: Stopping server: protocol=HTTP address=127.0.0.1:25087 network=tcp +33:28.063 [DEBUG] test-server.server.memberlist.lan: memberlist: Failed ping: test-client (timeout reached) +33:28.083 [INFO] test-server: Waiting for endpoints to shut down +33:28.083 [INFO] test-server: Endpoints down +--- PASS: TestAgent_AutoEncrypt (1.56s) === RUN TestSharedRPCRouter [WARN] freeport: 6 out of 20 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -41:46.378 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-7313686c-e594-6505-40ff-5c3fb3603b05 127.0.0.1 -2022-09-15T21:41:46.378Z [INFO] agent.router: Initializing LAN area manager -41:46.379 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13876 network=udp -41:46.379 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13876 network=tcp -41:46.380 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13877 network=tcp -41:46.380 [INFO] TestAgent: started state syncer -2022-09-15T21:41:46.381Z [WARN] agent.router.manager: No servers available -41:46.381 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13882 network=tcp -41:46.381 [INFO] TestAgent: Retry join is supported for the following discovery methods: cluster=LAN discovery_methods="aliyun aws gce mdns os packet" -41:46.381 [INFO] TestAgent: Joining cluster...: cluster=LAN -41:46.381 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:13872] -2022-09-15T21:41:46.381Z [WARN] agent.router.manager: No servers available -41:46.381 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" -41:46.381 [DEBUG] TestAgent.client.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:13872 -41:46.383 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-f14f5cae-2298-0f98-d4cd-7f4494cbf8ee 127.0.0.1 -41:46.383 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 -41:46.383 [DEBUG] TestAgent: systemd notify failed: error="No socket" -41:46.383 [INFO] TestAgent: Join cluster completed. Synced with initial agents: cluster=LAN num_agents=1 -41:46.383 [INFO] TestAgent.client: adding server: server="Node-f14f5cae-2298-0f98-d4cd-7f4494cbf8ee (Addr: tcp/127.0.0.1:13874) (DC: dc1)" -2022-09-15T21:41:46.215Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:46.236Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:46.239 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f14f5cae-2298-0f98-d4cd-7f4494cbf8ee Address:127.0.0.1:13874}]" -41:46.240 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f14f5cae-2298-0f98-d4cd-7f4494cbf8ee.dc1 127.0.0.1 -41:46.241 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f14f5cae-2298-0f98-d4cd-7f4494cbf8ee 127.0.0.1 -2022-09-15T21:41:46.241Z [INFO] agent.router: Initializing LAN area manager -41:46.241 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13869 network=udp -41:46.241 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13874 [Follower]" leader= -41:46.241 [INFO] TestAgent.server: Adding LAN server: server="Node-f14f5cae-2298-0f98-d4cd-7f4494cbf8ee (Addr: tcp/127.0.0.1:13874) (DC: dc1)" -41:46.242 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f14f5cae-2298-0f98-d4cd-7f4494cbf8ee.dc1 area=wan -41:46.242 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13869 network=tcp -41:46.243 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13870 network=tcp -41:46.243 [INFO] TestAgent: started state syncer -41:46.243 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13875 network=tcp -41:46.292 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:46.292 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13874 [Candidate]" term=2 -41:46.292 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:46.292 [DEBUG] TestAgent.server.raft: vote granted: from=f14f5cae-2298-0f98-d4cd-7f4494cbf8ee term=2 tally=1 -41:46.292 [INFO] TestAgent.server.raft: election won: tally=1 -41:46.292 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13874 [Leader]" -41:46.292 [INFO] TestAgent.server: cluster leadership acquired -41:46.293 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13874 -41:46.294 [INFO] TestAgent.server: New leader elected: payload=Node-f14f5cae-2298-0f98-d4cd-7f4494cbf8ee -41:46.294 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:46.296 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:46.296 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:46.296 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:46.296 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:46.296 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:46.296 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f14f5cae-2298-0f98-d4cd-7f4494cbf8ee -41:46.296 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f14f5cae-2298-0f98-d4cd-7f4494cbf8ee -41:46.298 [INFO] TestAgent.server: federation state anti-entropy synced -41:46.381 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:58118 -41:46.383 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7313686c-e594-6505-40ff-5c3fb3603b05 127.0.0.1 -41:46.383 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7313686c-e594-6505-40ff-5c3fb3603b05 -41:46.408 [INFO] TestAgent: Requesting shutdown -41:46.408 [INFO] TestAgent.server: shutting down server -41:46.408 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:46.408 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:46.408 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:46.408 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:46.408 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:46.409 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:46.409 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:46.409 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:46.409 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:46.409 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:46.409 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:46.409 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:46.409Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:46.409Z [INFO] agent.router.manager: shutting down -41:46.409 [INFO] TestAgent: consul server down -41:46.409 [INFO] TestAgent: shutdown complete -41:46.409 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13869 network=tcp -41:46.409 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13869 network=udp -41:46.409 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13870 network=tcp -41:46.909 [INFO] TestAgent: Waiting for endpoints to shut down -41:46.909 [INFO] TestAgent: Endpoints down ---- PASS: TestSharedRPCRouter (0.71s) +33:28.445 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-79f63cee-a1ee-ef5d-13d1-ea5787e30d75 127.0.0.1 +2021-08-13T15:33:28.445Z [INFO] agent.router: Initializing LAN area manager +33:28.446 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25107 network=udp +33:28.448 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25107 network=tcp +33:28.449 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25108 network=tcp +33:28.449 [INFO] TestAgent: started state syncer +2021-08-13T15:33:28.450Z [WARN] agent.router.manager: No servers available +33:28.451 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25113 network=tcp +33:28.451 [INFO] TestAgent: Retry join is supported for the following discovery methods: cluster=LAN discovery_methods="aliyun aws gce mdns os packet" +33:28.451 [INFO] TestAgent: Joining cluster...: cluster=LAN +33:28.451 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:25103] +2021-08-13T15:33:28.451Z [WARN] agent.router.manager: No servers available +33:28.451 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" +33:28.452 [DEBUG] TestAgent.client.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:25103 +33:28.455 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-aa7fcf07-97e5-4c89-ac99-ff4bd02c95b3 127.0.0.1 +33:28.455 [INFO] TestAgent.client: adding server: server="Node-aa7fcf07-97e5-4c89-ac99-ff4bd02c95b3 (Addr: tcp/127.0.0.1:25105) (DC: dc1)" +33:28.455 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 +33:28.455 [DEBUG] TestAgent: systemd notify failed: error="No socket" +33:28.455 [INFO] TestAgent: Join cluster completed. Synced with initial agents: cluster=LAN num_agents=1 +2021-08-13T15:33:28.099Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:28.114Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:28.117 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:aa7fcf07-97e5-4c89-ac99-ff4bd02c95b3 Address:127.0.0.1:25105}]" +33:28.117 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25105 [Follower]" leader= +33:28.118 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-aa7fcf07-97e5-4c89-ac99-ff4bd02c95b3.dc1 127.0.0.1 +33:28.119 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-aa7fcf07-97e5-4c89-ac99-ff4bd02c95b3 127.0.0.1 +2021-08-13T15:33:28.119Z [INFO] agent.router: Initializing LAN area manager +33:28.119 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25100 network=udp +33:28.119 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-aa7fcf07-97e5-4c89-ac99-ff4bd02c95b3.dc1 area=wan +33:28.119 [INFO] TestAgent.server: Adding LAN server: server="Node-aa7fcf07-97e5-4c89-ac99-ff4bd02c95b3 (Addr: tcp/127.0.0.1:25105) (DC: dc1)" +33:28.119 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25100 network=tcp +33:28.120 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25101 network=tcp +33:28.120 [INFO] TestAgent: started state syncer +33:28.120 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25106 network=tcp +33:28.161 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:28.161 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25105 [Candidate]" term=2 +33:28.161 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:28.161 [DEBUG] TestAgent.server.raft: vote granted: from=aa7fcf07-97e5-4c89-ac99-ff4bd02c95b3 term=2 tally=1 +33:28.161 [INFO] TestAgent.server.raft: election won: tally=1 +33:28.161 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25105 [Leader]" +33:28.161 [INFO] TestAgent.server: cluster leadership acquired +33:28.161 [INFO] TestAgent.server: New leader elected: payload=Node-aa7fcf07-97e5-4c89-ac99-ff4bd02c95b3 +33:28.162 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25105 +33:28.163 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:28.165 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:28.165 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:28.165 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:28.165 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:28.165 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:28.165 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-aa7fcf07-97e5-4c89-ac99-ff4bd02c95b3 +33:28.165 [INFO] TestAgent.server: member joined, marking health alive: member=Node-aa7fcf07-97e5-4c89-ac99-ff4bd02c95b3 +33:28.166 [INFO] TestAgent.server: federation state anti-entropy synced +33:28.262 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:28.262 [INFO] TestAgent: Synced node info +33:28.262 [DEBUG] TestAgent: Node info in sync +33:28.452 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:49074 +33:28.460 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-79f63cee-a1ee-ef5d-13d1-ea5787e30d75 127.0.0.1 +33:28.460 [INFO] TestAgent.server: member joined, marking health alive: member=Node-79f63cee-a1ee-ef5d-13d1-ea5787e30d75 +33:28.478 [INFO] TestAgent: Requesting shutdown +33:28.478 [INFO] TestAgent.server: shutting down server +33:28.478 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:28.478 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:28.478 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:28.478 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:28.478 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:28.478 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:28.478 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:28.478 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:28.478 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:28.478 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:28.478 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:28.478Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:28.478Z [INFO] agent.router.manager: shutting down +33:28.478 [INFO] TestAgent: consul server down +33:28.478 [INFO] TestAgent: shutdown complete +33:28.478 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25100 network=tcp +33:28.479 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25100 network=udp +33:28.479 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25101 network=tcp +33:28.979 [INFO] TestAgent: Waiting for endpoints to shut down +33:28.979 [INFO] TestAgent: Endpoints down +--- PASS: TestSharedRPCRouter (0.90s) === RUN TestCatalogRegister_Service_InvalidAddress === PAUSE TestCatalogRegister_Service_InvalidAddress === RUN TestCatalogDeregister @@ -13704,176 +13723,170 @@ [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:47.399Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:47.421Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:47.423 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:498995b0-70d6-c88a-4c99-115e8f7e8bb3 Address:127.0.0.1:13895}]" -41:47.424 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13895 [Follower]" leader= -41:47.424 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-498995b0-70d6-c88a-4c99-115e8f7e8bb3.dc2 127.0.0.1 -41:47.425 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-498995b0-70d6-c88a-4c99-115e8f7e8bb3 127.0.0.1 -2022-09-15T21:41:47.425Z [INFO] agent.router: Initializing LAN area manager -41:47.426 [INFO] TestAgent.server: Adding LAN server: server="Node-498995b0-70d6-c88a-4c99-115e8f7e8bb3 (Addr: tcp/127.0.0.1:13895) (DC: dc2)" -41:47.426 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-498995b0-70d6-c88a-4c99-115e8f7e8bb3.dc2 area=wan -41:47.426 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13890 network=udp -41:47.426 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13890 network=tcp -41:47.427 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13891 network=tcp -41:47.427 [INFO] TestAgent: started state syncer -41:47.427 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13896 network=tcp -41:47.427 [INFO] TestAgent: Retry join is supported for the following discovery methods: cluster=WAN discovery_methods="aliyun aws gce mdns os packet" -41:47.427 [INFO] TestAgent: Joining cluster...: cluster=WAN -41:47.427 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:13887] -41:47.427 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:13887 -41:47.429 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-27b79065-9dd7-6516-26ef-8061f07e033c.dc1 127.0.0.1 -41:47.429 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 -41:47.429 [INFO] TestAgent: Join cluster completed. Synced with initial agents: cluster=WAN num_agents=1 -41:47.429 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-27b79065-9dd7-6516-26ef-8061f07e033c.dc1 area=wan -41:47.491 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:47.491 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13895 [Candidate]" term=2 -41:47.491 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:47.491 [DEBUG] TestAgent.server.raft: vote granted: from=498995b0-70d6-c88a-4c99-115e8f7e8bb3 term=2 tally=1 -41:47.491 [INFO] TestAgent.server.raft: election won: tally=1 -41:47.491 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13895 [Leader]" -41:47.492 [INFO] TestAgent.server: cluster leadership acquired -41:47.492 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs, servers in acl datacenter are not yet upgraded: ACLDatacenter=dc1 mode=0 found=true -41:47.493 [INFO] TestAgent.server: New leader elected: payload=Node-498995b0-70d6-c88a-4c99-115e8f7e8bb3 -41:47.495 [DEBUG] connect.ca.consul: consul CA provider configured: id=ad:4a:c6:ab:ef:63:c9:60:1a:51:7f:19:62:e3:e9:d9:0e:76:55:10:6e:74:24:69:28:a1:6c:b8:b9:8f:fd:89 is_primary=false -41:47.501 [INFO] TestAgent.server.connect: received new intermediate certificate from primary datacenter -41:47.501 [INFO] TestAgent.server.connect: updated root certificates from primary datacenter -41:47.501 [INFO] TestAgent.server.connect: initialized secondary datacenter CA with provider: provider=consul -41:47.501 [INFO] TestAgent.leader: started routine: routine="config entry replication" -41:47.501 [INFO] TestAgent.leader: started routine: routine="federation state replication" -41:47.501 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:47.501 [INFO] TestAgent.leader: started routine: routine="secondary CA roots watch" -41:47.501 [INFO] TestAgent.leader: started routine: routine="intention replication" -41:47.501 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:47.501 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:47.501 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-498995b0-70d6-c88a-4c99-115e8f7e8bb3 -41:47.501 [INFO] TestAgent.server: member joined, marking health alive: member=Node-498995b0-70d6-c88a-4c99-115e8f7e8bb3 -41:47.502 [DEBUG] TestAgent.server.connect: starting Connect CA root replication from primary datacenter: primary=dc1 -41:47.502 [DEBUG] TestAgent.server.connect: starting Connect intention replication from primary datacenter: primary=dc1 -41:47.503 [DEBUG] TestAgent.server.replication.config_entry: finished fetching config entries: amount=0 -41:47.503 [DEBUG] TestAgent.server.replication.config_entry: Config Entry replication: local=0 remote=0 -41:47.503 [DEBUG] TestAgent.server.replication.config_entry: Config Entry replication: deletions=0 updates=0 -41:47.503 [DEBUG] TestAgent.server.replication.config_entry: replication completed through remote index: index=1 -41:47.504 [INFO] TestAgent.server.connect: updated root certificates from primary datacenter -41:47.504 [INFO] TestAgent.server: federation state anti-entropy synced -41:47.505 [DEBUG] TestAgent.server: finished fetching remote objects: amount=2 -41:47.505 [DEBUG] TestAgent.server: diffing replication state: local_amount=0 remote_amount=2 -41:47.505 [DEBUG] TestAgent.server: diffed replication state: deletions=0 updates=2 -41:47.505 [DEBUG] TestAgent.server: performing updates: updates=2 -41:47.506 [INFO] TestAgent.server: federation state anti-entropy synced -41:47.515 [DEBUG] TestAgent.server: finished updates -41:47.515 [DEBUG] TestAgent.replication.federation_state: replication completed through remote index: index=14 -41:47.516 [DEBUG] TestAgent.server: finished fetching remote objects: amount=2 -41:47.516 [DEBUG] TestAgent.server: diffing replication state: local_amount=2 remote_amount=2 -41:47.516 [DEBUG] TestAgent.server: diffed replication state: deletions=0 updates=1 -41:47.516 [DEBUG] TestAgent.server: performing updates: updates=1 -41:47.516 [DEBUG] TestAgent.server: finished updates -41:47.516 [DEBUG] TestAgent.replication.federation_state: replication completed through remote index: index=15 -41:47.565 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:47.566 [INFO] TestAgent: Synced node info -41:47.566 [DEBUG] TestAgent: Node info in sync -41:47.683 [INFO] TestAgent: Requesting shutdown -41:47.683 [INFO] TestAgent.server: shutting down server -41:47.683 [DEBUG] TestAgent.leader: stopping routine: routine="secondary CA roots watch" -41:47.683 [DEBUG] TestAgent.leader: stopping routine: routine="intention replication" -41:47.683 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:47.683 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:47.683 [DEBUG] TestAgent.leader: stopping routine: routine="config entry replication" -41:47.683 [DEBUG] TestAgent.leader: stopping routine: routine="federation state replication" -41:47.683 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:47.683 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:47.683 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:47.683 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:47.683 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:47.683 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:47.684 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:47.684Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:47.684Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:47.684Z [INFO] agent.router.manager: shutting down -41:47.685 [INFO] TestAgent: consul server down -41:47.685 [INFO] TestAgent: shutdown complete -41:47.685 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13890 network=tcp -41:47.685 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13890 network=udp -41:47.685 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13891 network=tcp -41:47.685 [ERROR] TestAgent.server.rpc: RPC failed to server in DC: server=127.0.0.1:13888 datacenter=dc1 method=ConnectCA.Roots error="rpc error making call: EOF" -41:47.685 [ERROR] TestAgent.server.connect: CA root replication failed, will retry: routine="secondary CA roots watch" error="Error retrieving the primary datacenter's roots: rpc error making call: EOF" -41:47.685 [DEBUG] TestAgent.leader: stopped routine: routine="secondary CA roots watch" -41:47.685 [ERROR] TestAgent.server.rpc: RPC failed to server in DC: server=127.0.0.1:13888 datacenter=dc1 method=ConfigEntry.ListAll error="rpc error making call: EOF" -41:47.685 [INFO] TestAgent.server.replication.config_entry: stopped replication -41:47.685 [DEBUG] TestAgent.leader: stopped routine: routine="config entry replication" -41:47.685 [ERROR] TestAgent.server.rpc: RPC failed to server in DC: server=127.0.0.1:13888 datacenter=dc1 method=Intention.List error="rpc error making call: EOF" -41:47.685 [ERROR] TestAgent.server.connect: error replicating intentions: routine="intention replication" error="rpc error making call: EOF" -41:47.685 [DEBUG] TestAgent.leader: stopped routine: routine="intention replication" -41:47.685 [ERROR] TestAgent.server.rpc: RPC failed to server in DC: server=127.0.0.1:13888 datacenter=dc1 method=FederationState.List error="rpc error making call: EOF" -41:47.685 [WARN] TestAgent.replication.federation_state: replication error (will retry if still leader): error="failed to retrieve federation states: rpc error making call: EOF" -41:47.685 [INFO] TestAgent.replication.federation_state: stopped replication -41:47.685 [DEBUG] TestAgent.leader: stopped routine: routine="federation state replication" -41:48.185 [INFO] TestAgent: Waiting for endpoints to shut down -41:48.185 [INFO] TestAgent: Endpoints down -2022-09-15T21:41:46.932Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:46.956Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:46.961 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:27b79065-9dd7-6516-26ef-8061f07e033c Address:127.0.0.1:13888}]" -41:46.962 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13888 [Follower]" leader= -41:46.964 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-27b79065-9dd7-6516-26ef-8061f07e033c.dc1 127.0.0.1 -41:46.977 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-27b79065-9dd7-6516-26ef-8061f07e033c 127.0.0.1 -2022-09-15T21:41:46.977Z [INFO] agent.router: Initializing LAN area manager -41:46.978 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13883 network=udp -41:46.978 [INFO] TestAgent.server: Adding LAN server: server="Node-27b79065-9dd7-6516-26ef-8061f07e033c (Addr: tcp/127.0.0.1:13888) (DC: dc1)" -41:46.978 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-27b79065-9dd7-6516-26ef-8061f07e033c.dc1 area=wan -41:46.978 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13883 network=tcp -41:46.979 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13884 network=tcp -41:46.979 [INFO] TestAgent: started state syncer -41:46.979 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13889 network=tcp -41:47.009 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:47.009 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13888 [Candidate]" term=2 -41:47.010 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:47.010 [DEBUG] TestAgent.server.raft: vote granted: from=27b79065-9dd7-6516-26ef-8061f07e033c term=2 tally=1 -41:47.010 [INFO] TestAgent.server.raft: election won: tally=1 -41:47.010 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13888 [Leader]" -41:47.010 [INFO] TestAgent.server: cluster leadership acquired -41:47.010 [INFO] TestAgent.server: New leader elected: payload=Node-27b79065-9dd7-6516-26ef-8061f07e033c -41:47.010 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13888 -41:47.011 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:47.014 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:47.014 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:47.014 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:47.014 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:47.014 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:47.014 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-27b79065-9dd7-6516-26ef-8061f07e033c -41:47.015 [INFO] TestAgent.server: member joined, marking health alive: member=Node-27b79065-9dd7-6516-26ef-8061f07e033c -41:47.016 [INFO] TestAgent.server: federation state anti-entropy synced -41:47.020 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:47.021 [INFO] TestAgent: Synced node info -41:47.427 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:55050 -41:47.428 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-498995b0-70d6-c88a-4c99-115e8f7e8bb3.dc2 127.0.0.1 -41:47.429 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-498995b0-70d6-c88a-4c99-115e8f7e8bb3.dc2 area=wan -41:47.702 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:47.702 [DEBUG] TestAgent: Node info in sync -41:47.702 [DEBUG] TestAgent: Node info in sync -41:48.185 [INFO] TestAgent: Requesting shutdown -41:48.185 [INFO] TestAgent.server: shutting down server -41:48.185 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:48.185 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:48.185 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:48.185 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:48.186 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:48.186 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:48.186 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:48.186 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:48.186 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -41:48.186 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:48.186 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -2022-09-15T21:41:48.186Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:48.186Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:48.186Z [INFO] agent.router.manager: shutting down -41:48.186 [INFO] TestAgent: consul server down -41:48.186 [INFO] TestAgent: shutdown complete -41:48.186 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13883 network=tcp -41:48.186 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13883 network=udp -41:48.186 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13884 network=tcp -41:48.687 [INFO] TestAgent: Waiting for endpoints to shut down -41:48.687 [INFO] TestAgent: Endpoints down ---- PASS: TestConnectCARoots_PEMEncoding (1.78s) +2021-08-13T15:33:29.458Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:29.474Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:29.477 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3146076e-42e5-c8a9-18ac-e73241092f63 Address:127.0.0.1:25126}]" +33:29.478 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3146076e-42e5-c8a9-18ac-e73241092f63.dc2 127.0.0.1 +33:29.479 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3146076e-42e5-c8a9-18ac-e73241092f63 127.0.0.1 +2021-08-13T15:33:29.479Z [INFO] agent.router: Initializing LAN area manager +33:29.479 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25126 [Follower]" leader= +33:29.483 [INFO] TestAgent.server: Adding LAN server: server="Node-3146076e-42e5-c8a9-18ac-e73241092f63 (Addr: tcp/127.0.0.1:25126) (DC: dc2)" +33:29.483 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3146076e-42e5-c8a9-18ac-e73241092f63.dc2 area=wan +33:29.484 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25121 network=udp +33:29.484 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25121 network=tcp +33:29.485 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25122 network=tcp +33:29.485 [INFO] TestAgent: started state syncer +33:29.485 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25127 network=tcp +33:29.485 [INFO] TestAgent: Retry join is supported for the following discovery methods: cluster=WAN discovery_methods="aliyun aws gce mdns os packet" +33:29.485 [INFO] TestAgent: Joining cluster...: cluster=WAN +33:29.485 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:25118] +33:29.485 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:25118 +33:29.487 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fdfa5d26-0857-2d64-8b2b-2bac455b6680.dc1 127.0.0.1 +33:29.487 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 +33:29.487 [INFO] TestAgent: Join cluster completed. Synced with initial agents: cluster=WAN num_agents=1 +33:29.487 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fdfa5d26-0857-2d64-8b2b-2bac455b6680.dc1 area=wan +33:29.532 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:29.532 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25126 [Candidate]" term=2 +33:29.532 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:29.532 [DEBUG] TestAgent.server.raft: vote granted: from=3146076e-42e5-c8a9-18ac-e73241092f63 term=2 tally=1 +33:29.532 [INFO] TestAgent.server.raft: election won: tally=1 +33:29.532 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25126 [Leader]" +33:29.532 [INFO] TestAgent.server: cluster leadership acquired +33:29.532 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs, servers in acl datacenter are not yet upgraded: ACLDatacenter=dc1 mode=0 found=true +33:29.533 [INFO] TestAgent.server: New leader elected: payload=Node-3146076e-42e5-c8a9-18ac-e73241092f63 +33:29.535 [DEBUG] connect.ca.consul: consul CA provider configured: id=ad:4a:c6:ab:ef:63:c9:60:1a:51:7f:19:62:e3:e9:d9:0e:76:55:10:6e:74:24:69:28:a1:6c:b8:b9:8f:fd:89 is_primary=false +33:29.539 [INFO] TestAgent.server.connect: received new intermediate certificate from primary datacenter +33:29.540 [INFO] TestAgent.server.connect: updated root certificates from primary datacenter +33:29.540 [INFO] TestAgent.server.connect: initialized secondary datacenter CA with provider: provider=consul +33:29.540 [INFO] TestAgent.leader: started routine: routine="config entry replication" +33:29.540 [INFO] TestAgent.leader: started routine: routine="federation state replication" +33:29.540 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:29.540 [INFO] TestAgent.leader: started routine: routine="secondary CA roots watch" +33:29.540 [INFO] TestAgent.leader: started routine: routine="intention replication" +33:29.540 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:29.540 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:29.540 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3146076e-42e5-c8a9-18ac-e73241092f63 +33:29.540 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3146076e-42e5-c8a9-18ac-e73241092f63 +33:29.541 [DEBUG] TestAgent.server.connect: starting Connect CA root replication from primary datacenter: primary=dc1 +33:29.541 [DEBUG] TestAgent.server.connect: starting Connect intention replication from primary datacenter: primary=dc1 +33:29.543 [DEBUG] TestAgent.server: finished fetching remote objects: amount=1 +33:29.543 [DEBUG] TestAgent.server: diffing replication state: local_amount=0 remote_amount=1 +33:29.543 [DEBUG] TestAgent.server: diffed replication state: deletions=0 updates=1 +33:29.543 [DEBUG] TestAgent.server: performing updates: updates=1 +33:29.543 [DEBUG] TestAgent.server: finished updates +33:29.543 [DEBUG] TestAgent.replication.federation_state: replication completed through remote index: index=11 +33:29.544 [DEBUG] TestAgent.server.replication.config_entry: finished fetching config entries: amount=0 +33:29.544 [DEBUG] TestAgent.server.replication.config_entry: Config Entry replication: local=0 remote=0 +33:29.544 [DEBUG] TestAgent.server.replication.config_entry: Config Entry replication: deletions=0 updates=0 +33:29.544 [DEBUG] TestAgent.server.replication.config_entry: replication completed through remote index: index=1 +33:29.545 [INFO] TestAgent.server.connect: updated root certificates from primary datacenter +33:29.546 [DEBUG] TestAgent.server: finished fetching remote objects: amount=2 +33:29.546 [DEBUG] TestAgent.server: diffing replication state: local_amount=1 remote_amount=2 +33:29.546 [DEBUG] TestAgent.server: diffed replication state: deletions=0 updates=1 +33:29.546 [DEBUG] TestAgent.server: performing updates: updates=1 +33:29.546 [DEBUG] TestAgent.server: finished updates +33:29.546 [DEBUG] TestAgent.replication.federation_state: replication completed through remote index: index=14 +33:29.546 [INFO] TestAgent.server: federation state anti-entropy synced +33:29.550 [INFO] TestAgent: Requesting shutdown +33:29.550 [INFO] TestAgent.server: shutting down server +33:29.550 [DEBUG] TestAgent.leader: stopping routine: routine="intention replication" +33:29.550 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:29.550 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:29.550 [DEBUG] TestAgent.leader: stopping routine: routine="config entry replication" +33:29.550 [DEBUG] TestAgent.leader: stopping routine: routine="federation state replication" +33:29.550 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:29.550 [DEBUG] TestAgent.leader: stopping routine: routine="secondary CA roots watch" +33:29.550 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:29.550 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:29.550 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:29.550 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:29.550 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:29.550 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:29.551 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:29.551Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:29.551Z [INFO] agent.router.manager: shutting down +33:29.551 [INFO] TestAgent: consul server down +33:29.551 [INFO] TestAgent: shutdown complete +33:29.551 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25121 network=tcp +33:29.551 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25121 network=udp +33:29.551 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25122 network=tcp +33:29.551 [ERROR] TestAgent.server.rpc: RPC failed to server in DC: server=127.0.0.1:25119 datacenter=dc1 method=ConfigEntry.ListAll error="rpc error making call: EOF" +33:29.551 [INFO] TestAgent.server.replication.config_entry: stopped replication +33:29.551 [DEBUG] TestAgent.leader: stopped routine: routine="config entry replication" +33:29.551 [ERROR] TestAgent.server.rpc: RPC failed to server in DC: server=127.0.0.1:25119 datacenter=dc1 method=FederationState.List error="rpc error making call: EOF" +33:29.551 [WARN] TestAgent.replication.federation_state: replication error (will retry if still leader): error="failed to retrieve federation states: rpc error making call: EOF" +33:29.551 [INFO] TestAgent.replication.federation_state: stopped replication +33:29.551 [DEBUG] TestAgent.leader: stopped routine: routine="federation state replication" +33:29.551 [ERROR] TestAgent.server.rpc: RPC failed to server in DC: server=127.0.0.1:25119 datacenter=dc1 method=ConnectCA.Roots error="rpc error making call: EOF" +33:29.551 [ERROR] TestAgent.server.connect: CA root replication failed, will retry: routine="secondary CA roots watch" error="Error retrieving the primary datacenter's roots: rpc error making call: EOF" +33:29.551 [DEBUG] TestAgent.leader: stopped routine: routine="secondary CA roots watch" +33:29.551 [ERROR] TestAgent.server.rpc: RPC failed to server in DC: server=127.0.0.1:25119 datacenter=dc1 method=Intention.List error="rpc error making call: EOF" +33:29.551 [ERROR] TestAgent.server.connect: error replicating intentions: routine="intention replication" error="rpc error making call: EOF" +33:29.551 [DEBUG] TestAgent.leader: stopped routine: routine="intention replication" +2021-08-13T15:33:29.551Z [INFO] agent.router.manager: shutting down +33:30.051 [INFO] TestAgent: Waiting for endpoints to shut down +33:30.051 [INFO] TestAgent: Endpoints down +2021-08-13T15:33:28.997Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:29.013Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:29.015 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fdfa5d26-0857-2d64-8b2b-2bac455b6680 Address:127.0.0.1:25119}]" +33:29.016 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fdfa5d26-0857-2d64-8b2b-2bac455b6680.dc1 127.0.0.1 +33:29.017 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fdfa5d26-0857-2d64-8b2b-2bac455b6680 127.0.0.1 +2021-08-13T15:33:29.017Z [INFO] agent.router: Initializing LAN area manager +33:29.017 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25114 network=udp +33:29.018 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25119 [Follower]" leader= +33:29.018 [INFO] TestAgent.server: Adding LAN server: server="Node-fdfa5d26-0857-2d64-8b2b-2bac455b6680 (Addr: tcp/127.0.0.1:25119) (DC: dc1)" +33:29.018 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fdfa5d26-0857-2d64-8b2b-2bac455b6680.dc1 area=wan +33:29.018 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25114 network=tcp +33:29.019 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25115 network=tcp +33:29.019 [INFO] TestAgent: started state syncer +33:29.019 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25120 network=tcp +33:29.077 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:29.077 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25119 [Candidate]" term=2 +33:29.077 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:29.077 [DEBUG] TestAgent.server.raft: vote granted: from=fdfa5d26-0857-2d64-8b2b-2bac455b6680 term=2 tally=1 +33:29.077 [INFO] TestAgent.server.raft: election won: tally=1 +33:29.077 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25119 [Leader]" +33:29.077 [INFO] TestAgent.server: cluster leadership acquired +33:29.078 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25119 +33:29.079 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:29.081 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:29.081 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:29.081 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:29.081 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:29.081 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:29.081 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fdfa5d26-0857-2d64-8b2b-2bac455b6680 +33:29.081 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fdfa5d26-0857-2d64-8b2b-2bac455b6680 +33:29.082 [INFO] TestAgent.server: New leader elected: payload=Node-fdfa5d26-0857-2d64-8b2b-2bac455b6680 +33:29.082 [INFO] TestAgent.server: federation state anti-entropy synced +33:29.334 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:29.335 [INFO] TestAgent: Synced node info +33:29.486 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:59854 +33:29.486 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3146076e-42e5-c8a9-18ac-e73241092f63.dc2 127.0.0.1 +33:29.486 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3146076e-42e5-c8a9-18ac-e73241092f63.dc2 area=wan +33:30.051 [INFO] TestAgent: Requesting shutdown +33:30.051 [INFO] TestAgent.server: shutting down server +33:30.051 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:30.051 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:30.051 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:30.051 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:30.051 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:30.052 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:30.052 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:30.052 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:30.052 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:30.052 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:30.052 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:30.052Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:30.052Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:30.052Z [INFO] agent.router.manager: shutting down +33:30.052 [INFO] TestAgent: consul server down +33:30.052 [INFO] TestAgent: shutdown complete +33:30.052 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25114 network=tcp +33:30.052 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25114 network=udp +33:30.053 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25115 network=tcp +33:30.553 [INFO] TestAgent: Waiting for endpoints to shut down +33:30.553 [INFO] TestAgent: Endpoints down +--- PASS: TestConnectCARoots_PEMEncoding (1.57s) === RUN TestCoordinate_Disabled_Response === PAUSE TestCoordinate_Disabled_Response === RUN TestCoordinate_Datacenters @@ -13917,317 +13930,314 @@ [WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:48.708Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:48.727Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:48.730 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4d17e08b-ed93-9502-9730-bf0e0d14642b Address:127.0.0.1:13902}]" -41:48.730 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13902 [Follower]" leader= -41:48.731 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4d17e08b-ed93-9502-9730-bf0e0d14642b.dc1 127.0.0.1 -41:48.732 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4d17e08b-ed93-9502-9730-bf0e0d14642b 127.0.0.1 -2022-09-15T21:41:48.732Z [INFO] agent.router: Initializing LAN area manager -41:48.732 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13897 network=udp -41:48.732 [INFO] TestAgent.server: Adding LAN server: server="Node-4d17e08b-ed93-9502-9730-bf0e0d14642b (Addr: tcp/127.0.0.1:13902) (DC: dc1)" -41:48.732 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4d17e08b-ed93-9502-9730-bf0e0d14642b.dc1 area=wan -41:48.733 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13897 network=tcp -41:48.734 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13898 network=tcp -41:48.734 [INFO] TestAgent: started state syncer -41:48.734 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13903 network=tcp -41:48.767 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:48.767 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13902 [Candidate]" term=2 -41:48.768 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:48.768 [DEBUG] TestAgent.server.raft: vote granted: from=4d17e08b-ed93-9502-9730-bf0e0d14642b term=2 tally=1 -41:48.768 [INFO] TestAgent.server.raft: election won: tally=1 -41:48.768 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13902 [Leader]" -41:48.768 [INFO] TestAgent.server: cluster leadership acquired -41:48.768 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13902 -41:48.768 [INFO] TestAgent.server: New leader elected: payload=Node-4d17e08b-ed93-9502-9730-bf0e0d14642b -41:48.770 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:48.774 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:48.774 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:48.774 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:48.774 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:48.774 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:48.774 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4d17e08b-ed93-9502-9730-bf0e0d14642b -41:48.774 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4d17e08b-ed93-9502-9730-bf0e0d14642b -41:48.775 [INFO] TestAgent.server: federation state anti-entropy synced -41:48.790 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:48.790 [INFO] TestAgent: Synced node info -41:48.905 [DEBUG] TestAgent.dns: request served from client: name=google.node.consul. type=TXT class=IN latency=278.04µs client=127.0.0.1:34031 client_network=udp -41:48.908 [INFO] TestAgent: Requesting shutdown -41:48.908 [INFO] TestAgent.server: shutting down server -41:48.908 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:48.909 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:48.909 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:48.909 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:48.909 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:48.909 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:48.909 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:48.909 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:48.909 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:48.909 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:48.909 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:48.909Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:48.909Z [INFO] agent.router.manager: shutting down -41:48.909 [INFO] TestAgent: consul server down -41:48.909 [INFO] TestAgent: shutdown complete -41:48.909 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13897 network=tcp -41:48.909 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13897 network=udp -41:48.909 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13898 network=tcp -41:49.410 [INFO] TestAgent: Waiting for endpoints to shut down -41:49.410 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_NodeLookup_TXT (0.72s) +2021-08-13T15:33:30.571Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:30.610Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:30.613 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:45bc71f1-b28b-40f5-75f0-f50fd4471b15 Address:127.0.0.1:25133}]" +33:30.613 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25133 [Follower]" leader= +33:30.614 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-45bc71f1-b28b-40f5-75f0-f50fd4471b15.dc1 127.0.0.1 +33:30.615 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-45bc71f1-b28b-40f5-75f0-f50fd4471b15 127.0.0.1 +2021-08-13T15:33:30.615Z [INFO] agent.router: Initializing LAN area manager +33:30.615 [INFO] TestAgent.server: Adding LAN server: server="Node-45bc71f1-b28b-40f5-75f0-f50fd4471b15 (Addr: tcp/127.0.0.1:25133) (DC: dc1)" +33:30.615 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-45bc71f1-b28b-40f5-75f0-f50fd4471b15.dc1 area=wan +33:30.616 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25128 network=tcp +33:30.616 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25128 network=udp +33:30.616 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25129 network=tcp +33:30.617 [INFO] TestAgent: started state syncer +33:30.617 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25134 network=tcp +33:30.678 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:30.678 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25133 [Candidate]" term=2 +33:30.679 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:30.679 [DEBUG] TestAgent.server.raft: vote granted: from=45bc71f1-b28b-40f5-75f0-f50fd4471b15 term=2 tally=1 +33:30.679 [INFO] TestAgent.server.raft: election won: tally=1 +33:30.679 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25133 [Leader]" +33:30.679 [INFO] TestAgent.server: cluster leadership acquired +33:30.679 [INFO] TestAgent.server: New leader elected: payload=Node-45bc71f1-b28b-40f5-75f0-f50fd4471b15 +33:30.679 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25133 +33:30.680 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:30.683 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:30.683 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:30.683 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:30.683 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:30.683 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:30.683 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-45bc71f1-b28b-40f5-75f0-f50fd4471b15 +33:30.683 [INFO] TestAgent.server: member joined, marking health alive: member=Node-45bc71f1-b28b-40f5-75f0-f50fd4471b15 +33:30.684 [INFO] TestAgent.server: federation state anti-entropy synced +33:30.780 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:30.780 [INFO] TestAgent: Synced node info +33:30.781 [DEBUG] TestAgent: Node info in sync +33:31.019 [DEBUG] TestAgent.dns: request served from client: name=google.node.consul. type=TXT class=IN latency=247.32µs client=127.0.0.1:52614 client_network=udp +33:31.019 [INFO] TestAgent: Requesting shutdown +33:31.019 [INFO] TestAgent.server: shutting down server +33:31.019 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:31.019 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:31.019 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:31.019 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:31.019 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:31.019 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:31.019 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:31.019 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:31.019 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:31.019 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:31.019 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:31.019Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:31.019Z [INFO] agent.router.manager: shutting down +33:31.019 [INFO] TestAgent: consul server down +33:31.019 [INFO] TestAgent: shutdown complete +33:31.019 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25128 network=tcp +33:31.019 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25128 network=udp +33:31.019 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25129 network=tcp +33:31.520 [INFO] TestAgent: Waiting for endpoints to shut down +33:31.520 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_NodeLookup_TXT (0.97s) === RUN TestDNS_NodeLookup_TXT_DontSuppress [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:49.465Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:49.493Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:49.496 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6e636aed-1089-982d-a1d5-e79e31dab9b2 Address:127.0.0.1:13909}]" -41:49.497 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6e636aed-1089-982d-a1d5-e79e31dab9b2.dc1 127.0.0.1 -41:49.497 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6e636aed-1089-982d-a1d5-e79e31dab9b2 127.0.0.1 -2022-09-15T21:41:49.497Z [INFO] agent.router: Initializing LAN area manager -41:49.498 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13909 [Follower]" leader= -41:49.498 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6e636aed-1089-982d-a1d5-e79e31dab9b2.dc1 area=wan -41:49.498 [INFO] TestAgent.server: Adding LAN server: server="Node-6e636aed-1089-982d-a1d5-e79e31dab9b2 (Addr: tcp/127.0.0.1:13909) (DC: dc1)" -41:49.498 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13904 network=udp -41:49.499 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13904 network=tcp -41:49.499 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13905 network=tcp -41:49.500 [INFO] TestAgent: started state syncer -41:49.500 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13910 network=tcp -41:49.539 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:49.539 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13909 [Candidate]" term=2 -41:49.539 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:49.539 [DEBUG] TestAgent.server.raft: vote granted: from=6e636aed-1089-982d-a1d5-e79e31dab9b2 term=2 tally=1 -41:49.539 [INFO] TestAgent.server.raft: election won: tally=1 -41:49.539 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13909 [Leader]" -41:49.540 [INFO] TestAgent.server: cluster leadership acquired -41:49.540 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13909 -41:49.541 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:49.542 [INFO] TestAgent.server: New leader elected: payload=Node-6e636aed-1089-982d-a1d5-e79e31dab9b2 -41:49.543 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:49.543 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:49.543 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:49.543 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:49.543 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:49.544 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6e636aed-1089-982d-a1d5-e79e31dab9b2 -41:49.544 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6e636aed-1089-982d-a1d5-e79e31dab9b2 -41:49.545 [INFO] TestAgent.server: federation state anti-entropy synced -41:49.913 [DEBUG] TestAgent.dns: request served from client: name=google.node.consul. type=TXT class=IN latency=211.18µs client=127.0.0.1:54518 client_network=udp -41:49.913 [INFO] TestAgent: Requesting shutdown -41:49.913 [INFO] TestAgent.server: shutting down server -41:49.913 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:49.913 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:49.913 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:49.913 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:49.913 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:49.914 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:49.914 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:49.914 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:49.914 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:49.914 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:49.914 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:49.914 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:49.914Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:49.914Z [INFO] agent.router.manager: shutting down -41:49.914 [INFO] TestAgent: consul server down -41:49.914 [INFO] TestAgent: shutdown complete -41:49.914 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13904 network=tcp -41:49.914 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13904 network=udp -41:49.914 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13905 network=tcp -41:50.414 [INFO] TestAgent: Waiting for endpoints to shut down -41:50.414 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_NodeLookup_TXT_DontSuppress (1.00s) +2021-08-13T15:33:31.538Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:31.558Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:31.561 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:49f83a7b-2d90-c034-df50-3a49b840b61e Address:127.0.0.1:25140}]" +33:31.562 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-49f83a7b-2d90-c034-df50-3a49b840b61e.dc1 127.0.0.1 +33:31.562 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-49f83a7b-2d90-c034-df50-3a49b840b61e 127.0.0.1 +2021-08-13T15:33:31.562Z [INFO] agent.router: Initializing LAN area manager +33:31.563 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25135 network=udp +33:31.563 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25135 network=tcp +33:31.564 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25136 network=tcp +33:31.564 [INFO] TestAgent: started state syncer +33:31.563 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25140 [Follower]" leader= +33:31.563 [INFO] TestAgent.server: Adding LAN server: server="Node-49f83a7b-2d90-c034-df50-3a49b840b61e (Addr: tcp/127.0.0.1:25140) (DC: dc1)" +33:31.564 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25141 network=tcp +33:31.563 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-49f83a7b-2d90-c034-df50-3a49b840b61e.dc1 area=wan +33:31.613 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:31.613 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25140 [Candidate]" term=2 +33:31.613 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:31.613 [DEBUG] TestAgent.server.raft: vote granted: from=49f83a7b-2d90-c034-df50-3a49b840b61e term=2 tally=1 +33:31.613 [INFO] TestAgent.server.raft: election won: tally=1 +33:31.613 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25140 [Leader]" +33:31.613 [INFO] TestAgent.server: cluster leadership acquired +33:31.614 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25140 +33:31.615 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:31.617 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:31.617 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:31.617 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:31.617 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:31.617 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:31.617 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-49f83a7b-2d90-c034-df50-3a49b840b61e +33:31.617 [INFO] TestAgent.server: member joined, marking health alive: member=Node-49f83a7b-2d90-c034-df50-3a49b840b61e +33:31.618 [INFO] TestAgent.server: New leader elected: payload=Node-49f83a7b-2d90-c034-df50-3a49b840b61e +33:31.618 [INFO] TestAgent.server: federation state anti-entropy synced +33:31.705 [DEBUG] TestAgent.dns: request served from client: name=google.node.consul. type=TXT class=IN latency=248.421µs client=127.0.0.1:36528 client_network=udp +33:31.705 [INFO] TestAgent: Requesting shutdown +33:31.705 [INFO] TestAgent.server: shutting down server +33:31.705 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:31.705 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:31.705 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:31.705 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:31.705 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:31.705 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:31.705 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:31.705 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:31.705 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:31.705 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:31.705 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:31.705 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:31.705Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:31.705Z [INFO] agent.router.manager: shutting down +33:31.705 [INFO] TestAgent: consul server down +33:31.705 [INFO] TestAgent: shutdown complete +33:31.705 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25135 network=tcp +33:31.705 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25135 network=udp +33:31.706 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25136 network=tcp +33:32.206 [INFO] TestAgent: Waiting for endpoints to shut down +33:32.206 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_NodeLookup_TXT_DontSuppress (0.69s) === RUN TestDNS_NodeLookup_ANY [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:50.432Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:50.452Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:50.455 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c63db9bc-43d0-e069-16ab-4498e6798407 Address:127.0.0.1:13916}]" -41:50.456 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c63db9bc-43d0-e069-16ab-4498e6798407.dc1 127.0.0.1 -41:50.457 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13916 [Follower]" leader= -41:50.457 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c63db9bc-43d0-e069-16ab-4498e6798407 127.0.0.1 -2022-09-15T21:41:50.457Z [INFO] agent.router: Initializing LAN area manager -41:50.457 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c63db9bc-43d0-e069-16ab-4498e6798407.dc1 area=wan -41:50.457 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13911 network=udp -41:50.457 [INFO] TestAgent.server: Adding LAN server: server="Node-c63db9bc-43d0-e069-16ab-4498e6798407 (Addr: tcp/127.0.0.1:13916) (DC: dc1)" -41:50.457 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13911 network=tcp -41:50.458 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13912 network=tcp -41:50.458 [INFO] TestAgent: started state syncer -41:50.458 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13917 network=tcp -41:50.521 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:50.521 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13916 [Candidate]" term=2 -41:50.522 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:50.522 [DEBUG] TestAgent.server.raft: vote granted: from=c63db9bc-43d0-e069-16ab-4498e6798407 term=2 tally=1 -41:50.522 [INFO] TestAgent.server.raft: election won: tally=1 -41:50.522 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13916 [Leader]" -41:50.522 [INFO] TestAgent.server: cluster leadership acquired -41:50.522 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13916 -41:50.523 [INFO] TestAgent.server: New leader elected: payload=Node-c63db9bc-43d0-e069-16ab-4498e6798407 -41:50.523 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:50.525 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:50.525 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:50.525 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:50.525 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:50.525 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:50.527 [INFO] TestAgent.server: federation state anti-entropy synced -41:50.527 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c63db9bc-43d0-e069-16ab-4498e6798407 -41:50.527 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c63db9bc-43d0-e069-16ab-4498e6798407 -41:50.732 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:50.733 [INFO] TestAgent: Synced node info -41:50.733 [DEBUG] TestAgent: Node info in sync -41:50.740 [DEBUG] TestAgent.dns: request served from client: name=bar.node.consul. type=ANY class=IN latency=216.4µs client=127.0.0.1:54306 client_network=udp -41:50.740 [INFO] TestAgent: Requesting shutdown -41:50.740 [INFO] TestAgent.server: shutting down server -41:50.740 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:50.740 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:50.740 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:50.740 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:50.740 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:50.741 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:50.741 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:50.741 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:50.741 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:50.741 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:50.741 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:50.741Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:50.741Z [INFO] agent.router.manager: shutting down -41:50.741 [INFO] TestAgent: consul server down -41:50.741 [INFO] TestAgent: shutdown complete -41:50.741 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13911 network=tcp -41:50.741 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13911 network=udp -41:50.741 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13912 network=tcp -41:51.242 [INFO] TestAgent: Waiting for endpoints to shut down -41:51.242 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_NodeLookup_ANY (0.83s) +2021-08-13T15:33:32.222Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:32.238Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:32.240 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fad46bcc-4776-7680-6251-cbcbfd8c6804 Address:127.0.0.1:25147}]" +33:32.241 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fad46bcc-4776-7680-6251-cbcbfd8c6804.dc1 127.0.0.1 +33:32.242 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fad46bcc-4776-7680-6251-cbcbfd8c6804 127.0.0.1 +2021-08-13T15:33:32.242Z [INFO] agent.router: Initializing LAN area manager +33:32.242 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25142 network=udp +33:32.242 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25147 [Follower]" leader= +33:32.243 [INFO] TestAgent.server: Adding LAN server: server="Node-fad46bcc-4776-7680-6251-cbcbfd8c6804 (Addr: tcp/127.0.0.1:25147) (DC: dc1)" +33:32.243 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fad46bcc-4776-7680-6251-cbcbfd8c6804.dc1 area=wan +33:32.243 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25142 network=tcp +33:32.244 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25143 network=tcp +33:32.244 [INFO] TestAgent: started state syncer +33:32.244 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25148 network=tcp +33:32.303 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:32.303 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25147 [Candidate]" term=2 +33:32.303 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:32.303 [DEBUG] TestAgent.server.raft: vote granted: from=fad46bcc-4776-7680-6251-cbcbfd8c6804 term=2 tally=1 +33:32.303 [INFO] TestAgent.server.raft: election won: tally=1 +33:32.303 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25147 [Leader]" +33:32.304 [INFO] TestAgent.server: cluster leadership acquired +33:32.304 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25147 +33:32.305 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:32.307 [INFO] TestAgent.server: New leader elected: payload=Node-fad46bcc-4776-7680-6251-cbcbfd8c6804 +33:32.307 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:32.307 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:32.307 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:32.307 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:32.307 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:32.307 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fad46bcc-4776-7680-6251-cbcbfd8c6804 +33:32.308 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fad46bcc-4776-7680-6251-cbcbfd8c6804 +33:32.309 [INFO] TestAgent.server: federation state anti-entropy synced +33:32.325 [DEBUG] TestAgent.dns: request served from client: name=bar.node.consul. type=ANY class=IN latency=190.821µs client=127.0.0.1:57478 client_network=udp +33:32.325 [INFO] TestAgent: Requesting shutdown +33:32.325 [INFO] TestAgent.server: shutting down server +33:32.325 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:32.325 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:32.325 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:32.325 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:32.325 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:32.325 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:32.325 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:32.325 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:32.325 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:32.325 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:32.325 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:32.325 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:32.325Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:32.326Z [INFO] agent.router.manager: shutting down +33:32.326 [INFO] TestAgent: consul server down +33:32.326 [INFO] TestAgent: shutdown complete +33:32.326 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25142 network=tcp +33:32.326 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25142 network=udp +33:32.326 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25143 network=tcp +33:32.826 [INFO] TestAgent: Waiting for endpoints to shut down +33:32.826 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_NodeLookup_ANY (0.62s) === RUN TestDNS_NodeLookup_ANY_DontSuppressTXT [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:51.266Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:51.286Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:51.289 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:30e1824d-c608-290c-f288-b7f8a1a0aa25 Address:127.0.0.1:13923}]" -41:51.290 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13923 [Follower]" leader= -41:51.290 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-30e1824d-c608-290c-f288-b7f8a1a0aa25.dc1 127.0.0.1 -41:51.291 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-30e1824d-c608-290c-f288-b7f8a1a0aa25 127.0.0.1 -2022-09-15T21:41:51.291Z [INFO] agent.router: Initializing LAN area manager -41:51.292 [INFO] TestAgent.server: Adding LAN server: server="Node-30e1824d-c608-290c-f288-b7f8a1a0aa25 (Addr: tcp/127.0.0.1:13923) (DC: dc1)" -41:51.292 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-30e1824d-c608-290c-f288-b7f8a1a0aa25.dc1 area=wan -41:51.292 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13918 network=udp -41:51.292 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13918 network=tcp -41:51.293 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13919 network=tcp -41:51.293 [INFO] TestAgent: started state syncer -41:51.293 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13924 network=tcp -41:51.338 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:51.338 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13923 [Candidate]" term=2 -41:51.338 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:51.338 [DEBUG] TestAgent.server.raft: vote granted: from=30e1824d-c608-290c-f288-b7f8a1a0aa25 term=2 tally=1 -41:51.338 [INFO] TestAgent.server.raft: election won: tally=1 -41:51.338 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13923 [Leader]" -41:51.338 [INFO] TestAgent.server: cluster leadership acquired -41:51.339 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13923 -41:51.339 [INFO] TestAgent.server: New leader elected: payload=Node-30e1824d-c608-290c-f288-b7f8a1a0aa25 -41:51.340 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:51.343 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:51.343 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:51.343 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:51.343 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:51.343 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:51.343 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-30e1824d-c608-290c-f288-b7f8a1a0aa25 -41:51.343 [INFO] TestAgent.server: member joined, marking health alive: member=Node-30e1824d-c608-290c-f288-b7f8a1a0aa25 -41:51.344 [INFO] TestAgent.server: federation state anti-entropy synced -41:51.473 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:51.474 [INFO] TestAgent: Synced node info -41:51.474 [DEBUG] TestAgent: Node info in sync -41:51.614 [DEBUG] TestAgent.dns: request served from client: name=bar.node.consul. type=ANY class=IN latency=391.44µs client=127.0.0.1:53020 client_network=udp -41:51.614 [INFO] TestAgent: Requesting shutdown -41:51.614 [INFO] TestAgent.server: shutting down server -41:51.614 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:51.614 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:51.614 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:51.614 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:51.614 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:51.614 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:51.614 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:51.614 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:51.614 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:51.614 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:51.615 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:51.615Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:51.615Z [INFO] agent.router.manager: shutting down -41:51.615 [INFO] TestAgent: consul server down -41:51.615 [INFO] TestAgent: shutdown complete -41:51.615 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13918 network=tcp -41:51.615 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13918 network=udp -41:51.615 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13919 network=tcp -41:52.116 [INFO] TestAgent: Waiting for endpoints to shut down -41:52.116 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_NodeLookup_ANY_DontSuppressTXT (0.87s) +2021-08-13T15:33:32.843Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:32.891Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:32.894 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:41acd356-7f97-8d0f-563d-8001fce96c70 Address:127.0.0.1:25154}]" +33:32.894 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25154 [Follower]" leader= +33:32.896 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-41acd356-7f97-8d0f-563d-8001fce96c70.dc1 127.0.0.1 +33:32.897 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-41acd356-7f97-8d0f-563d-8001fce96c70 127.0.0.1 +2021-08-13T15:33:32.897Z [INFO] agent.router: Initializing LAN area manager +33:32.897 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25149 network=udp +33:32.897 [INFO] TestAgent.server: Adding LAN server: server="Node-41acd356-7f97-8d0f-563d-8001fce96c70 (Addr: tcp/127.0.0.1:25154) (DC: dc1)" +33:32.897 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-41acd356-7f97-8d0f-563d-8001fce96c70.dc1 area=wan +33:32.897 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25149 network=tcp +33:32.899 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25150 network=tcp +33:32.900 [INFO] TestAgent: started state syncer +33:32.900 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25155 network=tcp +33:32.947 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:32.947 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25154 [Candidate]" term=2 +33:32.948 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:32.948 [DEBUG] TestAgent.server.raft: vote granted: from=41acd356-7f97-8d0f-563d-8001fce96c70 term=2 tally=1 +33:32.948 [INFO] TestAgent.server.raft: election won: tally=1 +33:32.948 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25154 [Leader]" +33:32.948 [INFO] TestAgent.server: cluster leadership acquired +33:32.948 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25154 +33:32.949 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:32.951 [INFO] TestAgent.server: New leader elected: payload=Node-41acd356-7f97-8d0f-563d-8001fce96c70 +33:32.952 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:32.952 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:32.952 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:32.952 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:32.952 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:32.952 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-41acd356-7f97-8d0f-563d-8001fce96c70 +33:32.952 [INFO] TestAgent.server: member joined, marking health alive: member=Node-41acd356-7f97-8d0f-563d-8001fce96c70 +33:32.953 [INFO] TestAgent.server: federation state anti-entropy synced +33:33.020 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:33.020 [INFO] TestAgent: Synced node info +33:33.020 [DEBUG] TestAgent: Node info in sync +33:33.280 [DEBUG] TestAgent.dns: request served from client: name=bar.node.consul. type=ANY class=IN latency=196.56µs client=127.0.0.1:38688 client_network=udp +33:33.280 [INFO] TestAgent: Requesting shutdown +33:33.280 [INFO] TestAgent.server: shutting down server +33:33.280 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:33.280 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:33.280 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:33.280 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:33.280 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:33.280 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:33.280 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:33.280 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:33.280 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:33.280 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:33.281 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:33.281Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:33.281Z [INFO] agent.router.manager: shutting down +33:33.281 [INFO] TestAgent: consul server down +33:33.281 [INFO] TestAgent: shutdown complete +33:33.281 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25149 network=tcp +33:33.281 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25149 network=udp +33:33.281 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25150 network=tcp +33:33.781 [INFO] TestAgent: Waiting for endpoints to shut down +33:33.781 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_NodeLookup_ANY_DontSuppressTXT (0.96s) === RUN TestDNS_NodeLookup_A_SuppressTXT [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:52.143Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:52.180Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:52.183 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d429d88c-c972-de32-7926-bdda6dc02d33 Address:127.0.0.1:13930}]" -41:52.183 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13930 [Follower]" leader= -41:52.184 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d429d88c-c972-de32-7926-bdda6dc02d33.dc1 127.0.0.1 -41:52.185 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d429d88c-c972-de32-7926-bdda6dc02d33 127.0.0.1 -2022-09-15T21:41:52.185Z [INFO] agent.router: Initializing LAN area manager -41:52.185 [INFO] TestAgent.server: Adding LAN server: server="Node-d429d88c-c972-de32-7926-bdda6dc02d33 (Addr: tcp/127.0.0.1:13930) (DC: dc1)" -41:52.185 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d429d88c-c972-de32-7926-bdda6dc02d33.dc1 area=wan -41:52.185 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13925 network=udp -41:52.185 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13925 network=tcp -41:52.186 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13926 network=tcp -41:52.187 [INFO] TestAgent: started state syncer -41:52.187 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13931 network=tcp -41:52.219 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:52.219 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13930 [Candidate]" term=2 -41:52.219 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:52.219 [DEBUG] TestAgent.server.raft: vote granted: from=d429d88c-c972-de32-7926-bdda6dc02d33 term=2 tally=1 -41:52.219 [INFO] TestAgent.server.raft: election won: tally=1 -41:52.219 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13930 [Leader]" -41:52.219 [INFO] TestAgent.server: cluster leadership acquired -41:52.219 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13930 -41:52.220 [INFO] TestAgent.server: New leader elected: payload=Node-d429d88c-c972-de32-7926-bdda6dc02d33 -41:52.221 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:52.223 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:52.223 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:52.223 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:52.223 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:52.223 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:52.224 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d429d88c-c972-de32-7926-bdda6dc02d33 -41:52.224 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d429d88c-c972-de32-7926-bdda6dc02d33 -41:52.224 [INFO] TestAgent.server: federation state anti-entropy synced -41:52.320 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:52.320 [INFO] TestAgent: Synced node info -41:52.321 [DEBUG] TestAgent: Node info in sync -41:52.591 [DEBUG] TestAgent.dns: request served from client: name=bar.node.consul. type=A class=IN latency=168.86µs client=127.0.0.1:43948 client_network=udp -41:52.591 [INFO] TestAgent: Requesting shutdown -41:52.591 [INFO] TestAgent.server: shutting down server -41:52.591 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:52.592 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:52.592 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:52.592 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:52.592 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:52.592 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:52.592 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:52.592 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:52.592 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:52.592 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -41:52.592 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -2022-09-15T21:41:52.592Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:52.592Z [INFO] agent.router.manager: shutting down -41:52.592 [INFO] TestAgent: consul server down -41:52.592 [INFO] TestAgent: shutdown complete -41:52.592 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13925 network=tcp -41:52.592 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13925 network=udp -41:52.592 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13926 network=tcp -41:53.092 [INFO] TestAgent: Waiting for endpoints to shut down -41:53.092 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_NodeLookup_A_SuppressTXT (0.98s) +2021-08-13T15:33:33.799Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:33.815Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:33.818 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:75e073f9-92cd-3006-7032-76a887ed89ed Address:127.0.0.1:25161}]" +33:33.823 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25161 [Follower]" leader= +33:33.824 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-75e073f9-92cd-3006-7032-76a887ed89ed.dc1 127.0.0.1 +33:33.825 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-75e073f9-92cd-3006-7032-76a887ed89ed 127.0.0.1 +2021-08-13T15:33:33.825Z [INFO] agent.router: Initializing LAN area manager +33:33.825 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25156 network=udp +33:33.825 [INFO] TestAgent.server: Adding LAN server: server="Node-75e073f9-92cd-3006-7032-76a887ed89ed (Addr: tcp/127.0.0.1:25161) (DC: dc1)" +33:33.825 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-75e073f9-92cd-3006-7032-76a887ed89ed.dc1 area=wan +33:33.826 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25156 network=tcp +33:33.826 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25157 network=tcp +33:33.827 [INFO] TestAgent: started state syncer +33:33.827 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25162 network=tcp +33:33.877 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:33.878 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25161 [Candidate]" term=2 +33:33.878 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:33.878 [DEBUG] TestAgent.server.raft: vote granted: from=75e073f9-92cd-3006-7032-76a887ed89ed term=2 tally=1 +33:33.878 [INFO] TestAgent.server.raft: election won: tally=1 +33:33.878 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25161 [Leader]" +33:33.878 [INFO] TestAgent.server: cluster leadership acquired +33:33.878 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25161 +33:33.879 [INFO] TestAgent.server: New leader elected: payload=Node-75e073f9-92cd-3006-7032-76a887ed89ed +33:33.880 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:33.882 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:33.882 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:33.882 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:33.882 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:33.882 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:33.882 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-75e073f9-92cd-3006-7032-76a887ed89ed +33:33.882 [INFO] TestAgent.server: member joined, marking health alive: member=Node-75e073f9-92cd-3006-7032-76a887ed89ed +33:33.883 [INFO] TestAgent.server: federation state anti-entropy synced +33:34.113 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:34.114 [INFO] TestAgent: Synced node info +33:34.114 [DEBUG] TestAgent: Node info in sync +33:34.183 [DEBUG] TestAgent.dns: request served from client: name=bar.node.consul. type=A class=IN latency=186.54µs client=127.0.0.1:38065 client_network=udp +33:34.183 [INFO] TestAgent: Requesting shutdown +33:34.183 [INFO] TestAgent.server: shutting down server +33:34.183 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:34.183 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:34.183 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:34.183 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:34.183 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:34.183 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:34.183 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:34.183 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:34.183 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:34.183 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:34.184 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:34.184Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:34.184Z [INFO] agent.router.manager: shutting down +33:34.184 [INFO] TestAgent: consul server down +33:34.184 [INFO] TestAgent: shutdown complete +33:34.184 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25156 network=tcp +33:34.184 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25156 network=udp +33:34.184 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25157 network=tcp +33:34.684 [INFO] TestAgent: Waiting for endpoints to shut down +33:34.684 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_NodeLookup_A_SuppressTXT (0.90s) === RUN TestDNS_EDNS0 === PAUSE TestDNS_EDNS0 === RUN TestDNS_EDNS0_ECS @@ -14348,126 +14358,125 @@ [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:53.117Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:53.134Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:53.137 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:78da9514-2f99-e70a-f53a-c314005f3e0f Address:127.0.0.1:13937}]" -41:53.137 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13937 [Follower]" leader= -41:53.138 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-78da9514-2f99-e70a-f53a-c314005f3e0f.dc1 127.0.0.1 -41:53.139 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-78da9514-2f99-e70a-f53a-c314005f3e0f 127.0.0.1 -2022-09-15T21:41:53.139Z [INFO] agent.router: Initializing LAN area manager -41:53.140 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13932 network=udp -41:53.140 [INFO] TestAgent.server: Adding LAN server: server="Node-78da9514-2f99-e70a-f53a-c314005f3e0f (Addr: tcp/127.0.0.1:13937) (DC: dc1)" -41:53.140 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-78da9514-2f99-e70a-f53a-c314005f3e0f.dc1 area=wan -41:53.140 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13932 network=tcp -41:53.141 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13933 network=tcp -41:53.141 [INFO] TestAgent: started state syncer -41:53.141 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13938 network=tcp -41:53.195 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:53.195 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13937 [Candidate]" term=2 -41:53.196 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:53.196 [DEBUG] TestAgent.server.raft: vote granted: from=78da9514-2f99-e70a-f53a-c314005f3e0f term=2 tally=1 -41:53.196 [INFO] TestAgent.server.raft: election won: tally=1 -41:53.196 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13937 [Leader]" -41:53.196 [INFO] TestAgent.server: cluster leadership acquired -41:53.196 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13937 -41:53.198 [INFO] TestAgent.server: New leader elected: payload=Node-78da9514-2f99-e70a-f53a-c314005f3e0f -41:53.198 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:53.201 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:53.201 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:53.201 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:53.201 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:53.201 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:53.201 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-78da9514-2f99-e70a-f53a-c314005f3e0f -41:53.201 [INFO] TestAgent.server: member joined, marking health alive: member=Node-78da9514-2f99-e70a-f53a-c314005f3e0f -41:53.202 [INFO] TestAgent.server: federation state anti-entropy synced -41:53.422 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:53.423 [INFO] TestAgent: Synced node info -41:53.423 [DEBUG] TestAgent: Node info in sync -41:53.585 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=6.1637ms client=127.0.0.1:36159 client_network=udp -41:53.585 [INFO] TestAgent: Requesting shutdown -41:53.585 [INFO] TestAgent.server: shutting down server -41:53.585 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:53.585 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:53.585 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:53.585 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:53.585 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:53.585 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:53.585 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:53.585 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:53.585 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:53.585 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:53.585 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:53.586Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:53.586Z [INFO] agent.router.manager: shutting down -41:53.586 [INFO] TestAgent: consul server down -41:53.586 [INFO] TestAgent: shutdown complete -41:53.586 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13932 network=tcp -41:53.586 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13932 network=udp -41:53.586 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13933 network=tcp -41:54.086 [INFO] TestAgent: Waiting for endpoints to shut down -41:54.086 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_MetaTXT (0.99s) +2021-08-13T15:33:34.706Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:34.724Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:34.727 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5fbc8838-65b6-880b-1e43-47b5eb9a421d Address:127.0.0.1:25168}]" +33:34.727 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25168 [Follower]" leader= +33:34.728 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5fbc8838-65b6-880b-1e43-47b5eb9a421d.dc1 127.0.0.1 +33:34.729 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5fbc8838-65b6-880b-1e43-47b5eb9a421d 127.0.0.1 +2021-08-13T15:33:34.729Z [INFO] agent.router: Initializing LAN area manager +33:34.729 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25163 network=udp +33:34.730 [INFO] TestAgent.server: Adding LAN server: server="Node-5fbc8838-65b6-880b-1e43-47b5eb9a421d (Addr: tcp/127.0.0.1:25168) (DC: dc1)" +33:34.730 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5fbc8838-65b6-880b-1e43-47b5eb9a421d.dc1 area=wan +33:34.730 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25163 network=tcp +33:34.731 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25164 network=tcp +33:34.731 [INFO] TestAgent: started state syncer +33:34.731 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25169 network=tcp +33:34.797 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:34.797 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25168 [Candidate]" term=2 +33:34.797 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:34.797 [DEBUG] TestAgent.server.raft: vote granted: from=5fbc8838-65b6-880b-1e43-47b5eb9a421d term=2 tally=1 +33:34.797 [INFO] TestAgent.server.raft: election won: tally=1 +33:34.797 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25168 [Leader]" +33:34.797 [INFO] TestAgent.server: cluster leadership acquired +33:34.797 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25168 +33:34.799 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:34.799 [INFO] TestAgent.server: New leader elected: payload=Node-5fbc8838-65b6-880b-1e43-47b5eb9a421d +33:34.801 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:34.801 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:34.801 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:34.801 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:34.801 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:34.801 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5fbc8838-65b6-880b-1e43-47b5eb9a421d +33:34.802 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5fbc8838-65b6-880b-1e43-47b5eb9a421d +33:34.803 [INFO] TestAgent.server: federation state anti-entropy synced +33:34.807 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=327.741µs client=127.0.0.1:48304 client_network=udp +33:34.807 [INFO] TestAgent: Requesting shutdown +33:34.807 [INFO] TestAgent.server: shutting down server +33:34.807 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:34.807 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:34.807 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:34.807 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:34.807 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:34.807 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:34.807 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:34.807 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:34.807 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:34.807 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:34.807 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:34.807 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:34.807Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:34.807Z [INFO] agent.router.manager: shutting down +33:34.808 [INFO] TestAgent: consul server down +33:34.808 [INFO] TestAgent: shutdown complete +33:34.808 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25163 network=tcp +33:34.808 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25163 network=udp +33:34.808 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25164 network=tcp +33:35.308 [INFO] TestAgent: Waiting for endpoints to shut down +33:35.308 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup_MetaTXT (0.62s) === RUN TestDNS_ServiceLookup_SuppressTXT [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:54.107Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:54.124Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:54.127 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f006a1ad-5be4-823f-6518-b68a867e9e07 Address:127.0.0.1:13944}]" -41:54.127 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13944 [Follower]" leader= -41:54.128 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f006a1ad-5be4-823f-6518-b68a867e9e07.dc1 127.0.0.1 -41:54.129 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f006a1ad-5be4-823f-6518-b68a867e9e07 127.0.0.1 -2022-09-15T21:41:54.129Z [INFO] agent.router: Initializing LAN area manager -41:54.129 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f006a1ad-5be4-823f-6518-b68a867e9e07.dc1 area=wan -41:54.129 [INFO] TestAgent.server: Adding LAN server: server="Node-f006a1ad-5be4-823f-6518-b68a867e9e07 (Addr: tcp/127.0.0.1:13944) (DC: dc1)" -41:54.129 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13939 network=udp -41:54.129 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13939 network=tcp -41:54.130 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13940 network=tcp -41:54.130 [INFO] TestAgent: started state syncer -41:54.130 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13945 network=tcp -41:54.172 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:54.172 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13944 [Candidate]" term=2 -41:54.173 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:54.173 [DEBUG] TestAgent.server.raft: vote granted: from=f006a1ad-5be4-823f-6518-b68a867e9e07 term=2 tally=1 -41:54.173 [INFO] TestAgent.server.raft: election won: tally=1 -41:54.173 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13944 [Leader]" -41:54.173 [INFO] TestAgent.server: cluster leadership acquired -41:54.173 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13944 -41:54.174 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:54.177 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:54.177 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:54.177 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:54.177 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:54.177 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:54.177 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f006a1ad-5be4-823f-6518-b68a867e9e07 -41:54.177 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f006a1ad-5be4-823f-6518-b68a867e9e07 -41:54.178 [INFO] TestAgent.server: New leader elected: payload=Node-f006a1ad-5be4-823f-6518-b68a867e9e07 -41:54.179 [INFO] TestAgent.server: federation state anti-entropy synced -41:54.252 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=295.38µs client=127.0.0.1:52228 client_network=udp -41:54.252 [INFO] TestAgent: Requesting shutdown -41:54.253 [INFO] TestAgent.server: shutting down server -41:54.253 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:54.253 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:54.253 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:54.253 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:54.253 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:54.253 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:54.253 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:54.253 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:54.253 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:54.253 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:54.253 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:54.253 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:54.253Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:54.253Z [INFO] agent.router.manager: shutting down -41:54.253 [INFO] TestAgent: consul server down -41:54.253 [INFO] TestAgent: shutdown complete -41:54.253 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13939 network=tcp -41:54.253 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13939 network=udp -41:54.253 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13940 network=tcp -41:54.754 [INFO] TestAgent: Waiting for endpoints to shut down -41:54.754 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_SuppressTXT (0.67s) +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:33:35.327Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:35.344Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:35.351 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:99a7280d-0d23-e5c2-c5e7-6259804dc4ab Address:127.0.0.1:25175}]" +33:35.352 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-99a7280d-0d23-e5c2-c5e7-6259804dc4ab.dc1 127.0.0.1 +33:35.353 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-99a7280d-0d23-e5c2-c5e7-6259804dc4ab 127.0.0.1 +2021-08-13T15:33:35.353Z [INFO] agent.router: Initializing LAN area manager +33:35.353 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25170 network=udp +33:35.353 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25175 [Follower]" leader= +33:35.354 [INFO] TestAgent.server: Adding LAN server: server="Node-99a7280d-0d23-e5c2-c5e7-6259804dc4ab (Addr: tcp/127.0.0.1:25175) (DC: dc1)" +33:35.354 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-99a7280d-0d23-e5c2-c5e7-6259804dc4ab.dc1 area=wan +33:35.354 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25170 network=tcp +33:35.355 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25171 network=tcp +33:35.355 [INFO] TestAgent: started state syncer +33:35.355 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25176 network=tcp +33:35.398 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:35.398 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25175 [Candidate]" term=2 +33:35.398 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:35.398 [DEBUG] TestAgent.server.raft: vote granted: from=99a7280d-0d23-e5c2-c5e7-6259804dc4ab term=2 tally=1 +33:35.398 [INFO] TestAgent.server.raft: election won: tally=1 +33:35.398 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25175 [Leader]" +33:35.398 [INFO] TestAgent.server: cluster leadership acquired +33:35.399 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25175 +33:35.399 [INFO] TestAgent.server: New leader elected: payload=Node-99a7280d-0d23-e5c2-c5e7-6259804dc4ab +33:35.400 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:35.412 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:35.412 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:35.412 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:35.412 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:35.412 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:35.412 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-99a7280d-0d23-e5c2-c5e7-6259804dc4ab +33:35.412 [INFO] TestAgent.server: member joined, marking health alive: member=Node-99a7280d-0d23-e5c2-c5e7-6259804dc4ab +33:35.415 [INFO] TestAgent.server: federation state anti-entropy synced +33:35.609 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:35.610 [INFO] TestAgent: Synced node info +33:35.718 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=301.161µs client=127.0.0.1:55002 client_network=udp +33:35.718 [INFO] TestAgent: Requesting shutdown +33:35.718 [INFO] TestAgent.server: shutting down server +33:35.718 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:35.718 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:35.718 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:35.718 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:35.718 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:35.718 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:35.718 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:35.718 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:35.718 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:35.718 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:35.718 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:35.718Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:35.718Z [INFO] agent.router.manager: shutting down +33:35.719 [INFO] TestAgent: consul server down +33:35.719 [INFO] TestAgent: shutdown complete +33:35.719 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25170 network=tcp +33:35.719 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25170 network=udp +33:35.719 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25171 network=tcp +33:36.219 [INFO] TestAgent: Waiting for endpoints to shut down +33:36.219 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup_SuppressTXT (0.91s) === RUN TestDNS_AddressLookup === PAUSE TestDNS_AddressLookup === RUN TestDNS_AddressLookupIPV6 @@ -14620,7 +14629,7 @@ === RUN TestDecodeACLToken/hashes_empty_string === RUN TestDecodeACLToken/hashes_null === RUN TestDecodeACLToken/hashes_numeric_value ---- PASS: TestDecodeACLToken (0.01s) +--- PASS: TestDecodeACLToken (0.00s) --- PASS: TestDecodeACLToken/timestamps_correctly_RFC3339_formatted (0.00s) --- PASS: TestDecodeACLToken/timestamps_incorrectly_formatted_(RFC822) (0.00s) --- PASS: TestDecodeACLToken/timestamps_incorrectly_formatted_(RFC850) (0.00s) @@ -14666,7 +14675,6 @@ === RUN TestDecodeAgentRegisterCheck/malformatted_map === RUN TestDecodeAgentRegisterCheck/not_a_map_(slice) === RUN TestDecodeAgentRegisterCheck/not_a_map_(int) -[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! === RUN TestDecodeAgentRegisterCheck/scriptArgs:_all_set === RUN TestDecodeAgentRegisterCheck/scriptArgs:_first_and_second_set === RUN TestDecodeAgentRegisterCheck/scriptArgs:_first_and_third_set @@ -14691,7 +14699,7 @@ === RUN TestDecodeAgentRegisterCheck/serviceID:_first_set === RUN TestDecodeAgentRegisterCheck/serviceID:_second_set === RUN TestDecodeAgentRegisterCheck/serviceID:_neither_set ---- PASS: TestDecodeAgentRegisterCheck (0.02s) +--- PASS: TestDecodeAgentRegisterCheck (0.00s) --- PASS: TestDecodeAgentRegisterCheck/durations_correctly_formatted (0.00s) --- PASS: TestDecodeAgentRegisterCheck/durations_small,_correctly_formatted (0.00s) --- PASS: TestDecodeAgentRegisterCheck/durations_incorrectly_formatted (0.00s) @@ -14700,7 +14708,7 @@ --- PASS: TestDecodeAgentRegisterCheck/durations_numeric (0.00s) --- PASS: TestDecodeAgentRegisterCheck/durations_negative (0.00s) --- PASS: TestDecodeAgentRegisterCheck/durations_numeric_and_negative (0.00s) - --- PASS: TestDecodeAgentRegisterCheck/filled_in_map (0.01s) + --- PASS: TestDecodeAgentRegisterCheck/filled_in_map (0.00s) --- PASS: TestDecodeAgentRegisterCheck/empty_map (0.00s) --- PASS: TestDecodeAgentRegisterCheck/empty_map#01 (0.00s) --- PASS: TestDecodeAgentRegisterCheck/malformatted_map (0.00s) @@ -14825,7 +14833,7 @@ === RUN TestDecodeAgentRegisterService/serviceID:_first_set === RUN TestDecodeAgentRegisterService/serviceID:_second_set === RUN TestDecodeAgentRegisterService/serviceID:_neither_set ---- PASS: TestDecodeAgentRegisterService (0.03s) +--- PASS: TestDecodeAgentRegisterService (0.01s) --- PASS: TestDecodeAgentRegisterService/translateEnableTagTCs:_both_set (0.00s) --- PASS: TestDecodeAgentRegisterService/translateEnableTagTCs:_first_set (0.00s) --- PASS: TestDecodeAgentRegisterService/translateEnableTagTCs:_second_set (0.00s) @@ -15031,19 +15039,8 @@ --- PASS: TestDecodeTxnConvertOps/durations_negative (0.00s) --- PASS: TestDecodeTxnConvertOps/durations_numeric_and_negative (0.00s) === RUN TestHTTPAPI_MethodNotAllowed_OSS +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/query/xxx/execute -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/query/xxx/execute -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/query/xxx/execute -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/query/xxx/execute -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/query/xxx/execute -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/query/xxx/execute -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/query/xxx/explain -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/query/xxx/explain -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/query/xxx/explain -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/query/xxx/explain -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/query/xxx/explain -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/query/xxx/explain === RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/query === RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/query === RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/query @@ -15056,1958 +15053,1956 @@ === RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/query/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/query/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/query/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/token/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/token/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/token/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/token/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/token/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/token/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/services -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/services -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/services -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/services -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/services -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/services -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/list -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/list -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/list -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/list -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/list -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/list -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/connect/ca/configuration -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/connect/ca/configuration -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/connect/ca/configuration -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/connect/ca/configuration -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/connect/ca/configuration -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/connect/ca/configuration -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/update -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/update -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/update -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/update -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/update -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/update -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/binding-rule -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/binding-rule -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/binding-rule -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/binding-rule -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/binding-rule -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/binding-rule -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/token -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/token -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/token -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/token -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/token -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/token -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/service/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/service/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/service/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/service/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/service/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/service/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/connect/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/connect/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/connect/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/connect/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/connect/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/connect/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/join/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/join/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/join/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/join/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/join/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/join/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/node-services/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/node-services/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/node-services/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/node-services/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/node-services/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/node-services/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/config/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/config/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/config/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/config/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/config/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/config/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/ingress/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/ingress/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/ingress/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/ingress/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/ingress/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/ingress/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/snapshot -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/snapshot -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/snapshot -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/snapshot -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/snapshot -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/snapshot -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/clone/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/clone/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/clone/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/clone/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/clone/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/clone/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/policy/name/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/policy/name/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/policy/name/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/policy/name/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/policy/name/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/policy/name/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/query/xxx/execute +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/query/xxx/execute +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/query/xxx/execute +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/query/xxx/execute +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/query/xxx/execute +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/query/xxx/execute +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/query/xxx/explain +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/query/xxx/explain +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/query/xxx/explain +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/query/xxx/explain +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/query/xxx/explain +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/query/xxx/explain +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/update/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/update/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/update/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/update/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/update/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/update/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/connect/intentions/match +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/connect/intentions/match +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/connect/intentions/match +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/connect/intentions/match +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/connect/intentions/match +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/connect/intentions/match +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/checks/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/checks/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/checks/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/checks/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/checks/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/checks/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/state/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/state/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/state/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/state/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/state/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/state/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/roles +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/roles +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/roles +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/roles +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/roles +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/roles +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/metrics +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/metrics +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/metrics +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/metrics +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/metrics +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/metrics +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/leave +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/leave +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/leave +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/leave +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/leave +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/leave === RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/role === RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/role === RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/role === RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/role === RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/role === RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/role -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/maintenance -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/maintenance -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/maintenance -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/maintenance -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/maintenance -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/maintenance -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/ui/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/ui/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/ui/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/ui/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/ui/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/ui/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/destroy/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/destroy/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/destroy/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/destroy/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/destroy/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/destroy/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/warn/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/warn/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/warn/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/warn/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/warn/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/warn/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/node/ -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/ui/gateway-services-nodes/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/ui/gateway-services-nodes/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/ui/gateway-services-nodes/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/ui/gateway-services-nodes/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/ui/gateway-services-nodes/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/ui/gateway-services-nodes/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/kv/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/kv/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/kv/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/kv/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/kv/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/kv/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/checks -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/checks -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/checks -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/checks -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/checks -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/checks +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/service/register +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/service/register +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/service/register +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/service/register +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/service/register +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/service/register === RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/connect/intentions === RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/connect/intentions === RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/connect/intentions === RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/connect/intentions === RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/connect/intentions === RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/connect/intentions -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/connect/intentions/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/connect/intentions/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/connect/intentions/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/connect/intentions/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/connect/intentions/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/connect/intentions/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/event/fire/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/event/fire/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/event/fire/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/event/fire/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/event/fire/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/event/fire/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/list -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/list -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/list -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/list -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/list -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/list -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/rules/translate/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/rules/translate/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/rules/translate/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/rules/translate/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/rules/translate/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/rules/translate/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/tokens -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/tokens -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/tokens -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/tokens -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/tokens -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/tokens -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/self -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/self -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/self -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/self -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/self -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/self -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/fail/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/fail/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/fail/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/fail/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/fail/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/fail/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/operator/raft/peer +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/operator/raft/peer +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/operator/raft/peer +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/operator/raft/peer +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/operator/raft/peer +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/operator/raft/peer +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/event/list +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/event/list +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/event/list +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/event/list +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/event/list +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/event/list +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/policies +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/policies +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/policies +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/policies +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/policies +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/policies +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/maintenance +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/maintenance +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/maintenance +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/maintenance +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/maintenance +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/maintenance +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/force-leave/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/force-leave/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/force-leave/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/force-leave/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/force-leave/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/force-leave/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/info/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/info/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/info/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/info/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/info/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/info/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/binding-rules -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/binding-rules -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/binding-rules -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/binding-rules -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/binding-rules -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/binding-rules -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/members -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/members -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/members -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/members -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/members -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/members === RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/bootstrap === RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/bootstrap === RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/bootstrap === RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/bootstrap === RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/bootstrap === RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/bootstrap +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/rules/translate/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/rules/translate/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/rules/translate/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/rules/translate/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/rules/translate/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/rules/translate/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/create +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/create +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/create +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/create +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/create +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/create === RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/info/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/info/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/info/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/info/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/info/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/info/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/token/self -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/token/self -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/token/self -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/token/self -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/token/self -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/token/self +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/coordinate/update +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/coordinate/update +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/coordinate/update +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/coordinate/update +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/coordinate/update +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/coordinate/update +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/operator/keyring +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/operator/keyring +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/operator/keyring +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/operator/keyring +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/operator/keyring +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/operator/keyring +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/service/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/service/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/service/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/service/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/service/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/service/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/connect/ca/roots === RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/connect/ca/roots === RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/connect/ca/roots === RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/connect/ca/roots === RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/connect/ca/roots === RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/connect/ca/roots -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/federation-state/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/federation-state/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/federation-state/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/federation-state/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/federation-state/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/federation-state/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/create -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/create -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/create -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/create -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/create -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/create -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/policy/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/policy/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/policy/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/policy/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/policy/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/policy/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/role/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/role/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/role/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/role/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/role/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/role/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/auth-method -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/auth-method -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/auth-method -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/auth-method -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/auth-method -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/auth-method -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/connect/intentions/check -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/connect/intentions/check -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/connect/intentions/check -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/connect/intentions/check -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/connect/intentions/check -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/connect/intentions/check -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/policies -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/policies -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/policies -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/policies -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/policies -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/policies -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/services -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/services -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/services -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/services -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/services -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/services -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/force-leave/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/force-leave/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/force-leave/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/force-leave/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/force-leave/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/force-leave/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/leave -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/leave -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/leave -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/leave -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/leave -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/leave -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/connect/ca/roots -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/connect/ca/roots -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/connect/ca/roots -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/connect/ca/roots -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/connect/ca/roots -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/connect/ca/roots -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/state/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/state/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/state/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/state/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/state/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/state/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/service/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/service/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/service/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/service/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/service/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/service/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/discovery-chain/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/discovery-chain/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/discovery-chain/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/discovery-chain/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/discovery-chain/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/discovery-chain/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/ui/services -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/ui/services -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/ui/services -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/ui/services -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/ui/services -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/ui/services -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/operator/raft/peer -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/operator/raft/peer -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/operator/raft/peer -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/operator/raft/peer -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/operator/raft/peer -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/operator/raft/peer -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/login -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/login -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/login -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/login -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/login -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/login -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/policy -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/policy -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/policy -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/policy -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/policy -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/policy +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/clone/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/clone/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/clone/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/clone/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/clone/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/clone/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/binding-rule/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/binding-rule/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/binding-rule/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/binding-rule/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/binding-rule/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/binding-rule/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/token/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/token/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/token/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/token/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/token/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/token/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/ui/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/ui/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/ui/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/ui/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/ui/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/ui/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/warn/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/warn/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/warn/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/warn/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/warn/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/warn/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/connect/ca/leaf/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/connect/ca/leaf/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/connect/ca/leaf/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/connect/ca/leaf/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/connect/ca/leaf/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/connect/ca/leaf/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/connect/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/connect/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/connect/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/connect/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/connect/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/connect/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/config === RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/config === RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/config === RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/config === RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/config === RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/config -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/coordinate/nodes -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/coordinate/nodes -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/coordinate/nodes -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/coordinate/nodes -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/coordinate/nodes -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/coordinate/nodes -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/logout -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/logout -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/logout -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/logout -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/logout -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/logout -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/pass/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/pass/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/pass/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/pass/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/pass/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/pass/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/connect/authorize -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/connect/authorize -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/connect/authorize -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/connect/authorize -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/connect/authorize -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/connect/authorize -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/service/maintenance/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/service/maintenance/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/service/maintenance/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/service/maintenance/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/service/maintenance/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/service/maintenance/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/deregister -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/deregister -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/deregister -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/deregister -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/deregister -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/deregister +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/federation-states/mesh-gateways +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/federation-states/mesh-gateways +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/federation-states/mesh-gateways +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/federation-states/mesh-gateways +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/federation-states/mesh-gateways +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/federation-states/mesh-gateways +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/federation-state/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/federation-state/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/federation-state/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/federation-state/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/federation-state/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/federation-state/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/operator/autopilot/configuration +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/operator/autopilot/configuration +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/operator/autopilot/configuration +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/operator/autopilot/configuration +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/operator/autopilot/configuration +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/operator/autopilot/configuration +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/destroy/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/destroy/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/destroy/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/destroy/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/destroy/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/destroy/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/deregister/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/deregister/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/deregister/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/deregister/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/deregister/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/deregister/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/datacenters === RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/datacenters === RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/datacenters === RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/datacenters === RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/datacenters === RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/datacenters -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/connect/intentions/match -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/connect/intentions/match -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/connect/intentions/match -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/connect/intentions/match -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/connect/intentions/match -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/connect/intentions/match -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/status/leader -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/status/leader -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/status/leader -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/status/leader -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/status/leader -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/status/leader -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/service/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/service/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/service/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/service/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/service/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/service/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/register -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/register -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/register -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/register -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/register -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/register -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/checks/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/checks/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/checks/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/checks/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/checks/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/checks/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/rules/translate -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/rules/translate -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/rules/translate -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/rules/translate -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/rules/translate -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/rules/translate -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/host -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/host -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/host -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/host -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/host -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/host -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/metrics -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/metrics -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/metrics -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/metrics -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/metrics -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/metrics -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/register -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/register -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/register -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/register -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/register -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/register -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/event/list -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/event/list -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/event/list -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/event/list -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/event/list -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/event/list +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/ui/services +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/ui/services +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/ui/services +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/ui/services +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/ui/services +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/ui/services +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/list +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/list +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/list +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/list +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/list +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/list +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/role/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/role/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/role/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/role/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/role/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/role/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/binding-rule +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/binding-rule +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/binding-rule +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/binding-rule +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/binding-rule +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/binding-rule +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/token/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/token/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/token/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/token/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/token/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/token/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/members +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/members +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/members +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/members +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/members +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/members +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/pass/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/pass/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/pass/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/pass/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/pass/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/pass/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/service/deregister/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/service/deregister/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/service/deregister/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/service/deregister/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/service/deregister/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/service/deregister/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/gateway-services/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/gateway-services/ +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/gateway-services/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/gateway-services/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/gateway-services/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/gateway-services/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/service/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/service/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/service/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/service/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/service/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/service/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/create +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/create +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/create +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/create +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/create +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/create === RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/replication === RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/replication === RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/replication === RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/replication === RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/replication === RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/replication +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/token/self +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/token/self +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/token/self +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/token/self +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/token/self +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/token/self +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/node-services/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/node-services/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/node-services/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/node-services/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/node-services/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/node-services/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/role/name/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/role/name/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/role/name/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/role/name/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/role/name/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/role/name/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/coordinate/nodes +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/coordinate/nodes +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/coordinate/nodes +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/coordinate/nodes +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/coordinate/nodes +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/coordinate/nodes +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/destroy/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/destroy/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/destroy/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/destroy/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/destroy/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/destroy/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/auth-methods +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/auth-methods +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/auth-methods +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/auth-methods +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/auth-methods +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/auth-methods === RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/health/service/name/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/health/service/name/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/health/service/name/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/health/service/name/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/health/service/name/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/health/service/name/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/federation-states/mesh-gateways -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/federation-states/mesh-gateways -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/federation-states/mesh-gateways -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/federation-states/mesh-gateways -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/federation-states/mesh-gateways -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/federation-states/mesh-gateways -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/acl/authorize -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/acl/authorize -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/acl/authorize -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/acl/authorize -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/acl/authorize -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/acl/authorize -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/roles -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/roles -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/roles -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/roles -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/roles -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/roles -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/auth-method/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/auth-method/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/auth-method/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/auth-method/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/auth-method/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/auth-method/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/service/register -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/service/register -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/service/register -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/service/register -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/service/register -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/service/register -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/service/deregister/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/service/deregister/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/service/deregister/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/service/deregister/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/service/deregister/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/service/deregister/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/federation-states +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/federation-states +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/federation-states +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/federation-states +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/federation-states +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/federation-states +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/connect/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/connect/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/connect/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/connect/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/connect/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/connect/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/policy +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/policy +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/policy +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/policy +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/policy +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/policy +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/policy/name/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/policy/name/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/policy/name/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/policy/name/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/policy/name/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/policy/name/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/services +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/services +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/services +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/services +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/services +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/services +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/connect/authorize +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/connect/authorize +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/connect/authorize +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/connect/authorize +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/connect/authorize +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/connect/authorize === RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/nodes === RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/nodes === RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/nodes === RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/nodes === RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/nodes === RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/nodes -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/operator/keyring -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/operator/keyring -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/operator/keyring -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/operator/keyring -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/operator/keyring -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/operator/keyring +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/renew/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/renew/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/renew/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/renew/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/renew/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/renew/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/txn === RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/txn === RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/txn === RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/txn === RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/txn === RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/txn -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/destroy/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/destroy/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/destroy/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/destroy/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/destroy/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/destroy/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/update/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/update/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/update/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/update/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/update/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/update/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/gateway-services/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/gateway-services/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/gateway-services/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/gateway-services/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/gateway-services/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/gateway-services/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/coordinate/datacenters -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/coordinate/datacenters -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/coordinate/datacenters -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/coordinate/datacenters -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/coordinate/datacenters -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/coordinate/datacenters +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/tokens +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/tokens +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/tokens +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/tokens +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/tokens +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/tokens +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/fail/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/fail/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/fail/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/fail/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/fail/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/fail/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/auth-method/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/auth-method/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/auth-method/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/auth-method/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/auth-method/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/auth-method/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/join/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/join/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/join/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/join/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/join/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/join/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/event/fire/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/event/fire/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/event/fire/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/event/fire/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/event/fire/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/event/fire/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/ui/nodes +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/ui/nodes +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/ui/nodes +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/ui/nodes +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/ui/nodes +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/ui/nodes +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/rules/translate +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/rules/translate +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/rules/translate +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/rules/translate +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/rules/translate +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/rules/translate +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/ui/gateway-services-nodes/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/ui/gateway-services-nodes/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/ui/gateway-services-nodes/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/ui/gateway-services-nodes/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/ui/gateway-services-nodes/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/ui/gateway-services-nodes/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/services +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/services +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/services +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/services +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/services +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/services +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/ingress/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/ingress/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/ingress/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/ingress/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/ingress/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/ingress/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/operator/autopilot/health +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/operator/autopilot/health +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/operator/autopilot/health +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/operator/autopilot/health +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/operator/autopilot/health +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/operator/autopilot/health +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/status/leader +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/status/leader +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/status/leader +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/status/leader +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/status/leader +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/status/leader +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/logout +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/logout +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/logout +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/logout +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/logout +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/logout +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/host +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/host +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/host +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/host +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/host +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/host +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/connect/ca/roots +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/connect/ca/roots +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/connect/ca/roots +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/connect/ca/roots +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/connect/ca/roots +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/connect/ca/roots +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/connect/intentions/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/connect/intentions/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/connect/intentions/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/connect/intentions/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/connect/intentions/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/connect/intentions/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/operator/raft/configuration === RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/operator/raft/configuration === RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/operator/raft/configuration === RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/operator/raft/configuration === RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/operator/raft/configuration === RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/operator/raft/configuration -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/operator/autopilot/configuration -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/operator/autopilot/configuration -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/operator/autopilot/configuration -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/operator/autopilot/configuration -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/operator/autopilot/configuration -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/operator/autopilot/configuration -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/connect/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/connect/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/connect/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/connect/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/connect/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/connect/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/list +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/list +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/list +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/list +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/list +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/list +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/policy/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/policy/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/policy/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/policy/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/policy/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/policy/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/token +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/token +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/token +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/token +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/token +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/token +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/connect/ca/configuration +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/connect/ca/configuration +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/connect/ca/configuration +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/connect/ca/configuration +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/connect/ca/configuration +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/connect/ca/configuration +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/service/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/service/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/service/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/service/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/service/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/service/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/register +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/register +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/register +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/register +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/register +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/register +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/discovery-chain/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/discovery-chain/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/discovery-chain/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/discovery-chain/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/discovery-chain/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/discovery-chain/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/self +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/self +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/self +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/self +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/self +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/self +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/deregister +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/deregister +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/deregister +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/deregister +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/deregister +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/deregister +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/connect/intentions/check +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/connect/intentions/check +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/connect/intentions/check +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/connect/intentions/check +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/connect/intentions/check +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/connect/intentions/check === RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/coordinate/node/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/coordinate/node/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/coordinate/node/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/coordinate/node/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/coordinate/node/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/coordinate/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/connect/ca/leaf/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/connect/ca/leaf/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/connect/ca/leaf/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/connect/ca/leaf/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/connect/ca/leaf/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/connect/ca/leaf/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/coordinate/update -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/coordinate/update -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/coordinate/update -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/coordinate/update -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/coordinate/update -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/coordinate/update -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/operator/autopilot/health -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/operator/autopilot/health -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/operator/autopilot/health -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/operator/autopilot/health -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/operator/autopilot/health -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/operator/autopilot/health -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/renew/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/renew/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/renew/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/renew/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/renew/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/renew/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/deregister/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/deregister/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/deregister/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/deregister/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/deregister/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/deregister/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/create -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/create -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/create -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/create -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/create -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/create -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/auth-methods -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/auth-methods -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/auth-methods -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/auth-methods -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/auth-methods -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/auth-methods -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/token/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/token/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/token/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/token/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/token/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/token/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/role/name/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/role/name/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/role/name/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/role/name/ -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/role/name/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/role/name/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/kv/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/kv/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/kv/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/kv/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/kv/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/kv/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/binding-rules +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/binding-rules +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/binding-rules +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/binding-rules +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/binding-rules +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/binding-rules +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/auth-method +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/auth-method +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/auth-method +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/auth-method +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/auth-method +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/auth-method +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/checks +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/checks +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/checks +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/checks +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/checks +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/checks +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/coordinate/datacenters +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/coordinate/datacenters +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/coordinate/datacenters +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/coordinate/datacenters +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/coordinate/datacenters +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/coordinate/datacenters +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/login +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/login +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/login +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/login +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/login +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/login === RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/health/service/id/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/health/service/id/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/health/service/id/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/health/service/id/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/health/service/id/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/health/service/id/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/ui/nodes -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/ui/nodes -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/ui/nodes -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/ui/nodes -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/ui/nodes -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/ui/nodes -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/federation-states -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/federation-states -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/federation-states -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/federation-states -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/federation-states -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/federation-states +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/acl/authorize +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/acl/authorize +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/acl/authorize +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/acl/authorize +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/acl/authorize +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/acl/authorize +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/update +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/update +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/update +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/update +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/update +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/update +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/register +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/register +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/register +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/register +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/register +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/register +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/service/maintenance/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/service/maintenance/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/service/maintenance/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/service/maintenance/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/service/maintenance/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/service/maintenance/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/config/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/config/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/config/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/config/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/config/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/config/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/snapshot +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/snapshot +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/snapshot +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/snapshot +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/snapshot +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/snapshot [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:54.842Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:54.861Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:54.865 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c35ccb5d-53e0-6f1c-9275-1acbe2742bd7 Address:127.0.0.1:13951}]" -41:54.865 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13951 [Follower]" leader= -41:54.866 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c35ccb5d-53e0-6f1c-9275-1acbe2742bd7.dc1 127.0.0.1 -41:54.867 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c35ccb5d-53e0-6f1c-9275-1acbe2742bd7 127.0.0.1 -2022-09-15T21:41:54.867Z [INFO] agent.router: Initializing LAN area manager -41:54.867 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13946 network=udp -41:54.867 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c35ccb5d-53e0-6f1c-9275-1acbe2742bd7.dc1 area=wan -41:54.867 [INFO] TestAgent.server: Adding LAN server: server="Node-c35ccb5d-53e0-6f1c-9275-1acbe2742bd7 (Addr: tcp/127.0.0.1:13951) (DC: dc1)" -41:54.868 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13946 network=tcp -41:54.869 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13947 network=tcp -41:54.869 [INFO] TestAgent: started state syncer -41:54.869 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13952 network=tcp -41:54.910 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:54.910 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13951 [Candidate]" term=2 -41:54.910 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:54.910 [DEBUG] TestAgent.server.raft: vote granted: from=c35ccb5d-53e0-6f1c-9275-1acbe2742bd7 term=2 tally=1 -41:54.910 [INFO] TestAgent.server.raft: election won: tally=1 -41:54.910 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13951 [Leader]" -41:54.910 [INFO] TestAgent.server: cluster leadership acquired -41:54.911 [INFO] TestAgent.server: initializing acls -41:54.911 [INFO] TestAgent.server: Created ACL 'global-management' policy -41:54.911 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -41:54.912 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -41:54.913 [INFO] TestAgent.server: Created ACL anonymous token from configuration -41:54.913 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -41:54.913 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -41:54.913 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-c35ccb5d-53e0-6f1c-9275-1acbe2742bd7 -41:54.913 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-c35ccb5d-53e0-6f1c-9275-1acbe2742bd7.dc1 -41:54.914 [INFO] TestAgent.server: New leader elected: payload=Node-c35ccb5d-53e0-6f1c-9275-1acbe2742bd7 -41:54.914 [INFO] TestAgent.server: Updating LAN server: server="Node-c35ccb5d-53e0-6f1c-9275-1acbe2742bd7 (Addr: tcp/127.0.0.1:13951) (DC: dc1)" -41:54.914 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-c35ccb5d-53e0-6f1c-9275-1acbe2742bd7.dc1 area=wan -41:54.919 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:54.921 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:54.921 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:54.921 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:54.921 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:54.921 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:54.921 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c35ccb5d-53e0-6f1c-9275-1acbe2742bd7 -41:54.921 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c35ccb5d-53e0-6f1c-9275-1acbe2742bd7 -41:54.922 [INFO] TestAgent.server: federation state anti-entropy synced -41:54.927 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c35ccb5d-53e0-6f1c-9275-1acbe2742bd7 -41:55.033 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:55.034 [INFO] TestAgent: Synced node info -41:55.055 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-c35ccb5d-53e0-6f1c-9275-1acbe2742bd7 -41:55.058 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/query/xxx/execute from=127.0.0.1:48890 latency=877.42µs -41:55.060 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/query/xxx/execute from=127.0.0.1:48892 error="method PUT not allowed" -41:55.060 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/query/xxx/execute from=127.0.0.1:48892 latency=426.58µs -41:55.061 [ERROR] TestAgent.http: Request error: method=POST url=/v1/query/xxx/execute from=127.0.0.1:48894 error="method POST not allowed" -41:55.062 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/query/xxx/execute from=127.0.0.1:48894 latency=544.34µs -41:55.063 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/query/xxx/execute from=127.0.0.1:48896 error="method DELETE not allowed" -41:55.063 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/query/xxx/execute from=127.0.0.1:48896 latency=202.4µs -41:55.065 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/query/xxx/execute from=127.0.0.1:48898 error="method HEAD not allowed" -41:55.065 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/query/xxx/execute from=127.0.0.1:48898 latency=53.72µs -41:55.065 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/query/xxx/execute from=127.0.0.1:48898 latency=262.4µs -41:55.067 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/query/xxx/explain from=127.0.0.1:48900 latency=425.38µs -41:55.068 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/query/xxx/explain from=127.0.0.1:48902 error="method PUT not allowed" -41:55.068 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/query/xxx/explain from=127.0.0.1:48902 latency=111.18µs -41:55.069 [ERROR] TestAgent.http: Request error: method=POST url=/v1/query/xxx/explain from=127.0.0.1:48904 error="method POST not allowed" -41:55.069 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/query/xxx/explain from=127.0.0.1:48904 latency=97.76µs -41:55.075 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/query/xxx/explain from=127.0.0.1:48906 error="method DELETE not allowed" -41:55.075 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/query/xxx/explain from=127.0.0.1:48906 latency=134.54µs -41:55.077 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/query/xxx/explain from=127.0.0.1:48908 error="method HEAD not allowed" -41:55.077 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/query/xxx/explain from=127.0.0.1:48908 latency=38.08µs -41:55.078 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/query/xxx/explain from=127.0.0.1:48908 latency=84.42µs -41:55.080 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/query from=127.0.0.1:48910 latency=954.02µs -41:55.081 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/query from=127.0.0.1:48912 error="method PUT not allowed" -41:55.081 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/query from=127.0.0.1:48912 latency=98.44µs -41:55.082 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/query from=127.0.0.1:48914 latency=78.08µs -41:55.084 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/query from=127.0.0.1:48916 error="method DELETE not allowed" -41:55.084 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/query from=127.0.0.1:48916 latency=91.44µs -41:55.085 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/query from=127.0.0.1:48918 error="method HEAD not allowed" -41:55.085 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/query from=127.0.0.1:48918 latency=45.46µs -41:55.085 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/query from=127.0.0.1:48918 latency=3.94µs -41:55.086 [ERROR] TestAgent.http: Request error: method=GET url=/v1/query/ from=127.0.0.1:48918 error="failed prepared query lookup: index error: UUID must be 36 characters" -41:55.086 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/query/ from=127.0.0.1:48918 latency=221.26µs -41:55.087 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/query/ from=127.0.0.1:48920 error="Prepared Query lookup failed: failed prepared query lookup: index error: UUID must be 36 characters" -41:55.088 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/query/ from=127.0.0.1:48920 latency=217.18µs -41:55.089 [ERROR] TestAgent.http: Request error: method=POST url=/v1/query/ from=127.0.0.1:48922 error="method POST not allowed" -41:55.089 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/query/ from=127.0.0.1:48922 latency=89.54µs -41:55.090 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/query/ from=127.0.0.1:48924 error="Prepared Query lookup failed: failed prepared query lookup: index error: UUID must be 36 characters" -41:55.090 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/query/ from=127.0.0.1:48924 latency=351.68µs -41:55.091 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/query/ from=127.0.0.1:48926 error="method HEAD not allowed" -41:55.091 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/query/ from=127.0.0.1:48926 latency=51.42µs -41:55.092 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/query/ from=127.0.0.1:48926 latency=67.8µs -41:55.093 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/token/ from=127.0.0.1:48928 error="Bad request: Missing token ID" -41:55.093 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/ from=127.0.0.1:48928 latency=105.96µs -41:55.094 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/token/ from=127.0.0.1:48930 error="Bad request: Token decoding failed: EOF" -41:55.094 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/ from=127.0.0.1:48930 latency=111µs -41:55.096 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/token/ from=127.0.0.1:48932 error="method POST not allowed" -41:55.096 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/token/ from=127.0.0.1:48932 latency=256.02µs -41:55.097 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/token/ from=127.0.0.1:48934 error="Bad request: Missing token ID" -41:55.097 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/token/ from=127.0.0.1:48934 latency=88.96µs -41:55.098 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/token/ from=127.0.0.1:48936 error="method HEAD not allowed" -41:55.098 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/token/ from=127.0.0.1:48936 latency=42.34µs -41:55.099 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/token/ from=127.0.0.1:48936 latency=4.2µs -41:55.099 [DEBUG] TestAgent.acl: dropping service from result due to ACLs: service=consul -41:55.099 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/services from=127.0.0.1:48936 latency=384.42µs -41:55.101 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/catalog/services from=127.0.0.1:48938 error="method PUT not allowed" -41:55.101 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/catalog/services from=127.0.0.1:48938 latency=101.06µs -41:55.104 [ERROR] TestAgent.http: Request error: method=POST url=/v1/catalog/services from=127.0.0.1:48940 error="method POST not allowed" -41:55.104 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/catalog/services from=127.0.0.1:48940 latency=116.7µs -41:55.105 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/catalog/services from=127.0.0.1:48942 error="method DELETE not allowed" -41:55.105 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/catalog/services from=127.0.0.1:48942 latency=87.74µs -41:55.106 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/catalog/services from=127.0.0.1:48944 error="method HEAD not allowed" -41:55.106 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/catalog/services from=127.0.0.1:48944 latency=47.64µs -41:55.108 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/catalog/services from=127.0.0.1:48944 latency=5.82µs -41:55.110 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/list from=127.0.0.1:48944 error="Permission denied" -41:55.110 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/list from=127.0.0.1:48944 latency=303.9µs -41:55.114 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/list from=127.0.0.1:48946 error="method PUT not allowed" -41:55.114 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/list from=127.0.0.1:48946 latency=127.5µs -41:55.116 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/list from=127.0.0.1:48948 error="method POST not allowed" -41:55.116 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/list from=127.0.0.1:48948 latency=122.28µs -41:55.118 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/list from=127.0.0.1:48950 error="method DELETE not allowed" -41:55.118 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/list from=127.0.0.1:48950 latency=110.64µs -41:55.126 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/list from=127.0.0.1:48952 error="method HEAD not allowed" -41:55.127 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/list from=127.0.0.1:48952 latency=71.28µs -41:55.127 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/list from=127.0.0.1:48952 latency=5.94µs -41:55.128 [ERROR] TestAgent.http: Request error: method=GET url=/v1/connect/ca/configuration from=127.0.0.1:48952 error="Permission denied" -41:55.128 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/ca/configuration from=127.0.0.1:48952 latency=248.6µs -41:55.130 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/connect/ca/configuration from=127.0.0.1:48954 error="Bad request: Request decode failed: EOF" -41:55.130 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/connect/ca/configuration from=127.0.0.1:48954 latency=133.88µs -41:55.131 [ERROR] TestAgent.http: Request error: method=POST url=/v1/connect/ca/configuration from=127.0.0.1:48956 error="method POST not allowed" -41:55.131 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/ca/configuration from=127.0.0.1:48956 latency=104.26µs -41:55.132 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/connect/ca/configuration from=127.0.0.1:48958 error="method DELETE not allowed" -41:55.132 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/connect/ca/configuration from=127.0.0.1:48958 latency=95.26µs -41:55.134 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/connect/ca/configuration from=127.0.0.1:48960 error="method HEAD not allowed" -41:55.134 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/connect/ca/configuration from=127.0.0.1:48960 latency=46.24µs -41:55.138 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/connect/ca/configuration from=127.0.0.1:48960 latency=6.08µs -41:55.138 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/update from=127.0.0.1:48960 error="method GET not allowed" -41:55.139 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/update from=127.0.0.1:48960 latency=222.1µs -41:55.140 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/update from=127.0.0.1:48962 latency=65.38µs -41:55.147 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/update from=127.0.0.1:48964 error="method POST not allowed" -41:55.147 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/update from=127.0.0.1:48964 latency=338.56µs -41:55.148 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/update from=127.0.0.1:48966 error="method DELETE not allowed" -41:55.148 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/update from=127.0.0.1:48966 latency=90.62µs -41:55.149 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/update from=127.0.0.1:48968 error="method HEAD not allowed" -41:55.149 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/update from=127.0.0.1:48968 latency=36.84µs -41:55.150 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/update from=127.0.0.1:48968 latency=4.66µs -41:55.150 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/binding-rule from=127.0.0.1:48968 error="method GET not allowed" -41:55.150 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule from=127.0.0.1:48968 latency=89.24µs -41:55.152 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:48970 error="Bad request: BindingRule decoding failed: EOF" -41:55.152 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:48970 latency=111.28µs -41:55.153 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/binding-rule from=127.0.0.1:48972 error="method POST not allowed" -41:55.153 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/binding-rule from=127.0.0.1:48972 latency=91.6µs -41:55.154 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/binding-rule from=127.0.0.1:48974 error="method DELETE not allowed" -41:55.154 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule from=127.0.0.1:48974 latency=87.92µs -41:55.155 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/binding-rule from=127.0.0.1:48976 error="method HEAD not allowed" -41:55.155 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/binding-rule from=127.0.0.1:48976 latency=41.74µs -41:55.155 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/binding-rule from=127.0.0.1:48976 latency=4.3µs -41:55.156 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/token from=127.0.0.1:48976 error="method GET not allowed" -41:55.156 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token from=127.0.0.1:48976 latency=204.98µs -41:55.157 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/token from=127.0.0.1:48978 error="Bad request: Token decoding failed: EOF" -41:55.157 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:48978 latency=108.1µs -41:55.158 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/token from=127.0.0.1:48980 error="method POST not allowed" -41:55.158 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/token from=127.0.0.1:48980 latency=298.28µs -41:55.159 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/token from=127.0.0.1:48982 error="method DELETE not allowed" -41:55.159 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/token from=127.0.0.1:48982 latency=90.66µs -41:55.160 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/token from=127.0.0.1:48984 error="method HEAD not allowed" -41:55.160 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/token from=127.0.0.1:48984 latency=43.36µs -41:55.161 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/token from=127.0.0.1:48984 latency=3.42µs -41:55.161 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/service/ from=127.0.0.1:48984 latency=80.36µs -41:55.162 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/catalog/service/ from=127.0.0.1:48986 error="method PUT not allowed" -41:55.162 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/catalog/service/ from=127.0.0.1:48986 latency=85.08µs -41:55.163 [ERROR] TestAgent.http: Request error: method=POST url=/v1/catalog/service/ from=127.0.0.1:48988 error="method POST not allowed" -41:55.163 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/catalog/service/ from=127.0.0.1:48988 latency=84.2µs -41:55.164 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/catalog/service/ from=127.0.0.1:48990 error="method DELETE not allowed" -41:55.164 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/catalog/service/ from=127.0.0.1:48990 latency=86.1µs -41:55.165 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/catalog/service/ from=127.0.0.1:48992 error="method HEAD not allowed" -41:55.165 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/catalog/service/ from=127.0.0.1:48992 latency=38.44µs -41:55.166 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/catalog/service/ from=127.0.0.1:48992 latency=3.9µs -41:55.166 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/health/connect/ from=127.0.0.1:48992 latency=82.96µs -41:55.167 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/health/connect/ from=127.0.0.1:48994 error="method PUT not allowed" -41:55.167 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/health/connect/ from=127.0.0.1:48994 latency=97.64µs -41:55.168 [ERROR] TestAgent.http: Request error: method=POST url=/v1/health/connect/ from=127.0.0.1:48996 error="method POST not allowed" -41:55.168 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/health/connect/ from=127.0.0.1:48996 latency=88.62µs -41:55.169 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/health/connect/ from=127.0.0.1:48998 error="method DELETE not allowed" -41:55.169 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/health/connect/ from=127.0.0.1:48998 latency=78µs -41:55.170 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/health/connect/ from=127.0.0.1:49000 error="method HEAD not allowed" -41:55.170 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/health/connect/ from=127.0.0.1:49000 latency=37.84µs -41:55.170 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/health/connect/ from=127.0.0.1:49000 latency=4.78µs -41:55.171 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/join/ from=127.0.0.1:49000 error="method GET not allowed" -41:55.171 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/join/ from=127.0.0.1:49000 latency=80.96µs -41:55.172 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/join/ from=127.0.0.1:49002 error="Permission denied" -41:55.172 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/join/ from=127.0.0.1:49002 latency=179.74µs -41:55.173 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/join/ from=127.0.0.1:49004 error="method POST not allowed" -41:55.173 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/join/ from=127.0.0.1:49004 latency=82.48µs -41:55.174 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/join/ from=127.0.0.1:49006 error="method DELETE not allowed" -41:55.174 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/join/ from=127.0.0.1:49006 latency=86.32µs -41:55.175 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/join/ from=127.0.0.1:49008 error="method HEAD not allowed" -41:55.175 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/join/ from=127.0.0.1:49008 latency=44.2µs -41:55.175 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/join/ from=127.0.0.1:49008 latency=4.6µs -41:55.176 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/node-services/ from=127.0.0.1:49008 latency=71.52µs -41:55.176 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/catalog/node-services/ from=127.0.0.1:49010 error="method PUT not allowed" -41:55.177 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/catalog/node-services/ from=127.0.0.1:49010 latency=81.1µs -41:55.177 [ERROR] TestAgent.http: Request error: method=POST url=/v1/catalog/node-services/ from=127.0.0.1:49012 error="method POST not allowed" -41:55.177 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/catalog/node-services/ from=127.0.0.1:49012 latency=73.1µs -41:55.179 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/catalog/node-services/ from=127.0.0.1:49014 error="method DELETE not allowed" -41:55.179 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/catalog/node-services/ from=127.0.0.1:49014 latency=94.78µs -41:55.180 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/catalog/node-services/ from=127.0.0.1:49016 error="method HEAD not allowed" -41:55.180 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/catalog/node-services/ from=127.0.0.1:49016 latency=40.02µs -41:55.180 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/catalog/node-services/ from=127.0.0.1:49016 latency=4.38µs -41:55.181 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ from=127.0.0.1:49016 latency=278.72µs -41:55.182 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/config/ from=127.0.0.1:49018 error="method PUT not allowed" -41:55.182 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config/ from=127.0.0.1:49018 latency=90.1µs -41:55.183 [ERROR] TestAgent.http: Request error: method=POST url=/v1/config/ from=127.0.0.1:49020 error="method POST not allowed" -41:55.183 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/config/ from=127.0.0.1:49020 latency=84.1µs -41:55.184 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/config/ from=127.0.0.1:49022 latency=71.1µs -41:55.185 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/config/ from=127.0.0.1:49024 error="method HEAD not allowed" -41:55.185 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/config/ from=127.0.0.1:49024 latency=87.44µs -41:55.186 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/config/ from=127.0.0.1:49024 latency=4.82µs -41:55.187 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/health/ingress/ from=127.0.0.1:49024 latency=95.58µs -41:55.188 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/health/ingress/ from=127.0.0.1:49026 error="method PUT not allowed" -41:55.188 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/health/ingress/ from=127.0.0.1:49026 latency=86.94µs -41:55.189 [ERROR] TestAgent.http: Request error: method=POST url=/v1/health/ingress/ from=127.0.0.1:49028 error="method POST not allowed" -41:55.189 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/health/ingress/ from=127.0.0.1:49028 latency=78.62µs -41:55.190 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/health/ingress/ from=127.0.0.1:49030 error="method DELETE not allowed" -41:55.190 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/health/ingress/ from=127.0.0.1:49030 latency=110.78µs -41:55.191 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/health/ingress/ from=127.0.0.1:49032 error="method HEAD not allowed" -41:55.191 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/health/ingress/ from=127.0.0.1:49032 latency=44.86µs -41:55.191 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/health/ingress/ from=127.0.0.1:49032 latency=3.94µs -41:55.192 [ERROR] TestAgent.http: Request error: method=GET url=/v1/snapshot from=127.0.0.1:49032 error="Permission denied" -41:55.192 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/snapshot from=127.0.0.1:49032 latency=169.18µs -41:55.193 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/snapshot from=127.0.0.1:49034 error="Permission denied" -41:55.193 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/snapshot from=127.0.0.1:49034 latency=130.12µs -41:55.194 [ERROR] TestAgent.http: Request error: method=POST url=/v1/snapshot from=127.0.0.1:49036 error="method POST not allowed" -41:55.194 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/snapshot from=127.0.0.1:49036 latency=84.42µs -41:55.195 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/snapshot from=127.0.0.1:49038 error="method DELETE not allowed" -41:55.195 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/snapshot from=127.0.0.1:49038 latency=87.92µs -41:55.196 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/snapshot from=127.0.0.1:49040 error="method HEAD not allowed" -41:55.196 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/snapshot from=127.0.0.1:49040 latency=39.34µs -41:55.196 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/snapshot from=127.0.0.1:49040 latency=3.5µs -41:55.197 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/clone/ from=127.0.0.1:49040 error="method GET not allowed" -41:55.197 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/clone/ from=127.0.0.1:49040 latency=83.36µs -41:55.200 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/clone/ from=127.0.0.1:49042 latency=104.4µs -41:55.202 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/clone/ from=127.0.0.1:49044 error="method POST not allowed" -41:55.202 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/clone/ from=127.0.0.1:49044 latency=106.36µs -41:55.205 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/clone/ from=127.0.0.1:49046 error="method DELETE not allowed" -41:55.205 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/clone/ from=127.0.0.1:49046 latency=103.52µs -41:55.209 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/clone/ from=127.0.0.1:49048 error="method HEAD not allowed" -41:55.209 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/clone/ from=127.0.0.1:49048 latency=54.04µs -41:55.209 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/clone/ from=127.0.0.1:49048 latency=3.88µs -41:55.210 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/policy/name/ from=127.0.0.1:49048 error="Bad request: Missing policy Name" -41:55.210 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policy/name/ from=127.0.0.1:49048 latency=94.28µs -41:55.211 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/policy/name/ from=127.0.0.1:49050 error="method PUT not allowed" -41:55.211 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy/name/ from=127.0.0.1:49050 latency=87.66µs -41:55.212 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/policy/name/ from=127.0.0.1:49052 error="method POST not allowed" -41:55.212 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/policy/name/ from=127.0.0.1:49052 latency=83.6µs -41:55.212 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/policy/name/ from=127.0.0.1:49054 error="method DELETE not allowed" -41:55.213 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/policy/name/ from=127.0.0.1:49054 latency=77.08µs -41:55.213 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/policy/name/ from=127.0.0.1:49056 error="method HEAD not allowed" -41:55.213 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/policy/name/ from=127.0.0.1:49056 latency=39.06µs -41:55.214 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/policy/name/ from=127.0.0.1:49056 latency=4.28µs -41:55.215 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/role from=127.0.0.1:49056 error="method GET not allowed" -41:55.215 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role from=127.0.0.1:49056 latency=88.9µs -41:55.216 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/role from=127.0.0.1:49058 error="Bad request: Role decoding failed: EOF" -41:55.216 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:49058 latency=111.28µs -41:55.217 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/role from=127.0.0.1:49060 error="method POST not allowed" -41:55.217 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/role from=127.0.0.1:49060 latency=92.06µs -41:55.218 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/role from=127.0.0.1:49062 error="method DELETE not allowed" -41:55.218 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/role from=127.0.0.1:49062 latency=111.42µs -41:55.219 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/role from=127.0.0.1:49064 error="method HEAD not allowed" -41:55.219 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/role from=127.0.0.1:49064 latency=43.7µs -41:55.220 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/role from=127.0.0.1:49064 latency=5.18µs -41:55.220 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/maintenance from=127.0.0.1:49064 error="method GET not allowed" -41:55.221 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/maintenance from=127.0.0.1:49064 latency=86.72µs -41:55.221 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/maintenance from=127.0.0.1:49066 latency=57.96µs -41:55.223 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/maintenance from=127.0.0.1:49068 error="method POST not allowed" -41:55.223 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/maintenance from=127.0.0.1:49068 latency=99.84µs -41:55.225 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/maintenance from=127.0.0.1:49070 error="method DELETE not allowed" -41:55.225 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/maintenance from=127.0.0.1:49070 latency=96.32µs -41:55.226 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/maintenance from=127.0.0.1:49072 error="method HEAD not allowed" -41:55.226 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/maintenance from=127.0.0.1:49072 latency=47.42µs -41:55.226 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/maintenance from=127.0.0.1:49072 latency=4.46µs -41:55.227 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/internal/ui/node/ from=127.0.0.1:49072 latency=70.46µs -41:55.228 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/internal/ui/node/ from=127.0.0.1:49074 error="method PUT not allowed" -41:55.228 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/internal/ui/node/ from=127.0.0.1:49074 latency=89.12µs -41:55.229 [ERROR] TestAgent.http: Request error: method=POST url=/v1/internal/ui/node/ from=127.0.0.1:49076 error="method POST not allowed" -41:55.229 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/internal/ui/node/ from=127.0.0.1:49076 latency=95.96µs -41:55.230 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/internal/ui/node/ from=127.0.0.1:49078 error="method DELETE not allowed" -41:55.230 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/internal/ui/node/ from=127.0.0.1:49078 latency=98.62µs -41:55.231 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/internal/ui/node/ from=127.0.0.1:49080 error="method HEAD not allowed" -41:55.231 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/internal/ui/node/ from=127.0.0.1:49080 latency=39.48µs -41:55.231 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/internal/ui/node/ from=127.0.0.1:49080 latency=4.34µs -41:55.232 [ERROR] TestAgent.http: Request error: method=GET url=/v1/session/destroy/ from=127.0.0.1:49080 error="method GET not allowed" -41:55.232 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/session/destroy/ from=127.0.0.1:49080 latency=79.4µs -41:55.233 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/ from=127.0.0.1:49082 latency=60.18µs -41:55.234 [ERROR] TestAgent.http: Request error: method=POST url=/v1/session/destroy/ from=127.0.0.1:49084 error="method POST not allowed" -41:55.234 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/session/destroy/ from=127.0.0.1:49084 latency=89.16µs -41:55.235 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/session/destroy/ from=127.0.0.1:49086 error="method DELETE not allowed" -41:55.235 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/session/destroy/ from=127.0.0.1:49086 latency=78.02µs -41:55.236 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/session/destroy/ from=127.0.0.1:49088 error="method HEAD not allowed" -41:55.236 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/session/destroy/ from=127.0.0.1:49088 latency=40.96µs -41:55.236 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/session/destroy/ from=127.0.0.1:49088 latency=4.3µs -41:55.237 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/check/warn/ from=127.0.0.1:49088 error="method GET not allowed" -41:55.237 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/check/warn/ from=127.0.0.1:49088 latency=82.16µs -41:55.238 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/check/warn/ from=127.0.0.1:49090 error="Unknown check """ -41:55.241 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/check/warn/ from=127.0.0.1:49090 latency=3.64502ms -41:55.252 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/check/warn/ from=127.0.0.1:49092 error="method POST not allowed" -41:55.253 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/check/warn/ from=127.0.0.1:49092 latency=153.98µs -41:55.254 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/check/warn/ from=127.0.0.1:49094 error="method DELETE not allowed" -41:55.254 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/check/warn/ from=127.0.0.1:49094 latency=343.3µs -41:55.256 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/check/warn/ from=127.0.0.1:49096 error="method HEAD not allowed" -41:55.256 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/check/warn/ from=127.0.0.1:49096 latency=80.3µs -41:55.258 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/check/warn/ from=127.0.0.1:49096 latency=6.34µs -41:55.259 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/node/ from=127.0.0.1:49096 latency=99.76µs -41:55.260 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/catalog/node/ from=127.0.0.1:49098 error="method PUT not allowed" -41:55.260 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/catalog/node/ from=127.0.0.1:49098 latency=126.68µs -41:55.262 [ERROR] TestAgent.http: Request error: method=POST url=/v1/catalog/node/ from=127.0.0.1:49100 error="method POST not allowed" -41:55.262 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/catalog/node/ from=127.0.0.1:49100 latency=422.68µs -41:55.265 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/catalog/node/ from=127.0.0.1:49102 error="method DELETE not allowed" -41:55.266 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/catalog/node/ from=127.0.0.1:49102 latency=126.02µs -41:55.274 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/catalog/node/ from=127.0.0.1:49104 error="method HEAD not allowed" -41:55.274 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/catalog/node/ from=127.0.0.1:49104 latency=70.74µs -41:55.275 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/catalog/node/ from=127.0.0.1:49104 latency=5.14µs -41:55.275 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/internal/ui/gateway-services-nodes/ from=127.0.0.1:49104 latency=82.22µs -41:55.276 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/internal/ui/gateway-services-nodes/ from=127.0.0.1:49106 error="method PUT not allowed" -41:55.277 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/internal/ui/gateway-services-nodes/ from=127.0.0.1:49106 latency=100.4µs -41:55.278 [ERROR] TestAgent.http: Request error: method=POST url=/v1/internal/ui/gateway-services-nodes/ from=127.0.0.1:49108 error="method POST not allowed" -41:55.278 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/internal/ui/gateway-services-nodes/ from=127.0.0.1:49108 latency=153.44µs -41:55.280 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/internal/ui/gateway-services-nodes/ from=127.0.0.1:49110 error="method DELETE not allowed" -41:55.280 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/internal/ui/gateway-services-nodes/ from=127.0.0.1:49110 latency=94.52µs -41:55.281 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/internal/ui/gateway-services-nodes/ from=127.0.0.1:49112 error="method HEAD not allowed" -41:55.281 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/internal/ui/gateway-services-nodes/ from=127.0.0.1:49112 latency=38.28µs -41:55.281 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/internal/ui/gateway-services-nodes/ from=127.0.0.1:49112 latency=3.92µs -41:55.282 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/ from=127.0.0.1:49112 latency=99.86µs -41:55.283 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/ from=127.0.0.1:49114 latency=66.66µs -41:55.284 [ERROR] TestAgent.http: Request error: method=POST url=/v1/kv/ from=127.0.0.1:49116 error="method POST not allowed" -41:55.284 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/kv/ from=127.0.0.1:49116 latency=94.64µs -41:55.285 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/ from=127.0.0.1:49118 latency=72.12µs -41:55.286 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/kv/ from=127.0.0.1:49120 error="method HEAD not allowed" -41:55.286 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/kv/ from=127.0.0.1:49120 latency=48.54µs -41:55.286 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/kv/ from=127.0.0.1:49120 latency=3.84µs -41:55.287 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/checks from=127.0.0.1:49120 latency=197.72µs -41:55.288 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/checks from=127.0.0.1:49122 error="method PUT not allowed" -41:55.288 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/checks from=127.0.0.1:49122 latency=83.56µs -41:55.289 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/checks from=127.0.0.1:49124 error="method POST not allowed" -41:55.289 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/checks from=127.0.0.1:49124 latency=83.24µs -41:55.290 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/checks from=127.0.0.1:49126 error="method DELETE not allowed" -41:55.290 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/checks from=127.0.0.1:49126 latency=75.5µs -41:55.291 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/checks from=127.0.0.1:49128 error="method HEAD not allowed" -41:55.291 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/checks from=127.0.0.1:49128 latency=33.04µs -41:55.291 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/checks from=127.0.0.1:49128 latency=3.8µs -41:55.292 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:49128 latency=451.88µs -41:55.294 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/connect/intentions from=127.0.0.1:49130 error="method PUT not allowed" -41:55.294 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/connect/intentions from=127.0.0.1:49130 latency=103.94µs -41:55.295 [ERROR] TestAgent.http: Request error: method=POST url=/v1/connect/intentions from=127.0.0.1:49132 error="Failed to decode request body: EOF" -41:55.295 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:49132 latency=99.86µs -41:55.296 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/connect/intentions from=127.0.0.1:49134 error="method DELETE not allowed" -41:55.296 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/connect/intentions from=127.0.0.1:49134 latency=84.66µs -41:55.297 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/connect/intentions from=127.0.0.1:49136 error="method HEAD not allowed" -41:55.297 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/connect/intentions from=127.0.0.1:49136 latency=38.18µs -41:55.297 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/connect/intentions from=127.0.0.1:49136 latency=3.78µs -41:55.298 [ERROR] TestAgent.http: Request error: method=GET url=/v1/connect/intentions/ from=127.0.0.1:49136 error="Bad request: failed intention lookup: index error: UUID must be 36 characters" -41:55.298 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions/ from=127.0.0.1:49136 latency=193.02µs -41:55.299 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/connect/intentions/ from=127.0.0.1:49138 error="Bad request: Request decode failed: EOF" -41:55.299 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/connect/intentions/ from=127.0.0.1:49138 latency=139.1µs -41:55.301 [ERROR] TestAgent.http: Request error: method=POST url=/v1/connect/intentions/ from=127.0.0.1:49140 error="method POST not allowed" -41:55.301 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions/ from=127.0.0.1:49140 latency=91.84µs -41:55.307 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/connect/intentions/ from=127.0.0.1:49142 error="Intention lookup failed: failed intention lookup: index error: UUID must be 36 characters" -41:55.307 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/connect/intentions/ from=127.0.0.1:49142 latency=350.54µs -41:55.308 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/connect/intentions/ from=127.0.0.1:49144 error="method HEAD not allowed" -41:55.308 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/connect/intentions/ from=127.0.0.1:49144 latency=44.28µs -41:55.309 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/connect/intentions/ from=127.0.0.1:49144 latency=3.98µs -41:55.309 [ERROR] TestAgent.http: Request error: method=GET url=/v1/event/fire/ from=127.0.0.1:49144 error="method GET not allowed" -41:55.309 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/event/fire/ from=127.0.0.1:49144 latency=86.88µs -41:55.310 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/event/fire/ from=127.0.0.1:49146 latency=61.78µs -41:55.311 [ERROR] TestAgent.http: Request error: method=POST url=/v1/event/fire/ from=127.0.0.1:49148 error="method POST not allowed" -41:55.311 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/event/fire/ from=127.0.0.1:49148 latency=93.82µs -41:55.312 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/event/fire/ from=127.0.0.1:49150 error="method DELETE not allowed" -41:55.312 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/event/fire/ from=127.0.0.1:49150 latency=89.8µs -41:55.314 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/event/fire/ from=127.0.0.1:49152 error="method HEAD not allowed" -41:55.314 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/event/fire/ from=127.0.0.1:49152 latency=77.98µs -41:55.316 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/event/fire/ from=127.0.0.1:49152 latency=6.84µs -41:55.317 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/session/node/ from=127.0.0.1:49152 latency=94.58µs -41:55.318 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/session/node/ from=127.0.0.1:49154 error="method PUT not allowed" -41:55.319 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/node/ from=127.0.0.1:49154 latency=106.04µs -41:55.319 [ERROR] TestAgent.http: Request error: method=POST url=/v1/session/node/ from=127.0.0.1:49156 error="method POST not allowed" -41:55.320 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/session/node/ from=127.0.0.1:49156 latency=88.94µs -41:55.320 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/session/node/ from=127.0.0.1:49158 error="method DELETE not allowed" -41:55.321 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/session/node/ from=127.0.0.1:49158 latency=97.8µs -41:55.321 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/session/node/ from=127.0.0.1:49160 error="method HEAD not allowed" -41:55.322 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/session/node/ from=127.0.0.1:49160 latency=73.56µs -41:55.322 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/session/node/ from=127.0.0.1:49160 latency=4.66µs -41:55.323 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/session/list from=127.0.0.1:49160 latency=453.88µs -41:55.324 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/session/list from=127.0.0.1:49162 error="method PUT not allowed" -41:55.324 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/list from=127.0.0.1:49162 latency=96.2µs -41:55.325 [ERROR] TestAgent.http: Request error: method=POST url=/v1/session/list from=127.0.0.1:49164 error="method POST not allowed" -41:55.326 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/session/list from=127.0.0.1:49164 latency=327.5µs -41:55.327 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/session/list from=127.0.0.1:49166 error="method DELETE not allowed" -41:55.327 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/session/list from=127.0.0.1:49166 latency=86.9µs -41:55.327 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/session/list from=127.0.0.1:49168 error="method HEAD not allowed" -41:55.328 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/session/list from=127.0.0.1:49168 latency=40.06µs -41:55.328 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/session/list from=127.0.0.1:49168 latency=3.6µs -41:55.328 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/rules/translate/ from=127.0.0.1:49168 error="Bad request: Missing token ID" -41:55.328 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/rules/translate/ from=127.0.0.1:49168 latency=78.76µs -41:55.330 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/rules/translate/ from=127.0.0.1:49170 error="method PUT not allowed" -41:55.330 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/rules/translate/ from=127.0.0.1:49170 latency=86.5µs -41:55.331 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/rules/translate/ from=127.0.0.1:49172 error="method POST not allowed" -41:55.331 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/rules/translate/ from=127.0.0.1:49172 latency=82.24µs -41:55.332 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/rules/translate/ from=127.0.0.1:49174 error="method DELETE not allowed" -41:55.332 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/rules/translate/ from=127.0.0.1:49174 latency=84.46µs -41:55.333 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/rules/translate/ from=127.0.0.1:49176 error="method HEAD not allowed" -41:55.333 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/rules/translate/ from=127.0.0.1:49176 latency=40.94µs -41:55.334 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/rules/translate/ from=127.0.0.1:49176 latency=3.22µs -41:55.334 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/tokens from=127.0.0.1:49176 error="Permission denied" -41:55.334 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/tokens from=127.0.0.1:49176 latency=213.6µs -41:55.335 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/tokens from=127.0.0.1:49178 error="method PUT not allowed" -41:55.335 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/tokens from=127.0.0.1:49178 latency=80µs -41:55.336 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/tokens from=127.0.0.1:49180 error="method POST not allowed" -41:55.336 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/tokens from=127.0.0.1:49180 latency=74.52µs -41:55.337 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/tokens from=127.0.0.1:49182 error="method DELETE not allowed" -41:55.337 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/tokens from=127.0.0.1:49182 latency=92.1µs -41:55.338 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/tokens from=127.0.0.1:49184 error="method HEAD not allowed" -41:55.338 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/tokens from=127.0.0.1:49184 latency=36.56µs -41:55.339 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/tokens from=127.0.0.1:49184 latency=3.52µs -41:55.339 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/self from=127.0.0.1:49184 error="Permission denied" -41:55.339 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:49184 latency=129.76µs -41:55.340 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/self from=127.0.0.1:49186 error="method PUT not allowed" -41:55.340 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/self from=127.0.0.1:49186 latency=89.6µs -41:55.341 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/self from=127.0.0.1:49188 error="method POST not allowed" -41:55.341 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/self from=127.0.0.1:49188 latency=89.46µs -41:55.343 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/self from=127.0.0.1:49190 error="method DELETE not allowed" -41:55.343 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/self from=127.0.0.1:49190 latency=94.4µs -41:55.344 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/self from=127.0.0.1:49192 error="method HEAD not allowed" -41:55.344 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/self from=127.0.0.1:49192 latency=42.78µs -41:55.344 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/self from=127.0.0.1:49192 latency=3.22µs -41:55.345 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/check/fail/ from=127.0.0.1:49192 error="method GET not allowed" -41:55.345 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/check/fail/ from=127.0.0.1:49192 latency=108.56µs -41:55.346 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/check/fail/ from=127.0.0.1:49194 error="Unknown check """ -41:55.346 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/check/fail/ from=127.0.0.1:49194 latency=186.26µs -41:55.347 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/check/fail/ from=127.0.0.1:49196 error="method POST not allowed" -41:55.347 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/check/fail/ from=127.0.0.1:49196 latency=99.12µs -41:55.348 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/check/fail/ from=127.0.0.1:49198 error="method DELETE not allowed" -41:55.348 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/check/fail/ from=127.0.0.1:49198 latency=77.32µs -41:55.349 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/check/fail/ from=127.0.0.1:49200 error="method HEAD not allowed" -41:55.349 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/check/fail/ from=127.0.0.1:49200 latency=40.52µs -41:55.350 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/check/fail/ from=127.0.0.1:49200 latency=3.5µs -41:55.350 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/session/info/ from=127.0.0.1:49200 latency=71.48µs -41:55.351 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/session/info/ from=127.0.0.1:49202 error="method PUT not allowed" -41:55.351 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/info/ from=127.0.0.1:49202 latency=85.9µs -41:55.352 [ERROR] TestAgent.http: Request error: method=POST url=/v1/session/info/ from=127.0.0.1:49204 error="method POST not allowed" -41:55.353 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/session/info/ from=127.0.0.1:49204 latency=88.8µs -41:55.354 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/session/info/ from=127.0.0.1:49206 error="method DELETE not allowed" -41:55.354 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/session/info/ from=127.0.0.1:49206 latency=90.78µs -41:55.355 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/session/info/ from=127.0.0.1:49208 error="method HEAD not allowed" -41:55.355 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/session/info/ from=127.0.0.1:49208 latency=47.68µs -41:55.356 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/session/info/ from=127.0.0.1:49208 latency=3.68µs -41:55.356 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/binding-rules from=127.0.0.1:49208 error="Permission denied" -41:55.356 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:49208 latency=245.98µs -41:55.357 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/binding-rules from=127.0.0.1:49210 error="method PUT not allowed" -41:55.357 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rules from=127.0.0.1:49210 latency=83.34µs -41:55.358 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/binding-rules from=127.0.0.1:49212 error="method POST not allowed" -41:55.358 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/binding-rules from=127.0.0.1:49212 latency=83.82µs -41:55.359 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/binding-rules from=127.0.0.1:49214 error="method DELETE not allowed" -41:55.359 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rules from=127.0.0.1:49214 latency=84µs -41:55.360 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/binding-rules from=127.0.0.1:49216 error="method HEAD not allowed" -41:55.360 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/binding-rules from=127.0.0.1:49216 latency=36.12µs -41:55.361 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/binding-rules from=127.0.0.1:49216 latency=3.76µs -41:55.361 [DEBUG] TestAgent: dropping node from result due to ACLs: node=Node-c35ccb5d-53e0-6f1c-9275-1acbe2742bd7 accessorID=00000000-0000-0000-0000-000000000002 -41:55.361 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/members from=127.0.0.1:49216 latency=360.06µs -41:55.362 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/members from=127.0.0.1:49218 error="method PUT not allowed" -41:55.362 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/members from=127.0.0.1:49218 latency=89.44µs -41:55.363 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/members from=127.0.0.1:49220 error="method POST not allowed" -41:55.363 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/members from=127.0.0.1:49220 latency=84.36µs -41:55.364 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/members from=127.0.0.1:49222 error="method DELETE not allowed" -41:55.364 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/members from=127.0.0.1:49222 latency=81.54µs -41:55.365 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/members from=127.0.0.1:49224 error="method HEAD not allowed" -41:55.365 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/members from=127.0.0.1:49224 latency=35.5µs -41:55.365 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/members from=127.0.0.1:49224 latency=3.94µs -41:55.366 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/bootstrap from=127.0.0.1:49224 error="method GET not allowed" -41:55.366 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/bootstrap from=127.0.0.1:49224 latency=90.22µs -41:55.367 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/bootstrap from=127.0.0.1:49226 latency=212.24µs -41:55.368 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/bootstrap from=127.0.0.1:49228 error="method POST not allowed" -41:55.368 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/bootstrap from=127.0.0.1:49228 latency=86.78µs -41:55.369 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/bootstrap from=127.0.0.1:49230 error="method DELETE not allowed" -41:55.369 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/bootstrap from=127.0.0.1:49230 latency=80.58µs -41:55.370 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/bootstrap from=127.0.0.1:49232 error="method HEAD not allowed" -41:55.370 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/bootstrap from=127.0.0.1:49232 latency=36.8µs -41:55.372 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/bootstrap from=127.0.0.1:49232 latency=5.42µs -41:55.373 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/info/ from=127.0.0.1:49232 latency=77.22µs -41:55.374 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/info/ from=127.0.0.1:49234 error="method PUT not allowed" -41:55.374 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/info/ from=127.0.0.1:49234 latency=120.76µs -41:55.374 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/info/ from=127.0.0.1:49236 error="method POST not allowed" -41:55.374 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/info/ from=127.0.0.1:49236 latency=81.06µs -41:55.375 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/info/ from=127.0.0.1:49238 error="method DELETE not allowed" -41:55.375 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/info/ from=127.0.0.1:49238 latency=82.46µs -41:55.376 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/info/ from=127.0.0.1:49240 error="method HEAD not allowed" -41:55.376 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/info/ from=127.0.0.1:49240 latency=37.16µs -41:55.377 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/info/ from=127.0.0.1:49240 latency=3.48µs -41:55.377 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/token/self from=127.0.0.1:49240 error="ACL not found" -41:55.377 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:49240 latency=216.32µs -41:55.378 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/token/self from=127.0.0.1:49242 error="method PUT not allowed" -41:55.379 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/self from=127.0.0.1:49242 latency=98.6µs -41:55.379 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/token/self from=127.0.0.1:49244 error="method POST not allowed" -41:55.379 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/token/self from=127.0.0.1:49244 latency=80.64µs -41:55.380 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/token/self from=127.0.0.1:49246 error="method DELETE not allowed" -41:55.380 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/token/self from=127.0.0.1:49246 latency=76.42µs -41:55.381 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/token/self from=127.0.0.1:49248 error="method HEAD not allowed" -41:55.381 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/token/self from=127.0.0.1:49248 latency=36.76µs -41:55.384 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/token/self from=127.0.0.1:49248 latency=4.74µs -41:55.385 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/ca/roots from=127.0.0.1:49248 latency=471.02µs -41:55.386 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/connect/ca/roots from=127.0.0.1:49250 error="method PUT not allowed" -41:55.386 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/connect/ca/roots from=127.0.0.1:49250 latency=98.96µs -41:55.387 [ERROR] TestAgent.http: Request error: method=POST url=/v1/connect/ca/roots from=127.0.0.1:49252 error="method POST not allowed" -41:55.387 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/ca/roots from=127.0.0.1:49252 latency=79.52µs -41:55.388 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/connect/ca/roots from=127.0.0.1:49254 error="method DELETE not allowed" -41:55.388 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/connect/ca/roots from=127.0.0.1:49254 latency=91.78µs -41:55.389 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/connect/ca/roots from=127.0.0.1:49256 error="method HEAD not allowed" -41:55.389 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/connect/ca/roots from=127.0.0.1:49256 latency=38.04µs -41:55.390 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/connect/ca/roots from=127.0.0.1:49256 latency=4.74µs -41:55.390 [ERROR] TestAgent.http: Request error: method=GET url=/v1/internal/federation-state/ from=127.0.0.1:49256 error="Bad request: Missing datacenter name" -41:55.390 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/internal/federation-state/ from=127.0.0.1:49256 latency=82.08µs -41:55.391 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/internal/federation-state/ from=127.0.0.1:49258 error="method PUT not allowed" -41:55.391 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/internal/federation-state/ from=127.0.0.1:49258 latency=80.44µs -41:55.392 [ERROR] TestAgent.http: Request error: method=POST url=/v1/internal/federation-state/ from=127.0.0.1:49260 error="method POST not allowed" -41:55.392 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/internal/federation-state/ from=127.0.0.1:49260 latency=76.96µs -41:55.393 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/internal/federation-state/ from=127.0.0.1:49262 error="method DELETE not allowed" -41:55.393 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/internal/federation-state/ from=127.0.0.1:49262 latency=81.48µs -41:55.394 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/internal/federation-state/ from=127.0.0.1:49264 error="method HEAD not allowed" -41:55.394 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/internal/federation-state/ from=127.0.0.1:49264 latency=39.1µs -41:55.394 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/internal/federation-state/ from=127.0.0.1:49264 latency=3.88µs -41:55.395 [ERROR] TestAgent.http: Request error: method=GET url=/v1/session/create from=127.0.0.1:49264 error="method GET not allowed" -41:55.395 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/session/create from=127.0.0.1:49264 latency=85.2µs -41:55.396 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/session/create from=127.0.0.1:49266 error="Permission denied" -41:55.396 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:49266 latency=334.64µs -41:55.397 [ERROR] TestAgent.http: Request error: method=POST url=/v1/session/create from=127.0.0.1:49268 error="method POST not allowed" -41:55.397 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/session/create from=127.0.0.1:49268 latency=103.72µs -41:55.398 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/session/create from=127.0.0.1:49270 error="method DELETE not allowed" -41:55.398 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/session/create from=127.0.0.1:49270 latency=85.78µs -41:55.399 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/session/create from=127.0.0.1:49272 error="method HEAD not allowed" -41:55.399 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/session/create from=127.0.0.1:49272 latency=40.4µs -41:55.400 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/session/create from=127.0.0.1:49272 latency=4.1µs -41:55.400 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/policy/ from=127.0.0.1:49272 error="Bad request: Missing policy ID" -41:55.400 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policy/ from=127.0.0.1:49272 latency=80.6µs -41:55.401 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/policy/ from=127.0.0.1:49274 error="Bad request: Policy decoding failed: EOF" -41:55.401 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy/ from=127.0.0.1:49274 latency=103.46µs -41:55.402 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/policy/ from=127.0.0.1:49276 error="method POST not allowed" -41:55.402 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/policy/ from=127.0.0.1:49276 latency=92.7µs -41:55.403 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/policy/ from=127.0.0.1:49278 error="Bad request: Missing policy ID" -41:55.403 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/policy/ from=127.0.0.1:49278 latency=80.52µs -41:55.406 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/policy/ from=127.0.0.1:49280 error="method HEAD not allowed" -41:55.406 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/policy/ from=127.0.0.1:49280 latency=108.96µs -41:55.407 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/policy/ from=127.0.0.1:49280 latency=4.12µs -41:55.407 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/role/ from=127.0.0.1:49280 error="Bad request: Missing role ID" -41:55.408 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/ from=127.0.0.1:49280 latency=101.08µs -41:55.408 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/role/ from=127.0.0.1:49282 error="Bad request: Role decoding failed: EOF" -41:55.409 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/ from=127.0.0.1:49282 latency=106.74µs -41:55.409 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/role/ from=127.0.0.1:49284 error="method POST not allowed" -41:55.409 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/role/ from=127.0.0.1:49284 latency=86.32µs -41:55.411 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/role/ from=127.0.0.1:49286 error="Bad request: Missing role ID" -41:55.411 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/role/ from=127.0.0.1:49286 latency=94.38µs -41:55.412 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/role/ from=127.0.0.1:49288 error="method HEAD not allowed" -41:55.412 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/role/ from=127.0.0.1:49288 latency=48.74µs -41:55.412 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/role/ from=127.0.0.1:49288 latency=4.62µs -41:55.412 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/auth-method from=127.0.0.1:49288 error="method GET not allowed" -41:55.413 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method from=127.0.0.1:49288 latency=100.44µs -41:55.413 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/auth-method from=127.0.0.1:49290 error="Bad request: AuthMethod decoding failed: EOF" -41:55.413 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:49290 latency=100.04µs -41:55.418 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/auth-method from=127.0.0.1:49292 error="method POST not allowed" -41:55.418 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/auth-method from=127.0.0.1:49292 latency=108.34µs -41:55.419 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/auth-method from=127.0.0.1:49294 error="method DELETE not allowed" -41:55.419 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/auth-method from=127.0.0.1:49294 latency=95.02µs -41:55.420 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/auth-method from=127.0.0.1:49296 error="method HEAD not allowed" -41:55.420 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/auth-method from=127.0.0.1:49296 latency=46.1µs -41:55.420 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/auth-method from=127.0.0.1:49296 latency=4.38µs -41:55.421 [ERROR] TestAgent.http: Request error: method=GET url=/v1/connect/intentions/check from=127.0.0.1:49296 error="required query parameter 'source' not set" -41:55.421 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions/check from=127.0.0.1:49296 latency=95.22µs -41:55.422 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/connect/intentions/check from=127.0.0.1:49298 error="method PUT not allowed" -41:55.422 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/connect/intentions/check from=127.0.0.1:49298 latency=89.74µs -41:55.423 [ERROR] TestAgent.http: Request error: method=POST url=/v1/connect/intentions/check from=127.0.0.1:49300 error="method POST not allowed" -41:55.423 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions/check from=127.0.0.1:49300 latency=90.76µs -41:55.424 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/connect/intentions/check from=127.0.0.1:49302 error="method DELETE not allowed" -41:55.424 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/connect/intentions/check from=127.0.0.1:49302 latency=88.84µs -41:55.426 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/connect/intentions/check from=127.0.0.1:49304 error="method HEAD not allowed" -41:55.426 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/connect/intentions/check from=127.0.0.1:49304 latency=60.04µs -41:55.426 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/connect/intentions/check from=127.0.0.1:49304 latency=3.8µs -41:55.427 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/policies from=127.0.0.1:49304 error="Permission denied" -41:55.427 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policies from=127.0.0.1:49304 latency=260.5µs -41:55.428 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/policies from=127.0.0.1:49306 error="method PUT not allowed" -41:55.428 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policies from=127.0.0.1:49306 latency=91.7µs -41:55.429 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/policies from=127.0.0.1:49308 error="method POST not allowed" -41:55.429 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/policies from=127.0.0.1:49308 latency=88.32µs -41:55.430 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/policies from=127.0.0.1:49310 error="method DELETE not allowed" -41:55.431 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/policies from=127.0.0.1:49310 latency=84.72µs -41:55.431 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/policies from=127.0.0.1:49312 error="method HEAD not allowed" -41:55.431 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/policies from=127.0.0.1:49312 latency=38µs -41:55.432 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/policies from=127.0.0.1:49312 latency=4.34µs -41:55.432 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:49312 latency=148.92µs -41:55.433 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/services from=127.0.0.1:49314 error="method PUT not allowed" -41:55.433 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/services from=127.0.0.1:49314 latency=94.1µs -41:55.435 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/services from=127.0.0.1:49316 error="method POST not allowed" -41:55.435 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/services from=127.0.0.1:49316 latency=100.08µs -41:55.436 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/services from=127.0.0.1:49318 error="method DELETE not allowed" -41:55.436 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/services from=127.0.0.1:49318 latency=85.32µs -41:55.437 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/services from=127.0.0.1:49320 error="method HEAD not allowed" -41:55.437 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/services from=127.0.0.1:49320 latency=34.52µs -41:55.437 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/services from=127.0.0.1:49320 latency=4.02µs -41:55.438 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/force-leave/ from=127.0.0.1:49320 error="method GET not allowed" -41:55.438 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/force-leave/ from=127.0.0.1:49320 latency=107.26µs -41:55.439 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/force-leave/ from=127.0.0.1:49322 error="Permission denied" -41:55.439 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/force-leave/ from=127.0.0.1:49322 latency=141.98µs -41:55.439 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/force-leave/ from=127.0.0.1:49324 error="method POST not allowed" -41:55.440 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/force-leave/ from=127.0.0.1:49324 latency=82.12µs -41:55.440 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/force-leave/ from=127.0.0.1:49326 error="method DELETE not allowed" -41:55.440 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/force-leave/ from=127.0.0.1:49326 latency=91.1µs -41:55.441 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/force-leave/ from=127.0.0.1:49328 error="method HEAD not allowed" -41:55.441 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/force-leave/ from=127.0.0.1:49328 latency=36.04µs -41:55.442 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/force-leave/ from=127.0.0.1:49328 latency=4.46µs -41:55.442 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/leave from=127.0.0.1:49328 error="method GET not allowed" -41:55.442 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/leave from=127.0.0.1:49328 latency=91.64µs -41:55.444 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/leave from=127.0.0.1:49330 error="Permission denied" -41:55.444 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/leave from=127.0.0.1:49330 latency=141.26µs -41:55.445 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/leave from=127.0.0.1:49332 error="method POST not allowed" -41:55.445 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/leave from=127.0.0.1:49332 latency=87.68µs -41:55.446 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/leave from=127.0.0.1:49334 error="method DELETE not allowed" -41:55.446 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/leave from=127.0.0.1:49334 latency=120.8µs -41:55.454 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/leave from=127.0.0.1:49336 error="method HEAD not allowed" -41:55.454 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/leave from=127.0.0.1:49336 latency=73.68µs -41:55.455 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/leave from=127.0.0.1:49336 latency=5.16µs -41:55.456 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/connect/ca/roots from=127.0.0.1:49336 latency=565.24µs -41:55.457 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/connect/ca/roots from=127.0.0.1:49338 error="method PUT not allowed" -41:55.457 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/connect/ca/roots from=127.0.0.1:49338 latency=91µs -41:55.458 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/connect/ca/roots from=127.0.0.1:49340 error="method POST not allowed" -41:55.458 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/connect/ca/roots from=127.0.0.1:49340 latency=90µs -41:55.459 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/connect/ca/roots from=127.0.0.1:49342 error="method DELETE not allowed" -41:55.459 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/connect/ca/roots from=127.0.0.1:49342 latency=97.6µs -41:55.460 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/connect/ca/roots from=127.0.0.1:49344 error="method HEAD not allowed" -41:55.460 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/connect/ca/roots from=127.0.0.1:49344 latency=38.16µs -41:55.461 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/connect/ca/roots from=127.0.0.1:49344 latency=3.52µs -41:55.461 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/health/state/ from=127.0.0.1:49344 latency=80.9µs -41:55.462 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/health/state/ from=127.0.0.1:49346 error="method PUT not allowed" -41:55.462 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/health/state/ from=127.0.0.1:49346 latency=81.8µs -41:55.463 [ERROR] TestAgent.http: Request error: method=POST url=/v1/health/state/ from=127.0.0.1:49348 error="method POST not allowed" -41:55.463 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/health/state/ from=127.0.0.1:49348 latency=82.76µs -41:55.464 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/health/state/ from=127.0.0.1:49350 error="method DELETE not allowed" -41:55.464 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/health/state/ from=127.0.0.1:49350 latency=81.16µs -41:55.465 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/health/state/ from=127.0.0.1:49352 error="method HEAD not allowed" -41:55.465 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/health/state/ from=127.0.0.1:49352 latency=37.1µs -41:55.465 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/health/state/ from=127.0.0.1:49352 latency=4.1µs -41:55.465 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/health/service/ from=127.0.0.1:49352 latency=73.52µs -41:55.466 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/health/service/ from=127.0.0.1:49354 error="method PUT not allowed" -41:55.467 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/health/service/ from=127.0.0.1:49354 latency=94.72µs -41:55.467 [ERROR] TestAgent.http: Request error: method=POST url=/v1/health/service/ from=127.0.0.1:49356 error="method POST not allowed" -41:55.468 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/health/service/ from=127.0.0.1:49356 latency=87.04µs -41:55.470 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/health/service/ from=127.0.0.1:49358 error="method DELETE not allowed" -41:55.470 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/health/service/ from=127.0.0.1:49358 latency=128.92µs -41:55.472 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/health/service/ from=127.0.0.1:49360 error="method HEAD not allowed" -41:55.472 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/health/service/ from=127.0.0.1:49360 latency=45.32µs -41:55.472 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/health/service/ from=127.0.0.1:49360 latency=4.48µs -41:55.473 [ERROR] TestAgent.http: Request error: method=GET url=/v1/discovery-chain/ from=127.0.0.1:49360 error="Bad request: Missing chain name" -41:55.473 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/discovery-chain/ from=127.0.0.1:49360 latency=101.3µs -41:55.478 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/discovery-chain/ from=127.0.0.1:49362 error="method PUT not allowed" -41:55.478 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/discovery-chain/ from=127.0.0.1:49362 latency=117.18µs -41:55.479 [ERROR] TestAgent.http: Request error: method=POST url=/v1/discovery-chain/ from=127.0.0.1:49364 error="Bad request: Missing chain name" -41:55.479 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/discovery-chain/ from=127.0.0.1:49364 latency=120.5µs -41:55.481 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/discovery-chain/ from=127.0.0.1:49366 error="method DELETE not allowed" -41:55.481 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/discovery-chain/ from=127.0.0.1:49366 latency=85.46µs -41:55.482 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/discovery-chain/ from=127.0.0.1:49368 error="method HEAD not allowed" -41:55.482 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/discovery-chain/ from=127.0.0.1:49368 latency=37.98µs -41:55.482 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/discovery-chain/ from=127.0.0.1:49368 latency=4.2µs -41:55.483 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-c35ccb5d-53e0-6f1c-9275-1acbe2742bd7 -41:55.483 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/internal/ui/services from=127.0.0.1:49368 latency=647.94µs -41:55.485 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/internal/ui/services from=127.0.0.1:49370 error="method PUT not allowed" -41:55.485 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/internal/ui/services from=127.0.0.1:49370 latency=93.74µs -41:55.486 [ERROR] TestAgent.http: Request error: method=POST url=/v1/internal/ui/services from=127.0.0.1:49372 error="method POST not allowed" -41:55.486 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/internal/ui/services from=127.0.0.1:49372 latency=92.64µs -41:55.487 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/internal/ui/services from=127.0.0.1:49374 error="method DELETE not allowed" -41:55.487 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/internal/ui/services from=127.0.0.1:49374 latency=98.36µs -41:55.488 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/internal/ui/services from=127.0.0.1:49376 error="method HEAD not allowed" -41:55.488 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/internal/ui/services from=127.0.0.1:49376 latency=38.56µs -41:55.489 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/internal/ui/services from=127.0.0.1:49376 latency=3.98µs -41:55.489 [ERROR] TestAgent.http: Request error: method=GET url=/v1/operator/raft/peer from=127.0.0.1:49376 error="method GET not allowed" -41:55.489 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/raft/peer from=127.0.0.1:49376 latency=80.26µs -41:55.490 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/operator/raft/peer from=127.0.0.1:49378 error="method PUT not allowed" -41:55.491 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/operator/raft/peer from=127.0.0.1:49378 latency=96.72µs -41:55.494 [ERROR] TestAgent.http: Request error: method=POST url=/v1/operator/raft/peer from=127.0.0.1:49380 error="method POST not allowed" -41:55.494 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/operator/raft/peer from=127.0.0.1:49380 latency=108.32µs -41:55.495 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/operator/raft/peer from=127.0.0.1:49382 error="Bad request: Must specify either ?id with the server's ID or ?address with IP:port of peer to remove" -41:55.495 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/operator/raft/peer from=127.0.0.1:49382 latency=109.5µs -41:55.496 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/operator/raft/peer from=127.0.0.1:49384 error="method HEAD not allowed" -41:55.496 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/operator/raft/peer from=127.0.0.1:49384 latency=41.64µs -41:55.497 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/operator/raft/peer from=127.0.0.1:49384 latency=29.42µs -41:55.497 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:55.497 [DEBUG] TestAgent: Node info in sync -41:55.497 [DEBUG] TestAgent: Node info in sync -41:55.497 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/login from=127.0.0.1:49384 error="method GET not allowed" -41:55.497 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/login from=127.0.0.1:49384 latency=85.9µs -41:55.499 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/login from=127.0.0.1:49386 error="method PUT not allowed" -41:55.499 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/login from=127.0.0.1:49386 latency=94.88µs -41:55.500 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/login from=127.0.0.1:49388 error="Bad request: Failed to decode request body: EOF" -41:55.500 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/login from=127.0.0.1:49388 latency=111.4µs -41:55.501 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/login from=127.0.0.1:49390 error="method DELETE not allowed" -41:55.501 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/login from=127.0.0.1:49390 latency=83.44µs -41:55.502 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/login from=127.0.0.1:49392 error="method HEAD not allowed" -41:55.502 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/login from=127.0.0.1:49392 latency=41.9µs -41:55.502 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/login from=127.0.0.1:49392 latency=3.96µs -41:55.503 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/policy from=127.0.0.1:49392 error="method GET not allowed" -41:55.503 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policy from=127.0.0.1:49392 latency=86.52µs -41:55.504 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/policy from=127.0.0.1:49394 error="Bad request: Policy decoding failed: EOF" -41:55.504 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:49394 latency=106.64µs -41:55.505 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/policy from=127.0.0.1:49396 error="method POST not allowed" -41:55.505 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/policy from=127.0.0.1:49396 latency=93.2µs -41:55.506 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/policy from=127.0.0.1:49398 error="method DELETE not allowed" -41:55.506 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/policy from=127.0.0.1:49398 latency=83.56µs -41:55.507 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/policy from=127.0.0.1:49400 error="method HEAD not allowed" -41:55.507 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/policy from=127.0.0.1:49400 latency=40.86µs -41:55.508 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/policy from=127.0.0.1:49400 latency=3.48µs -41:55.508 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/binding-rule/ from=127.0.0.1:49400 error="Bad request: Missing binding rule ID" -41:55.508 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/ from=127.0.0.1:49400 latency=76.94µs -41:55.509 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/binding-rule/ from=127.0.0.1:49402 error="Bad request: BindingRule decoding failed: EOF" -41:55.509 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/ from=127.0.0.1:49402 latency=94.4µs -41:55.510 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/binding-rule/ from=127.0.0.1:49404 error="method POST not allowed" -41:55.510 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/binding-rule/ from=127.0.0.1:49404 latency=81.3µs -41:55.511 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/binding-rule/ from=127.0.0.1:49406 error="Bad request: Missing binding rule ID" -41:55.511 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/ from=127.0.0.1:49406 latency=96.48µs -41:55.512 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/binding-rule/ from=127.0.0.1:49408 error="method HEAD not allowed" -41:55.512 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/binding-rule/ from=127.0.0.1:49408 latency=37.16µs -41:55.512 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/binding-rule/ from=127.0.0.1:49408 latency=4.42µs -41:55.513 [ERROR] TestAgent.http: Request error: method=GET url=/v1/config from=127.0.0.1:49408 error="method GET not allowed" -41:55.513 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config from=127.0.0.1:49408 latency=85.34µs -41:55.514 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/config from=127.0.0.1:49410 error="Bad request: Request decoding failed: EOF" -41:55.514 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:49410 latency=120.92µs -41:55.514 [ERROR] TestAgent.http: Request error: method=POST url=/v1/config from=127.0.0.1:49412 error="method POST not allowed" -41:55.515 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/config from=127.0.0.1:49412 latency=89.06µs -41:55.515 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/config from=127.0.0.1:49414 error="method DELETE not allowed" -41:55.515 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/config from=127.0.0.1:49414 latency=87.64µs -41:55.516 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/config from=127.0.0.1:49416 error="method HEAD not allowed" -41:55.516 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/config from=127.0.0.1:49416 latency=39.54µs -41:55.517 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/config from=127.0.0.1:49416 latency=3.12µs -41:55.518 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/nodes from=127.0.0.1:49416 latency=416.2µs -41:55.518 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/coordinate/nodes from=127.0.0.1:49418 error="method PUT not allowed" -41:55.519 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/coordinate/nodes from=127.0.0.1:49418 latency=87.6µs -41:55.519 [ERROR] TestAgent.http: Request error: method=POST url=/v1/coordinate/nodes from=127.0.0.1:49420 error="method POST not allowed" -41:55.519 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/coordinate/nodes from=127.0.0.1:49420 latency=82.36µs -41:55.520 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/coordinate/nodes from=127.0.0.1:49422 error="method DELETE not allowed" -41:55.520 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/coordinate/nodes from=127.0.0.1:49422 latency=86.08µs -41:55.521 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/coordinate/nodes from=127.0.0.1:49424 error="method HEAD not allowed" -41:55.521 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/coordinate/nodes from=127.0.0.1:49424 latency=36.78µs -41:55.522 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/coordinate/nodes from=127.0.0.1:49424 latency=4.5µs -41:55.522 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/logout from=127.0.0.1:49424 error="method GET not allowed" -41:55.522 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/logout from=127.0.0.1:49424 latency=82.74µs -41:55.523 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/logout from=127.0.0.1:49426 error="method PUT not allowed" -41:55.523 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/logout from=127.0.0.1:49426 latency=79.72µs -41:55.525 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/logout from=127.0.0.1:49428 error="ACL not found" -41:55.525 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/logout from=127.0.0.1:49428 latency=143.9µs -41:55.527 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/logout from=127.0.0.1:49430 error="method DELETE not allowed" -41:55.527 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/logout from=127.0.0.1:49430 latency=228.5µs -41:55.529 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/logout from=127.0.0.1:49432 error="method HEAD not allowed" -41:55.529 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/logout from=127.0.0.1:49432 latency=52.42µs -41:55.529 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/logout from=127.0.0.1:49432 latency=4.92µs -41:55.530 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/check/pass/ from=127.0.0.1:49432 error="method GET not allowed" -41:55.530 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/check/pass/ from=127.0.0.1:49432 latency=92.28µs -41:55.531 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/check/pass/ from=127.0.0.1:49434 error="Unknown check """ -41:55.531 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/check/pass/ from=127.0.0.1:49434 latency=178.98µs -41:55.532 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/check/pass/ from=127.0.0.1:49436 error="method POST not allowed" -41:55.532 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/check/pass/ from=127.0.0.1:49436 latency=83.5µs -41:55.533 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/check/pass/ from=127.0.0.1:49438 error="method DELETE not allowed" -41:55.533 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/check/pass/ from=127.0.0.1:49438 latency=83.78µs -41:55.534 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/check/pass/ from=127.0.0.1:49440 error="method HEAD not allowed" -41:55.534 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/check/pass/ from=127.0.0.1:49440 latency=39.42µs -41:55.535 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/check/pass/ from=127.0.0.1:49440 latency=3.92µs -41:55.535 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/connect/authorize from=127.0.0.1:49440 error="method GET not allowed" -41:55.535 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/connect/authorize from=127.0.0.1:49440 latency=81.56µs -41:55.536 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/connect/authorize from=127.0.0.1:49442 error="method PUT not allowed" -41:55.537 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/connect/authorize from=127.0.0.1:49442 latency=80.78µs -41:55.537 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/connect/authorize from=127.0.0.1:49444 error="Bad request: Request decode failed: EOF" -41:55.537 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/connect/authorize from=127.0.0.1:49444 latency=96.22µs -41:55.538 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/connect/authorize from=127.0.0.1:49446 error="method DELETE not allowed" -41:55.538 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/connect/authorize from=127.0.0.1:49446 latency=78.22µs -41:55.539 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/connect/authorize from=127.0.0.1:49448 error="method HEAD not allowed" -41:55.539 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/connect/authorize from=127.0.0.1:49448 latency=41.5µs -41:55.540 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/connect/authorize from=127.0.0.1:49448 latency=3.26µs -41:55.540 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/service/maintenance/ from=127.0.0.1:49448 error="method GET not allowed" -41:55.540 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/maintenance/ from=127.0.0.1:49448 latency=80.1µs -41:55.541 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/maintenance/ from=127.0.0.1:49450 latency=81.1µs -41:55.542 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/service/maintenance/ from=127.0.0.1:49452 error="method POST not allowed" -41:55.542 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/service/maintenance/ from=127.0.0.1:49452 latency=86.62µs -41:55.543 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/service/maintenance/ from=127.0.0.1:49454 error="method DELETE not allowed" -41:55.543 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/service/maintenance/ from=127.0.0.1:49454 latency=79.84µs -41:55.544 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/service/maintenance/ from=127.0.0.1:49456 error="method HEAD not allowed" -41:55.544 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/service/maintenance/ from=127.0.0.1:49456 latency=41.02µs -41:55.545 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/service/maintenance/ from=127.0.0.1:49456 latency=5.14µs -41:55.545 [ERROR] TestAgent.http: Request error: method=GET url=/v1/catalog/deregister from=127.0.0.1:49456 error="method GET not allowed" -41:55.545 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/deregister from=127.0.0.1:49456 latency=88.34µs -41:55.547 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/catalog/deregister from=127.0.0.1:49458 latency=78.28µs -41:55.548 [ERROR] TestAgent.http: Request error: method=POST url=/v1/catalog/deregister from=127.0.0.1:49460 error="method POST not allowed" -41:55.548 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/catalog/deregister from=127.0.0.1:49460 latency=96.94µs -41:55.549 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/catalog/deregister from=127.0.0.1:49462 error="method DELETE not allowed" -41:55.549 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/catalog/deregister from=127.0.0.1:49462 latency=84.38µs -41:55.550 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/catalog/deregister from=127.0.0.1:49464 error="method HEAD not allowed" -41:55.550 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/catalog/deregister from=127.0.0.1:49464 latency=38.24µs -41:55.550 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/catalog/deregister from=127.0.0.1:49464 latency=3.8µs -41:55.551 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/datacenters from=127.0.0.1:49464 latency=304.78µs -41:55.552 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/catalog/datacenters from=127.0.0.1:49466 error="method PUT not allowed" -41:55.552 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/catalog/datacenters from=127.0.0.1:49466 latency=92.36µs -41:55.553 [ERROR] TestAgent.http: Request error: method=POST url=/v1/catalog/datacenters from=127.0.0.1:49468 error="method POST not allowed" -41:55.553 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/catalog/datacenters from=127.0.0.1:49468 latency=90.7µs -41:55.554 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/catalog/datacenters from=127.0.0.1:49470 error="method DELETE not allowed" -41:55.554 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/catalog/datacenters from=127.0.0.1:49470 latency=82.54µs -41:55.555 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/catalog/datacenters from=127.0.0.1:49472 error="method HEAD not allowed" -41:55.555 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/catalog/datacenters from=127.0.0.1:49472 latency=60.18µs -41:55.556 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/catalog/datacenters from=127.0.0.1:49472 latency=5.26µs -41:55.556 [ERROR] TestAgent.http: Request error: method=GET url=/v1/connect/intentions/match from=127.0.0.1:49472 error="required query parameter 'by' not set" -41:55.556 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions/match from=127.0.0.1:49472 latency=92.58µs -41:55.558 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/connect/intentions/match from=127.0.0.1:49474 error="method PUT not allowed" -41:55.558 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/connect/intentions/match from=127.0.0.1:49474 latency=94.88µs -41:55.559 [ERROR] TestAgent.http: Request error: method=POST url=/v1/connect/intentions/match from=127.0.0.1:49476 error="method POST not allowed" -41:55.559 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions/match from=127.0.0.1:49476 latency=147.62µs -41:55.560 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/connect/intentions/match from=127.0.0.1:49478 error="method DELETE not allowed" -41:55.560 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/connect/intentions/match from=127.0.0.1:49478 latency=84.12µs -41:55.561 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/connect/intentions/match from=127.0.0.1:49480 error="method HEAD not allowed" -41:55.561 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/connect/intentions/match from=127.0.0.1:49480 latency=35.34µs -41:55.561 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/connect/intentions/match from=127.0.0.1:49480 latency=3.6µs -41:55.561 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/status/leader from=127.0.0.1:49480 latency=139.08µs -41:55.562 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/status/leader from=127.0.0.1:49482 error="method PUT not allowed" -41:55.562 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/status/leader from=127.0.0.1:49482 latency=83.88µs -41:55.563 [ERROR] TestAgent.http: Request error: method=POST url=/v1/status/leader from=127.0.0.1:49484 error="method POST not allowed" -41:55.563 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/status/leader from=127.0.0.1:49484 latency=95.18µs -41:55.564 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/status/leader from=127.0.0.1:49486 error="method DELETE not allowed" -41:55.564 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/status/leader from=127.0.0.1:49486 latency=81.54µs -41:55.565 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/status/leader from=127.0.0.1:49488 error="method HEAD not allowed" -41:55.565 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/status/leader from=127.0.0.1:49488 latency=35.64µs -41:55.565 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/status/leader from=127.0.0.1:49488 latency=3.38µs -41:55.566 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/ from=127.0.0.1:49488 latency=134.52µs -41:55.567 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/service/ from=127.0.0.1:49490 error="method PUT not allowed" -41:55.567 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/ from=127.0.0.1:49490 latency=86.92µs -41:55.568 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/service/ from=127.0.0.1:49492 error="method POST not allowed" -41:55.568 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/service/ from=127.0.0.1:49492 latency=84.08µs -41:55.569 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/service/ from=127.0.0.1:49494 error="method DELETE not allowed" -41:55.569 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/service/ from=127.0.0.1:49494 latency=89.8µs -41:55.570 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/service/ from=127.0.0.1:49496 error="method HEAD not allowed" -41:55.570 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/service/ from=127.0.0.1:49496 latency=42.54µs -41:55.570 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/service/ from=127.0.0.1:49496 latency=4.08µs -41:55.571 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/check/register from=127.0.0.1:49496 error="method GET not allowed" -41:55.571 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/check/register from=127.0.0.1:49496 latency=81.18µs -41:55.572 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/check/register from=127.0.0.1:49498 latency=68.64µs -41:55.573 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/check/register from=127.0.0.1:49500 error="method POST not allowed" -41:55.573 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/check/register from=127.0.0.1:49500 latency=93.8µs -41:55.574 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/check/register from=127.0.0.1:49502 error="method DELETE not allowed" -41:55.574 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/check/register from=127.0.0.1:49502 latency=84.2µs -41:55.575 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/check/register from=127.0.0.1:49504 error="method HEAD not allowed" -41:55.575 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/check/register from=127.0.0.1:49504 latency=35.1µs -41:55.575 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/check/register from=127.0.0.1:49504 latency=4.38µs -41:55.576 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/health/checks/ from=127.0.0.1:49504 latency=74.4µs -41:55.577 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/health/checks/ from=127.0.0.1:49506 error="method PUT not allowed" -41:55.577 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/health/checks/ from=127.0.0.1:49506 latency=84.26µs -41:55.578 [ERROR] TestAgent.http: Request error: method=POST url=/v1/health/checks/ from=127.0.0.1:49508 error="method POST not allowed" -41:55.578 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/health/checks/ from=127.0.0.1:49508 latency=102.9µs -41:55.579 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/health/checks/ from=127.0.0.1:49510 error="method DELETE not allowed" -41:55.579 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/health/checks/ from=127.0.0.1:49510 latency=80.54µs -41:55.580 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/health/checks/ from=127.0.0.1:49512 error="method HEAD not allowed" -41:55.580 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/health/checks/ from=127.0.0.1:49512 latency=39.48µs -41:55.581 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/health/checks/ from=127.0.0.1:49512 latency=3.48µs -41:55.582 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/rules/translate from=127.0.0.1:49512 error="method GET not allowed" -41:55.582 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/rules/translate from=127.0.0.1:49512 latency=88.74µs -41:55.583 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/rules/translate from=127.0.0.1:49514 error="method PUT not allowed" -41:55.583 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/rules/translate from=127.0.0.1:49514 latency=80.06µs -41:55.584 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/rules/translate from=127.0.0.1:49516 error="Permission denied" -41:55.584 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/rules/translate from=127.0.0.1:49516 latency=145.02µs -41:55.585 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/rules/translate from=127.0.0.1:49518 error="method DELETE not allowed" -41:55.585 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/rules/translate from=127.0.0.1:49518 latency=87.9µs -41:55.586 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/rules/translate from=127.0.0.1:49520 error="method HEAD not allowed" -41:55.586 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/rules/translate from=127.0.0.1:49520 latency=34.86µs -41:55.587 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/rules/translate from=127.0.0.1:49520 latency=4.52µs -41:55.587 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/host from=127.0.0.1:49520 error="Permission denied" -41:55.587 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/host from=127.0.0.1:49520 latency=136.5µs -41:55.588 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/host from=127.0.0.1:49522 error="method PUT not allowed" -41:55.588 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/host from=127.0.0.1:49522 latency=94.38µs -41:55.589 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/host from=127.0.0.1:49524 error="method POST not allowed" -41:55.589 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/host from=127.0.0.1:49524 latency=88.52µs -41:55.591 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/host from=127.0.0.1:49526 error="method DELETE not allowed" -41:55.591 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/host from=127.0.0.1:49526 latency=85.54µs -41:55.592 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/host from=127.0.0.1:49528 error="method HEAD not allowed" -41:55.592 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/host from=127.0.0.1:49528 latency=39.64µs -41:55.592 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/host from=127.0.0.1:49528 latency=19.98µs -41:55.593 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/metrics from=127.0.0.1:49528 error="Permission denied" -41:55.593 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/metrics from=127.0.0.1:49528 latency=142.6µs -41:55.600 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/metrics from=127.0.0.1:49530 error="method PUT not allowed" -41:55.600 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/metrics from=127.0.0.1:49530 latency=112.06µs -41:55.601 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/metrics from=127.0.0.1:49532 error="method POST not allowed" -41:55.601 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/metrics from=127.0.0.1:49532 latency=92.32µs -41:55.602 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/metrics from=127.0.0.1:49534 error="method DELETE not allowed" -41:55.602 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/metrics from=127.0.0.1:49534 latency=81.94µs -41:55.603 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/metrics from=127.0.0.1:49536 error="method HEAD not allowed" -41:55.603 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/metrics from=127.0.0.1:49536 latency=35.38µs -41:55.604 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/metrics from=127.0.0.1:49536 latency=4.68µs -41:55.604 [ERROR] TestAgent.http: Request error: method=GET url=/v1/catalog/register from=127.0.0.1:49536 error="method GET not allowed" -41:55.605 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/register from=127.0.0.1:49536 latency=87.6µs -41:55.605 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/catalog/register from=127.0.0.1:49538 latency=74.16µs -41:55.606 [ERROR] TestAgent.http: Request error: method=POST url=/v1/catalog/register from=127.0.0.1:49540 error="method POST not allowed" -41:55.606 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/catalog/register from=127.0.0.1:49540 latency=92.06µs -41:55.607 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/catalog/register from=127.0.0.1:49542 error="method DELETE not allowed" -41:55.607 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/catalog/register from=127.0.0.1:49542 latency=86.06µs -41:55.608 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/catalog/register from=127.0.0.1:49544 error="method HEAD not allowed" -41:55.608 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/catalog/register from=127.0.0.1:49544 latency=36.4µs -41:55.609 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/catalog/register from=127.0.0.1:49544 latency=4.1µs -41:55.610 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/event/list from=127.0.0.1:49544 latency=361.86µs -41:55.611 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/event/list from=127.0.0.1:49546 error="method PUT not allowed" -41:55.611 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/event/list from=127.0.0.1:49546 latency=94.3µs -41:55.611 [ERROR] TestAgent.http: Request error: method=POST url=/v1/event/list from=127.0.0.1:49548 error="method POST not allowed" -41:55.611 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/event/list from=127.0.0.1:49548 latency=80.5µs -41:55.612 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/event/list from=127.0.0.1:49550 error="method DELETE not allowed" -41:55.612 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/event/list from=127.0.0.1:49550 latency=82.76µs -41:55.613 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/event/list from=127.0.0.1:49552 error="method HEAD not allowed" -41:55.613 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/event/list from=127.0.0.1:49552 latency=47.68µs -41:55.614 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/event/list from=127.0.0.1:49552 latency=11.12µs -41:55.615 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/replication from=127.0.0.1:49552 latency=275µs -41:55.615 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/replication from=127.0.0.1:49554 error="method PUT not allowed" -41:55.616 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/replication from=127.0.0.1:49554 latency=84.62µs -41:55.616 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/replication from=127.0.0.1:49556 error="method POST not allowed" -41:55.616 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/replication from=127.0.0.1:49556 latency=87.36µs -41:55.617 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/replication from=127.0.0.1:49558 error="method DELETE not allowed" -41:55.617 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/replication from=127.0.0.1:49558 latency=92.32µs -41:55.619 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/replication from=127.0.0.1:49560 error="method HEAD not allowed" -41:55.619 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/replication from=127.0.0.1:49560 latency=43.86µs -41:55.619 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/replication from=127.0.0.1:49560 latency=3.86µs -41:55.620 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/health/service/name/ from=127.0.0.1:49560 error="Bad request: Missing service Name" -41:55.620 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/health/service/name/ from=127.0.0.1:49560 latency=98.76µs -41:55.621 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/health/service/name/ from=127.0.0.1:49562 error="method PUT not allowed" -41:55.621 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/health/service/name/ from=127.0.0.1:49562 latency=96.88µs -41:55.622 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/health/service/name/ from=127.0.0.1:49564 error="method POST not allowed" -41:55.622 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/health/service/name/ from=127.0.0.1:49564 latency=96.46µs -41:55.623 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/health/service/name/ from=127.0.0.1:49566 error="method DELETE not allowed" -41:55.623 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/health/service/name/ from=127.0.0.1:49566 latency=91.68µs -41:55.624 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/health/service/name/ from=127.0.0.1:49568 error="method HEAD not allowed" -41:55.624 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/health/service/name/ from=127.0.0.1:49568 latency=40.68µs -41:55.625 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/health/service/name/ from=127.0.0.1:49568 latency=4µs -41:55.626 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/internal/federation-states/mesh-gateways from=127.0.0.1:49568 latency=383.5µs -41:55.627 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/internal/federation-states/mesh-gateways from=127.0.0.1:49570 error="method PUT not allowed" -41:55.627 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/internal/federation-states/mesh-gateways from=127.0.0.1:49570 latency=85.54µs -41:55.628 [ERROR] TestAgent.http: Request error: method=POST url=/v1/internal/federation-states/mesh-gateways from=127.0.0.1:49572 error="method POST not allowed" -41:55.628 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/internal/federation-states/mesh-gateways from=127.0.0.1:49572 latency=89.68µs -41:55.629 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/internal/federation-states/mesh-gateways from=127.0.0.1:49574 error="method DELETE not allowed" -41:55.629 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/internal/federation-states/mesh-gateways from=127.0.0.1:49574 latency=90.98µs -41:55.630 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/internal/federation-states/mesh-gateways from=127.0.0.1:49576 error="method HEAD not allowed" -41:55.630 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/internal/federation-states/mesh-gateways from=127.0.0.1:49576 latency=40.58µs -41:55.630 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/internal/federation-states/mesh-gateways from=127.0.0.1:49576 latency=4.48µs -41:55.631 [ERROR] TestAgent.http: Request error: method=GET url=/v1/internal/acl/authorize from=127.0.0.1:49576 error="method GET not allowed" -41:55.631 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/internal/acl/authorize from=127.0.0.1:49576 latency=84.28µs -41:55.631 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/internal/acl/authorize from=127.0.0.1:49578 error="method PUT not allowed" -41:55.632 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/internal/acl/authorize from=127.0.0.1:49578 latency=79.82µs -41:55.632 [ERROR] TestAgent.http: Request error: method=POST url=/v1/internal/acl/authorize from=127.0.0.1:49580 error="Bad request: Failed to decode request body: EOF" -41:55.632 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/internal/acl/authorize from=127.0.0.1:49580 latency=95.34µs -41:55.633 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/internal/acl/authorize from=127.0.0.1:49582 error="method DELETE not allowed" -41:55.633 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/internal/acl/authorize from=127.0.0.1:49582 latency=75.58µs -41:55.635 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/internal/acl/authorize from=127.0.0.1:49584 error="method HEAD not allowed" -41:55.635 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/internal/acl/authorize from=127.0.0.1:49584 latency=37.92µs -41:55.635 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/internal/acl/authorize from=127.0.0.1:49584 latency=4.38µs -41:55.636 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/roles from=127.0.0.1:49584 error="Permission denied" -41:55.636 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/roles from=127.0.0.1:49584 latency=228.64µs -41:55.637 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/roles from=127.0.0.1:49586 error="method PUT not allowed" -41:55.637 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/roles from=127.0.0.1:49586 latency=86.5µs -41:55.638 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/roles from=127.0.0.1:49588 error="method POST not allowed" -41:55.638 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/roles from=127.0.0.1:49588 latency=90.7µs -41:55.639 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/roles from=127.0.0.1:49590 error="method DELETE not allowed" -41:55.639 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/roles from=127.0.0.1:49590 latency=82.72µs -41:55.640 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/roles from=127.0.0.1:49592 error="method HEAD not allowed" -41:55.640 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/roles from=127.0.0.1:49592 latency=39.88µs -41:55.640 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/roles from=127.0.0.1:49592 latency=3.72µs -41:55.641 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/auth-method/ from=127.0.0.1:49592 error="Bad request: Missing auth method name" -41:55.641 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/ from=127.0.0.1:49592 latency=88.54µs -41:55.642 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/auth-method/ from=127.0.0.1:49594 error="Bad request: AuthMethod decoding failed: EOF" -41:55.642 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/ from=127.0.0.1:49594 latency=98.24µs -41:55.643 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/auth-method/ from=127.0.0.1:49596 error="method POST not allowed" -41:55.643 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/auth-method/ from=127.0.0.1:49596 latency=90.64µs -41:55.647 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/auth-method/ from=127.0.0.1:49598 error="Bad request: Missing auth method name" -41:55.647 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/auth-method/ from=127.0.0.1:49598 latency=102.6µs -41:55.648 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/auth-method/ from=127.0.0.1:49600 error="method HEAD not allowed" -41:55.648 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/auth-method/ from=127.0.0.1:49600 latency=38µs -41:55.648 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/auth-method/ from=127.0.0.1:49600 latency=4.1µs -41:55.649 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/service/register from=127.0.0.1:49600 error="method GET not allowed" -41:55.649 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/register from=127.0.0.1:49600 latency=81.44µs -41:55.650 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:49602 latency=68.04µs -41:55.651 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/service/register from=127.0.0.1:49604 error="method POST not allowed" -41:55.651 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/service/register from=127.0.0.1:49604 latency=94.6µs -41:55.655 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/service/register from=127.0.0.1:49606 error="method DELETE not allowed" -41:55.655 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/service/register from=127.0.0.1:49606 latency=117.34µs -41:55.656 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/service/register from=127.0.0.1:49608 error="method HEAD not allowed" -41:55.656 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/service/register from=127.0.0.1:49608 latency=408.52µs -41:55.657 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/service/register from=127.0.0.1:49608 latency=5.42µs -41:55.657 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/service/deregister/ from=127.0.0.1:49608 error="method GET not allowed" -41:55.657 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/deregister/ from=127.0.0.1:49608 latency=105.94µs -41:55.659 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/service/deregister/ from=127.0.0.1:49610 error="Unknown service {"" {}}" -41:55.659 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/deregister/ from=127.0.0.1:49610 latency=190.66µs -41:55.660 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/service/deregister/ from=127.0.0.1:49612 error="method POST not allowed" -41:55.660 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/service/deregister/ from=127.0.0.1:49612 latency=97.08µs -41:55.661 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/service/deregister/ from=127.0.0.1:49614 error="method DELETE not allowed" -41:55.661 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/service/deregister/ from=127.0.0.1:49614 latency=101.76µs -41:55.662 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/service/deregister/ from=127.0.0.1:49616 error="method HEAD not allowed" -41:55.662 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/service/deregister/ from=127.0.0.1:49616 latency=48.06µs -41:55.662 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/service/deregister/ from=127.0.0.1:49616 latency=4.32µs -41:55.663 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-c35ccb5d-53e0-6f1c-9275-1acbe2742bd7 -41:55.663 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/nodes from=127.0.0.1:49616 latency=361.86µs -41:55.664 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/catalog/nodes from=127.0.0.1:49618 error="method PUT not allowed" -41:55.664 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/catalog/nodes from=127.0.0.1:49618 latency=89.28µs -41:55.665 [ERROR] TestAgent.http: Request error: method=POST url=/v1/catalog/nodes from=127.0.0.1:49620 error="method POST not allowed" -41:55.665 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/catalog/nodes from=127.0.0.1:49620 latency=87.44µs -41:55.666 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/catalog/nodes from=127.0.0.1:49622 error="method DELETE not allowed" -41:55.667 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/catalog/nodes from=127.0.0.1:49622 latency=87.54µs -41:55.667 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/catalog/nodes from=127.0.0.1:49624 error="method HEAD not allowed" -41:55.667 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/catalog/nodes from=127.0.0.1:49624 latency=52.56µs -41:55.668 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/catalog/nodes from=127.0.0.1:49624 latency=4.4µs -41:55.669 [ERROR] TestAgent.http: Request error: method=GET url=/v1/operator/keyring from=127.0.0.1:49624 error="Reading keyring denied by ACLs" -41:55.669 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/keyring from=127.0.0.1:49624 latency=226.06µs -41:55.670 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/operator/keyring from=127.0.0.1:49626 error="Bad request: Request decode failed: EOF" -41:55.670 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/operator/keyring from=127.0.0.1:49626 latency=98.36µs -41:55.671 [ERROR] TestAgent.http: Request error: method=POST url=/v1/operator/keyring from=127.0.0.1:49628 error="Bad request: Request decode failed: EOF" -41:55.671 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/operator/keyring from=127.0.0.1:49628 latency=95.74µs -41:55.672 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/operator/keyring from=127.0.0.1:49630 error="Bad request: Request decode failed: EOF" -41:55.672 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/operator/keyring from=127.0.0.1:49630 latency=85.28µs -41:55.673 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/operator/keyring from=127.0.0.1:49632 error="method HEAD not allowed" -41:55.673 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/operator/keyring from=127.0.0.1:49632 latency=43.06µs -41:55.673 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/operator/keyring from=127.0.0.1:49632 latency=5.02µs -41:55.673 [ERROR] TestAgent.http: Request error: method=GET url=/v1/txn from=127.0.0.1:49632 error="method GET not allowed" -41:55.673 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/txn from=127.0.0.1:49632 latency=77.1µs -41:55.675 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/txn from=127.0.0.1:49634 latency=68.64µs -41:55.675 [ERROR] TestAgent.http: Request error: method=POST url=/v1/txn from=127.0.0.1:49636 error="method POST not allowed" -41:55.676 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/txn from=127.0.0.1:49636 latency=81.16µs -41:55.676 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/txn from=127.0.0.1:49638 error="method DELETE not allowed" -41:55.676 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/txn from=127.0.0.1:49638 latency=85.04µs -41:55.677 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/txn from=127.0.0.1:49640 error="method HEAD not allowed" -41:55.677 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/txn from=127.0.0.1:49640 latency=31.66µs -41:55.677 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/txn from=127.0.0.1:49640 latency=3.38µs -41:55.678 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/destroy/ from=127.0.0.1:49640 error="method GET not allowed" -41:55.678 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/destroy/ from=127.0.0.1:49640 latency=79.28µs -41:55.679 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/destroy/ from=127.0.0.1:49642 latency=56.8µs -41:55.680 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/destroy/ from=127.0.0.1:49644 error="method POST not allowed" -41:55.680 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/destroy/ from=127.0.0.1:49644 latency=76.88µs -41:55.680 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/destroy/ from=127.0.0.1:49646 error="method DELETE not allowed" -41:55.681 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/destroy/ from=127.0.0.1:49646 latency=77.96µs -41:55.681 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/destroy/ from=127.0.0.1:49648 error="method HEAD not allowed" -41:55.681 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/destroy/ from=127.0.0.1:49648 latency=32.84µs -41:55.682 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/destroy/ from=127.0.0.1:49648 latency=3.78µs -41:55.682 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/check/update/ from=127.0.0.1:49648 error="method GET not allowed" -41:55.682 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/check/update/ from=127.0.0.1:49648 latency=81.82µs -41:55.683 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/check/update/ from=127.0.0.1:49650 latency=66.1µs -41:55.684 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/check/update/ from=127.0.0.1:49652 error="method POST not allowed" -41:55.684 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/check/update/ from=127.0.0.1:49652 latency=85.76µs -41:55.685 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/check/update/ from=127.0.0.1:49654 error="method DELETE not allowed" -41:55.685 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/check/update/ from=127.0.0.1:49654 latency=79.52µs -41:55.686 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/check/update/ from=127.0.0.1:49656 error="method HEAD not allowed" -41:55.686 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/check/update/ from=127.0.0.1:49656 latency=42.52µs -41:55.687 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/check/update/ from=127.0.0.1:49656 latency=3.92µs -41:55.687 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/gateway-services/ from=127.0.0.1:49656 latency=80.3µs -41:55.688 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/catalog/gateway-services/ from=127.0.0.1:49658 error="method PUT not allowed" -41:55.688 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/catalog/gateway-services/ from=127.0.0.1:49658 latency=97.4µs -41:55.689 [ERROR] TestAgent.http: Request error: method=POST url=/v1/catalog/gateway-services/ from=127.0.0.1:49660 error="method POST not allowed" -41:55.689 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/catalog/gateway-services/ from=127.0.0.1:49660 latency=85.74µs -41:55.690 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/catalog/gateway-services/ from=127.0.0.1:49662 error="method DELETE not allowed" -41:55.690 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/catalog/gateway-services/ from=127.0.0.1:49662 latency=87.52µs -41:55.691 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/catalog/gateway-services/ from=127.0.0.1:49664 error="method HEAD not allowed" -41:55.691 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/catalog/gateway-services/ from=127.0.0.1:49664 latency=36.22µs -41:55.692 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/catalog/gateway-services/ from=127.0.0.1:49664 latency=3.86µs -41:55.693 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/datacenters from=127.0.0.1:49664 latency=401.38µs -41:55.693 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/coordinate/datacenters from=127.0.0.1:49666 error="method PUT not allowed" -41:55.694 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/coordinate/datacenters from=127.0.0.1:49666 latency=110.74µs -41:55.694 [ERROR] TestAgent.http: Request error: method=POST url=/v1/coordinate/datacenters from=127.0.0.1:49668 error="method POST not allowed" -41:55.695 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/coordinate/datacenters from=127.0.0.1:49668 latency=78.94µs -41:55.695 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/coordinate/datacenters from=127.0.0.1:49670 error="method DELETE not allowed" -41:55.695 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/coordinate/datacenters from=127.0.0.1:49670 latency=94.74µs -41:55.696 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/coordinate/datacenters from=127.0.0.1:49672 error="method HEAD not allowed" -41:55.696 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/coordinate/datacenters from=127.0.0.1:49672 latency=40.58µs -41:55.697 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/coordinate/datacenters from=127.0.0.1:49672 latency=3.48µs -41:55.697 [ERROR] TestAgent.http: Request error: method=GET url=/v1/operator/raft/configuration from=127.0.0.1:49672 error="Permission denied" -41:55.697 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/raft/configuration from=127.0.0.1:49672 latency=291.4µs -41:55.698 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/operator/raft/configuration from=127.0.0.1:49674 error="method PUT not allowed" -41:55.699 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/operator/raft/configuration from=127.0.0.1:49674 latency=83.46µs -41:55.699 [ERROR] TestAgent.http: Request error: method=POST url=/v1/operator/raft/configuration from=127.0.0.1:49676 error="method POST not allowed" -41:55.699 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/operator/raft/configuration from=127.0.0.1:49676 latency=81.76µs -41:55.700 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/operator/raft/configuration from=127.0.0.1:49678 error="method DELETE not allowed" -41:55.700 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/operator/raft/configuration from=127.0.0.1:49678 latency=86.24µs -41:55.701 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/operator/raft/configuration from=127.0.0.1:49680 error="method HEAD not allowed" -41:55.701 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/operator/raft/configuration from=127.0.0.1:49680 latency=41.04µs -41:55.701 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/operator/raft/configuration from=127.0.0.1:49680 latency=4.1µs -41:55.702 [ERROR] TestAgent.http: Request error: method=GET url=/v1/operator/autopilot/configuration from=127.0.0.1:49680 error="Permission denied" -41:55.702 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/autopilot/configuration from=127.0.0.1:49680 latency=209.84µs -41:55.703 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/operator/autopilot/configuration from=127.0.0.1:49682 error="Bad request: Error parsing autopilot config: EOF" -41:55.703 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/operator/autopilot/configuration from=127.0.0.1:49682 latency=102.42µs -41:55.704 [ERROR] TestAgent.http: Request error: method=POST url=/v1/operator/autopilot/configuration from=127.0.0.1:49684 error="method POST not allowed" -41:55.704 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/operator/autopilot/configuration from=127.0.0.1:49684 latency=85.08µs -41:55.705 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/operator/autopilot/configuration from=127.0.0.1:49686 error="method DELETE not allowed" -41:55.705 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/operator/autopilot/configuration from=127.0.0.1:49686 latency=87.14µs -41:55.706 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/operator/autopilot/configuration from=127.0.0.1:49688 error="method HEAD not allowed" -41:55.706 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/operator/autopilot/configuration from=127.0.0.1:49688 latency=42.76µs -41:55.706 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/operator/autopilot/configuration from=127.0.0.1:49688 latency=3.9µs -41:55.707 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/connect/ from=127.0.0.1:49688 latency=83.08µs -41:55.708 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/catalog/connect/ from=127.0.0.1:49690 error="method PUT not allowed" -41:55.708 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/catalog/connect/ from=127.0.0.1:49690 latency=88.56µs -41:55.709 [ERROR] TestAgent.http: Request error: method=POST url=/v1/catalog/connect/ from=127.0.0.1:49692 error="method POST not allowed" -41:55.709 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/catalog/connect/ from=127.0.0.1:49692 latency=77.72µs -41:55.709 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/catalog/connect/ from=127.0.0.1:49694 error="method DELETE not allowed" -41:55.709 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/catalog/connect/ from=127.0.0.1:49694 latency=75.92µs -41:55.711 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/catalog/connect/ from=127.0.0.1:49696 error="method HEAD not allowed" -41:55.711 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/catalog/connect/ from=127.0.0.1:49696 latency=48.94µs -41:55.712 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/catalog/connect/ from=127.0.0.1:49696 latency=4.08µs -41:55.712 [ERROR] TestAgent.http: Request error: method=GET url=/v1/coordinate/node/ from=127.0.0.1:49696 error="Permission denied" -41:55.713 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/node/ from=127.0.0.1:49696 latency=237.72µs -41:55.713 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/coordinate/node/ from=127.0.0.1:49698 error="method PUT not allowed" -41:55.714 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/coordinate/node/ from=127.0.0.1:49698 latency=92.24µs -41:55.714 [ERROR] TestAgent.http: Request error: method=POST url=/v1/coordinate/node/ from=127.0.0.1:49700 error="method POST not allowed" -41:55.715 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/coordinate/node/ from=127.0.0.1:49700 latency=92.76µs -41:55.717 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/coordinate/node/ from=127.0.0.1:49702 error="method DELETE not allowed" -41:55.718 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/coordinate/node/ from=127.0.0.1:49702 latency=121.04µs -41:55.718 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/coordinate/node/ from=127.0.0.1:49704 error="method HEAD not allowed" -41:55.718 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/coordinate/node/ from=127.0.0.1:49704 latency=41.08µs -41:55.719 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/coordinate/node/ from=127.0.0.1:49704 latency=4.14µs -41:55.720 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:49704 error="URI must be either service or agent" -41:55.720 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:49704 latency=306.38µs -41:55.721 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:49706 error="method PUT not allowed" -41:55.721 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:49706 latency=90.7µs -41:55.727 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:49708 error="method POST not allowed" -41:55.727 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:49708 latency=122.78µs -41:55.728 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:49710 error="method DELETE not allowed" -41:55.728 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:49710 latency=95.7µs -41:55.729 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:49712 error="method HEAD not allowed" -41:55.729 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:49712 latency=37.56µs -41:55.730 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:49712 latency=4.84µs -41:55.731 [ERROR] TestAgent.http: Request error: method=GET url=/v1/coordinate/update from=127.0.0.1:49712 error="method GET not allowed" -41:55.731 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/update from=127.0.0.1:49712 latency=91.04µs -41:55.734 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/coordinate/update from=127.0.0.1:49714 latency=82.6µs -41:55.736 [ERROR] TestAgent.http: Request error: method=POST url=/v1/coordinate/update from=127.0.0.1:49716 error="method POST not allowed" -41:55.736 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/coordinate/update from=127.0.0.1:49716 latency=102.94µs -41:55.737 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/coordinate/update from=127.0.0.1:49718 error="method DELETE not allowed" -41:55.737 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/coordinate/update from=127.0.0.1:49718 latency=88.46µs -41:55.738 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/coordinate/update from=127.0.0.1:49720 error="method HEAD not allowed" -41:55.738 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/coordinate/update from=127.0.0.1:49720 latency=46.32µs -41:55.738 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/coordinate/update from=127.0.0.1:49720 latency=5.04µs -41:55.739 [ERROR] TestAgent.http: Request error: method=GET url=/v1/operator/autopilot/health from=127.0.0.1:49720 error="Permission denied" -41:55.739 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/autopilot/health from=127.0.0.1:49720 latency=254.94µs -41:55.740 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/operator/autopilot/health from=127.0.0.1:49722 error="method PUT not allowed" -41:55.740 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/operator/autopilot/health from=127.0.0.1:49722 latency=90.14µs -41:55.741 [ERROR] TestAgent.http: Request error: method=POST url=/v1/operator/autopilot/health from=127.0.0.1:49724 error="method POST not allowed" -41:55.741 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/operator/autopilot/health from=127.0.0.1:49724 latency=93.36µs -41:55.742 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/operator/autopilot/health from=127.0.0.1:49726 error="method DELETE not allowed" -41:55.742 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/operator/autopilot/health from=127.0.0.1:49726 latency=92.88µs -41:55.743 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/operator/autopilot/health from=127.0.0.1:49728 error="method HEAD not allowed" -41:55.743 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/operator/autopilot/health from=127.0.0.1:49728 latency=52.34µs -41:55.744 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/operator/autopilot/health from=127.0.0.1:49728 latency=4.62µs -41:55.744 [ERROR] TestAgent.http: Request error: method=GET url=/v1/session/renew/ from=127.0.0.1:49728 error="method GET not allowed" -41:55.745 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/session/renew/ from=127.0.0.1:49728 latency=90.12µs -41:55.746 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/renew/ from=127.0.0.1:49730 latency=128.6µs -41:55.746 [ERROR] TestAgent.http: Request error: method=POST url=/v1/session/renew/ from=127.0.0.1:49732 error="method POST not allowed" -41:55.747 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/session/renew/ from=127.0.0.1:49732 latency=89.76µs -41:55.747 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/session/renew/ from=127.0.0.1:49734 error="method DELETE not allowed" -41:55.747 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/session/renew/ from=127.0.0.1:49734 latency=86.1µs -41:55.748 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/session/renew/ from=127.0.0.1:49736 error="method HEAD not allowed" -41:55.748 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/session/renew/ from=127.0.0.1:49736 latency=42.08µs -41:55.749 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/session/renew/ from=127.0.0.1:49736 latency=3.66µs -41:55.749 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/check/deregister/ from=127.0.0.1:49736 error="method GET not allowed" -41:55.749 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/check/deregister/ from=127.0.0.1:49736 latency=79.88µs -41:55.751 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/check/deregister/ from=127.0.0.1:49738 error="Unknown check """ -41:55.751 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/check/deregister/ from=127.0.0.1:49738 latency=161.76µs -41:55.752 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/check/deregister/ from=127.0.0.1:49740 error="method POST not allowed" -41:55.752 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/check/deregister/ from=127.0.0.1:49740 latency=84.8µs -41:55.752 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/check/deregister/ from=127.0.0.1:49742 error="method DELETE not allowed" -41:55.752 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/check/deregister/ from=127.0.0.1:49742 latency=77.1µs -41:55.753 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/check/deregister/ from=127.0.0.1:49744 error="method HEAD not allowed" -41:55.753 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/check/deregister/ from=127.0.0.1:49744 latency=34.76µs -41:55.754 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/check/deregister/ from=127.0.0.1:49744 latency=3.66µs -41:55.754 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/create from=127.0.0.1:49744 error="method GET not allowed" -41:55.754 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/create from=127.0.0.1:49744 latency=88.88µs -41:55.755 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/create from=127.0.0.1:49746 error="Permission denied" -41:55.755 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/create from=127.0.0.1:49746 latency=207.84µs -41:55.756 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/create from=127.0.0.1:49748 error="method POST not allowed" -41:55.756 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/create from=127.0.0.1:49748 latency=77.94µs -41:55.757 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/create from=127.0.0.1:49750 error="method DELETE not allowed" -41:55.757 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/create from=127.0.0.1:49750 latency=110.1µs -41:55.758 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/create from=127.0.0.1:49752 error="method HEAD not allowed" -41:55.758 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/create from=127.0.0.1:49752 latency=39.64µs -41:55.758 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/create from=127.0.0.1:49752 latency=3.42µs -41:55.759 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/auth-methods from=127.0.0.1:49752 error="Permission denied" -41:55.759 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-methods from=127.0.0.1:49752 latency=216.94µs -41:55.762 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/auth-methods from=127.0.0.1:49754 error="method PUT not allowed" -41:55.762 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-methods from=127.0.0.1:49754 latency=111.44µs -41:55.763 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/auth-methods from=127.0.0.1:49756 error="method POST not allowed" -41:55.763 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/auth-methods from=127.0.0.1:49756 latency=88.46µs -41:55.764 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/auth-methods from=127.0.0.1:49758 error="method DELETE not allowed" -41:55.764 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/auth-methods from=127.0.0.1:49758 latency=76.74µs -41:55.765 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/auth-methods from=127.0.0.1:49760 error="method HEAD not allowed" -41:55.765 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/auth-methods from=127.0.0.1:49760 latency=32.62µs -41:55.765 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/auth-methods from=127.0.0.1:49760 latency=3.6µs -41:55.766 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/token/ from=127.0.0.1:49760 error="method GET not allowed" -41:55.766 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/token/ from=127.0.0.1:49760 latency=85.22µs -41:55.767 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/token/ from=127.0.0.1:49762 error="Permission denied" -41:55.767 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/token/ from=127.0.0.1:49762 latency=142.18µs -41:55.768 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/token/ from=127.0.0.1:49764 error="method POST not allowed" -41:55.768 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/token/ from=127.0.0.1:49764 latency=82.38µs -41:55.769 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/token/ from=127.0.0.1:49766 error="method DELETE not allowed" -41:55.769 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/token/ from=127.0.0.1:49766 latency=78.82µs -41:55.769 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/token/ from=127.0.0.1:49768 error="method HEAD not allowed" -41:55.769 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/token/ from=127.0.0.1:49768 latency=38.26µs -41:55.770 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/token/ from=127.0.0.1:49768 latency=4.04µs -41:55.771 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/role/name/ from=127.0.0.1:49768 error="Bad request: Missing role Name" -41:55.771 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/name/ from=127.0.0.1:49768 latency=87.2µs -41:55.772 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/role/name/ from=127.0.0.1:49770 error="method PUT not allowed" -41:55.772 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/name/ from=127.0.0.1:49770 latency=95.76µs -41:55.773 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/role/name/ from=127.0.0.1:49772 error="method POST not allowed" -41:55.773 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/role/name/ from=127.0.0.1:49772 latency=88.98µs -41:55.774 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/role/name/ from=127.0.0.1:49774 error="method DELETE not allowed" -41:55.774 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/role/name/ from=127.0.0.1:49774 latency=93.66µs -41:55.776 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/role/name/ from=127.0.0.1:49776 error="method HEAD not allowed" -41:55.776 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/role/name/ from=127.0.0.1:49776 latency=43.38µs -41:55.776 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/role/name/ from=127.0.0.1:49776 latency=4µs -41:55.776 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/health/service/id/ from=127.0.0.1:49776 error="Bad request: Missing serviceID" -41:55.777 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/health/service/id/ from=127.0.0.1:49776 latency=83.14µs -41:55.777 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/health/service/id/ from=127.0.0.1:49778 error="method PUT not allowed" -41:55.777 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/health/service/id/ from=127.0.0.1:49778 latency=90.26µs -41:55.778 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/health/service/id/ from=127.0.0.1:49780 error="method POST not allowed" -41:55.779 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/health/service/id/ from=127.0.0.1:49780 latency=90.58µs -41:55.779 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/health/service/id/ from=127.0.0.1:49782 error="method DELETE not allowed" -41:55.779 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/health/service/id/ from=127.0.0.1:49782 latency=83.86µs -41:55.780 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/health/service/id/ from=127.0.0.1:49784 error="method HEAD not allowed" -41:55.780 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/health/service/id/ from=127.0.0.1:49784 latency=38.36µs -41:55.781 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/health/service/id/ from=127.0.0.1:49784 latency=9.4µs -41:55.781 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/health/node/ from=127.0.0.1:49784 latency=67.84µs -41:55.782 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/health/node/ from=127.0.0.1:49786 error="method PUT not allowed" -41:55.782 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/health/node/ from=127.0.0.1:49786 latency=97.78µs -41:55.783 [ERROR] TestAgent.http: Request error: method=POST url=/v1/health/node/ from=127.0.0.1:49788 error="method POST not allowed" -41:55.784 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/health/node/ from=127.0.0.1:49788 latency=89.74µs -41:55.785 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/health/node/ from=127.0.0.1:49790 error="method DELETE not allowed" -41:55.785 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/health/node/ from=127.0.0.1:49790 latency=89.08µs -41:55.786 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/health/node/ from=127.0.0.1:49792 error="method HEAD not allowed" -41:55.786 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/health/node/ from=127.0.0.1:49792 latency=45.4µs -41:55.786 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/health/node/ from=127.0.0.1:49792 latency=4.12µs -41:55.787 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-c35ccb5d-53e0-6f1c-9275-1acbe2742bd7 -41:55.787 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/internal/ui/nodes from=127.0.0.1:49792 latency=491.54µs -41:55.788 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/internal/ui/nodes from=127.0.0.1:49794 error="method PUT not allowed" -41:55.788 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/internal/ui/nodes from=127.0.0.1:49794 latency=89.52µs -41:55.789 [ERROR] TestAgent.http: Request error: method=POST url=/v1/internal/ui/nodes from=127.0.0.1:49796 error="method POST not allowed" -41:55.789 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/internal/ui/nodes from=127.0.0.1:49796 latency=78.28µs -41:55.790 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/internal/ui/nodes from=127.0.0.1:49798 error="method DELETE not allowed" -41:55.790 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/internal/ui/nodes from=127.0.0.1:49798 latency=74.66µs -41:55.791 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/internal/ui/nodes from=127.0.0.1:49800 error="method HEAD not allowed" -41:55.791 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/internal/ui/nodes from=127.0.0.1:49800 latency=38.44µs -41:55.791 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/internal/ui/nodes from=127.0.0.1:49800 latency=3.74µs -41:55.792 [ERROR] TestAgent.http: Request error: method=GET url=/v1/internal/federation-states from=127.0.0.1:49800 error="Permission denied" -41:55.792 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/internal/federation-states from=127.0.0.1:49800 latency=182.18µs -41:55.793 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/internal/federation-states from=127.0.0.1:49802 error="method PUT not allowed" -41:55.793 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/internal/federation-states from=127.0.0.1:49802 latency=83.04µs -41:55.793 [ERROR] TestAgent.http: Request error: method=POST url=/v1/internal/federation-states from=127.0.0.1:49804 error="method POST not allowed" -41:55.794 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/internal/federation-states from=127.0.0.1:49804 latency=107.86µs -41:55.794 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/internal/federation-states from=127.0.0.1:49806 error="method DELETE not allowed" -41:55.794 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/internal/federation-states from=127.0.0.1:49806 latency=78.66µs -41:55.795 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/internal/federation-states from=127.0.0.1:49808 error="method HEAD not allowed" -41:55.795 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/internal/federation-states from=127.0.0.1:49808 latency=36.36µs -41:55.796 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/internal/federation-states from=127.0.0.1:49808 latency=3.62µs -41:55.796 [INFO] TestAgent: Requesting shutdown -41:55.796 [INFO] TestAgent.server: shutting down server -41:55.796 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -41:55.796 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -41:55.796 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:55.796 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:55.796 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:55.796 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:55.796 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:55.796 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:55.796 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:55.796 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -41:55.796 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -41:55.796 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:55.796 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:55.796 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:55.796 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:55.797Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:55.797Z [INFO] agent.router.manager: shutting down -41:55.797 [INFO] TestAgent: consul server down -41:55.797 [INFO] TestAgent: shutdown complete -41:55.797 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13946 network=tcp -41:55.797 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13946 network=udp -41:55.797 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13947 network=tcp -41:56.297 [INFO] TestAgent: Waiting for endpoints to shut down -41:56.297 [INFO] TestAgent: Endpoints down ---- PASS: TestHTTPAPI_MethodNotAllowed_OSS (1.47s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/query/xxx/execute (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/query/xxx/execute (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/query/xxx/execute (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/query/xxx/execute (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/query/xxx/execute (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/query/xxx/execute (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/query/xxx/explain (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/query/xxx/explain (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/query/xxx/explain (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/query/xxx/explain (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/query/xxx/explain (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/query/xxx/explain (0.00s) +2021-08-13T15:33:36.260Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:36.276Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:36.279 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:05b1ba5a-0998-b2db-f378-5ab5ce213906 Address:127.0.0.1:25182}]" +33:36.279 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25182 [Follower]" leader= +33:36.280 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-05b1ba5a-0998-b2db-f378-5ab5ce213906.dc1 127.0.0.1 +33:36.281 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-05b1ba5a-0998-b2db-f378-5ab5ce213906 127.0.0.1 +2021-08-13T15:33:36.281Z [INFO] agent.router: Initializing LAN area manager +33:36.281 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25177 network=udp +33:36.282 [INFO] TestAgent.server: Adding LAN server: server="Node-05b1ba5a-0998-b2db-f378-5ab5ce213906 (Addr: tcp/127.0.0.1:25182) (DC: dc1)" +33:36.282 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-05b1ba5a-0998-b2db-f378-5ab5ce213906.dc1 area=wan +33:36.282 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25177 network=tcp +33:36.283 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25178 network=tcp +33:36.283 [INFO] TestAgent: started state syncer +33:36.283 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25183 network=tcp +33:36.326 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:36.326 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25182 [Candidate]" term=2 +33:36.326 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:36.326 [DEBUG] TestAgent.server.raft: vote granted: from=05b1ba5a-0998-b2db-f378-5ab5ce213906 term=2 tally=1 +33:36.326 [INFO] TestAgent.server.raft: election won: tally=1 +33:36.326 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25182 [Leader]" +33:36.326 [INFO] TestAgent.server: cluster leadership acquired +33:36.327 [INFO] TestAgent.server: initializing acls +33:36.327 [INFO] TestAgent.server: New leader elected: payload=Node-05b1ba5a-0998-b2db-f378-5ab5ce213906 +33:36.327 [INFO] TestAgent.server: Created ACL 'global-management' policy +33:36.327 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +33:36.328 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +33:36.328 [INFO] TestAgent.server: Created ACL anonymous token from configuration +33:36.328 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +33:36.329 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +33:36.329 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-05b1ba5a-0998-b2db-f378-5ab5ce213906 +33:36.329 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-05b1ba5a-0998-b2db-f378-5ab5ce213906.dc1 +33:36.329 [INFO] TestAgent.server: Updating LAN server: server="Node-05b1ba5a-0998-b2db-f378-5ab5ce213906 (Addr: tcp/127.0.0.1:25182) (DC: dc1)" +33:36.329 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-05b1ba5a-0998-b2db-f378-5ab5ce213906.dc1 area=wan +33:36.330 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:36.332 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:36.332 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:36.332 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:36.332 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:36.332 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:36.332 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-05b1ba5a-0998-b2db-f378-5ab5ce213906 +33:36.332 [INFO] TestAgent.server: member joined, marking health alive: member=Node-05b1ba5a-0998-b2db-f378-5ab5ce213906 +33:36.333 [INFO] TestAgent.server: federation state anti-entropy synced +33:36.333 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-05b1ba5a-0998-b2db-f378-5ab5ce213906 +33:36.403 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:36.403 [INFO] TestAgent: Synced node info +33:36.403 [DEBUG] TestAgent: Node info in sync +33:36.708 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-05b1ba5a-0998-b2db-f378-5ab5ce213906 +33:36.710 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/query from=127.0.0.1:49780 latency=661.841µs +33:36.711 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/query from=127.0.0.1:49782 error="method PUT not allowed" +33:36.711 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/query from=127.0.0.1:49782 latency=330.961µs +33:36.712 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/query from=127.0.0.1:49784 latency=71.42µs +33:36.713 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/query from=127.0.0.1:49786 error="method DELETE not allowed" +33:36.713 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/query from=127.0.0.1:49786 latency=72.32µs +33:36.714 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/query from=127.0.0.1:49788 error="method HEAD not allowed" +33:36.714 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/query from=127.0.0.1:49788 latency=33.16µs +33:36.715 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/query from=127.0.0.1:49788 latency=3.26µs +33:36.715 [ERROR] TestAgent.http: Request error: method=GET url=/v1/query/ from=127.0.0.1:49788 error="failed prepared query lookup: index error: UUID must be 36 characters" +33:36.716 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/query/ from=127.0.0.1:49788 latency=318.26µs +33:36.717 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/query/ from=127.0.0.1:49790 error="Prepared Query lookup failed: failed prepared query lookup: index error: UUID must be 36 characters" +33:36.717 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/query/ from=127.0.0.1:49790 latency=321.841µs +33:36.718 [ERROR] TestAgent.http: Request error: method=POST url=/v1/query/ from=127.0.0.1:49792 error="method POST not allowed" +33:36.718 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/query/ from=127.0.0.1:49792 latency=80.04µs +33:36.719 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/query/ from=127.0.0.1:49794 error="Prepared Query lookup failed: failed prepared query lookup: index error: UUID must be 36 characters" +33:36.719 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/query/ from=127.0.0.1:49794 latency=142.3µs +33:36.720 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/query/ from=127.0.0.1:49796 error="method HEAD not allowed" +33:36.720 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/query/ from=127.0.0.1:49796 latency=34.86µs +33:36.721 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/query/ from=127.0.0.1:49796 latency=303.421µs +33:36.722 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/query/xxx/execute from=127.0.0.1:49798 latency=400.321µs +33:36.723 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/query/xxx/execute from=127.0.0.1:49800 error="method PUT not allowed" +33:36.724 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/query/xxx/execute from=127.0.0.1:49800 latency=71.88µs +33:36.724 [ERROR] TestAgent.http: Request error: method=POST url=/v1/query/xxx/execute from=127.0.0.1:49802 error="method POST not allowed" +33:36.724 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/query/xxx/execute from=127.0.0.1:49802 latency=86.06µs +33:36.725 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/query/xxx/execute from=127.0.0.1:49804 error="method DELETE not allowed" +33:36.725 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/query/xxx/execute from=127.0.0.1:49804 latency=70.8µs +33:36.726 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/query/xxx/execute from=127.0.0.1:49806 error="method HEAD not allowed" +33:36.726 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/query/xxx/execute from=127.0.0.1:49806 latency=30.28µs +33:36.727 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/query/xxx/execute from=127.0.0.1:49806 latency=52.12µs +33:36.736 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/query/xxx/explain from=127.0.0.1:49808 latency=134.501µs +33:36.737 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/query/xxx/explain from=127.0.0.1:49810 error="method PUT not allowed" +33:36.737 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/query/xxx/explain from=127.0.0.1:49810 latency=71.54µs +33:36.738 [ERROR] TestAgent.http: Request error: method=POST url=/v1/query/xxx/explain from=127.0.0.1:49812 error="method POST not allowed" +33:36.738 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/query/xxx/explain from=127.0.0.1:49812 latency=70.9µs +33:36.739 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/query/xxx/explain from=127.0.0.1:49814 error="method DELETE not allowed" +33:36.739 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/query/xxx/explain from=127.0.0.1:49814 latency=71.76µs +33:36.740 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/query/xxx/explain from=127.0.0.1:49816 error="method HEAD not allowed" +33:36.740 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/query/xxx/explain from=127.0.0.1:49816 latency=31µs +33:36.741 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/query/xxx/explain from=127.0.0.1:49816 latency=56.14µs +33:36.741 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/check/update/ from=127.0.0.1:49818 error="method GET not allowed" +33:36.742 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/check/update/ from=127.0.0.1:49818 latency=73.26µs +33:36.742 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/check/update/ from=127.0.0.1:49820 latency=71.22µs +33:36.744 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/check/update/ from=127.0.0.1:49822 error="method POST not allowed" +33:36.744 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/check/update/ from=127.0.0.1:49822 latency=73.301µs +33:36.745 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/check/update/ from=127.0.0.1:49824 error="method DELETE not allowed" +33:36.745 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/check/update/ from=127.0.0.1:49824 latency=70.061µs +33:36.746 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/check/update/ from=127.0.0.1:49826 error="method HEAD not allowed" +33:36.746 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/check/update/ from=127.0.0.1:49826 latency=29.86µs +33:36.746 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/check/update/ from=127.0.0.1:49826 latency=3.26µs +33:36.746 [ERROR] TestAgent.http: Request error: method=GET url=/v1/connect/intentions/match from=127.0.0.1:49826 error="required query parameter 'by' not set" +33:36.746 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions/match from=127.0.0.1:49826 latency=74.26µs +33:36.748 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/connect/intentions/match from=127.0.0.1:49828 error="method PUT not allowed" +33:36.748 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/connect/intentions/match from=127.0.0.1:49828 latency=73.701µs +33:36.748 [ERROR] TestAgent.http: Request error: method=POST url=/v1/connect/intentions/match from=127.0.0.1:49830 error="method POST not allowed" +33:36.749 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions/match from=127.0.0.1:49830 latency=69.641µs +33:36.749 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/connect/intentions/match from=127.0.0.1:49832 error="method DELETE not allowed" +33:36.749 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/connect/intentions/match from=127.0.0.1:49832 latency=69.72µs +33:36.750 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/connect/intentions/match from=127.0.0.1:49834 error="method HEAD not allowed" +33:36.750 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/connect/intentions/match from=127.0.0.1:49834 latency=32.12µs +33:36.751 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/connect/intentions/match from=127.0.0.1:49834 latency=3.48µs +33:36.752 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/health/checks/ from=127.0.0.1:49834 latency=60.94µs +33:36.752 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/health/checks/ from=127.0.0.1:49836 error="method PUT not allowed" +33:36.752 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/health/checks/ from=127.0.0.1:49836 latency=72.821µs +33:36.753 [ERROR] TestAgent.http: Request error: method=POST url=/v1/health/checks/ from=127.0.0.1:49838 error="method POST not allowed" +33:36.753 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/health/checks/ from=127.0.0.1:49838 latency=68.98µs +33:36.754 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/health/checks/ from=127.0.0.1:49840 error="method DELETE not allowed" +33:36.754 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/health/checks/ from=127.0.0.1:49840 latency=68.98µs +33:36.756 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/health/checks/ from=127.0.0.1:49842 error="method HEAD not allowed" +33:36.756 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/health/checks/ from=127.0.0.1:49842 latency=32.26µs +33:36.756 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/health/checks/ from=127.0.0.1:49842 latency=3.28µs +33:36.756 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/health/state/ from=127.0.0.1:49842 latency=63.64µs +33:36.757 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/health/state/ from=127.0.0.1:49844 error="method PUT not allowed" +33:36.757 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/health/state/ from=127.0.0.1:49844 latency=70.68µs +33:36.758 [ERROR] TestAgent.http: Request error: method=POST url=/v1/health/state/ from=127.0.0.1:49846 error="method POST not allowed" +33:36.758 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/health/state/ from=127.0.0.1:49846 latency=68.96µs +33:36.759 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/health/state/ from=127.0.0.1:49848 error="method DELETE not allowed" +33:36.759 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/health/state/ from=127.0.0.1:49848 latency=224.201µs +33:36.760 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/health/state/ from=127.0.0.1:49850 error="method HEAD not allowed" +33:36.760 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/health/state/ from=127.0.0.1:49850 latency=31.5µs +33:36.761 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/health/state/ from=127.0.0.1:49850 latency=3.1µs +33:36.761 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/roles from=127.0.0.1:49850 error="Permission denied" +33:36.761 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/roles from=127.0.0.1:49850 latency=381.101µs +33:36.762 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/roles from=127.0.0.1:49852 error="method PUT not allowed" +33:36.762 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/roles from=127.0.0.1:49852 latency=72.601µs +33:36.764 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/roles from=127.0.0.1:49854 error="method POST not allowed" +33:36.764 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/roles from=127.0.0.1:49854 latency=71.46µs +33:36.765 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/roles from=127.0.0.1:49856 error="method DELETE not allowed" +33:36.765 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/roles from=127.0.0.1:49856 latency=69.56µs +33:36.765 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/roles from=127.0.0.1:49858 error="method HEAD not allowed" +33:36.766 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/roles from=127.0.0.1:49858 latency=30.52µs +33:36.766 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/roles from=127.0.0.1:49858 latency=3.3µs +33:36.766 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/metrics from=127.0.0.1:49858 error="Permission denied" +33:36.766 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/metrics from=127.0.0.1:49858 latency=107.24µs +33:36.768 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/metrics from=127.0.0.1:49860 error="method PUT not allowed" +33:36.768 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/metrics from=127.0.0.1:49860 latency=73.06µs +33:36.769 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/metrics from=127.0.0.1:49862 error="method POST not allowed" +33:36.769 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/metrics from=127.0.0.1:49862 latency=69.32µs +33:36.769 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/metrics from=127.0.0.1:49864 error="method DELETE not allowed" +33:36.770 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/metrics from=127.0.0.1:49864 latency=68.58µs +33:36.770 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/metrics from=127.0.0.1:49866 error="method HEAD not allowed" +33:36.770 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/metrics from=127.0.0.1:49866 latency=31.54µs +33:36.771 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/metrics from=127.0.0.1:49866 latency=3.16µs +33:36.772 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/leave from=127.0.0.1:49866 error="method GET not allowed" +33:36.772 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/leave from=127.0.0.1:49866 latency=71.08µs +33:36.772 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/leave from=127.0.0.1:49868 error="Permission denied" +33:36.772 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/leave from=127.0.0.1:49868 latency=107.88µs +33:36.773 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/leave from=127.0.0.1:49870 error="method POST not allowed" +33:36.773 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/leave from=127.0.0.1:49870 latency=69.4µs +33:36.774 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/leave from=127.0.0.1:49872 error="method DELETE not allowed" +33:36.774 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/leave from=127.0.0.1:49872 latency=69.44µs +33:36.776 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/leave from=127.0.0.1:49874 error="method HEAD not allowed" +33:36.776 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/leave from=127.0.0.1:49874 latency=38.021µs +33:36.776 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/leave from=127.0.0.1:49874 latency=3.16µs +33:36.777 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/role from=127.0.0.1:49874 error="method GET not allowed" +33:36.777 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role from=127.0.0.1:49874 latency=72.58µs +33:36.777 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/role from=127.0.0.1:49876 error="Bad request: Role decoding failed: EOF" +33:36.778 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:49876 latency=84.92µs +33:36.778 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/role from=127.0.0.1:49878 error="method POST not allowed" +33:36.778 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/role from=127.0.0.1:49878 latency=70.161µs +33:36.780 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/role from=127.0.0.1:49880 error="method DELETE not allowed" +33:36.780 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/role from=127.0.0.1:49880 latency=72.921µs +33:36.781 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/role from=127.0.0.1:49882 error="method HEAD not allowed" +33:36.781 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/role from=127.0.0.1:49882 latency=30.52µs +33:36.781 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/role from=127.0.0.1:49882 latency=3.74µs +33:36.781 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/service/register from=127.0.0.1:49882 error="method GET not allowed" +33:36.781 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/register from=127.0.0.1:49882 latency=68.621µs +33:36.782 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:49884 latency=55.281µs +33:36.783 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/service/register from=127.0.0.1:49886 error="method POST not allowed" +33:36.784 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/service/register from=127.0.0.1:49886 latency=70.66µs +33:36.784 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/service/register from=127.0.0.1:49888 error="method DELETE not allowed" +33:36.785 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/service/register from=127.0.0.1:49888 latency=69.541µs +33:36.785 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/service/register from=127.0.0.1:49890 error="method HEAD not allowed" +33:36.785 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/service/register from=127.0.0.1:49890 latency=31µs +33:36.786 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/service/register from=127.0.0.1:49890 latency=3.461µs +33:36.786 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:49890 latency=362.061µs +33:36.788 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/connect/intentions from=127.0.0.1:49892 error="method PUT not allowed" +33:36.788 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/connect/intentions from=127.0.0.1:49892 latency=72.78µs +33:36.789 [ERROR] TestAgent.http: Request error: method=POST url=/v1/connect/intentions from=127.0.0.1:49894 error="Failed to decode request body: EOF" +33:36.789 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:49894 latency=73.1µs +33:36.790 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/connect/intentions from=127.0.0.1:49896 error="method DELETE not allowed" +33:36.790 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/connect/intentions from=127.0.0.1:49896 latency=69.1µs +33:36.790 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/connect/intentions from=127.0.0.1:49898 error="method HEAD not allowed" +33:36.790 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/connect/intentions from=127.0.0.1:49898 latency=30.08µs +33:36.791 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/connect/intentions from=127.0.0.1:49898 latency=3.24µs +33:36.791 [ERROR] TestAgent.http: Request error: method=GET url=/v1/operator/raft/peer from=127.0.0.1:49898 error="method GET not allowed" +33:36.791 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/raft/peer from=127.0.0.1:49898 latency=75.96µs +33:36.792 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/operator/raft/peer from=127.0.0.1:49900 error="method PUT not allowed" +33:36.792 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/operator/raft/peer from=127.0.0.1:49900 latency=68.561µs +33:36.793 [ERROR] TestAgent.http: Request error: method=POST url=/v1/operator/raft/peer from=127.0.0.1:49902 error="method POST not allowed" +33:36.793 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/operator/raft/peer from=127.0.0.1:49902 latency=72.84µs +33:36.794 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/operator/raft/peer from=127.0.0.1:49904 error="Bad request: Must specify either ?id with the server's ID or ?address with IP:port of peer to remove" +33:36.794 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/operator/raft/peer from=127.0.0.1:49904 latency=74.9µs +33:36.795 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/operator/raft/peer from=127.0.0.1:49906 error="method HEAD not allowed" +33:36.795 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/operator/raft/peer from=127.0.0.1:49906 latency=37.62µs +33:36.796 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/operator/raft/peer from=127.0.0.1:49906 latency=3.3µs +33:36.796 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/event/list from=127.0.0.1:49906 latency=346.201µs +33:36.797 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/event/list from=127.0.0.1:49908 error="method PUT not allowed" +33:36.797 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/event/list from=127.0.0.1:49908 latency=327.66µs +33:36.798 [ERROR] TestAgent.http: Request error: method=POST url=/v1/event/list from=127.0.0.1:49910 error="method POST not allowed" +33:36.798 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/event/list from=127.0.0.1:49910 latency=73.56µs +33:36.800 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/event/list from=127.0.0.1:49912 error="method DELETE not allowed" +33:36.800 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/event/list from=127.0.0.1:49912 latency=93.741µs +33:36.800 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/event/list from=127.0.0.1:49914 error="method HEAD not allowed" +33:36.800 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/event/list from=127.0.0.1:49914 latency=30.24µs +33:36.801 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/event/list from=127.0.0.1:49914 latency=3.28µs +33:36.801 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/policies from=127.0.0.1:49914 error="Permission denied" +33:36.801 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policies from=127.0.0.1:49914 latency=158.381µs +33:36.802 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/policies from=127.0.0.1:49916 error="method PUT not allowed" +33:36.802 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policies from=127.0.0.1:49916 latency=75.281µs +33:36.803 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/policies from=127.0.0.1:49918 error="method POST not allowed" +33:36.804 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/policies from=127.0.0.1:49918 latency=179.541µs +33:36.805 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/policies from=127.0.0.1:49920 error="method DELETE not allowed" +33:36.805 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/policies from=127.0.0.1:49920 latency=276.14µs +33:36.806 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/policies from=127.0.0.1:49922 error="method HEAD not allowed" +33:36.806 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/policies from=127.0.0.1:49922 latency=30.741µs +33:36.806 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/policies from=127.0.0.1:49922 latency=3.46µs +33:36.806 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/maintenance from=127.0.0.1:49922 error="method GET not allowed" +33:36.807 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/maintenance from=127.0.0.1:49922 latency=71.22µs +33:36.808 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/maintenance from=127.0.0.1:49924 latency=49.46µs +33:36.809 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/maintenance from=127.0.0.1:49926 error="method POST not allowed" +33:36.809 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/maintenance from=127.0.0.1:49926 latency=70.94µs +33:36.809 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/maintenance from=127.0.0.1:49928 error="method DELETE not allowed" +33:36.809 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/maintenance from=127.0.0.1:49928 latency=69.28µs +33:36.810 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/maintenance from=127.0.0.1:49930 error="method HEAD not allowed" +33:36.810 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/maintenance from=127.0.0.1:49930 latency=29.52µs +33:36.815 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/maintenance from=127.0.0.1:49930 latency=6.44µs +33:36.815 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/force-leave/ from=127.0.0.1:49930 error="method GET not allowed" +33:36.815 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/force-leave/ from=127.0.0.1:49930 latency=79.08µs +33:36.816 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/force-leave/ from=127.0.0.1:49932 error="Permission denied" +33:36.817 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/force-leave/ from=127.0.0.1:49932 latency=111.261µs +33:36.818 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/force-leave/ from=127.0.0.1:49934 error="method POST not allowed" +33:36.818 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/force-leave/ from=127.0.0.1:49934 latency=163.18µs +33:36.819 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/force-leave/ from=127.0.0.1:49936 error="method DELETE not allowed" +33:36.827 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/force-leave/ from=127.0.0.1:49936 latency=8.35762ms +33:36.828 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/force-leave/ from=127.0.0.1:49938 error="method HEAD not allowed" +33:36.828 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/force-leave/ from=127.0.0.1:49938 latency=40.48µs +33:36.828 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/force-leave/ from=127.0.0.1:49938 latency=11.401µs +33:36.829 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/session/info/ from=127.0.0.1:49938 latency=58.061µs +33:36.829 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/session/info/ from=127.0.0.1:49940 error="method PUT not allowed" +33:36.829 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/info/ from=127.0.0.1:49940 latency=71.84µs +33:36.830 [ERROR] TestAgent.http: Request error: method=POST url=/v1/session/info/ from=127.0.0.1:49942 error="method POST not allowed" +33:36.830 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/session/info/ from=127.0.0.1:49942 latency=76.76µs +33:36.831 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/session/info/ from=127.0.0.1:49944 error="method DELETE not allowed" +33:36.831 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/session/info/ from=127.0.0.1:49944 latency=74.84µs +33:36.832 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/session/info/ from=127.0.0.1:49946 error="method HEAD not allowed" +33:36.832 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/session/info/ from=127.0.0.1:49946 latency=31.6µs +33:36.832 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/session/info/ from=127.0.0.1:49946 latency=3.04µs +33:36.833 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/bootstrap from=127.0.0.1:49946 error="method GET not allowed" +33:36.833 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/bootstrap from=127.0.0.1:49946 latency=70.54µs +33:36.834 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/bootstrap from=127.0.0.1:49948 latency=176.041µs +33:36.834 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/bootstrap from=127.0.0.1:49950 error="method POST not allowed" +33:36.834 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/bootstrap from=127.0.0.1:49950 latency=76.54µs +33:36.835 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/bootstrap from=127.0.0.1:49952 error="method DELETE not allowed" +33:36.835 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/bootstrap from=127.0.0.1:49952 latency=72.181µs +33:36.836 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/bootstrap from=127.0.0.1:49954 error="method HEAD not allowed" +33:36.836 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/bootstrap from=127.0.0.1:49954 latency=30.44µs +33:36.836 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/bootstrap from=127.0.0.1:49954 latency=3.24µs +33:36.837 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/rules/translate/ from=127.0.0.1:49954 error="Bad request: Missing token ID" +33:36.837 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/rules/translate/ from=127.0.0.1:49954 latency=70.68µs +33:36.838 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/rules/translate/ from=127.0.0.1:49956 error="method PUT not allowed" +33:36.838 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/rules/translate/ from=127.0.0.1:49956 latency=73.44µs +33:36.838 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/rules/translate/ from=127.0.0.1:49958 error="method POST not allowed" +33:36.838 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/rules/translate/ from=127.0.0.1:49958 latency=75.32µs +33:36.839 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/rules/translate/ from=127.0.0.1:49960 error="method DELETE not allowed" +33:36.839 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/rules/translate/ from=127.0.0.1:49960 latency=70.18µs +33:36.840 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/rules/translate/ from=127.0.0.1:49962 error="method HEAD not allowed" +33:36.840 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/rules/translate/ from=127.0.0.1:49962 latency=42.081µs +33:36.840 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/rules/translate/ from=127.0.0.1:49962 latency=3.12µs +33:36.841 [ERROR] TestAgent.http: Request error: method=GET url=/v1/session/create from=127.0.0.1:49962 error="method GET not allowed" +33:36.841 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/session/create from=127.0.0.1:49962 latency=68.1µs +33:36.842 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/session/create from=127.0.0.1:49964 error="Permission denied" +33:36.842 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:49964 latency=237.101µs +33:36.842 [ERROR] TestAgent.http: Request error: method=POST url=/v1/session/create from=127.0.0.1:49966 error="method POST not allowed" +33:36.842 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/session/create from=127.0.0.1:49966 latency=76.18µs +33:36.843 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/session/create from=127.0.0.1:49968 error="method DELETE not allowed" +33:36.843 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/session/create from=127.0.0.1:49968 latency=82.02µs +33:36.844 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/session/create from=127.0.0.1:49970 error="method HEAD not allowed" +33:36.844 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/session/create from=127.0.0.1:49970 latency=35.32µs +33:36.844 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/session/create from=127.0.0.1:49970 latency=3.36µs +33:36.845 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/info/ from=127.0.0.1:49970 latency=61.74µs +33:36.846 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/info/ from=127.0.0.1:49972 error="method PUT not allowed" +33:36.846 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/info/ from=127.0.0.1:49972 latency=76.3µs +33:36.847 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/info/ from=127.0.0.1:49974 error="method POST not allowed" +33:36.847 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/info/ from=127.0.0.1:49974 latency=78.36µs +33:36.847 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/info/ from=127.0.0.1:49976 error="method DELETE not allowed" +33:36.848 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/info/ from=127.0.0.1:49976 latency=75.56µs +33:36.848 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/info/ from=127.0.0.1:49978 error="method HEAD not allowed" +33:36.848 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/info/ from=127.0.0.1:49978 latency=32.94µs +33:36.849 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/info/ from=127.0.0.1:49978 latency=2.96µs +33:36.849 [ERROR] TestAgent.http: Request error: method=GET url=/v1/coordinate/update from=127.0.0.1:49978 error="method GET not allowed" +33:36.849 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/update from=127.0.0.1:49978 latency=74.12µs +33:36.850 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/coordinate/update from=127.0.0.1:49980 latency=70.9µs +33:36.851 [ERROR] TestAgent.http: Request error: method=POST url=/v1/coordinate/update from=127.0.0.1:49982 error="method POST not allowed" +33:36.851 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/coordinate/update from=127.0.0.1:49982 latency=98.601µs +33:36.852 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/coordinate/update from=127.0.0.1:49984 error="method DELETE not allowed" +33:36.852 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/coordinate/update from=127.0.0.1:49984 latency=81.06µs +33:36.853 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/coordinate/update from=127.0.0.1:49986 error="method HEAD not allowed" +33:36.853 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/coordinate/update from=127.0.0.1:49986 latency=38.161µs +33:36.854 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/coordinate/update from=127.0.0.1:49986 latency=3.56µs +33:36.854 [ERROR] TestAgent.http: Request error: method=GET url=/v1/operator/keyring from=127.0.0.1:49986 error="Reading keyring denied by ACLs" +33:36.854 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/keyring from=127.0.0.1:49986 latency=206.1µs +33:36.856 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/operator/keyring from=127.0.0.1:49988 error="Bad request: Request decode failed: EOF" +33:36.856 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/operator/keyring from=127.0.0.1:49988 latency=121.94µs +33:36.857 [ERROR] TestAgent.http: Request error: method=POST url=/v1/operator/keyring from=127.0.0.1:49990 error="Bad request: Request decode failed: EOF" +33:36.857 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/operator/keyring from=127.0.0.1:49990 latency=87.82µs +33:36.858 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/operator/keyring from=127.0.0.1:49992 error="Bad request: Request decode failed: EOF" +33:36.858 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/operator/keyring from=127.0.0.1:49992 latency=95.04µs +33:36.863 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/operator/keyring from=127.0.0.1:49994 error="method HEAD not allowed" +33:36.863 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/operator/keyring from=127.0.0.1:49994 latency=55.98µs +33:36.864 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/operator/keyring from=127.0.0.1:49994 latency=3.92µs +33:36.864 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/service/ from=127.0.0.1:49994 latency=84.3µs +33:36.865 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/catalog/service/ from=127.0.0.1:49996 error="method PUT not allowed" +33:36.865 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/catalog/service/ from=127.0.0.1:49996 latency=82.12µs +33:36.866 [ERROR] TestAgent.http: Request error: method=POST url=/v1/catalog/service/ from=127.0.0.1:49998 error="method POST not allowed" +33:36.866 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/catalog/service/ from=127.0.0.1:49998 latency=80.6µs +33:36.867 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/catalog/service/ from=127.0.0.1:50000 error="method DELETE not allowed" +33:36.867 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/catalog/service/ from=127.0.0.1:50000 latency=80.46µs +33:36.868 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/catalog/service/ from=127.0.0.1:50002 error="method HEAD not allowed" +33:36.868 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/catalog/service/ from=127.0.0.1:50002 latency=33.741µs +33:36.868 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/catalog/service/ from=127.0.0.1:50002 latency=4.04µs +33:36.870 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/ca/roots from=127.0.0.1:50002 latency=1.463284ms +33:36.876 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/connect/ca/roots from=127.0.0.1:50004 error="method PUT not allowed" +33:36.876 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/connect/ca/roots from=127.0.0.1:50004 latency=136.04µs +33:36.877 [ERROR] TestAgent.http: Request error: method=POST url=/v1/connect/ca/roots from=127.0.0.1:50006 error="method POST not allowed" +33:36.878 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/ca/roots from=127.0.0.1:50006 latency=107.6µs +33:36.891 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/connect/ca/roots from=127.0.0.1:50008 error="method DELETE not allowed" +33:36.892 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/connect/ca/roots from=127.0.0.1:50008 latency=126.98µs +33:36.893 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/connect/ca/roots from=127.0.0.1:50010 error="method HEAD not allowed" +33:36.893 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/connect/ca/roots from=127.0.0.1:50010 latency=74µs +33:36.893 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/connect/ca/roots from=127.0.0.1:50010 latency=5.12µs +33:36.894 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/clone/ from=127.0.0.1:50010 error="method GET not allowed" +33:36.894 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/clone/ from=127.0.0.1:50010 latency=110.701µs +33:36.899 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/clone/ from=127.0.0.1:50012 latency=88.28µs +33:36.902 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/clone/ from=127.0.0.1:50014 error="method POST not allowed" +33:36.902 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/clone/ from=127.0.0.1:50014 latency=183.881µs +33:36.904 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/clone/ from=127.0.0.1:50016 error="method DELETE not allowed" +33:36.904 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/clone/ from=127.0.0.1:50016 latency=114.6µs +33:36.906 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/clone/ from=127.0.0.1:50018 error="method HEAD not allowed" +33:36.906 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/clone/ from=127.0.0.1:50018 latency=43.28µs +33:36.906 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/clone/ from=127.0.0.1:50018 latency=3.6µs +33:36.907 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/binding-rule/ from=127.0.0.1:50018 error="Bad request: Missing binding rule ID" +33:36.907 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/ from=127.0.0.1:50018 latency=156.26µs +33:36.908 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/binding-rule/ from=127.0.0.1:50020 error="Bad request: BindingRule decoding failed: EOF" +33:36.908 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/ from=127.0.0.1:50020 latency=107.94µs +33:36.909 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/binding-rule/ from=127.0.0.1:50022 error="method POST not allowed" +33:36.909 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/binding-rule/ from=127.0.0.1:50022 latency=77.2µs +33:36.910 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/binding-rule/ from=127.0.0.1:50024 error="Bad request: Missing binding rule ID" +33:36.910 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/ from=127.0.0.1:50024 latency=99.901µs +33:36.912 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/binding-rule/ from=127.0.0.1:50026 error="method HEAD not allowed" +33:36.912 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/binding-rule/ from=127.0.0.1:50026 latency=57.14µs +33:36.912 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/binding-rule/ from=127.0.0.1:50026 latency=5.08µs +33:36.913 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/token/ from=127.0.0.1:50026 error="method GET not allowed" +33:36.913 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/token/ from=127.0.0.1:50026 latency=130.92µs +33:36.915 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/token/ from=127.0.0.1:50028 error="Permission denied" +33:36.915 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/token/ from=127.0.0.1:50028 latency=292.76µs +33:36.916 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/token/ from=127.0.0.1:50030 error="method POST not allowed" +33:36.916 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/token/ from=127.0.0.1:50030 latency=83.54µs +33:36.917 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/token/ from=127.0.0.1:50032 error="method DELETE not allowed" +33:36.917 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/token/ from=127.0.0.1:50032 latency=74.141µs +33:36.918 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/token/ from=127.0.0.1:50034 error="method HEAD not allowed" +33:36.918 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/token/ from=127.0.0.1:50034 latency=31.52µs +33:36.918 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/token/ from=127.0.0.1:50034 latency=3.28µs +33:36.919 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/internal/ui/node/ from=127.0.0.1:50034 latency=295.881µs +33:36.920 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/internal/ui/node/ from=127.0.0.1:50036 error="method PUT not allowed" +33:36.920 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/internal/ui/node/ from=127.0.0.1:50036 latency=76.4µs +33:36.921 [ERROR] TestAgent.http: Request error: method=POST url=/v1/internal/ui/node/ from=127.0.0.1:50038 error="method POST not allowed" +33:36.921 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/internal/ui/node/ from=127.0.0.1:50038 latency=72.72µs +33:36.921 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/internal/ui/node/ from=127.0.0.1:50040 error="method DELETE not allowed" +33:36.921 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/internal/ui/node/ from=127.0.0.1:50040 latency=74.44µs +33:36.922 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/internal/ui/node/ from=127.0.0.1:50042 error="method HEAD not allowed" +33:36.922 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/internal/ui/node/ from=127.0.0.1:50042 latency=40.78µs +33:36.923 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/internal/ui/node/ from=127.0.0.1:50042 latency=3.84µs +33:36.923 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/check/warn/ from=127.0.0.1:50042 error="method GET not allowed" +33:36.923 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/check/warn/ from=127.0.0.1:50042 latency=78.14µs +33:36.924 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/check/warn/ from=127.0.0.1:50044 error="Unknown check """ +33:36.924 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/check/warn/ from=127.0.0.1:50044 latency=137.78µs +33:36.925 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/check/warn/ from=127.0.0.1:50046 error="method POST not allowed" +33:36.925 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/check/warn/ from=127.0.0.1:50046 latency=75.72µs +33:36.926 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/check/warn/ from=127.0.0.1:50048 error="method DELETE not allowed" +33:36.926 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/check/warn/ from=127.0.0.1:50048 latency=84.161µs +33:36.927 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/check/warn/ from=127.0.0.1:50050 error="method HEAD not allowed" +33:36.927 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/check/warn/ from=127.0.0.1:50050 latency=35.72µs +33:36.927 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/check/warn/ from=127.0.0.1:50050 latency=3.4µs +33:36.928 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:50050 error="URI must be either service or agent" +33:36.928 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:50050 latency=659.281µs +33:36.929 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:50052 error="method PUT not allowed" +33:36.929 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:50052 latency=82.321µs +33:36.930 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:50054 error="method POST not allowed" +33:36.930 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:50054 latency=75.321µs +33:36.931 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:50056 error="method DELETE not allowed" +33:36.931 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:50056 latency=94.821µs +33:36.932 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:50058 error="method HEAD not allowed" +33:36.932 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:50058 latency=35.86µs +33:36.932 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:50058 latency=3.18µs +33:36.933 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/connect/ from=127.0.0.1:50058 latency=257.301µs +33:36.934 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/catalog/connect/ from=127.0.0.1:50060 error="method PUT not allowed" +33:36.934 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/catalog/connect/ from=127.0.0.1:50060 latency=72.48µs +33:36.934 [ERROR] TestAgent.http: Request error: method=POST url=/v1/catalog/connect/ from=127.0.0.1:50062 error="method POST not allowed" +33:36.934 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/catalog/connect/ from=127.0.0.1:50062 latency=76.42µs +33:36.935 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/catalog/connect/ from=127.0.0.1:50064 error="method DELETE not allowed" +33:36.935 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/catalog/connect/ from=127.0.0.1:50064 latency=117.42µs +33:36.936 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/catalog/connect/ from=127.0.0.1:50066 error="method HEAD not allowed" +33:36.936 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/catalog/connect/ from=127.0.0.1:50066 latency=35.84µs +33:36.936 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/catalog/connect/ from=127.0.0.1:50066 latency=3.58µs +33:36.937 [ERROR] TestAgent.http: Request error: method=GET url=/v1/config from=127.0.0.1:50066 error="method GET not allowed" +33:36.937 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config from=127.0.0.1:50066 latency=74.541µs +33:36.938 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/config from=127.0.0.1:50068 error="Bad request: Request decoding failed: EOF" +33:36.938 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:50068 latency=91.6µs +33:36.938 [ERROR] TestAgent.http: Request error: method=POST url=/v1/config from=127.0.0.1:50070 error="method POST not allowed" +33:36.939 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/config from=127.0.0.1:50070 latency=82.34µs +33:36.939 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/config from=127.0.0.1:50072 error="method DELETE not allowed" +33:36.939 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/config from=127.0.0.1:50072 latency=76.78µs +33:36.940 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/config from=127.0.0.1:50074 error="method HEAD not allowed" +33:36.940 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/config from=127.0.0.1:50074 latency=41.74µs +33:36.941 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/config from=127.0.0.1:50074 latency=3.32µs +33:36.941 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/internal/federation-states/mesh-gateways from=127.0.0.1:50074 latency=330.881µs +33:36.942 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/internal/federation-states/mesh-gateways from=127.0.0.1:50076 error="method PUT not allowed" +33:36.942 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/internal/federation-states/mesh-gateways from=127.0.0.1:50076 latency=78.46µs +33:36.943 [ERROR] TestAgent.http: Request error: method=POST url=/v1/internal/federation-states/mesh-gateways from=127.0.0.1:50078 error="method POST not allowed" +33:36.943 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/internal/federation-states/mesh-gateways from=127.0.0.1:50078 latency=78.96µs +33:36.944 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/internal/federation-states/mesh-gateways from=127.0.0.1:50080 error="method DELETE not allowed" +33:36.944 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/internal/federation-states/mesh-gateways from=127.0.0.1:50080 latency=79.54µs +33:36.945 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/internal/federation-states/mesh-gateways from=127.0.0.1:50082 error="method HEAD not allowed" +33:36.945 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/internal/federation-states/mesh-gateways from=127.0.0.1:50082 latency=47.36µs +33:36.945 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/internal/federation-states/mesh-gateways from=127.0.0.1:50082 latency=3.42µs +33:36.946 [ERROR] TestAgent.http: Request error: method=GET url=/v1/internal/federation-state/ from=127.0.0.1:50082 error="Bad request: Missing datacenter name" +33:36.946 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/internal/federation-state/ from=127.0.0.1:50082 latency=76.56µs +33:36.947 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/internal/federation-state/ from=127.0.0.1:50084 error="method PUT not allowed" +33:36.947 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/internal/federation-state/ from=127.0.0.1:50084 latency=73.58µs +33:36.947 [ERROR] TestAgent.http: Request error: method=POST url=/v1/internal/federation-state/ from=127.0.0.1:50086 error="method POST not allowed" +33:36.948 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/internal/federation-state/ from=127.0.0.1:50086 latency=78.1µs +33:36.948 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/internal/federation-state/ from=127.0.0.1:50088 error="method DELETE not allowed" +33:36.948 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/internal/federation-state/ from=127.0.0.1:50088 latency=80.68µs +33:36.949 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/internal/federation-state/ from=127.0.0.1:50090 error="method HEAD not allowed" +33:36.949 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/internal/federation-state/ from=127.0.0.1:50090 latency=32.68µs +33:36.950 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/internal/federation-state/ from=127.0.0.1:50090 latency=2.94µs +33:36.950 [ERROR] TestAgent.http: Request error: method=GET url=/v1/operator/autopilot/configuration from=127.0.0.1:50090 error="Permission denied" +33:36.950 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/autopilot/configuration from=127.0.0.1:50090 latency=192.76µs +33:36.951 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/operator/autopilot/configuration from=127.0.0.1:50092 error="Bad request: Error parsing autopilot config: EOF" +33:36.951 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/operator/autopilot/configuration from=127.0.0.1:50092 latency=96.58µs +33:36.952 [ERROR] TestAgent.http: Request error: method=POST url=/v1/operator/autopilot/configuration from=127.0.0.1:50094 error="method POST not allowed" +33:36.952 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/operator/autopilot/configuration from=127.0.0.1:50094 latency=78.481µs +33:36.953 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/operator/autopilot/configuration from=127.0.0.1:50096 error="method DELETE not allowed" +33:36.953 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/operator/autopilot/configuration from=127.0.0.1:50096 latency=80.88µs +33:36.954 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/operator/autopilot/configuration from=127.0.0.1:50098 error="method HEAD not allowed" +33:36.954 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/operator/autopilot/configuration from=127.0.0.1:50098 latency=37.18µs +33:36.954 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/operator/autopilot/configuration from=127.0.0.1:50098 latency=3.16µs +33:36.955 [ERROR] TestAgent.http: Request error: method=GET url=/v1/session/destroy/ from=127.0.0.1:50098 error="method GET not allowed" +33:36.955 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/session/destroy/ from=127.0.0.1:50098 latency=77.02µs +33:36.956 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/ from=127.0.0.1:50100 latency=57.941µs +33:36.956 [ERROR] TestAgent.http: Request error: method=POST url=/v1/session/destroy/ from=127.0.0.1:50102 error="method POST not allowed" +33:36.956 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/session/destroy/ from=127.0.0.1:50102 latency=78.961µs +33:36.957 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/session/destroy/ from=127.0.0.1:50104 error="method DELETE not allowed" +33:36.957 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/session/destroy/ from=127.0.0.1:50104 latency=71.94µs +33:36.958 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/session/destroy/ from=127.0.0.1:50106 error="method HEAD not allowed" +33:36.958 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/session/destroy/ from=127.0.0.1:50106 latency=53.6µs +33:36.958 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/session/destroy/ from=127.0.0.1:50106 latency=3.82µs +33:36.959 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/check/deregister/ from=127.0.0.1:50106 error="method GET not allowed" +33:36.959 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/check/deregister/ from=127.0.0.1:50106 latency=77.5µs +33:36.960 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/check/deregister/ from=127.0.0.1:50108 error="Unknown check """ +33:36.960 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/check/deregister/ from=127.0.0.1:50108 latency=136.501µs +33:36.961 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/check/deregister/ from=127.0.0.1:50110 error="method POST not allowed" +33:36.961 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/check/deregister/ from=127.0.0.1:50110 latency=76.02µs +33:36.962 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/check/deregister/ from=127.0.0.1:50112 error="method DELETE not allowed" +33:36.962 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/check/deregister/ from=127.0.0.1:50112 latency=84.24µs +33:36.962 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/check/deregister/ from=127.0.0.1:50114 error="method HEAD not allowed" +33:36.962 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/check/deregister/ from=127.0.0.1:50114 latency=35.52µs +33:36.963 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/check/deregister/ from=127.0.0.1:50114 latency=3.46µs +33:36.964 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/datacenters from=127.0.0.1:50114 latency=272.2µs +33:36.964 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/catalog/datacenters from=127.0.0.1:50116 error="method PUT not allowed" +33:36.964 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/catalog/datacenters from=127.0.0.1:50116 latency=76.22µs +33:36.965 [ERROR] TestAgent.http: Request error: method=POST url=/v1/catalog/datacenters from=127.0.0.1:50118 error="method POST not allowed" +33:36.965 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/catalog/datacenters from=127.0.0.1:50118 latency=77.18µs +33:36.966 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/catalog/datacenters from=127.0.0.1:50120 error="method DELETE not allowed" +33:36.966 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/catalog/datacenters from=127.0.0.1:50120 latency=76.58µs +33:36.971 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/catalog/datacenters from=127.0.0.1:50122 error="method HEAD not allowed" +33:36.971 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/catalog/datacenters from=127.0.0.1:50122 latency=63.58µs +33:36.972 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/catalog/datacenters from=127.0.0.1:50122 latency=4.38µs +33:36.972 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-05b1ba5a-0998-b2db-f378-5ab5ce213906 +33:36.973 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/internal/ui/services from=127.0.0.1:50122 latency=563.781µs +33:36.974 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/internal/ui/services from=127.0.0.1:50124 error="method PUT not allowed" +33:36.974 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/internal/ui/services from=127.0.0.1:50124 latency=93.4µs +33:36.983 [ERROR] TestAgent.http: Request error: method=POST url=/v1/internal/ui/services from=127.0.0.1:50126 error="method POST not allowed" +33:36.983 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/internal/ui/services from=127.0.0.1:50126 latency=173.72µs +33:36.984 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/internal/ui/services from=127.0.0.1:50128 error="method DELETE not allowed" +33:36.984 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/internal/ui/services from=127.0.0.1:50128 latency=79.66µs +33:36.985 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/internal/ui/services from=127.0.0.1:50130 error="method HEAD not allowed" +33:36.985 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/internal/ui/services from=127.0.0.1:50130 latency=32.441µs +33:36.986 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/internal/ui/services from=127.0.0.1:50130 latency=3.6µs +33:36.986 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/list from=127.0.0.1:50130 error="Permission denied" +33:36.986 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/list from=127.0.0.1:50130 latency=212.241µs +33:36.989 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/list from=127.0.0.1:50132 error="method PUT not allowed" +33:36.989 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/list from=127.0.0.1:50132 latency=315.68µs +33:36.990 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/list from=127.0.0.1:50134 error="method POST not allowed" +33:36.991 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/list from=127.0.0.1:50134 latency=94.3µs +33:36.992 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/list from=127.0.0.1:50136 error="method DELETE not allowed" +33:36.992 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/list from=127.0.0.1:50136 latency=88.02µs +33:36.993 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/list from=127.0.0.1:50138 error="method HEAD not allowed" +33:36.993 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/list from=127.0.0.1:50138 latency=36.121µs +33:36.993 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/list from=127.0.0.1:50138 latency=3.72µs +33:36.994 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/role/ from=127.0.0.1:50138 error="Bad request: Missing role ID" +33:36.994 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/ from=127.0.0.1:50138 latency=80.14µs +33:36.995 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/role/ from=127.0.0.1:50140 error="Bad request: Role decoding failed: EOF" +33:36.995 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/ from=127.0.0.1:50140 latency=118.34µs +33:36.997 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/role/ from=127.0.0.1:50142 error="method POST not allowed" +33:36.997 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/role/ from=127.0.0.1:50142 latency=81.801µs +33:36.998 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/role/ from=127.0.0.1:50144 error="Bad request: Missing role ID" +33:36.998 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/role/ from=127.0.0.1:50144 latency=76.541µs +33:36.998 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/role/ from=127.0.0.1:50146 error="method HEAD not allowed" +33:36.998 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/role/ from=127.0.0.1:50146 latency=33.22µs +33:36.999 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/role/ from=127.0.0.1:50146 latency=3.12µs +33:36.999 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/binding-rule from=127.0.0.1:50146 error="method GET not allowed" +33:36.999 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule from=127.0.0.1:50146 latency=87.281µs +33:37.000 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:50148 error="Bad request: BindingRule decoding failed: EOF" +33:37.000 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:50148 latency=94.601µs +33:37.001 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/binding-rule from=127.0.0.1:50150 error="method POST not allowed" +33:37.001 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/binding-rule from=127.0.0.1:50150 latency=76.001µs +33:37.002 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/binding-rule from=127.0.0.1:50152 error="method DELETE not allowed" +33:37.002 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule from=127.0.0.1:50152 latency=74.321µs +33:37.003 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/binding-rule from=127.0.0.1:50154 error="method HEAD not allowed" +33:37.003 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/binding-rule from=127.0.0.1:50154 latency=33.96µs +33:37.003 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/binding-rule from=127.0.0.1:50154 latency=3.14µs +33:37.004 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/token/ from=127.0.0.1:50154 error="Bad request: Missing token ID" +33:37.004 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/ from=127.0.0.1:50154 latency=77.341µs +33:37.004 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/token/ from=127.0.0.1:50156 error="Bad request: Token decoding failed: EOF" +33:37.004 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/ from=127.0.0.1:50156 latency=90.541µs +33:37.005 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/token/ from=127.0.0.1:50158 error="method POST not allowed" +33:37.005 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/token/ from=127.0.0.1:50158 latency=84.801µs +33:37.006 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/token/ from=127.0.0.1:50160 error="Bad request: Missing token ID" +33:37.006 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/token/ from=127.0.0.1:50160 latency=76.961µs +33:37.007 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/token/ from=127.0.0.1:50162 error="method HEAD not allowed" +33:37.007 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/token/ from=127.0.0.1:50162 latency=36.84µs +33:37.007 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/token/ from=127.0.0.1:50162 latency=3.32µs +33:37.008 [DEBUG] TestAgent: dropping node from result due to ACLs: node=Node-05b1ba5a-0998-b2db-f378-5ab5ce213906 accessorID=00000000-0000-0000-0000-000000000002 +33:37.008 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/members from=127.0.0.1:50162 latency=357.141µs +33:37.009 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/members from=127.0.0.1:50164 error="method PUT not allowed" +33:37.009 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/members from=127.0.0.1:50164 latency=79.76µs +33:37.010 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/members from=127.0.0.1:50166 error="method POST not allowed" +33:37.010 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/members from=127.0.0.1:50166 latency=86.56µs +33:37.011 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/members from=127.0.0.1:50168 error="method DELETE not allowed" +33:37.011 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/members from=127.0.0.1:50168 latency=85.361µs +33:37.012 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/members from=127.0.0.1:50170 error="method HEAD not allowed" +33:37.012 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/members from=127.0.0.1:50170 latency=37.68µs +33:37.012 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/members from=127.0.0.1:50170 latency=3.44µs +33:37.013 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/check/pass/ from=127.0.0.1:50170 error="method GET not allowed" +33:37.013 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/check/pass/ from=127.0.0.1:50170 latency=80.48µs +33:37.014 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/check/pass/ from=127.0.0.1:50172 error="Unknown check """ +33:37.014 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/check/pass/ from=127.0.0.1:50172 latency=132.3µs +33:37.015 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/check/pass/ from=127.0.0.1:50174 error="method POST not allowed" +33:37.015 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/check/pass/ from=127.0.0.1:50174 latency=79.68µs +33:37.016 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/check/pass/ from=127.0.0.1:50176 error="method DELETE not allowed" +33:37.016 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/check/pass/ from=127.0.0.1:50176 latency=80.24µs +33:37.016 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/check/pass/ from=127.0.0.1:50178 error="method HEAD not allowed" +33:37.016 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/check/pass/ from=127.0.0.1:50178 latency=37.64µs +33:37.017 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/check/pass/ from=127.0.0.1:50178 latency=4.02µs +33:37.017 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/service/deregister/ from=127.0.0.1:50178 error="method GET not allowed" +33:37.017 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/deregister/ from=127.0.0.1:50178 latency=77.34µs +33:37.018 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/service/deregister/ from=127.0.0.1:50180 error="Unknown service {"" {}}" +33:37.018 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/deregister/ from=127.0.0.1:50180 latency=144.341µs +33:37.019 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/service/deregister/ from=127.0.0.1:50182 error="method POST not allowed" +33:37.019 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/service/deregister/ from=127.0.0.1:50182 latency=83.3µs +33:37.020 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/service/deregister/ from=127.0.0.1:50184 error="method DELETE not allowed" +33:37.020 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/service/deregister/ from=127.0.0.1:50184 latency=76.88µs +33:37.021 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/service/deregister/ from=127.0.0.1:50186 error="method HEAD not allowed" +33:37.021 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/service/deregister/ from=127.0.0.1:50186 latency=34.14µs +33:37.021 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/service/deregister/ from=127.0.0.1:50186 latency=3.6µs +33:37.022 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/gateway-services/ from=127.0.0.1:50186 latency=68.801µs +33:37.023 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/catalog/gateway-services/ from=127.0.0.1:50188 error="method PUT not allowed" +33:37.023 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/catalog/gateway-services/ from=127.0.0.1:50188 latency=76.741µs +33:37.023 [ERROR] TestAgent.http: Request error: method=POST url=/v1/catalog/gateway-services/ from=127.0.0.1:50190 error="method POST not allowed" +33:37.023 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/catalog/gateway-services/ from=127.0.0.1:50190 latency=73.18µs +33:37.024 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/catalog/gateway-services/ from=127.0.0.1:50192 error="method DELETE not allowed" +33:37.025 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/catalog/gateway-services/ from=127.0.0.1:50192 latency=82.96µs +33:37.025 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/catalog/gateway-services/ from=127.0.0.1:50194 error="method HEAD not allowed" +33:37.025 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/catalog/gateway-services/ from=127.0.0.1:50194 latency=39.62µs +33:37.026 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/catalog/gateway-services/ from=127.0.0.1:50194 latency=3.4µs +33:37.026 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/health/service/ from=127.0.0.1:50194 latency=72.36µs +33:37.027 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/health/service/ from=127.0.0.1:50196 error="method PUT not allowed" +33:37.027 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/health/service/ from=127.0.0.1:50196 latency=78.64µs +33:37.028 [ERROR] TestAgent.http: Request error: method=POST url=/v1/health/service/ from=127.0.0.1:50198 error="method POST not allowed" +33:37.028 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/health/service/ from=127.0.0.1:50198 latency=73.481µs +33:37.029 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/health/service/ from=127.0.0.1:50200 error="method DELETE not allowed" +33:37.029 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/health/service/ from=127.0.0.1:50200 latency=71.901µs +33:37.029 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/health/service/ from=127.0.0.1:50202 error="method HEAD not allowed" +33:37.029 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/health/service/ from=127.0.0.1:50202 latency=34µs +33:37.030 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/health/service/ from=127.0.0.1:50202 latency=3.38µs +33:37.030 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/create from=127.0.0.1:50202 error="method GET not allowed" +33:37.030 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/create from=127.0.0.1:50202 latency=75.36µs +33:37.031 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/create from=127.0.0.1:50204 error="Permission denied" +33:37.031 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/create from=127.0.0.1:50204 latency=198µs +33:37.032 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/create from=127.0.0.1:50206 error="method POST not allowed" +33:37.032 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/create from=127.0.0.1:50206 latency=76.12µs +33:37.033 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/create from=127.0.0.1:50208 error="method DELETE not allowed" +33:37.033 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/create from=127.0.0.1:50208 latency=78.72µs +33:37.034 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/create from=127.0.0.1:50210 error="method HEAD not allowed" +33:37.034 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/create from=127.0.0.1:50210 latency=31.08µs +33:37.034 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/create from=127.0.0.1:50210 latency=3.9µs +33:37.035 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/replication from=127.0.0.1:50210 latency=245.1µs +33:37.035 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/replication from=127.0.0.1:50212 error="method PUT not allowed" +33:37.036 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/replication from=127.0.0.1:50212 latency=77.28µs +33:37.036 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/replication from=127.0.0.1:50214 error="method POST not allowed" +33:37.036 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/replication from=127.0.0.1:50214 latency=80µs +33:37.037 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/replication from=127.0.0.1:50216 error="method DELETE not allowed" +33:37.037 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/replication from=127.0.0.1:50216 latency=72.76µs +33:37.038 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/replication from=127.0.0.1:50218 error="method HEAD not allowed" +33:37.038 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/replication from=127.0.0.1:50218 latency=34.82µs +33:37.038 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/replication from=127.0.0.1:50218 latency=3.58µs +33:37.039 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/token/self from=127.0.0.1:50218 error="ACL not found" +33:37.039 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:50218 latency=188.92µs +33:37.040 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/token/self from=127.0.0.1:50220 error="method PUT not allowed" +33:37.040 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/self from=127.0.0.1:50220 latency=79.44µs +33:37.040 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/token/self from=127.0.0.1:50222 error="method POST not allowed" +33:37.041 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/token/self from=127.0.0.1:50222 latency=71.48µs +33:37.041 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/token/self from=127.0.0.1:50224 error="method DELETE not allowed" +33:37.041 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/token/self from=127.0.0.1:50224 latency=81.04µs +33:37.042 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/token/self from=127.0.0.1:50226 error="method HEAD not allowed" +33:37.042 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/token/self from=127.0.0.1:50226 latency=35µs +33:37.043 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/token/self from=127.0.0.1:50226 latency=3.56µs +33:37.043 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/node-services/ from=127.0.0.1:50226 latency=66.361µs +33:37.044 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/catalog/node-services/ from=127.0.0.1:50228 error="method PUT not allowed" +33:37.044 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/catalog/node-services/ from=127.0.0.1:50228 latency=77.381µs +33:37.045 [ERROR] TestAgent.http: Request error: method=POST url=/v1/catalog/node-services/ from=127.0.0.1:50230 error="method POST not allowed" +33:37.045 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/catalog/node-services/ from=127.0.0.1:50230 latency=72.34µs +33:37.045 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/catalog/node-services/ from=127.0.0.1:50232 error="method DELETE not allowed" +33:37.045 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/catalog/node-services/ from=127.0.0.1:50232 latency=75.04µs +33:37.046 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/catalog/node-services/ from=127.0.0.1:50234 error="method HEAD not allowed" +33:37.046 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/catalog/node-services/ from=127.0.0.1:50234 latency=33.94µs +33:37.047 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/catalog/node-services/ from=127.0.0.1:50234 latency=3.52µs +33:37.047 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/health/node/ from=127.0.0.1:50234 latency=62.7µs +33:37.048 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/health/node/ from=127.0.0.1:50236 error="method PUT not allowed" +33:37.048 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/health/node/ from=127.0.0.1:50236 latency=75.72µs +33:37.049 [ERROR] TestAgent.http: Request error: method=POST url=/v1/health/node/ from=127.0.0.1:50238 error="method POST not allowed" +33:37.049 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/health/node/ from=127.0.0.1:50238 latency=74.48µs +33:37.049 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/health/node/ from=127.0.0.1:50240 error="method DELETE not allowed" +33:37.050 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/health/node/ from=127.0.0.1:50240 latency=69.981µs +33:37.050 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/health/node/ from=127.0.0.1:50242 error="method HEAD not allowed" +33:37.050 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/health/node/ from=127.0.0.1:50242 latency=36.46µs +33:37.051 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/health/node/ from=127.0.0.1:50242 latency=3.64µs +33:37.051 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/role/name/ from=127.0.0.1:50242 error="Bad request: Missing role Name" +33:37.051 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/name/ from=127.0.0.1:50242 latency=74.74µs +33:37.052 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/role/name/ from=127.0.0.1:50244 error="method PUT not allowed" +33:37.052 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/name/ from=127.0.0.1:50244 latency=76.5µs +33:37.053 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/role/name/ from=127.0.0.1:50246 error="method POST not allowed" +33:37.053 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/role/name/ from=127.0.0.1:50246 latency=83.02µs +33:37.054 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/role/name/ from=127.0.0.1:50248 error="method DELETE not allowed" +33:37.054 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/role/name/ from=127.0.0.1:50248 latency=76µs +33:37.054 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/role/name/ from=127.0.0.1:50250 error="method HEAD not allowed" +33:37.054 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/role/name/ from=127.0.0.1:50250 latency=36.14µs +33:37.055 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/role/name/ from=127.0.0.1:50250 latency=3.4µs +33:37.056 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/nodes from=127.0.0.1:50250 latency=363.761µs +33:37.056 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/coordinate/nodes from=127.0.0.1:50252 error="method PUT not allowed" +33:37.057 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/coordinate/nodes from=127.0.0.1:50252 latency=79.76µs +33:37.057 [ERROR] TestAgent.http: Request error: method=POST url=/v1/coordinate/nodes from=127.0.0.1:50254 error="method POST not allowed" +33:37.057 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/coordinate/nodes from=127.0.0.1:50254 latency=77µs +33:37.058 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/coordinate/nodes from=127.0.0.1:50256 error="method DELETE not allowed" +33:37.058 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/coordinate/nodes from=127.0.0.1:50256 latency=77.86µs +33:37.059 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/coordinate/nodes from=127.0.0.1:50258 error="method HEAD not allowed" +33:37.059 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/coordinate/nodes from=127.0.0.1:50258 latency=33.4µs +33:37.060 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/coordinate/nodes from=127.0.0.1:50258 latency=3.26µs +33:37.060 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/destroy/ from=127.0.0.1:50258 error="method GET not allowed" +33:37.060 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/destroy/ from=127.0.0.1:50258 latency=78.34µs +33:37.061 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/destroy/ from=127.0.0.1:50260 latency=53.38µs +33:37.062 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/destroy/ from=127.0.0.1:50262 error="method POST not allowed" +33:37.062 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/destroy/ from=127.0.0.1:50262 latency=84.82µs +33:37.062 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/destroy/ from=127.0.0.1:50264 error="method DELETE not allowed" +33:37.063 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/destroy/ from=127.0.0.1:50264 latency=74.36µs +33:37.063 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/destroy/ from=127.0.0.1:50266 error="method HEAD not allowed" +33:37.063 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/destroy/ from=127.0.0.1:50266 latency=31.461µs +33:37.064 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/destroy/ from=127.0.0.1:50266 latency=3.56µs +33:37.064 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/auth-methods from=127.0.0.1:50266 error="Permission denied" +33:37.064 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-methods from=127.0.0.1:50266 latency=186.921µs +33:37.065 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/auth-methods from=127.0.0.1:50268 error="method PUT not allowed" +33:37.065 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-methods from=127.0.0.1:50268 latency=80.54µs +33:37.066 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/auth-methods from=127.0.0.1:50270 error="method POST not allowed" +33:37.066 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/auth-methods from=127.0.0.1:50270 latency=77.54µs +33:37.067 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/auth-methods from=127.0.0.1:50272 error="method DELETE not allowed" +33:37.067 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/auth-methods from=127.0.0.1:50272 latency=80.08µs +33:37.068 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/auth-methods from=127.0.0.1:50274 error="method HEAD not allowed" +33:37.068 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/auth-methods from=127.0.0.1:50274 latency=41.4µs +33:37.068 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/auth-methods from=127.0.0.1:50274 latency=3.34µs +33:37.069 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/health/service/name/ from=127.0.0.1:50274 error="Bad request: Missing service Name" +33:37.069 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/health/service/name/ from=127.0.0.1:50274 latency=80.6µs +33:37.070 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/health/service/name/ from=127.0.0.1:50276 error="method PUT not allowed" +33:37.070 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/health/service/name/ from=127.0.0.1:50276 latency=78.94µs +33:37.070 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/health/service/name/ from=127.0.0.1:50278 error="method POST not allowed" +33:37.070 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/health/service/name/ from=127.0.0.1:50278 latency=73.58µs +33:37.071 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/health/service/name/ from=127.0.0.1:50280 error="method DELETE not allowed" +33:37.071 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/health/service/name/ from=127.0.0.1:50280 latency=75.481µs +33:37.072 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/health/service/name/ from=127.0.0.1:50282 error="method HEAD not allowed" +33:37.072 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/health/service/name/ from=127.0.0.1:50282 latency=39.76µs +33:37.072 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/health/service/name/ from=127.0.0.1:50282 latency=3.4µs +33:37.073 [ERROR] TestAgent.http: Request error: method=GET url=/v1/internal/federation-states from=127.0.0.1:50282 error="Permission denied" +33:37.073 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/internal/federation-states from=127.0.0.1:50282 latency=179.921µs +33:37.074 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/internal/federation-states from=127.0.0.1:50284 error="method PUT not allowed" +33:37.074 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/internal/federation-states from=127.0.0.1:50284 latency=79.36µs +33:37.075 [ERROR] TestAgent.http: Request error: method=POST url=/v1/internal/federation-states from=127.0.0.1:50286 error="method POST not allowed" +33:37.075 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/internal/federation-states from=127.0.0.1:50286 latency=77.82µs +33:37.075 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/internal/federation-states from=127.0.0.1:50288 error="method DELETE not allowed" +33:37.076 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/internal/federation-states from=127.0.0.1:50288 latency=81.94µs +33:37.076 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/internal/federation-states from=127.0.0.1:50290 error="method HEAD not allowed" +33:37.076 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/internal/federation-states from=127.0.0.1:50290 latency=35.26µs +33:37.077 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/internal/federation-states from=127.0.0.1:50290 latency=3.76µs +33:37.077 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/health/connect/ from=127.0.0.1:50290 latency=72.96µs +33:37.078 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/health/connect/ from=127.0.0.1:50292 error="method PUT not allowed" +33:37.078 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/health/connect/ from=127.0.0.1:50292 latency=78.32µs +33:37.079 [ERROR] TestAgent.http: Request error: method=POST url=/v1/health/connect/ from=127.0.0.1:50294 error="method POST not allowed" +33:37.079 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/health/connect/ from=127.0.0.1:50294 latency=78.88µs +33:37.080 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/health/connect/ from=127.0.0.1:50296 error="method DELETE not allowed" +33:37.080 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/health/connect/ from=127.0.0.1:50296 latency=84.9µs +33:37.081 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/health/connect/ from=127.0.0.1:50298 error="method HEAD not allowed" +33:37.081 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/health/connect/ from=127.0.0.1:50298 latency=36.36µs +33:37.081 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/health/connect/ from=127.0.0.1:50298 latency=3.06µs +33:37.082 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/policy from=127.0.0.1:50298 error="method GET not allowed" +33:37.082 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policy from=127.0.0.1:50298 latency=78.22µs +33:37.082 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/policy from=127.0.0.1:50300 error="Bad request: Policy decoding failed: EOF" +33:37.083 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:50300 latency=90.14µs +33:37.083 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/policy from=127.0.0.1:50302 error="method POST not allowed" +33:37.084 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/policy from=127.0.0.1:50302 latency=81.9µs +33:37.084 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/policy from=127.0.0.1:50304 error="method DELETE not allowed" +33:37.084 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/policy from=127.0.0.1:50304 latency=80.86µs +33:37.085 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/policy from=127.0.0.1:50306 error="method HEAD not allowed" +33:37.085 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/policy from=127.0.0.1:50306 latency=35.74µs +33:37.085 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/policy from=127.0.0.1:50306 latency=3.5µs +33:37.086 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/policy/name/ from=127.0.0.1:50306 error="Bad request: Missing policy Name" +33:37.086 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policy/name/ from=127.0.0.1:50306 latency=80.46µs +33:37.087 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/policy/name/ from=127.0.0.1:50308 error="method PUT not allowed" +33:37.087 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy/name/ from=127.0.0.1:50308 latency=77.12µs +33:37.088 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/policy/name/ from=127.0.0.1:50310 error="method POST not allowed" +33:37.088 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/policy/name/ from=127.0.0.1:50310 latency=77.2µs +33:37.088 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/policy/name/ from=127.0.0.1:50312 error="method DELETE not allowed" +33:37.088 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/policy/name/ from=127.0.0.1:50312 latency=73.061µs +33:37.089 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/policy/name/ from=127.0.0.1:50314 error="method HEAD not allowed" +33:37.089 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/policy/name/ from=127.0.0.1:50314 latency=32.32µs +33:37.090 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/policy/name/ from=127.0.0.1:50314 latency=3.42µs +33:37.090 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:50314 latency=159.52µs +33:37.091 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/services from=127.0.0.1:50316 error="method PUT not allowed" +33:37.091 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/services from=127.0.0.1:50316 latency=79.94µs +33:37.092 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/services from=127.0.0.1:50318 error="method POST not allowed" +33:37.092 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/services from=127.0.0.1:50318 latency=82µs +33:37.093 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/services from=127.0.0.1:50320 error="method DELETE not allowed" +33:37.093 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/services from=127.0.0.1:50320 latency=80.36µs +33:37.093 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/services from=127.0.0.1:50322 error="method HEAD not allowed" +33:37.093 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/services from=127.0.0.1:50322 latency=37.62µs +33:37.094 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/services from=127.0.0.1:50322 latency=3.14µs +33:37.094 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/connect/authorize from=127.0.0.1:50322 error="method GET not allowed" +33:37.094 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/connect/authorize from=127.0.0.1:50322 latency=70.32µs +33:37.095 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/connect/authorize from=127.0.0.1:50324 error="method PUT not allowed" +33:37.095 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/connect/authorize from=127.0.0.1:50324 latency=73.84µs +33:37.096 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/connect/authorize from=127.0.0.1:50326 error="Bad request: Request decode failed: EOF" +33:37.096 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/connect/authorize from=127.0.0.1:50326 latency=86.201µs +33:37.097 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/connect/authorize from=127.0.0.1:50328 error="method DELETE not allowed" +33:37.097 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/connect/authorize from=127.0.0.1:50328 latency=78.4µs +33:37.097 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/connect/authorize from=127.0.0.1:50330 error="method HEAD not allowed" +33:37.097 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/connect/authorize from=127.0.0.1:50330 latency=32.2µs +33:37.098 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/connect/authorize from=127.0.0.1:50330 latency=3.38µs +33:37.098 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-05b1ba5a-0998-b2db-f378-5ab5ce213906 +33:37.099 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/nodes from=127.0.0.1:50330 latency=312.541µs +33:37.099 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/catalog/nodes from=127.0.0.1:50332 error="method PUT not allowed" +33:37.099 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/catalog/nodes from=127.0.0.1:50332 latency=81.46µs +33:37.100 [ERROR] TestAgent.http: Request error: method=POST url=/v1/catalog/nodes from=127.0.0.1:50334 error="method POST not allowed" +33:37.100 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/catalog/nodes from=127.0.0.1:50334 latency=73.6µs +33:37.101 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/catalog/nodes from=127.0.0.1:50336 error="method DELETE not allowed" +33:37.101 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/catalog/nodes from=127.0.0.1:50336 latency=72.321µs +33:37.102 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/catalog/nodes from=127.0.0.1:50338 error="method HEAD not allowed" +33:37.102 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/catalog/nodes from=127.0.0.1:50338 latency=33.5µs +33:37.102 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/catalog/nodes from=127.0.0.1:50338 latency=3.88µs +33:37.103 [ERROR] TestAgent.http: Request error: method=GET url=/v1/session/renew/ from=127.0.0.1:50338 error="method GET not allowed" +33:37.103 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/session/renew/ from=127.0.0.1:50338 latency=78.22µs +33:37.104 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/renew/ from=127.0.0.1:50340 latency=61.66µs +33:37.104 [ERROR] TestAgent.http: Request error: method=POST url=/v1/session/renew/ from=127.0.0.1:50342 error="method POST not allowed" +33:37.104 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/session/renew/ from=127.0.0.1:50342 latency=77.6µs +33:37.105 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/session/renew/ from=127.0.0.1:50344 error="method DELETE not allowed" +33:37.105 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/session/renew/ from=127.0.0.1:50344 latency=79.24µs +33:37.106 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/session/renew/ from=127.0.0.1:50346 error="method HEAD not allowed" +33:37.106 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/session/renew/ from=127.0.0.1:50346 latency=32.14µs +33:37.107 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/session/renew/ from=127.0.0.1:50346 latency=3.28µs +33:37.107 [ERROR] TestAgent.http: Request error: method=GET url=/v1/txn from=127.0.0.1:50346 error="method GET not allowed" +33:37.107 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/txn from=127.0.0.1:50346 latency=74.781µs +33:37.108 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/txn from=127.0.0.1:50348 latency=73.341µs +33:37.109 [ERROR] TestAgent.http: Request error: method=POST url=/v1/txn from=127.0.0.1:50350 error="method POST not allowed" +33:37.109 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/txn from=127.0.0.1:50350 latency=74.52µs +33:37.109 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/txn from=127.0.0.1:50352 error="method DELETE not allowed" +33:37.110 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/txn from=127.0.0.1:50352 latency=83.52µs +33:37.110 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/txn from=127.0.0.1:50354 error="method HEAD not allowed" +33:37.110 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/txn from=127.0.0.1:50354 latency=31.08µs +33:37.111 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/txn from=127.0.0.1:50354 latency=3.3µs +33:37.111 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/tokens from=127.0.0.1:50354 error="Permission denied" +33:37.111 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/tokens from=127.0.0.1:50354 latency=213.06µs +33:37.112 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/tokens from=127.0.0.1:50356 error="method PUT not allowed" +33:37.112 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/tokens from=127.0.0.1:50356 latency=78.68µs +33:37.113 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/tokens from=127.0.0.1:50358 error="method POST not allowed" +33:37.113 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/tokens from=127.0.0.1:50358 latency=75.82µs +33:37.115 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/tokens from=127.0.0.1:50360 error="method DELETE not allowed" +33:37.115 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/tokens from=127.0.0.1:50360 latency=229.3µs +33:37.117 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/tokens from=127.0.0.1:50362 error="method HEAD not allowed" +33:37.117 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/tokens from=127.0.0.1:50362 latency=68.441µs +33:37.117 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/tokens from=127.0.0.1:50362 latency=4.46µs +33:37.118 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/check/fail/ from=127.0.0.1:50362 error="method GET not allowed" +33:37.118 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/check/fail/ from=127.0.0.1:50362 latency=89.94µs +33:37.119 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/check/fail/ from=127.0.0.1:50364 error="Unknown check """ +33:37.119 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/check/fail/ from=127.0.0.1:50364 latency=151.7µs +33:37.119 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/check/fail/ from=127.0.0.1:50366 error="method POST not allowed" +33:37.119 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/check/fail/ from=127.0.0.1:50366 latency=79.3µs +33:37.120 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/check/fail/ from=127.0.0.1:50368 error="method DELETE not allowed" +33:37.120 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/check/fail/ from=127.0.0.1:50368 latency=74.32µs +33:37.121 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/check/fail/ from=127.0.0.1:50370 error="method HEAD not allowed" +33:37.121 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/check/fail/ from=127.0.0.1:50370 latency=30.58µs +33:37.121 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/check/fail/ from=127.0.0.1:50370 latency=3.64µs +33:37.122 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/session/node/ from=127.0.0.1:50370 latency=63.4µs +33:37.123 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/session/node/ from=127.0.0.1:50372 error="method PUT not allowed" +33:37.123 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/node/ from=127.0.0.1:50372 latency=167.38µs +33:37.124 [ERROR] TestAgent.http: Request error: method=POST url=/v1/session/node/ from=127.0.0.1:50374 error="method POST not allowed" +33:37.124 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/session/node/ from=127.0.0.1:50374 latency=72.06µs +33:37.124 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/session/node/ from=127.0.0.1:50376 error="method DELETE not allowed" +33:37.124 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/session/node/ from=127.0.0.1:50376 latency=75.92µs +33:37.125 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/session/node/ from=127.0.0.1:50378 error="method HEAD not allowed" +33:37.125 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/session/node/ from=127.0.0.1:50378 latency=30.461µs +33:37.126 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/session/node/ from=127.0.0.1:50378 latency=3.98µs +33:37.126 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/auth-method/ from=127.0.0.1:50378 error="Bad request: Missing auth method name" +33:37.126 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/ from=127.0.0.1:50378 latency=76.401µs +33:37.127 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/auth-method/ from=127.0.0.1:50380 error="Bad request: AuthMethod decoding failed: EOF" +33:37.127 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/ from=127.0.0.1:50380 latency=97.141µs +33:37.128 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/auth-method/ from=127.0.0.1:50382 error="method POST not allowed" +33:37.128 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/auth-method/ from=127.0.0.1:50382 latency=73.62µs +33:37.129 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/auth-method/ from=127.0.0.1:50384 error="Bad request: Missing auth method name" +33:37.129 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/auth-method/ from=127.0.0.1:50384 latency=70.46µs +33:37.129 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/auth-method/ from=127.0.0.1:50386 error="method HEAD not allowed" +33:37.129 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/auth-method/ from=127.0.0.1:50386 latency=32.98µs +33:37.130 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/auth-method/ from=127.0.0.1:50386 latency=3.76µs +33:37.130 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/join/ from=127.0.0.1:50386 error="method GET not allowed" +33:37.130 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/join/ from=127.0.0.1:50386 latency=87.34µs +33:37.131 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/join/ from=127.0.0.1:50388 error="Permission denied" +33:37.131 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/join/ from=127.0.0.1:50388 latency=134.28µs +33:37.132 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/join/ from=127.0.0.1:50390 error="method POST not allowed" +33:37.132 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/join/ from=127.0.0.1:50390 latency=79.96µs +33:37.133 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/join/ from=127.0.0.1:50392 error="method DELETE not allowed" +33:37.133 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/join/ from=127.0.0.1:50392 latency=78.5µs +33:37.133 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/join/ from=127.0.0.1:50394 error="method HEAD not allowed" +33:37.134 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/join/ from=127.0.0.1:50394 latency=32.12µs +33:37.134 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/join/ from=127.0.0.1:50394 latency=3.58µs +33:37.134 [ERROR] TestAgent.http: Request error: method=GET url=/v1/event/fire/ from=127.0.0.1:50394 error="method GET not allowed" +33:37.134 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/event/fire/ from=127.0.0.1:50394 latency=72.84µs +33:37.135 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/event/fire/ from=127.0.0.1:50396 latency=61.44µs +33:37.136 [ERROR] TestAgent.http: Request error: method=POST url=/v1/event/fire/ from=127.0.0.1:50398 error="method POST not allowed" +33:37.136 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/event/fire/ from=127.0.0.1:50398 latency=71.48µs +33:37.137 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/event/fire/ from=127.0.0.1:50400 error="method DELETE not allowed" +33:37.137 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/event/fire/ from=127.0.0.1:50400 latency=129.8µs +33:37.138 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/event/fire/ from=127.0.0.1:50402 error="method HEAD not allowed" +33:37.138 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/event/fire/ from=127.0.0.1:50402 latency=53µs +33:37.139 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/event/fire/ from=127.0.0.1:50402 latency=4.52µs +33:37.140 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-05b1ba5a-0998-b2db-f378-5ab5ce213906 +33:37.140 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/internal/ui/nodes from=127.0.0.1:50402 latency=553.101µs +33:37.141 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/internal/ui/nodes from=127.0.0.1:50404 error="method PUT not allowed" +33:37.141 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/internal/ui/nodes from=127.0.0.1:50404 latency=88.68µs +33:37.142 [ERROR] TestAgent.http: Request error: method=POST url=/v1/internal/ui/nodes from=127.0.0.1:50406 error="method POST not allowed" +33:37.142 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/internal/ui/nodes from=127.0.0.1:50406 latency=76.54µs +33:37.143 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/internal/ui/nodes from=127.0.0.1:50408 error="method DELETE not allowed" +33:37.143 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/internal/ui/nodes from=127.0.0.1:50408 latency=88.6µs +33:37.144 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/internal/ui/nodes from=127.0.0.1:50410 error="method HEAD not allowed" +33:37.144 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/internal/ui/nodes from=127.0.0.1:50410 latency=32.86µs +33:37.144 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/internal/ui/nodes from=127.0.0.1:50410 latency=4.22µs +33:37.145 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/rules/translate from=127.0.0.1:50410 error="method GET not allowed" +33:37.145 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/rules/translate from=127.0.0.1:50410 latency=78.24µs +33:37.146 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/rules/translate from=127.0.0.1:50412 error="method PUT not allowed" +33:37.146 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/rules/translate from=127.0.0.1:50412 latency=73µs +33:37.147 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/rules/translate from=127.0.0.1:50414 error="Permission denied" +33:37.147 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/rules/translate from=127.0.0.1:50414 latency=140.48µs +33:37.147 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/rules/translate from=127.0.0.1:50416 error="method DELETE not allowed" +33:37.147 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/rules/translate from=127.0.0.1:50416 latency=77.5µs +33:37.148 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/rules/translate from=127.0.0.1:50418 error="method HEAD not allowed" +33:37.148 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/rules/translate from=127.0.0.1:50418 latency=31.701µs +33:37.149 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/rules/translate from=127.0.0.1:50418 latency=3.3µs +33:37.149 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/internal/ui/gateway-services-nodes/ from=127.0.0.1:50418 latency=62.561µs +33:37.150 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/internal/ui/gateway-services-nodes/ from=127.0.0.1:50420 error="method PUT not allowed" +33:37.150 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/internal/ui/gateway-services-nodes/ from=127.0.0.1:50420 latency=82.08µs +33:37.151 [ERROR] TestAgent.http: Request error: method=POST url=/v1/internal/ui/gateway-services-nodes/ from=127.0.0.1:50422 error="method POST not allowed" +33:37.151 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/internal/ui/gateway-services-nodes/ from=127.0.0.1:50422 latency=177.521µs +33:37.152 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/internal/ui/gateway-services-nodes/ from=127.0.0.1:50424 error="method DELETE not allowed" +33:37.152 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/internal/ui/gateway-services-nodes/ from=127.0.0.1:50424 latency=81.74µs +33:37.153 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/internal/ui/gateway-services-nodes/ from=127.0.0.1:50426 error="method HEAD not allowed" +33:37.153 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/internal/ui/gateway-services-nodes/ from=127.0.0.1:50426 latency=36.68µs +33:37.153 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/internal/ui/gateway-services-nodes/ from=127.0.0.1:50426 latency=3.56µs +33:37.154 [DEBUG] TestAgent.acl: dropping service from result due to ACLs: service=consul +33:37.154 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/services from=127.0.0.1:50426 latency=319.84µs +33:37.155 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/catalog/services from=127.0.0.1:50428 error="method PUT not allowed" +33:37.155 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/catalog/services from=127.0.0.1:50428 latency=78.36µs +33:37.155 [ERROR] TestAgent.http: Request error: method=POST url=/v1/catalog/services from=127.0.0.1:50430 error="method POST not allowed" +33:37.155 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/catalog/services from=127.0.0.1:50430 latency=77.54µs +33:37.156 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/catalog/services from=127.0.0.1:50432 error="method DELETE not allowed" +33:37.156 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/catalog/services from=127.0.0.1:50432 latency=76.08µs +33:37.157 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/catalog/services from=127.0.0.1:50434 error="method HEAD not allowed" +33:37.157 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/catalog/services from=127.0.0.1:50434 latency=31.42µs +33:37.157 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/catalog/services from=127.0.0.1:50434 latency=3.62µs +33:37.158 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/health/ingress/ from=127.0.0.1:50434 latency=68.56µs +33:37.159 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/health/ingress/ from=127.0.0.1:50436 error="method PUT not allowed" +33:37.159 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/health/ingress/ from=127.0.0.1:50436 latency=78.34µs +33:37.160 [ERROR] TestAgent.http: Request error: method=POST url=/v1/health/ingress/ from=127.0.0.1:50438 error="method POST not allowed" +33:37.160 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/health/ingress/ from=127.0.0.1:50438 latency=81.86µs +33:37.160 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/health/ingress/ from=127.0.0.1:50440 error="method DELETE not allowed" +33:37.160 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/health/ingress/ from=127.0.0.1:50440 latency=78.04µs +33:37.161 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/health/ingress/ from=127.0.0.1:50442 error="method HEAD not allowed" +33:37.161 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/health/ingress/ from=127.0.0.1:50442 latency=33.021µs +33:37.162 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/health/ingress/ from=127.0.0.1:50442 latency=3.54µs +33:37.162 [ERROR] TestAgent.http: Request error: method=GET url=/v1/operator/autopilot/health from=127.0.0.1:50442 error="Permission denied" +33:37.162 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/autopilot/health from=127.0.0.1:50442 latency=204.62µs +33:37.163 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/operator/autopilot/health from=127.0.0.1:50444 error="method PUT not allowed" +33:37.163 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/operator/autopilot/health from=127.0.0.1:50444 latency=75.86µs +33:37.164 [ERROR] TestAgent.http: Request error: method=POST url=/v1/operator/autopilot/health from=127.0.0.1:50446 error="method POST not allowed" +33:37.164 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/operator/autopilot/health from=127.0.0.1:50446 latency=72.68µs +33:37.165 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/operator/autopilot/health from=127.0.0.1:50448 error="method DELETE not allowed" +33:37.165 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/operator/autopilot/health from=127.0.0.1:50448 latency=72.481µs +33:37.165 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/operator/autopilot/health from=127.0.0.1:50450 error="method HEAD not allowed" +33:37.165 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/operator/autopilot/health from=127.0.0.1:50450 latency=33.42µs +33:37.166 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/operator/autopilot/health from=127.0.0.1:50450 latency=3.76µs +33:37.166 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/status/leader from=127.0.0.1:50450 latency=109.74µs +33:37.167 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/status/leader from=127.0.0.1:50452 error="method PUT not allowed" +33:37.167 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/status/leader from=127.0.0.1:50452 latency=77.4µs +33:37.168 [ERROR] TestAgent.http: Request error: method=POST url=/v1/status/leader from=127.0.0.1:50454 error="method POST not allowed" +33:37.168 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/status/leader from=127.0.0.1:50454 latency=71.98µs +33:37.169 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/status/leader from=127.0.0.1:50456 error="method DELETE not allowed" +33:37.169 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/status/leader from=127.0.0.1:50456 latency=71.58µs +33:37.169 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/status/leader from=127.0.0.1:50458 error="method HEAD not allowed" +33:37.169 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/status/leader from=127.0.0.1:50458 latency=37.4µs +33:37.170 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/status/leader from=127.0.0.1:50458 latency=3.44µs +33:37.170 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/logout from=127.0.0.1:50458 error="method GET not allowed" +33:37.170 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/logout from=127.0.0.1:50458 latency=78.461µs +33:37.171 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/logout from=127.0.0.1:50460 error="method PUT not allowed" +33:37.171 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/logout from=127.0.0.1:50460 latency=75.48µs +33:37.172 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/logout from=127.0.0.1:50462 error="ACL not found" +33:37.172 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/logout from=127.0.0.1:50462 latency=78.42µs +33:37.173 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/logout from=127.0.0.1:50464 error="method DELETE not allowed" +33:37.173 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/logout from=127.0.0.1:50464 latency=72.6µs +33:37.174 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/logout from=127.0.0.1:50466 error="method HEAD not allowed" +33:37.174 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/logout from=127.0.0.1:50466 latency=36.96µs +33:37.174 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/logout from=127.0.0.1:50466 latency=4.1µs +33:37.174 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/host from=127.0.0.1:50466 error="Permission denied" +33:37.174 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/host from=127.0.0.1:50466 latency=127.48µs +33:37.175 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/host from=127.0.0.1:50468 error="method PUT not allowed" +33:37.175 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/host from=127.0.0.1:50468 latency=79.06µs +33:37.176 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/host from=127.0.0.1:50470 error="method POST not allowed" +33:37.176 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/host from=127.0.0.1:50470 latency=73.96µs +33:37.177 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/host from=127.0.0.1:50472 error="method DELETE not allowed" +33:37.177 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/host from=127.0.0.1:50472 latency=83.56µs +33:37.178 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/host from=127.0.0.1:50474 error="method HEAD not allowed" +33:37.178 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/host from=127.0.0.1:50474 latency=36.92µs +33:37.178 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/host from=127.0.0.1:50474 latency=3.46µs +33:37.179 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/connect/ca/roots from=127.0.0.1:50474 latency=286.981µs +33:37.180 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/connect/ca/roots from=127.0.0.1:50476 error="method PUT not allowed" +33:37.180 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/connect/ca/roots from=127.0.0.1:50476 latency=75.3µs +33:37.181 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/connect/ca/roots from=127.0.0.1:50478 error="method POST not allowed" +33:37.181 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/connect/ca/roots from=127.0.0.1:50478 latency=72.14µs +33:37.182 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/connect/ca/roots from=127.0.0.1:50480 error="method DELETE not allowed" +33:37.182 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/connect/ca/roots from=127.0.0.1:50480 latency=73.92µs +33:37.183 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/connect/ca/roots from=127.0.0.1:50482 error="method HEAD not allowed" +33:37.183 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/connect/ca/roots from=127.0.0.1:50482 latency=66.96µs +33:37.183 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/connect/ca/roots from=127.0.0.1:50482 latency=3.22µs +33:37.184 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/node/ from=127.0.0.1:50482 latency=63.061µs +33:37.185 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/catalog/node/ from=127.0.0.1:50484 error="method PUT not allowed" +33:37.185 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/catalog/node/ from=127.0.0.1:50484 latency=78.841µs +33:37.185 [ERROR] TestAgent.http: Request error: method=POST url=/v1/catalog/node/ from=127.0.0.1:50486 error="method POST not allowed" +33:37.185 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/catalog/node/ from=127.0.0.1:50486 latency=71.76µs +33:37.186 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/catalog/node/ from=127.0.0.1:50488 error="method DELETE not allowed" +33:37.186 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/catalog/node/ from=127.0.0.1:50488 latency=72.26µs +33:37.187 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/catalog/node/ from=127.0.0.1:50490 error="method HEAD not allowed" +33:37.187 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/catalog/node/ from=127.0.0.1:50490 latency=34.12µs +33:37.187 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/catalog/node/ from=127.0.0.1:50490 latency=3.28µs +33:37.188 [ERROR] TestAgent.http: Request error: method=GET url=/v1/connect/intentions/ from=127.0.0.1:50490 error="Bad request: failed intention lookup: index error: UUID must be 36 characters" +33:37.188 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions/ from=127.0.0.1:50490 latency=180.24µs +33:37.189 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/connect/intentions/ from=127.0.0.1:50492 error="Bad request: Request decode failed: EOF" +33:37.189 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/connect/intentions/ from=127.0.0.1:50492 latency=89.8µs +33:37.190 [ERROR] TestAgent.http: Request error: method=POST url=/v1/connect/intentions/ from=127.0.0.1:50494 error="method POST not allowed" +33:37.190 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions/ from=127.0.0.1:50494 latency=79.74µs +33:37.190 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/connect/intentions/ from=127.0.0.1:50496 error="Intention lookup failed: failed intention lookup: index error: UUID must be 36 characters" +33:37.191 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/connect/intentions/ from=127.0.0.1:50496 latency=273.56µs +33:37.191 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/connect/intentions/ from=127.0.0.1:50498 error="method HEAD not allowed" +33:37.191 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/connect/intentions/ from=127.0.0.1:50498 latency=37.92µs +33:37.192 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/connect/intentions/ from=127.0.0.1:50498 latency=3.28µs +33:37.192 [ERROR] TestAgent.http: Request error: method=GET url=/v1/operator/raft/configuration from=127.0.0.1:50498 error="Permission denied" +33:37.193 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/raft/configuration from=127.0.0.1:50498 latency=226.161µs +33:37.193 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/operator/raft/configuration from=127.0.0.1:50500 error="method PUT not allowed" +33:37.193 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/operator/raft/configuration from=127.0.0.1:50500 latency=74.72µs +33:37.194 [ERROR] TestAgent.http: Request error: method=POST url=/v1/operator/raft/configuration from=127.0.0.1:50502 error="method POST not allowed" +33:37.194 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/operator/raft/configuration from=127.0.0.1:50502 latency=77.101µs +33:37.195 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/operator/raft/configuration from=127.0.0.1:50504 error="method DELETE not allowed" +33:37.195 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/operator/raft/configuration from=127.0.0.1:50504 latency=78.041µs +33:37.196 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/operator/raft/configuration from=127.0.0.1:50506 error="method HEAD not allowed" +33:37.196 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/operator/raft/configuration from=127.0.0.1:50506 latency=32.54µs +33:37.196 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/operator/raft/configuration from=127.0.0.1:50506 latency=3.16µs +33:37.197 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/session/list from=127.0.0.1:50506 latency=371.341µs +33:37.198 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/session/list from=127.0.0.1:50508 error="method PUT not allowed" +33:37.198 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/list from=127.0.0.1:50508 latency=74.44µs +33:37.198 [ERROR] TestAgent.http: Request error: method=POST url=/v1/session/list from=127.0.0.1:50510 error="method POST not allowed" +33:37.198 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/session/list from=127.0.0.1:50510 latency=78.301µs +33:37.199 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/session/list from=127.0.0.1:50512 error="method DELETE not allowed" +33:37.199 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/session/list from=127.0.0.1:50512 latency=71.96µs +33:37.200 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/session/list from=127.0.0.1:50514 error="method HEAD not allowed" +33:37.200 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/session/list from=127.0.0.1:50514 latency=30.36µs +33:37.200 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/session/list from=127.0.0.1:50514 latency=3.1µs +33:37.201 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/policy/ from=127.0.0.1:50514 error="Bad request: Missing policy ID" +33:37.201 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policy/ from=127.0.0.1:50514 latency=75.88µs +33:37.202 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/policy/ from=127.0.0.1:50516 error="Bad request: Policy decoding failed: EOF" +33:37.202 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy/ from=127.0.0.1:50516 latency=105.28µs +33:37.202 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/policy/ from=127.0.0.1:50518 error="method POST not allowed" +33:37.202 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/policy/ from=127.0.0.1:50518 latency=77.24µs +33:37.203 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/policy/ from=127.0.0.1:50520 error="Bad request: Missing policy ID" +33:37.203 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/policy/ from=127.0.0.1:50520 latency=72.64µs +33:37.204 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/policy/ from=127.0.0.1:50522 error="method HEAD not allowed" +33:37.204 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/policy/ from=127.0.0.1:50522 latency=33.161µs +33:37.204 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/policy/ from=127.0.0.1:50522 latency=3.58µs +33:37.205 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/token from=127.0.0.1:50522 error="method GET not allowed" +33:37.205 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token from=127.0.0.1:50522 latency=76.501µs +33:37.206 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/token from=127.0.0.1:50524 error="Bad request: Token decoding failed: EOF" +33:37.206 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:50524 latency=85.581µs +33:37.207 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/token from=127.0.0.1:50526 error="method POST not allowed" +33:37.207 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/token from=127.0.0.1:50526 latency=74.52µs +33:37.207 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/token from=127.0.0.1:50528 error="method DELETE not allowed" +33:37.207 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/token from=127.0.0.1:50528 latency=78.16µs +33:37.208 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/token from=127.0.0.1:50530 error="method HEAD not allowed" +33:37.208 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/token from=127.0.0.1:50530 latency=32.14µs +33:37.208 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/token from=127.0.0.1:50530 latency=3.36µs +33:37.209 [ERROR] TestAgent.http: Request error: method=GET url=/v1/connect/ca/configuration from=127.0.0.1:50530 error="Permission denied" +33:37.209 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/ca/configuration from=127.0.0.1:50530 latency=199.36µs +33:37.210 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/connect/ca/configuration from=127.0.0.1:50532 error="Bad request: Request decode failed: EOF" +33:37.210 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/connect/ca/configuration from=127.0.0.1:50532 latency=87.08µs +33:37.211 [ERROR] TestAgent.http: Request error: method=POST url=/v1/connect/ca/configuration from=127.0.0.1:50534 error="method POST not allowed" +33:37.211 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/ca/configuration from=127.0.0.1:50534 latency=79.76µs +33:37.211 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/connect/ca/configuration from=127.0.0.1:50536 error="method DELETE not allowed" +33:37.211 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/connect/ca/configuration from=127.0.0.1:50536 latency=79.721µs +33:37.212 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/connect/ca/configuration from=127.0.0.1:50538 error="method HEAD not allowed" +33:37.212 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/connect/ca/configuration from=127.0.0.1:50538 latency=30.96µs +33:37.213 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/connect/ca/configuration from=127.0.0.1:50538 latency=3.88µs +33:37.213 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/ from=127.0.0.1:50538 latency=104.46µs +33:37.214 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/service/ from=127.0.0.1:50540 error="method PUT not allowed" +33:37.214 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/ from=127.0.0.1:50540 latency=75.221µs +33:37.215 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/service/ from=127.0.0.1:50542 error="method POST not allowed" +33:37.215 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/service/ from=127.0.0.1:50542 latency=78.22µs +33:37.216 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/service/ from=127.0.0.1:50544 error="method DELETE not allowed" +33:37.216 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/service/ from=127.0.0.1:50544 latency=90.1µs +33:37.217 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/service/ from=127.0.0.1:50546 error="method HEAD not allowed" +33:37.217 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/service/ from=127.0.0.1:50546 latency=35µs +33:37.217 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/service/ from=127.0.0.1:50546 latency=3.62µs +33:37.217 [ERROR] TestAgent.http: Request error: method=GET url=/v1/catalog/register from=127.0.0.1:50546 error="method GET not allowed" +33:37.218 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/register from=127.0.0.1:50546 latency=77.141µs +33:37.218 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/catalog/register from=127.0.0.1:50548 latency=66.44µs +33:37.219 [ERROR] TestAgent.http: Request error: method=POST url=/v1/catalog/register from=127.0.0.1:50550 error="method POST not allowed" +33:37.219 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/catalog/register from=127.0.0.1:50550 latency=78.64µs +33:37.220 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/catalog/register from=127.0.0.1:50552 error="method DELETE not allowed" +33:37.220 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/catalog/register from=127.0.0.1:50552 latency=71.601µs +33:37.221 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/catalog/register from=127.0.0.1:50554 error="method HEAD not allowed" +33:37.221 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/catalog/register from=127.0.0.1:50554 latency=31.3µs +33:37.221 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/catalog/register from=127.0.0.1:50554 latency=3.42µs +33:37.222 [ERROR] TestAgent.http: Request error: method=GET url=/v1/discovery-chain/ from=127.0.0.1:50554 error="Bad request: Missing chain name" +33:37.222 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/discovery-chain/ from=127.0.0.1:50554 latency=88.8µs +33:37.222 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/discovery-chain/ from=127.0.0.1:50556 error="method PUT not allowed" +33:37.223 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/discovery-chain/ from=127.0.0.1:50556 latency=76.4µs +33:37.223 [ERROR] TestAgent.http: Request error: method=POST url=/v1/discovery-chain/ from=127.0.0.1:50558 error="Bad request: Missing chain name" +33:37.223 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/discovery-chain/ from=127.0.0.1:50558 latency=87.76µs +33:37.224 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/discovery-chain/ from=127.0.0.1:50560 error="method DELETE not allowed" +33:37.224 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/discovery-chain/ from=127.0.0.1:50560 latency=80.68µs +33:37.225 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/discovery-chain/ from=127.0.0.1:50562 error="method HEAD not allowed" +33:37.225 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/discovery-chain/ from=127.0.0.1:50562 latency=33.28µs +33:37.225 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/discovery-chain/ from=127.0.0.1:50562 latency=3.42µs +33:37.226 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/self from=127.0.0.1:50562 error="Permission denied" +33:37.226 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:50562 latency=129.4µs +33:37.227 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/self from=127.0.0.1:50564 error="method PUT not allowed" +33:37.227 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/self from=127.0.0.1:50564 latency=77.72µs +33:37.228 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/self from=127.0.0.1:50566 error="method POST not allowed" +33:37.228 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/self from=127.0.0.1:50566 latency=75.82µs +33:37.228 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/self from=127.0.0.1:50568 error="method DELETE not allowed" +33:37.229 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/self from=127.0.0.1:50568 latency=79.98µs +33:37.229 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/self from=127.0.0.1:50570 error="method HEAD not allowed" +33:37.229 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/self from=127.0.0.1:50570 latency=33.54µs +33:37.230 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/self from=127.0.0.1:50570 latency=3.5µs +33:37.230 [ERROR] TestAgent.http: Request error: method=GET url=/v1/catalog/deregister from=127.0.0.1:50570 error="method GET not allowed" +33:37.230 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/deregister from=127.0.0.1:50570 latency=73.12µs +33:37.231 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/catalog/deregister from=127.0.0.1:50572 latency=81.14µs +33:37.232 [ERROR] TestAgent.http: Request error: method=POST url=/v1/catalog/deregister from=127.0.0.1:50574 error="method POST not allowed" +33:37.232 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/catalog/deregister from=127.0.0.1:50574 latency=74.781µs +33:37.233 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/catalog/deregister from=127.0.0.1:50576 error="method DELETE not allowed" +33:37.233 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/catalog/deregister from=127.0.0.1:50576 latency=74.14µs +33:37.233 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/catalog/deregister from=127.0.0.1:50578 error="method HEAD not allowed" +33:37.233 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/catalog/deregister from=127.0.0.1:50578 latency=33.42µs +33:37.234 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/catalog/deregister from=127.0.0.1:50578 latency=2.94µs +33:37.234 [ERROR] TestAgent.http: Request error: method=GET url=/v1/connect/intentions/check from=127.0.0.1:50578 error="required query parameter 'source' not set" +33:37.234 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions/check from=127.0.0.1:50578 latency=81.32µs +33:37.235 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/connect/intentions/check from=127.0.0.1:50580 error="method PUT not allowed" +33:37.235 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/connect/intentions/check from=127.0.0.1:50580 latency=74.64µs +33:37.236 [ERROR] TestAgent.http: Request error: method=POST url=/v1/connect/intentions/check from=127.0.0.1:50582 error="method POST not allowed" +33:37.236 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions/check from=127.0.0.1:50582 latency=76.06µs +33:37.237 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/connect/intentions/check from=127.0.0.1:50584 error="method DELETE not allowed" +33:37.237 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/connect/intentions/check from=127.0.0.1:50584 latency=76.86µs +33:37.238 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/connect/intentions/check from=127.0.0.1:50586 error="method HEAD not allowed" +33:37.238 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/connect/intentions/check from=127.0.0.1:50586 latency=35µs +33:37.238 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/connect/intentions/check from=127.0.0.1:50586 latency=3.7µs +33:37.239 [ERROR] TestAgent.http: Request error: method=GET url=/v1/coordinate/node/ from=127.0.0.1:50586 error="Permission denied" +33:37.239 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/node/ from=127.0.0.1:50586 latency=216.66µs +33:37.239 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/coordinate/node/ from=127.0.0.1:50588 error="method PUT not allowed" +33:37.240 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/coordinate/node/ from=127.0.0.1:50588 latency=77.46µs +33:37.240 [ERROR] TestAgent.http: Request error: method=POST url=/v1/coordinate/node/ from=127.0.0.1:50590 error="method POST not allowed" +33:37.240 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/coordinate/node/ from=127.0.0.1:50590 latency=80.681µs +33:37.241 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/coordinate/node/ from=127.0.0.1:50592 error="method DELETE not allowed" +33:37.241 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/coordinate/node/ from=127.0.0.1:50592 latency=80.38µs +33:37.242 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/coordinate/node/ from=127.0.0.1:50594 error="method HEAD not allowed" +33:37.242 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/coordinate/node/ from=127.0.0.1:50594 latency=32.58µs +33:37.242 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/coordinate/node/ from=127.0.0.1:50594 latency=13.08µs +33:37.243 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/ from=127.0.0.1:50594 latency=63.84µs +33:37.244 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/ from=127.0.0.1:50596 latency=57.76µs +33:37.245 [ERROR] TestAgent.http: Request error: method=POST url=/v1/kv/ from=127.0.0.1:50598 error="method POST not allowed" +33:37.245 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/kv/ from=127.0.0.1:50598 latency=79.08µs +33:37.245 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/ from=127.0.0.1:50600 latency=66.06µs +33:37.246 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/kv/ from=127.0.0.1:50602 error="method HEAD not allowed" +33:37.246 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/kv/ from=127.0.0.1:50602 latency=33.98µs +33:37.247 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/kv/ from=127.0.0.1:50602 latency=4.14µs +33:37.247 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/binding-rules from=127.0.0.1:50602 error="Permission denied" +33:37.247 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:50602 latency=193.54µs +33:37.248 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/binding-rules from=127.0.0.1:50604 error="method PUT not allowed" +33:37.248 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rules from=127.0.0.1:50604 latency=76.94µs +33:37.249 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/binding-rules from=127.0.0.1:50606 error="method POST not allowed" +33:37.249 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/binding-rules from=127.0.0.1:50606 latency=71.6µs +33:37.249 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/binding-rules from=127.0.0.1:50608 error="method DELETE not allowed" +33:37.250 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rules from=127.0.0.1:50608 latency=74.701µs +33:37.250 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/binding-rules from=127.0.0.1:50610 error="method HEAD not allowed" +33:37.250 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/binding-rules from=127.0.0.1:50610 latency=30.3µs +33:37.251 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/binding-rules from=127.0.0.1:50610 latency=3.22µs +33:37.251 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/auth-method from=127.0.0.1:50610 error="method GET not allowed" +33:37.251 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method from=127.0.0.1:50610 latency=74.52µs +33:37.252 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/auth-method from=127.0.0.1:50612 error="Bad request: AuthMethod decoding failed: EOF" +33:37.252 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:50612 latency=90.44µs +33:37.253 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/auth-method from=127.0.0.1:50614 error="method POST not allowed" +33:37.253 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/auth-method from=127.0.0.1:50614 latency=73.68µs +33:37.253 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/auth-method from=127.0.0.1:50616 error="method DELETE not allowed" +33:37.254 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/auth-method from=127.0.0.1:50616 latency=73.94µs +33:37.254 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/auth-method from=127.0.0.1:50618 error="method HEAD not allowed" +33:37.254 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/auth-method from=127.0.0.1:50618 latency=31.981µs +33:37.255 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/auth-method from=127.0.0.1:50618 latency=3.28µs +33:37.255 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/checks from=127.0.0.1:50618 latency=157.601µs +33:37.256 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/checks from=127.0.0.1:50620 error="method PUT not allowed" +33:37.256 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/checks from=127.0.0.1:50620 latency=76.2µs +33:37.257 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/checks from=127.0.0.1:50622 error="method POST not allowed" +33:37.257 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/checks from=127.0.0.1:50622 latency=87.84µs +33:37.258 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/checks from=127.0.0.1:50624 error="method DELETE not allowed" +33:37.258 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/checks from=127.0.0.1:50624 latency=80.86µs +33:37.258 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/checks from=127.0.0.1:50626 error="method HEAD not allowed" +33:37.258 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/checks from=127.0.0.1:50626 latency=33.46µs +33:37.259 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/checks from=127.0.0.1:50626 latency=3.38µs +33:37.260 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/datacenters from=127.0.0.1:50626 latency=354.781µs +33:37.261 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/coordinate/datacenters from=127.0.0.1:50628 error="method PUT not allowed" +33:37.261 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/coordinate/datacenters from=127.0.0.1:50628 latency=83.94µs +33:37.261 [ERROR] TestAgent.http: Request error: method=POST url=/v1/coordinate/datacenters from=127.0.0.1:50630 error="method POST not allowed" +33:37.262 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/coordinate/datacenters from=127.0.0.1:50630 latency=77.42µs +33:37.262 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/coordinate/datacenters from=127.0.0.1:50632 error="method DELETE not allowed" +33:37.262 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/coordinate/datacenters from=127.0.0.1:50632 latency=75.7µs +33:37.263 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/coordinate/datacenters from=127.0.0.1:50634 error="method HEAD not allowed" +33:37.263 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/coordinate/datacenters from=127.0.0.1:50634 latency=36.26µs +33:37.264 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/coordinate/datacenters from=127.0.0.1:50634 latency=3.52µs +33:37.264 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/login from=127.0.0.1:50634 error="method GET not allowed" +33:37.264 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/login from=127.0.0.1:50634 latency=78.84µs +33:37.265 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/login from=127.0.0.1:50636 error="method PUT not allowed" +33:37.265 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/login from=127.0.0.1:50636 latency=73.5µs +33:37.266 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/login from=127.0.0.1:50638 error="Bad request: Failed to decode request body: EOF" +33:37.266 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/login from=127.0.0.1:50638 latency=90.72µs +33:37.267 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/login from=127.0.0.1:50640 error="method DELETE not allowed" +33:37.267 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/login from=127.0.0.1:50640 latency=96.881µs +33:37.268 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/login from=127.0.0.1:50642 error="method HEAD not allowed" +33:37.268 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/login from=127.0.0.1:50642 latency=36.76µs +33:37.268 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/login from=127.0.0.1:50642 latency=3.08µs +33:37.269 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/health/service/id/ from=127.0.0.1:50642 error="Bad request: Missing serviceID" +33:37.269 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/health/service/id/ from=127.0.0.1:50642 latency=77.42µs +33:37.269 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/health/service/id/ from=127.0.0.1:50644 error="method PUT not allowed" +33:37.269 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/health/service/id/ from=127.0.0.1:50644 latency=73.761µs +33:37.270 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/health/service/id/ from=127.0.0.1:50646 error="method POST not allowed" +33:37.270 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/health/service/id/ from=127.0.0.1:50646 latency=77.74µs +33:37.271 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/health/service/id/ from=127.0.0.1:50648 error="method DELETE not allowed" +33:37.271 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/health/service/id/ from=127.0.0.1:50648 latency=73.14µs +33:37.272 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/health/service/id/ from=127.0.0.1:50650 error="method HEAD not allowed" +33:37.272 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/health/service/id/ from=127.0.0.1:50650 latency=34.94µs +33:37.272 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/health/service/id/ from=127.0.0.1:50650 latency=3.98µs +33:37.273 [ERROR] TestAgent.http: Request error: method=GET url=/v1/internal/acl/authorize from=127.0.0.1:50650 error="method GET not allowed" +33:37.273 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/internal/acl/authorize from=127.0.0.1:50650 latency=76.02µs +33:37.274 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/internal/acl/authorize from=127.0.0.1:50652 error="method PUT not allowed" +33:37.274 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/internal/acl/authorize from=127.0.0.1:50652 latency=72.62µs +33:37.274 [ERROR] TestAgent.http: Request error: method=POST url=/v1/internal/acl/authorize from=127.0.0.1:50654 error="Bad request: Failed to decode request body: EOF" +33:37.274 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/internal/acl/authorize from=127.0.0.1:50654 latency=98.1µs +33:37.275 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/internal/acl/authorize from=127.0.0.1:50656 error="method DELETE not allowed" +33:37.275 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/internal/acl/authorize from=127.0.0.1:50656 latency=77.48µs +33:37.276 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/internal/acl/authorize from=127.0.0.1:50658 error="method HEAD not allowed" +33:37.276 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/internal/acl/authorize from=127.0.0.1:50658 latency=36.16µs +33:37.276 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/internal/acl/authorize from=127.0.0.1:50658 latency=3.44µs +33:37.277 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/update from=127.0.0.1:50658 error="method GET not allowed" +33:37.277 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/update from=127.0.0.1:50658 latency=75.36µs +33:37.278 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/update from=127.0.0.1:50660 latency=60.32µs +33:37.279 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/update from=127.0.0.1:50662 error="method POST not allowed" +33:37.279 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/update from=127.0.0.1:50662 latency=76.68µs +33:37.279 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/update from=127.0.0.1:50664 error="method DELETE not allowed" +33:37.280 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/update from=127.0.0.1:50664 latency=73.04µs +33:37.280 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/update from=127.0.0.1:50666 error="method HEAD not allowed" +33:37.280 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/update from=127.0.0.1:50666 latency=36.88µs +33:37.281 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/update from=127.0.0.1:50666 latency=3.52µs +33:37.281 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/check/register from=127.0.0.1:50666 error="method GET not allowed" +33:37.281 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/check/register from=127.0.0.1:50666 latency=73.3µs +33:37.282 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/check/register from=127.0.0.1:50668 latency=62.92µs +33:37.283 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/check/register from=127.0.0.1:50670 error="method POST not allowed" +33:37.283 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/check/register from=127.0.0.1:50670 latency=79.48µs +33:37.284 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/check/register from=127.0.0.1:50672 error="method DELETE not allowed" +33:37.284 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/check/register from=127.0.0.1:50672 latency=77.9µs +33:37.285 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/check/register from=127.0.0.1:50674 error="method HEAD not allowed" +33:37.285 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/check/register from=127.0.0.1:50674 latency=33.26µs +33:37.285 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/check/register from=127.0.0.1:50674 latency=4µs +33:37.285 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/service/maintenance/ from=127.0.0.1:50674 error="method GET not allowed" +33:37.286 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/maintenance/ from=127.0.0.1:50674 latency=72.96µs +33:37.286 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/maintenance/ from=127.0.0.1:50676 latency=51.04µs +33:37.287 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/service/maintenance/ from=127.0.0.1:50678 error="method POST not allowed" +33:37.287 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/service/maintenance/ from=127.0.0.1:50678 latency=78.221µs +33:37.288 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/service/maintenance/ from=127.0.0.1:50680 error="method DELETE not allowed" +33:37.288 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/service/maintenance/ from=127.0.0.1:50680 latency=78.58µs +33:37.289 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/service/maintenance/ from=127.0.0.1:50682 error="method HEAD not allowed" +33:37.289 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/service/maintenance/ from=127.0.0.1:50682 latency=38.26µs +33:37.289 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/service/maintenance/ from=127.0.0.1:50682 latency=3.74µs +33:37.290 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ from=127.0.0.1:50682 latency=271.721µs +33:37.291 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/config/ from=127.0.0.1:50684 error="method PUT not allowed" +33:37.291 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config/ from=127.0.0.1:50684 latency=80.3µs +33:37.291 [ERROR] TestAgent.http: Request error: method=POST url=/v1/config/ from=127.0.0.1:50686 error="method POST not allowed" +33:37.291 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/config/ from=127.0.0.1:50686 latency=73.48µs +33:37.292 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/config/ from=127.0.0.1:50688 latency=53.28µs +33:37.293 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/config/ from=127.0.0.1:50690 error="method HEAD not allowed" +33:37.293 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/config/ from=127.0.0.1:50690 latency=42.02µs +33:37.293 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/config/ from=127.0.0.1:50690 latency=3.32µs +33:37.294 [ERROR] TestAgent.http: Request error: method=GET url=/v1/snapshot from=127.0.0.1:50690 error="Permission denied" +33:37.294 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/snapshot from=127.0.0.1:50690 latency=123.521µs +33:37.295 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/snapshot from=127.0.0.1:50692 error="Permission denied" +33:37.295 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/snapshot from=127.0.0.1:50692 latency=113.201µs +33:37.295 [ERROR] TestAgent.http: Request error: method=POST url=/v1/snapshot from=127.0.0.1:50694 error="method POST not allowed" +33:37.295 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/snapshot from=127.0.0.1:50694 latency=78.441µs +33:37.296 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/snapshot from=127.0.0.1:50696 error="method DELETE not allowed" +33:37.296 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/snapshot from=127.0.0.1:50696 latency=73.34µs +33:37.297 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/snapshot from=127.0.0.1:50698 error="method HEAD not allowed" +33:37.297 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/snapshot from=127.0.0.1:50698 latency=34µs +33:37.297 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/snapshot from=127.0.0.1:50698 latency=3.44µs +33:37.298 [INFO] TestAgent: Requesting shutdown +33:37.298 [INFO] TestAgent.server: shutting down server +33:37.298 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:37.298 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:37.298 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +33:37.298 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +33:37.298 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:37.298 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:37.298 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:37.298 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:37.298 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:37.298 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:37.298 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +33:37.298 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +33:37.298 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:37.298 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +33:37.298 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +2021-08-13T15:33:37.298Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:37.298Z [INFO] agent.router.manager: shutting down +33:37.298 [INFO] TestAgent: consul server down +33:37.298 [INFO] TestAgent: shutdown complete +33:37.298 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25177 network=tcp +33:37.298 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25177 network=udp +33:37.298 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25178 network=tcp +33:37.799 [INFO] TestAgent: Waiting for endpoints to shut down +33:37.799 [INFO] TestAgent: Endpoints down +--- PASS: TestHTTPAPI_MethodNotAllowed_OSS (1.56s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/query (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/query (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/query (0.00s) @@ -17020,1411 +17015,1420 @@ --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/query/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/query/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/query/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/token/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/token/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/token/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/token/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/token/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/token/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/services (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/services (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/services (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/services (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/services (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/services (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/list (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/list (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/list (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/list (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/list (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/list (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/connect/ca/configuration (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/connect/ca/configuration (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/connect/ca/configuration (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/connect/ca/configuration (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/connect/ca/configuration (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/connect/ca/configuration (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/update (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/update (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/update (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/update (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/update (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/update (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/binding-rule (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/binding-rule (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/binding-rule (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/binding-rule (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/binding-rule (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/binding-rule (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/token (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/token (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/token (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/token (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/token (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/token (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/service/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/service/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/service/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/service/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/service/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/service/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/connect/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/connect/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/connect/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/connect/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/connect/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/connect/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/join/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/join/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/join/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/join/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/join/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/join/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/node-services/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/node-services/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/node-services/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/node-services/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/node-services/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/node-services/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/config/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/config/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/config/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/config/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/config/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/config/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/ingress/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/ingress/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/ingress/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/ingress/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/ingress/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/ingress/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/snapshot (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/snapshot (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/snapshot (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/snapshot (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/snapshot (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/snapshot (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/clone/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/clone/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/clone/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/clone/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/clone/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/clone/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/policy/name/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/policy/name/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/policy/name/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/policy/name/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/policy/name/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/policy/name/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/query/xxx/execute (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/query/xxx/execute (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/query/xxx/execute (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/query/xxx/execute (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/query/xxx/execute (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/query/xxx/execute (0.01s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/query/xxx/explain (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/query/xxx/explain (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/query/xxx/explain (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/query/xxx/explain (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/query/xxx/explain (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/query/xxx/explain (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/update/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/update/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/update/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/update/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/update/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/update/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/connect/intentions/match (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/connect/intentions/match (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/connect/intentions/match (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/connect/intentions/match (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/connect/intentions/match (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/connect/intentions/match (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/checks/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/checks/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/checks/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/checks/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/checks/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/checks/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/state/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/state/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/state/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/state/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/state/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/state/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/roles (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/roles (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/roles (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/roles (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/roles (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/roles (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/metrics (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/metrics (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/metrics (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/metrics (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/metrics (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/metrics (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/leave (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/leave (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/leave (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/leave (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/leave (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/leave (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/role (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/role (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/role (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/role (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/role (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/role (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/maintenance (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/maintenance (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/maintenance (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/maintenance (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/maintenance (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/maintenance (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/ui/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/ui/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/ui/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/ui/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/ui/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/ui/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/destroy/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/destroy/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/destroy/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/destroy/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/destroy/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/destroy/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/warn/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/warn/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/warn/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/warn/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/warn/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/warn/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/node/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/ui/gateway-services-nodes/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/ui/gateway-services-nodes/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/ui/gateway-services-nodes/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/ui/gateway-services-nodes/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/ui/gateway-services-nodes/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/ui/gateway-services-nodes/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/kv/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/kv/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/kv/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/kv/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/kv/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/kv/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/checks (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/checks (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/checks (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/checks (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/checks (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/checks (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/service/register (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/service/register (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/service/register (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/service/register (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/service/register (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/service/register (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/connect/intentions (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/connect/intentions (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/connect/intentions (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/connect/intentions (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/connect/intentions (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/connect/intentions (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/connect/intentions/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/connect/intentions/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/connect/intentions/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/connect/intentions/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/connect/intentions/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/connect/intentions/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/event/fire/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/event/fire/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/event/fire/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/event/fire/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/event/fire/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/event/fire/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/list (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/list (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/list (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/list (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/list (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/list (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/rules/translate/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/rules/translate/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/rules/translate/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/rules/translate/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/rules/translate/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/rules/translate/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/tokens (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/tokens (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/tokens (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/tokens (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/tokens (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/tokens (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/self (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/self (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/self (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/self (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/self (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/self (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/fail/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/fail/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/fail/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/fail/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/fail/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/fail/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/operator/raft/peer (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/operator/raft/peer (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/operator/raft/peer (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/operator/raft/peer (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/operator/raft/peer (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/operator/raft/peer (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/event/list (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/event/list (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/event/list (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/event/list (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/event/list (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/event/list (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/policies (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/policies (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/policies (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/policies (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/policies (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/policies (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/maintenance (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/maintenance (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/maintenance (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/maintenance (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/maintenance (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/maintenance (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/force-leave/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/force-leave/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/force-leave/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/force-leave/ (0.01s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/force-leave/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/force-leave/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/info/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/info/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/info/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/info/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/info/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/info/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/binding-rules (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/binding-rules (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/binding-rules (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/binding-rules (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/binding-rules (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/binding-rules (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/members (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/members (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/members (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/members (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/members (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/members (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/bootstrap (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/bootstrap (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/bootstrap (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/bootstrap (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/bootstrap (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/bootstrap (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/rules/translate/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/rules/translate/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/rules/translate/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/rules/translate/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/rules/translate/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/rules/translate/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/create (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/create (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/create (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/create (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/create (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/create (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/info/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/info/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/info/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/info/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/info/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/info/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/token/self (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/token/self (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/token/self (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/token/self (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/token/self (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/token/self (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/connect/ca/roots (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/coordinate/update (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/coordinate/update (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/coordinate/update (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/coordinate/update (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/coordinate/update (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/coordinate/update (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/operator/keyring (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/operator/keyring (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/operator/keyring (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/operator/keyring (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/operator/keyring (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/operator/keyring (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/service/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/service/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/service/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/service/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/service/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/service/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/connect/ca/roots (0.01s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/connect/ca/roots (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/connect/ca/roots (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/connect/ca/roots (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/connect/ca/roots (0.01s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/connect/ca/roots (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/connect/ca/roots (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/federation-state/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/federation-state/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/federation-state/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/federation-state/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/federation-state/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/federation-state/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/create (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/create (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/create (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/create (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/create (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/create (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/policy/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/policy/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/policy/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/policy/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/policy/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/policy/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/role/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/role/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/role/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/role/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/role/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/role/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/auth-method (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/auth-method (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/auth-method (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/auth-method (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/auth-method (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/auth-method (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/connect/intentions/check (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/connect/intentions/check (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/connect/intentions/check (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/connect/intentions/check (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/connect/intentions/check (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/connect/intentions/check (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/policies (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/policies (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/policies (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/policies (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/policies (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/policies (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/services (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/services (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/services (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/services (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/services (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/services (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/force-leave/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/force-leave/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/force-leave/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/force-leave/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/force-leave/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/force-leave/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/leave (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/leave (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/leave (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/leave (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/leave (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/leave (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/connect/ca/roots (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/connect/ca/roots (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/connect/ca/roots (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/connect/ca/roots (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/connect/ca/roots (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/connect/ca/roots (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/state/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/state/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/state/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/state/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/state/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/state/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/service/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/service/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/service/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/service/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/service/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/service/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/discovery-chain/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/discovery-chain/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/discovery-chain/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/discovery-chain/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/discovery-chain/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/discovery-chain/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/ui/services (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/ui/services (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/ui/services (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/ui/services (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/ui/services (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/ui/services (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/operator/raft/peer (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/operator/raft/peer (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/operator/raft/peer (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/operator/raft/peer (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/operator/raft/peer (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/operator/raft/peer (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/login (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/login (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/login (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/login (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/login (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/login (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/policy (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/policy (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/policy (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/policy (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/policy (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/policy (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/clone/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/clone/ (0.01s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/clone/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/clone/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/clone/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/clone/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/binding-rule/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/binding-rule/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/binding-rule/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/binding-rule/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/binding-rule/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/binding-rule/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/token/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/token/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/token/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/token/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/token/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/token/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/ui/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/ui/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/ui/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/ui/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/ui/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/ui/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/warn/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/warn/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/warn/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/warn/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/warn/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/warn/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/connect/ca/leaf/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/connect/ca/leaf/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/connect/ca/leaf/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/connect/ca/leaf/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/connect/ca/leaf/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/connect/ca/leaf/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/connect/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/connect/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/connect/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/connect/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/connect/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/connect/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/config (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/config (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/config (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/config (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/config (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/config (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/coordinate/nodes (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/coordinate/nodes (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/coordinate/nodes (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/coordinate/nodes (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/coordinate/nodes (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/coordinate/nodes (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/logout (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/logout (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/logout (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/logout (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/logout (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/logout (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/pass/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/pass/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/pass/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/pass/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/pass/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/pass/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/connect/authorize (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/connect/authorize (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/connect/authorize (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/connect/authorize (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/connect/authorize (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/connect/authorize (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/service/maintenance/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/service/maintenance/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/service/maintenance/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/service/maintenance/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/service/maintenance/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/service/maintenance/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/deregister (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/deregister (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/deregister (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/deregister (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/deregister (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/deregister (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/federation-states/mesh-gateways (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/federation-states/mesh-gateways (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/federation-states/mesh-gateways (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/federation-states/mesh-gateways (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/federation-states/mesh-gateways (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/federation-states/mesh-gateways (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/federation-state/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/federation-state/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/federation-state/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/federation-state/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/federation-state/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/federation-state/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/operator/autopilot/configuration (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/operator/autopilot/configuration (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/operator/autopilot/configuration (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/operator/autopilot/configuration (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/operator/autopilot/configuration (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/operator/autopilot/configuration (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/destroy/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/destroy/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/destroy/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/destroy/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/destroy/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/destroy/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/deregister/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/deregister/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/deregister/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/deregister/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/deregister/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/deregister/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/datacenters (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/datacenters (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/datacenters (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/datacenters (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/datacenters (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/datacenters (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/connect/intentions/match (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/connect/intentions/match (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/connect/intentions/match (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/connect/intentions/match (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/connect/intentions/match (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/connect/intentions/match (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/status/leader (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/status/leader (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/status/leader (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/status/leader (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/status/leader (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/status/leader (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/service/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/service/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/service/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/service/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/service/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/service/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/register (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/register (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/register (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/register (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/register (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/register (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/checks/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/checks/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/checks/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/checks/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/checks/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/checks/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/rules/translate (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/rules/translate (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/rules/translate (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/rules/translate (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/rules/translate (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/rules/translate (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/host (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/host (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/host (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/host (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/host (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/host (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/metrics (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/metrics (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/metrics (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/metrics (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/metrics (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/metrics (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/register (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/register (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/register (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/register (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/register (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/register (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/event/list (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/event/list (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/event/list (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/event/list (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/event/list (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/event/list (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/ui/services (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/ui/services (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/ui/services (0.01s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/ui/services (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/ui/services (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/ui/services (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/list (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/list (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/list (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/list (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/list (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/list (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/role/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/role/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/role/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/role/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/role/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/role/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/binding-rule (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/binding-rule (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/binding-rule (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/binding-rule (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/binding-rule (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/binding-rule (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/token/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/token/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/token/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/token/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/token/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/token/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/members (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/members (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/members (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/members (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/members (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/members (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/pass/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/pass/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/pass/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/pass/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/pass/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/pass/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/service/deregister/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/service/deregister/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/service/deregister/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/service/deregister/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/service/deregister/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/service/deregister/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/gateway-services/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/gateway-services/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/gateway-services/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/gateway-services/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/gateway-services/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/gateway-services/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/service/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/service/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/service/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/service/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/service/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/service/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/create (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/create (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/create (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/create (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/create (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/create (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/replication (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/replication (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/replication (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/replication (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/replication (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/replication (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/token/self (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/token/self (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/token/self (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/token/self (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/token/self (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/token/self (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/node-services/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/node-services/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/node-services/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/node-services/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/node-services/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/node-services/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/role/name/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/role/name/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/role/name/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/role/name/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/role/name/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/role/name/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/coordinate/nodes (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/coordinate/nodes (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/coordinate/nodes (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/coordinate/nodes (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/coordinate/nodes (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/coordinate/nodes (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/destroy/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/destroy/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/destroy/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/destroy/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/destroy/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/destroy/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/auth-methods (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/auth-methods (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/auth-methods (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/auth-methods (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/auth-methods (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/auth-methods (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/health/service/name/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/health/service/name/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/health/service/name/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/health/service/name/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/health/service/name/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/health/service/name/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/federation-states/mesh-gateways (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/federation-states/mesh-gateways (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/federation-states/mesh-gateways (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/federation-states/mesh-gateways (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/federation-states/mesh-gateways (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/federation-states/mesh-gateways (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/acl/authorize (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/acl/authorize (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/acl/authorize (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/acl/authorize (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/acl/authorize (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/acl/authorize (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/roles (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/roles (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/roles (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/roles (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/roles (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/roles (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/auth-method/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/auth-method/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/auth-method/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/auth-method/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/auth-method/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/auth-method/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/service/register (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/service/register (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/service/register (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/service/register (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/service/register (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/service/register (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/service/deregister/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/service/deregister/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/service/deregister/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/service/deregister/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/service/deregister/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/service/deregister/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/federation-states (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/federation-states (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/federation-states (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/federation-states (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/federation-states (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/federation-states (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/connect/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/connect/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/connect/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/connect/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/connect/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/connect/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/policy (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/policy (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/policy (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/policy (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/policy (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/policy (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/policy/name/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/policy/name/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/policy/name/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/policy/name/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/policy/name/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/policy/name/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/services (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/services (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/services (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/services (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/services (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/services (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/connect/authorize (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/connect/authorize (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/connect/authorize (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/connect/authorize (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/connect/authorize (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/connect/authorize (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/nodes (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/nodes (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/nodes (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/nodes (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/nodes (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/nodes (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/operator/keyring (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/operator/keyring (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/operator/keyring (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/operator/keyring (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/operator/keyring (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/operator/keyring (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/renew/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/renew/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/renew/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/renew/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/renew/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/renew/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/txn (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/txn (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/txn (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/txn (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/txn (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/txn (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/destroy/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/destroy/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/destroy/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/destroy/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/destroy/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/destroy/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/update/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/update/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/update/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/update/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/update/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/update/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/gateway-services/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/gateway-services/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/gateway-services/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/gateway-services/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/gateway-services/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/gateway-services/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/coordinate/datacenters (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/coordinate/datacenters (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/coordinate/datacenters (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/coordinate/datacenters (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/coordinate/datacenters (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/coordinate/datacenters (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/tokens (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/tokens (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/tokens (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/tokens (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/tokens (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/tokens (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/fail/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/fail/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/fail/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/fail/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/fail/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/fail/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/auth-method/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/auth-method/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/auth-method/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/auth-method/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/auth-method/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/auth-method/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/join/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/join/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/join/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/join/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/join/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/join/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/event/fire/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/event/fire/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/event/fire/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/event/fire/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/event/fire/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/event/fire/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/ui/nodes (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/ui/nodes (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/ui/nodes (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/ui/nodes (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/ui/nodes (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/ui/nodes (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/rules/translate (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/rules/translate (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/rules/translate (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/rules/translate (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/rules/translate (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/rules/translate (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/ui/gateway-services-nodes/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/ui/gateway-services-nodes/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/ui/gateway-services-nodes/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/ui/gateway-services-nodes/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/ui/gateway-services-nodes/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/ui/gateway-services-nodes/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/services (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/services (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/services (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/services (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/services (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/services (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/ingress/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/ingress/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/ingress/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/ingress/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/ingress/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/ingress/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/operator/autopilot/health (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/operator/autopilot/health (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/operator/autopilot/health (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/operator/autopilot/health (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/operator/autopilot/health (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/operator/autopilot/health (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/status/leader (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/status/leader (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/status/leader (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/status/leader (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/status/leader (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/status/leader (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/logout (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/logout (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/logout (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/logout (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/logout (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/logout (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/host (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/host (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/host (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/host (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/host (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/host (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/connect/ca/roots (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/connect/ca/roots (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/connect/ca/roots (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/connect/ca/roots (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/connect/ca/roots (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/connect/ca/roots (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/connect/intentions/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/connect/intentions/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/connect/intentions/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/connect/intentions/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/connect/intentions/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/connect/intentions/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/operator/raft/configuration (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/operator/raft/configuration (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/operator/raft/configuration (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/operator/raft/configuration (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/operator/raft/configuration (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/operator/raft/configuration (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/operator/autopilot/configuration (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/operator/autopilot/configuration (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/operator/autopilot/configuration (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/operator/autopilot/configuration (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/operator/autopilot/configuration (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/operator/autopilot/configuration (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/connect/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/connect/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/connect/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/connect/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/connect/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/connect/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/list (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/list (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/list (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/list (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/list (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/list (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/policy/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/policy/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/policy/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/policy/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/policy/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/policy/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/token (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/token (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/token (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/token (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/token (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/token (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/connect/ca/configuration (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/connect/ca/configuration (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/connect/ca/configuration (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/connect/ca/configuration (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/connect/ca/configuration (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/connect/ca/configuration (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/service/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/service/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/service/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/service/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/service/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/service/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/register (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/register (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/register (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/register (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/register (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/register (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/discovery-chain/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/discovery-chain/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/discovery-chain/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/discovery-chain/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/discovery-chain/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/discovery-chain/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/self (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/self (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/self (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/self (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/self (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/self (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/deregister (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/deregister (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/deregister (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/deregister (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/deregister (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/deregister (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/connect/intentions/check (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/connect/intentions/check (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/connect/intentions/check (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/connect/intentions/check (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/connect/intentions/check (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/connect/intentions/check (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/coordinate/node/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/coordinate/node/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/coordinate/node/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/coordinate/node/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/coordinate/node/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/coordinate/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/connect/ca/leaf/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/connect/ca/leaf/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/connect/ca/leaf/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/connect/ca/leaf/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/connect/ca/leaf/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/connect/ca/leaf/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/coordinate/update (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/coordinate/update (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/coordinate/update (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/coordinate/update (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/coordinate/update (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/coordinate/update (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/operator/autopilot/health (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/operator/autopilot/health (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/operator/autopilot/health (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/operator/autopilot/health (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/operator/autopilot/health (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/operator/autopilot/health (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/renew/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/renew/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/renew/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/renew/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/renew/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/renew/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/deregister/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/deregister/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/deregister/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/deregister/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/deregister/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/deregister/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/create (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/create (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/create (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/create (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/create (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/create (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/auth-methods (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/auth-methods (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/auth-methods (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/auth-methods (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/auth-methods (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/auth-methods (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/token/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/token/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/token/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/token/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/token/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/token/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/role/name/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/role/name/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/role/name/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/role/name/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/role/name/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/role/name/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/kv/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/kv/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/kv/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/kv/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/kv/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/kv/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/binding-rules (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/binding-rules (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/binding-rules (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/binding-rules (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/binding-rules (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/binding-rules (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/auth-method (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/auth-method (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/auth-method (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/auth-method (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/auth-method (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/auth-method (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/checks (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/checks (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/checks (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/checks (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/checks (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/checks (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/coordinate/datacenters (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/coordinate/datacenters (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/coordinate/datacenters (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/coordinate/datacenters (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/coordinate/datacenters (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/coordinate/datacenters (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/login (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/login (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/login (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/login (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/login (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/login (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/health/service/id/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/health/service/id/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/health/service/id/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/health/service/id/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/health/service/id/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/health/service/id/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/ui/nodes (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/ui/nodes (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/ui/nodes (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/ui/nodes (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/ui/nodes (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/ui/nodes (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/federation-states (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/federation-states (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/federation-states (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/federation-states (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/federation-states (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/federation-states (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/acl/authorize (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/acl/authorize (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/acl/authorize (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/acl/authorize (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/acl/authorize (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/acl/authorize (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/update (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/update (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/update (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/update (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/update (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/update (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/register (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/register (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/register (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/register (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/register (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/register (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/service/maintenance/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/service/maintenance/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/service/maintenance/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/service/maintenance/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/service/maintenance/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/service/maintenance/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/config/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/config/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/config/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/config/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/config/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/config/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/snapshot (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/snapshot (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/snapshot (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/snapshot (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/snapshot (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/snapshot (0.00s) === RUN TestHTTPAPI_OptionMethod_OSS [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! === RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/query === RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/query/ === RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/query/xxx/execute === RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/query/xxx/explain -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/service/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/register +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/update/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/intentions/match === RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/health/checks/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/health/state/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/roles === RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/metrics -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/register +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/leave +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/role +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/service/register +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/intentions +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/operator/raft/peer === RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/event/list -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/rules/translate -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/host -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/acl/authorize -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/replication -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/health/service/name/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/federation-states/mesh-gateways -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/service/deregister/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/nodes +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/policies +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/maintenance +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/force-leave/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/info/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/bootstrap +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/rules/translate/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/create +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/info/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/coordinate/update === RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/operator/keyring -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/txn -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/roles -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/auth-method/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/service/register -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/coordinate/datacenters -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/operator/raft/configuration -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/operator/autopilot/configuration -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/destroy/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/update/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/gateway-services/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/connect/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/coordinate/node/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/service/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/ca/roots +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/clone/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/binding-rule/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/token/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/ui/node/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/destroy/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/warn/ === RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/connect/ca/leaf/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/coordinate/update -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/operator/autopilot/health -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/renew/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/connect/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/config +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/federation-states/mesh-gateways +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/federation-state/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/operator/autopilot/configuration === RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/deregister/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/datacenters +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/ui/services +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/gateway-services/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/list +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/role/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/binding-rule +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/token/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/members +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/pass/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/service/deregister/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/health/service/ === RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/create -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/auth-methods -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/token/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/role/name/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/health/service/id/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/replication +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/token/self +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/node-services/ === RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/health/node/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/ui/nodes +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/role/name/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/coordinate/nodes +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/destroy/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/auth-methods +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/health/service/name/ === RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/federation-states -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/token/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/services -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/list -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/ca/configuration -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/service/ === RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/health/connect/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/update -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/binding-rule -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/token -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/health/ingress/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/snapshot -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/join/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/node-services/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/config/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/maintenance -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/ui/node/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/destroy/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/clone/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/policy === RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/policy/name/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/role -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/kv/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/warn/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/node/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/ui/gateway-services-nodes/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/services +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/connect/authorize +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/nodes +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/renew/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/txn +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/tokens +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/fail/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/node/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/auth-method/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/join/ === RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/event/fire/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/checks -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/intentions +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/ui/nodes +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/rules/translate +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/ui/gateway-services-nodes/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/services +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/health/ingress/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/operator/autopilot/health +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/status/leader +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/logout +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/host +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/connect/ca/roots +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/node/ === RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/intentions/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/node/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/operator/raft/configuration === RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/list -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/rules/translate/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/tokens +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/policy/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/token +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/ca/configuration +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/service/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/register +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/discovery-chain/ === RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/self -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/fail/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/info/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/binding-rules -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/members -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/ca/roots -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/federation-state/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/create -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/bootstrap -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/info/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/token/self +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/deregister === RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/intentions/check -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/policy/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/role/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/coordinate/node/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/kv/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/binding-rules === RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/auth-method -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/policies -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/services -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/force-leave/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/health/service/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/leave -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/connect/ca/roots -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/health/state/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/discovery-chain/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/ui/services -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/operator/raft/peer -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/config -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/coordinate/nodes +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/checks +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/coordinate/datacenters === RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/login -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/policy -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/binding-rule/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/health/service/id/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/acl/authorize +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/update +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/register === RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/service/maintenance/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/deregister -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/datacenters -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/intentions/match -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/status/leader -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/logout -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/pass/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/connect/authorize +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/config/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/snapshot [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:56.317Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:41:56.317Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:56.343Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:41:56.343Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:56.346 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c6c9c608-e12e-1950-c778-954bc4800c84 Address:127.0.0.1:13958}]" -41:56.348 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13958 [Follower]" leader= -41:56.349 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c6c9c608-e12e-1950-c778-954bc4800c84.dc1 127.0.0.1 -41:56.350 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c6c9c608-e12e-1950-c778-954bc4800c84 127.0.0.1 -2022-09-15T21:41:56.350Z [INFO] agent.router: Initializing LAN area manager -41:56.350 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13953 network=udp -41:56.350 [INFO] TestAgent.server: Adding LAN server: server="Node-c6c9c608-e12e-1950-c778-954bc4800c84 (Addr: tcp/127.0.0.1:13958) (DC: dc1)" -41:56.350 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c6c9c608-e12e-1950-c778-954bc4800c84.dc1 area=wan -41:56.351 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13953 network=tcp -41:56.351 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13954 network=tcp -41:56.352 [INFO] TestAgent: started state syncer -41:56.352 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13959 network=tcp -41:56.401 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -41:56.402 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:56.402 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13958 [Candidate]" term=2 -41:56.402 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:56.402 [DEBUG] TestAgent.server.raft: vote granted: from=c6c9c608-e12e-1950-c778-954bc4800c84 term=2 tally=1 -41:56.402 [INFO] TestAgent.server.raft: election won: tally=1 -41:56.402 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13958 [Leader]" -41:56.403 [INFO] TestAgent.server: cluster leadership acquired -41:56.403 [INFO] TestAgent.server: initializing acls -41:56.403 [INFO] TestAgent.server: Created ACL 'global-management' policy -41:56.404 [INFO] TestAgent.server: Created ACL anonymous token from configuration -41:56.404 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -41:56.404 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -41:56.404 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-c6c9c608-e12e-1950-c778-954bc4800c84 -41:56.404 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-c6c9c608-e12e-1950-c778-954bc4800c84.dc1 -41:56.406 [INFO] TestAgent.server: New leader elected: payload=Node-c6c9c608-e12e-1950-c778-954bc4800c84 -41:56.406 [INFO] TestAgent.server: Updating LAN server: server="Node-c6c9c608-e12e-1950-c778-954bc4800c84 (Addr: tcp/127.0.0.1:13958) (DC: dc1)" -41:56.406 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-c6c9c608-e12e-1950-c778-954bc4800c84.dc1 area=wan -41:56.406 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:56.409 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:56.409 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:56.409 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:56.409 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:56.409 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:56.409 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c6c9c608-e12e-1950-c778-954bc4800c84 -41:56.409 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c6c9c608-e12e-1950-c778-954bc4800c84 -41:56.410 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c6c9c608-e12e-1950-c778-954bc4800c84 -41:56.411 [INFO] TestAgent.server: federation state anti-entropy synced -41:56.628 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:56.629 [INFO] TestAgent: Synced node info -41:56.683 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/query from= latency=7.26µs -41:56.684 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/query/ from= latency=233.96µs -41:56.684 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/query/xxx/execute from= latency=26.56µs -41:56.684 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/query/xxx/explain from= latency=21.62µs -41:56.684 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/agent/service/ from= latency=3µs -41:56.684 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/agent/check/register from= latency=3.06µs -41:56.684 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/health/checks/ from= latency=2.36µs -41:56.684 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/agent/metrics from= latency=2.1µs -41:56.685 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/catalog/register from= latency=2.04µs -41:56.685 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/event/list from= latency=2.72µs -41:56.685 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/acl/rules/translate from= latency=3.16µs -41:56.685 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/agent/host from= latency=3.7µs -41:56.685 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/internal/acl/authorize from= latency=2.46µs -41:56.685 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/acl/replication from= latency=2.06µs -41:56.685 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/agent/health/service/name/ from= latency=1.94µs -41:56.685 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/internal/federation-states/mesh-gateways from= latency=2.16µs -41:56.685 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/agent/service/deregister/ from= latency=2.44µs -41:56.685 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/catalog/nodes from= latency=2.72µs -41:56.685 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/operator/keyring from= latency=2.58µs -41:56.686 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/txn from= latency=5.78µs -41:56.686 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/acl/roles from= latency=3.06µs -41:56.686 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/acl/auth-method/ from= latency=2.52µs -41:56.686 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/agent/service/register from= latency=3.02µs -41:56.686 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/coordinate/datacenters from= latency=2.84µs -41:56.686 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/operator/raft/configuration from= latency=2.7µs -41:56.686 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/operator/autopilot/configuration from= latency=3.06µs -41:56.687 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/acl/destroy/ from= latency=3.28µs -41:56.687 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/agent/check/update/ from= latency=2.8µs -41:56.687 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/catalog/gateway-services/ from= latency=2.88µs -41:56.687 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/catalog/connect/ from= latency=2.3µs -41:56.687 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/coordinate/node/ from= latency=2.56µs -41:56.687 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/agent/connect/ca/leaf/ from= latency=2.54µs -41:56.687 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/coordinate/update from= latency=2.46µs -41:56.687 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/operator/autopilot/health from= latency=2.46µs -41:56.687 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/session/renew/ from= latency=2.94µs -41:56.688 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/agent/check/deregister/ from= latency=2.24µs -41:56.688 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/acl/create from= latency=1.94µs -41:56.688 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/acl/auth-methods from= latency=2.48µs -41:56.688 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/agent/token/ from= latency=2.06µs -41:56.688 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/acl/role/name/ from= latency=2µs -41:56.688 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/agent/health/service/id/ from= latency=2.1µs -41:56.688 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/health/node/ from= latency=2.14µs -41:56.688 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/internal/ui/nodes from= latency=2.74µs -41:56.688 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/internal/federation-states from= latency=2.52µs -41:56.688 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/acl/token/ from= latency=2.02µs -41:56.688 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/catalog/services from= latency=2.48µs -41:56.689 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/acl/list from= latency=2.42µs -41:56.689 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/connect/ca/configuration from= latency=2.46µs -41:56.689 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/catalog/service/ from= latency=2.02µs -41:56.689 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/health/connect/ from= latency=2.02µs -41:56.689 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/acl/update from= latency=1.84µs -41:56.689 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/acl/binding-rule from= latency=2.5µs -41:56.689 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/acl/token from= latency=1.96µs -41:56.689 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/health/ingress/ from= latency=2.12µs -41:56.689 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/snapshot from= latency=2.36µs -41:56.689 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/agent/join/ from= latency=2.26µs -41:56.689 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/catalog/node-services/ from= latency=1.88µs -41:56.689 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/config/ from= latency=2.14µs -41:56.690 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/agent/maintenance from= latency=5.2µs -41:56.690 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/internal/ui/node/ from= latency=2.88µs -41:56.690 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/session/destroy/ from= latency=3.44µs -41:56.690 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/acl/clone/ from= latency=2.66µs -41:56.690 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/acl/policy/name/ from= latency=2.54µs -41:56.690 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/acl/role from= latency=2.72µs -41:56.690 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/kv/ from= latency=2.2µs -41:56.691 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/agent/check/warn/ from= latency=2.84µs -41:56.691 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/catalog/node/ from= latency=2.74µs -41:56.691 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/internal/ui/gateway-services-nodes/ from= latency=2.52µs -41:56.691 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/event/fire/ from= latency=2.5µs -41:56.691 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/agent/checks from= latency=2.12µs -41:56.691 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/connect/intentions from= latency=2.74µs -41:56.691 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/connect/intentions/ from= latency=2.58µs -41:56.691 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/session/node/ from= latency=2.38µs -41:56.691 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/session/list from= latency=2.14µs -41:56.691 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/acl/rules/translate/ from= latency=2.48µs -41:56.692 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/acl/tokens from= latency=2.42µs -41:56.692 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/agent/self from= latency=2.32µs -41:56.692 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/agent/check/fail/ from= latency=1.96µs -41:56.692 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/session/info/ from= latency=2.1µs -41:56.692 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/acl/binding-rules from= latency=2.2µs -41:56.692 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/agent/members from= latency=2.36µs -41:56.692 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/connect/ca/roots from= latency=2.36µs -41:56.692 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/internal/federation-state/ from= latency=14.74µs -41:56.692 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/session/create from= latency=2.1µs -41:56.692 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/acl/bootstrap from= latency=2.42µs -41:56.693 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/acl/info/ from= latency=2.42µs -41:56.693 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/acl/token/self from= latency=2.28µs -41:56.693 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/connect/intentions/check from= latency=2.08µs -41:56.693 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/acl/policy/ from= latency=2.54µs -41:56.693 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/acl/role/ from= latency=2.08µs -41:56.693 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/acl/auth-method from= latency=2.3µs -41:56.693 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/acl/policies from= latency=3.06µs -41:56.693 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/agent/services from= latency=1.86µs -41:56.693 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/agent/force-leave/ from= latency=2.46µs -41:56.693 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/health/service/ from= latency=2.04µs -41:56.694 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/agent/leave from= latency=3.42µs -41:56.694 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/agent/connect/ca/roots from= latency=2.6µs -41:56.694 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/health/state/ from= latency=2.16µs -41:56.694 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/discovery-chain/ from= latency=2.7µs -41:56.694 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/internal/ui/services from= latency=2.46µs -41:56.694 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/operator/raft/peer from= latency=2.6µs -41:56.694 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/config from= latency=2.76µs -41:56.694 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/coordinate/nodes from= latency=2.24µs -41:56.695 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/acl/login from= latency=2.52µs -41:56.695 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/acl/policy from= latency=2.56µs -41:56.695 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/acl/binding-rule/ from= latency=3.54µs -41:56.695 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/agent/service/maintenance/ from= latency=2.4µs -41:56.695 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/catalog/deregister from= latency=2.16µs -41:56.695 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/catalog/datacenters from= latency=2.66µs -41:56.695 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/connect/intentions/match from= latency=2.22µs -41:56.695 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/status/leader from= latency=2.7µs -41:56.695 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/acl/logout from= latency=2.88µs -41:56.696 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/agent/check/pass/ from= latency=2.34µs -41:56.696 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:13954/v1/agent/connect/authorize from= latency=2.7µs -41:56.696 [INFO] TestAgent: Requesting shutdown -41:56.696 [INFO] TestAgent.server: shutting down server -41:56.696 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -41:56.696 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -41:56.696 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:56.696 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:56.696 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:56.696 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:56.696 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:56.696 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:56.696 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:56.696 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -41:56.696 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -41:56.696 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:56.696 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:56.696 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:56.696 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:56.696Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:56.696Z [INFO] agent.router.manager: shutting down -41:56.697 [INFO] TestAgent: consul server down -41:56.697 [INFO] TestAgent: shutdown complete -41:56.697 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13953 network=tcp -41:56.697 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13953 network=udp -41:56.697 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13954 network=tcp -41:57.198 [INFO] TestAgent: Waiting for endpoints to shut down -41:57.198 [INFO] TestAgent: Endpoints down ---- PASS: TestHTTPAPI_OptionMethod_OSS (0.90s) +2021-08-13T15:33:37.817Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:33:37.817Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:37.833Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:33:37.833Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:37.835 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:63df547e-f111-c0f3-5501-30646db6ad2d Address:127.0.0.1:25189}]" +33:37.836 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-63df547e-f111-c0f3-5501-30646db6ad2d.dc1 127.0.0.1 +33:37.837 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-63df547e-f111-c0f3-5501-30646db6ad2d 127.0.0.1 +2021-08-13T15:33:37.837Z [INFO] agent.router: Initializing LAN area manager +33:37.837 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25184 network=udp +33:37.837 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25189 [Follower]" leader= +33:37.838 [INFO] TestAgent.server: Adding LAN server: server="Node-63df547e-f111-c0f3-5501-30646db6ad2d (Addr: tcp/127.0.0.1:25189) (DC: dc1)" +33:37.838 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-63df547e-f111-c0f3-5501-30646db6ad2d.dc1 area=wan +33:37.838 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25184 network=tcp +33:37.839 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25185 network=tcp +33:37.839 [INFO] TestAgent: started state syncer +33:37.839 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25190 network=tcp +33:37.888 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +33:37.893 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:37.894 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25189 [Candidate]" term=2 +33:37.894 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:37.894 [DEBUG] TestAgent.server.raft: vote granted: from=63df547e-f111-c0f3-5501-30646db6ad2d term=2 tally=1 +33:37.894 [INFO] TestAgent.server.raft: election won: tally=1 +33:37.894 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25189 [Leader]" +33:37.894 [INFO] TestAgent.server: cluster leadership acquired +33:37.894 [INFO] TestAgent.server: New leader elected: payload=Node-63df547e-f111-c0f3-5501-30646db6ad2d +33:37.894 [INFO] TestAgent.server: initializing acls +33:37.895 [INFO] TestAgent.server: Created ACL 'global-management' policy +33:37.895 [INFO] TestAgent.server: Created ACL anonymous token from configuration +33:37.895 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +33:37.895 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +33:37.896 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-63df547e-f111-c0f3-5501-30646db6ad2d +33:37.896 [INFO] TestAgent.server: Updating LAN server: server="Node-63df547e-f111-c0f3-5501-30646db6ad2d (Addr: tcp/127.0.0.1:25189) (DC: dc1)" +33:37.896 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-63df547e-f111-c0f3-5501-30646db6ad2d.dc1 +33:37.896 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-63df547e-f111-c0f3-5501-30646db6ad2d.dc1 area=wan +33:37.897 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:37.899 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:37.899 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:37.899 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:37.899 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:37.899 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:37.899 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-63df547e-f111-c0f3-5501-30646db6ad2d +33:37.899 [INFO] TestAgent.server: member joined, marking health alive: member=Node-63df547e-f111-c0f3-5501-30646db6ad2d +33:37.900 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-63df547e-f111-c0f3-5501-30646db6ad2d +33:37.901 [INFO] TestAgent.server: federation state anti-entropy synced +33:38.114 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/query from= latency=6.08µs +33:38.115 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/query/ from= latency=195.661µs +33:38.115 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/query/xxx/execute from= latency=28.9µs +33:38.115 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/query/xxx/explain from= latency=24.26µs +33:38.115 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/agent/check/update/ from= latency=3µs +33:38.115 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/connect/intentions/match from= latency=3.02µs +33:38.115 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/health/checks/ from= latency=2.74µs +33:38.116 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/health/state/ from= latency=2.66µs +33:38.116 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/acl/roles from= latency=2.82µs +33:38.116 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/agent/metrics from= latency=2.52µs +33:38.116 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/agent/leave from= latency=3µs +33:38.116 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/acl/role from= latency=3.24µs +33:38.116 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/agent/service/register from= latency=2.82µs +33:38.116 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/connect/intentions from= latency=2.88µs +33:38.116 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/operator/raft/peer from= latency=2.34µs +33:38.116 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/event/list from= latency=2.86µs +33:38.117 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/acl/policies from= latency=2.62µs +33:38.117 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/agent/maintenance from= latency=3µs +33:38.117 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/agent/force-leave/ from= latency=2.88µs +33:38.117 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/session/info/ from= latency=2.98µs +33:38.117 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/acl/bootstrap from= latency=2.52µs +33:38.117 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/acl/rules/translate/ from= latency=2.82µs +33:38.117 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/session/create from= latency=2.72µs +33:38.117 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/acl/info/ from= latency=3.36µs +33:38.117 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/coordinate/update from= latency=2.56µs +33:38.118 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/operator/keyring from= latency=3.28µs +33:38.118 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/catalog/service/ from= latency=2.9µs +33:38.118 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/connect/ca/roots from= latency=2.74µs +33:38.118 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/acl/clone/ from= latency=2.64µs +33:38.118 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/acl/binding-rule/ from= latency=2.62µs +33:38.118 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/agent/token/ from= latency=2.681µs +33:38.118 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/internal/ui/node/ from= latency=2.94µs +33:38.118 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/session/destroy/ from= latency=2.52µs +33:38.118 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/agent/check/warn/ from= latency=2.5µs +33:38.119 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/agent/connect/ca/leaf/ from= latency=3.02µs +33:38.119 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/catalog/connect/ from= latency=2.64µs +33:38.119 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/config from= latency=2.5µs +33:38.119 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/internal/federation-states/mesh-gateways from= latency=2.68µs +33:38.119 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/internal/federation-state/ from= latency=2.56µs +33:38.119 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/operator/autopilot/configuration from= latency=2.76µs +33:38.119 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/agent/check/deregister/ from= latency=2.94µs +33:38.119 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/catalog/datacenters from= latency=2.42µs +33:38.120 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/internal/ui/services from= latency=2.98µs +33:38.120 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/catalog/gateway-services/ from= latency=2.66µs +33:38.120 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/acl/list from= latency=2.76µs +33:38.120 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/acl/role/ from= latency=2.68µs +33:38.120 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/acl/binding-rule from= latency=2.86µs +33:38.120 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/acl/token/ from= latency=2.88µs +33:38.120 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/agent/members from= latency=2.7µs +33:38.120 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/agent/check/pass/ from= latency=2.74µs +33:38.121 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/agent/service/deregister/ from= latency=2.52µs +33:38.121 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/health/service/ from= latency=2.32µs +33:38.121 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/acl/create from= latency=2.84µs +33:38.121 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/acl/replication from= latency=2.32µs +33:38.121 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/acl/token/self from= latency=2.16µs +33:38.121 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/catalog/node-services/ from= latency=2.48µs +33:38.121 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/health/node/ from= latency=2.02µs +33:38.121 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/acl/role/name/ from= latency=2.76µs +33:38.121 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/coordinate/nodes from= latency=2.48µs +33:38.122 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/acl/destroy/ from= latency=2.36µs +33:38.122 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/acl/auth-methods from= latency=2.18µs +33:38.122 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/agent/health/service/name/ from= latency=2.54µs +33:38.122 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/internal/federation-states from= latency=2.24µs +33:38.122 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/health/connect/ from= latency=2.68µs +33:38.122 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/acl/policy from= latency=2.92µs +33:38.122 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/acl/policy/name/ from= latency=2.34µs +33:38.122 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/agent/services from= latency=2.38µs +33:38.122 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/agent/connect/authorize from= latency=2.92µs +33:38.122 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/catalog/nodes from= latency=2.58µs +33:38.123 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/session/renew/ from= latency=2.66µs +33:38.123 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/txn from= latency=2.54µs +33:38.123 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/acl/tokens from= latency=4.94µs +33:38.123 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/agent/check/fail/ from= latency=2.92µs +33:38.123 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/session/node/ from= latency=2.7µs +33:38.123 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/acl/auth-method/ from= latency=2.62µs +33:38.123 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/agent/join/ from= latency=2.58µs +33:38.123 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/event/fire/ from= latency=2.58µs +33:38.124 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/internal/ui/nodes from= latency=2.96µs +33:38.124 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/acl/rules/translate from= latency=2.48µs +33:38.124 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/internal/ui/gateway-services-nodes/ from= latency=2.66µs +33:38.124 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/catalog/services from= latency=2.94µs +33:38.124 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/health/ingress/ from= latency=2.42µs +33:38.124 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/operator/autopilot/health from= latency=2.28µs +33:38.124 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/status/leader from= latency=2.96µs +33:38.124 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/acl/logout from= latency=2.38µs +33:38.125 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/agent/host from= latency=2.68µs +33:38.125 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/agent/connect/ca/roots from= latency=2.4µs +33:38.125 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/catalog/node/ from= latency=2.18µs +33:38.125 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/connect/intentions/ from= latency=2.52µs +33:38.125 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/operator/raft/configuration from= latency=2.48µs +33:38.125 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/session/list from= latency=2.46µs +33:38.125 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/acl/policy/ from= latency=2.86µs +33:38.125 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/acl/token from= latency=2.72µs +33:38.125 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/connect/ca/configuration from= latency=2.72µs +33:38.126 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/agent/service/ from= latency=2.52µs +33:38.126 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/catalog/register from= latency=2.66µs +33:38.126 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/discovery-chain/ from= latency=2.94µs +33:38.126 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/agent/self from= latency=2.7µs +33:38.126 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/catalog/deregister from= latency=2.5µs +33:38.126 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/connect/intentions/check from= latency=2.6µs +33:38.126 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/coordinate/node/ from= latency=2.16µs +33:38.126 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/kv/ from= latency=2.88µs +33:38.126 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/acl/binding-rules from= latency=2.58µs +33:38.126 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/acl/auth-method from= latency=2.5µs +33:38.127 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/agent/checks from= latency=2.44µs +33:38.127 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/coordinate/datacenters from= latency=2.8µs +33:38.127 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/acl/login from= latency=3.94µs +33:38.127 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/agent/health/service/id/ from= latency=5.06µs +33:38.127 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/internal/acl/authorize from= latency=2.72µs +33:38.127 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/acl/update from= latency=2.68µs +33:38.127 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/agent/check/register from= latency=2.28µs +33:38.127 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/agent/service/maintenance/ from= latency=2.5µs +33:38.128 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/config/ from= latency=2.6µs +33:38.128 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:25185/v1/snapshot from= latency=2.86µs +33:38.128 [INFO] TestAgent: Requesting shutdown +33:38.128 [INFO] TestAgent.server: shutting down server +33:38.128 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:38.128 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:38.128 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:38.128 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:38.128 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +33:38.128 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +33:38.128 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:38.128 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:38.128 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:38.128 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:38.128 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:38.128 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:38.128 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:38.128 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +33:38.128 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +33:38.128 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +2021-08-13T15:33:38.128Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:38.128Z [INFO] agent.router.manager: shutting down +33:38.129 [INFO] TestAgent: consul server down +33:38.129 [INFO] TestAgent: shutdown complete +33:38.129 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25184 network=tcp +33:38.129 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25184 network=udp +33:38.129 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25185 network=tcp +33:38.629 [INFO] TestAgent: Waiting for endpoints to shut down +33:38.629 [INFO] TestAgent: Endpoints down +--- PASS: TestHTTPAPI_OptionMethod_OSS (0.83s) --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/query (0.00s) --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/query/ (0.00s) --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/query/xxx/execute (0.00s) --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/query/xxx/explain (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/service/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/register (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/update/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/intentions/match (0.00s) --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/health/checks/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/health/state/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/roles (0.00s) --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/metrics (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/register (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/leave (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/role (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/service/register (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/intentions (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/operator/raft/peer (0.00s) --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/event/list (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/rules/translate (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/host (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/acl/authorize (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/replication (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/health/service/name/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/federation-states/mesh-gateways (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/service/deregister/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/nodes (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/policies (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/maintenance (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/force-leave/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/info/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/bootstrap (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/rules/translate/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/create (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/info/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/coordinate/update (0.00s) --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/operator/keyring (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/txn (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/roles (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/auth-method/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/service/register (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/coordinate/datacenters (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/operator/raft/configuration (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/operator/autopilot/configuration (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/destroy/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/update/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/gateway-services/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/connect/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/coordinate/node/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/service/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/ca/roots (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/clone/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/binding-rule/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/token/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/ui/node/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/destroy/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/warn/ (0.00s) --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/connect/ca/leaf/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/coordinate/update (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/operator/autopilot/health (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/renew/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/connect/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/config (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/federation-states/mesh-gateways (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/federation-state/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/operator/autopilot/configuration (0.00s) --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/deregister/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/datacenters (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/ui/services (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/gateway-services/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/list (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/role/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/binding-rule (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/token/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/members (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/pass/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/service/deregister/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/health/service/ (0.00s) --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/create (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/auth-methods (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/token/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/role/name/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/health/service/id/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/replication (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/token/self (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/node-services/ (0.00s) --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/health/node/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/ui/nodes (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/role/name/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/coordinate/nodes (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/destroy/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/auth-methods (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/health/service/name/ (0.00s) --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/federation-states (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/token/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/services (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/list (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/ca/configuration (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/service/ (0.00s) --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/health/connect/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/update (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/binding-rule (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/token (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/health/ingress/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/snapshot (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/join/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/node-services/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/config/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/maintenance (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/ui/node/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/destroy/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/clone/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/policy (0.00s) --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/policy/name/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/role (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/kv/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/warn/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/node/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/ui/gateway-services-nodes/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/services (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/connect/authorize (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/nodes (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/renew/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/txn (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/tokens (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/fail/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/node/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/auth-method/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/join/ (0.00s) --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/event/fire/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/checks (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/intentions (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/ui/nodes (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/rules/translate (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/ui/gateway-services-nodes/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/services (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/health/ingress/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/operator/autopilot/health (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/status/leader (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/logout (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/host (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/connect/ca/roots (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/node/ (0.00s) --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/intentions/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/node/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/operator/raft/configuration (0.00s) --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/list (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/rules/translate/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/tokens (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/policy/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/token (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/ca/configuration (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/service/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/register (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/discovery-chain/ (0.00s) --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/self (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/fail/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/info/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/binding-rules (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/members (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/ca/roots (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/federation-state/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/create (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/bootstrap (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/info/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/token/self (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/deregister (0.00s) --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/intentions/check (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/policy/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/role/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/coordinate/node/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/kv/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/binding-rules (0.00s) --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/auth-method (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/policies (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/services (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/force-leave/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/health/service/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/leave (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/connect/ca/roots (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/health/state/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/discovery-chain/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/ui/services (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/operator/raft/peer (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/config (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/coordinate/nodes (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/checks (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/coordinate/datacenters (0.00s) --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/login (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/policy (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/binding-rule/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/health/service/id/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/acl/authorize (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/update (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/register (0.00s) --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/service/maintenance/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/deregister (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/datacenters (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/intentions/match (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/status/leader (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/logout (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/pass/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/connect/authorize (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/config/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/snapshot (0.00s) === RUN TestHTTPAPI_AllowedNets_OSS [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/session/renew/ -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/deregister/ -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/create -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/token/ -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/token/ -=== RUN TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/acl/token/ -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/connect/ca/configuration === RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/update -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/binding-rule -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/token -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/snapshot -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/join/ +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/register +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/service/maintenance/ === RUN TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/config/ +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/snapshot +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/update/ +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/leave +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/role +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/service/register +=== RUN TestHTTPAPI_AllowedNets_OSS/POST_/v1/connect/intentions +=== RUN TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/operator/raft/peer === RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/maintenance -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/session/destroy/ +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/force-leave/ +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/bootstrap +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/session/create +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/coordinate/update +=== RUN TestHTTPAPI_AllowedNets_OSS/POST_/v1/operator/keyring +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/operator/keyring +=== RUN TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/operator/keyring === RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/clone/ -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/role -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/kv/ -=== RUN TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/kv/ +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/binding-rule/ +=== RUN TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/acl/binding-rule/ +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/token/ +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/operator/autopilot/configuration +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/session/destroy/ === RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/warn/ +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/config +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/deregister/ +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/pass/ +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/service/deregister/ +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/role/ +=== RUN TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/acl/role/ +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/binding-rule +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/token/ +=== RUN TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/acl/token/ +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/create +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/destroy/ +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/session/renew/ +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/txn +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/policy +=== RUN TestHTTPAPI_AllowedNets_OSS/POST_/v1/agent/connect/authorize +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/fail/ +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/auth-method/ +=== RUN TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/acl/auth-method/ +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/join/ === RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/event/fire/ -=== RUN TestHTTPAPI_AllowedNets_OSS/POST_/v1/connect/intentions +=== RUN TestHTTPAPI_AllowedNets_OSS/POST_/v1/acl/rules/translate +=== RUN TestHTTPAPI_AllowedNets_OSS/POST_/v1/acl/logout === RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/connect/intentions/ === RUN TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/connect/intentions/ -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/fail/ -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/session/create -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/bootstrap === RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/policy/ === RUN TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/acl/policy/ -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/role/ -=== RUN TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/acl/role/ -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/auth-method -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/force-leave/ -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/leave +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/token +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/connect/ca/configuration +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/catalog/register === RUN TestHTTPAPI_AllowedNets_OSS/POST_/v1/discovery-chain/ -=== RUN TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/operator/raft/peer -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/config -=== RUN TestHTTPAPI_AllowedNets_OSS/POST_/v1/acl/login -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/policy -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/binding-rule/ -=== RUN TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/acl/binding-rule/ -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/service/maintenance/ === RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/catalog/deregister -=== RUN TestHTTPAPI_AllowedNets_OSS/POST_/v1/acl/logout -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/pass/ -=== RUN TestHTTPAPI_AllowedNets_OSS/POST_/v1/agent/connect/authorize -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/register -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/catalog/register -=== RUN TestHTTPAPI_AllowedNets_OSS/POST_/v1/acl/rules/translate +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/kv/ +=== RUN TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/kv/ +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/auth-method +=== RUN TestHTTPAPI_AllowedNets_OSS/POST_/v1/acl/login === RUN TestHTTPAPI_AllowedNets_OSS/POST_/v1/internal/acl/authorize -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/service/deregister/ -=== RUN TestHTTPAPI_AllowedNets_OSS/POST_/v1/operator/keyring -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/operator/keyring -=== RUN TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/operator/keyring -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/txn -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/auth-method/ -=== RUN TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/acl/auth-method/ -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/service/register -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/operator/autopilot/configuration -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/destroy/ -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/update/ -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/coordinate/update [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:57.225Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:41:57.225Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:57.274Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:41:57.274Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:57.280 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:83d598c7-8a46-8cec-2b18-13dc57a4a60e Address:127.0.0.1:13965}]" -41:57.280 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13965 [Follower]" leader= -41:57.281 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-83d598c7-8a46-8cec-2b18-13dc57a4a60e.dc1 127.0.0.1 -41:57.284 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-83d598c7-8a46-8cec-2b18-13dc57a4a60e 127.0.0.1 -2022-09-15T21:41:57.284Z [INFO] agent.router: Initializing LAN area manager -41:57.284 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13960 network=udp -41:57.284 [INFO] TestAgent.server: Adding LAN server: server="Node-83d598c7-8a46-8cec-2b18-13dc57a4a60e (Addr: tcp/127.0.0.1:13965) (DC: dc1)" -41:57.284 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-83d598c7-8a46-8cec-2b18-13dc57a4a60e.dc1 area=wan -41:57.284 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13960 network=tcp -41:57.286 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13961 network=tcp -41:57.286 [INFO] TestAgent: started state syncer -41:57.287 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13966 network=tcp -41:57.335 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -41:57.342 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:57.342 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13965 [Candidate]" term=2 -41:57.342 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:57.342 [DEBUG] TestAgent.server.raft: vote granted: from=83d598c7-8a46-8cec-2b18-13dc57a4a60e term=2 tally=1 -41:57.342 [INFO] TestAgent.server.raft: election won: tally=1 -41:57.342 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13965 [Leader]" -41:57.342 [INFO] TestAgent.server: cluster leadership acquired -41:57.343 [INFO] TestAgent.server: initializing acls -41:57.343 [INFO] TestAgent.server: Created ACL 'global-management' policy -41:57.344 [INFO] TestAgent.server: Created ACL anonymous token from configuration -41:57.344 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -41:57.344 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -41:57.344 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-83d598c7-8a46-8cec-2b18-13dc57a4a60e -41:57.344 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-83d598c7-8a46-8cec-2b18-13dc57a4a60e.dc1 -41:57.346 [INFO] TestAgent.server: New leader elected: payload=Node-83d598c7-8a46-8cec-2b18-13dc57a4a60e -41:57.346 [INFO] TestAgent.server: Updating LAN server: server="Node-83d598c7-8a46-8cec-2b18-13dc57a4a60e (Addr: tcp/127.0.0.1:13965) (DC: dc1)" -41:57.346 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-83d598c7-8a46-8cec-2b18-13dc57a4a60e.dc1 area=wan -41:57.358 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:57.361 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:57.361 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:57.361 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:57.361 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:57.361 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:57.361 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-83d598c7-8a46-8cec-2b18-13dc57a4a60e -41:57.361 [INFO] TestAgent.server: member joined, marking health alive: member=Node-83d598c7-8a46-8cec-2b18-13dc57a4a60e -41:57.362 [INFO] TestAgent.server: federation state anti-entropy synced -41:57.364 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-83d598c7-8a46-8cec-2b18-13dc57a4a60e -41:57.385 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:57.385 [INFO] TestAgent: Synced node info -41:57.706 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/session/renew/ from=192.168.1.2:5555 error="Access is restricted" -41:57.706 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/session/renew/ from=192.168.1.2:5555 latency=64.16µs -41:57.707 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/agent/check/deregister/ from=192.168.1.2:5555 error="Access is restricted" -41:57.707 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/agent/check/deregister/ from=192.168.1.2:5555 latency=26.02µs -41:57.707 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/acl/create from=192.168.1.2:5555 error="Access is restricted" -41:57.707 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/acl/create from=192.168.1.2:5555 latency=24.4µs -41:57.707 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/agent/token/ from=192.168.1.2:5555 error="Access is restricted" -41:57.707 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/agent/token/ from=192.168.1.2:5555 latency=23.2µs -41:57.707 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/acl/token/ from=192.168.1.2:5555 error="Access is restricted" -41:57.707 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/acl/token/ from=192.168.1.2:5555 latency=23.06µs -41:57.707 [ERROR] TestAgent.http: Request error: method=DELETE url=http://127.0.0.1:13961/v1/acl/token/ from=192.168.1.2:5555 error="Access is restricted" -41:57.707 [DEBUG] TestAgent.http: Request finished: method=DELETE url=http://127.0.0.1:13961/v1/acl/token/ from=192.168.1.2:5555 latency=23.58µs -41:57.707 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/connect/ca/configuration from=192.168.1.2:5555 error="Access is restricted" -41:57.707 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/connect/ca/configuration from=192.168.1.2:5555 latency=23.54µs -41:57.708 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/acl/update from=192.168.1.2:5555 error="Access is restricted" -41:57.708 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/acl/update from=192.168.1.2:5555 latency=28.2µs -41:57.708 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/acl/binding-rule from=192.168.1.2:5555 error="Access is restricted" -41:57.708 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/acl/binding-rule from=192.168.1.2:5555 latency=21.34µs -41:57.708 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/acl/token from=192.168.1.2:5555 error="Access is restricted" -41:57.708 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/acl/token from=192.168.1.2:5555 latency=23.72µs -41:57.708 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/snapshot from=192.168.1.2:5555 error="Access is restricted" -41:57.708 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/snapshot from=192.168.1.2:5555 latency=22.94µs -41:57.708 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/agent/join/ from=192.168.1.2:5555 error="Access is restricted" -41:57.708 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/agent/join/ from=192.168.1.2:5555 latency=20.76µs -41:57.708 [ERROR] TestAgent.http: Request error: method=DELETE url=http://127.0.0.1:13961/v1/config/ from=192.168.1.2:5555 error="Access is restricted" -41:57.708 [DEBUG] TestAgent.http: Request finished: method=DELETE url=http://127.0.0.1:13961/v1/config/ from=192.168.1.2:5555 latency=21.78µs -41:57.708 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/agent/maintenance from=192.168.1.2:5555 error="Access is restricted" -41:57.708 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/agent/maintenance from=192.168.1.2:5555 latency=25.42µs -41:57.709 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/session/destroy/ from=192.168.1.2:5555 error="Access is restricted" -41:57.709 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/session/destroy/ from=192.168.1.2:5555 latency=22.46µs -41:57.709 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/acl/clone/ from=192.168.1.2:5555 error="Access is restricted" -41:57.709 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/acl/clone/ from=192.168.1.2:5555 latency=22.94µs -41:57.709 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/acl/role from=192.168.1.2:5555 error="Access is restricted" -41:57.709 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/acl/role from=192.168.1.2:5555 latency=22.8µs -41:57.709 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/kv/ from=192.168.1.2:5555 error="Access is restricted" -41:57.709 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/kv/ from=192.168.1.2:5555 latency=22.88µs -41:57.709 [ERROR] TestAgent.http: Request error: method=DELETE url=http://127.0.0.1:13961/v1/kv/ from=192.168.1.2:5555 error="Access is restricted" -41:57.709 [DEBUG] TestAgent.http: Request finished: method=DELETE url=http://127.0.0.1:13961/v1/kv/ from=192.168.1.2:5555 latency=23µs -41:57.709 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/agent/check/warn/ from=192.168.1.2:5555 error="Access is restricted" -41:57.709 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/agent/check/warn/ from=192.168.1.2:5555 latency=21.1µs -41:57.714 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/event/fire/ from=192.168.1.2:5555 error="Access is restricted" -41:57.714 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/event/fire/ from=192.168.1.2:5555 latency=52.02µs -41:57.714 [ERROR] TestAgent.http: Request error: method=POST url=http://127.0.0.1:13961/v1/connect/intentions from=192.168.1.2:5555 error="Access is restricted" -41:57.714 [DEBUG] TestAgent.http: Request finished: method=POST url=http://127.0.0.1:13961/v1/connect/intentions from=192.168.1.2:5555 latency=26.5µs -41:57.714 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/connect/intentions/ from=192.168.1.2:5555 error="Access is restricted" -41:57.714 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/connect/intentions/ from=192.168.1.2:5555 latency=20.04µs -41:57.714 [ERROR] TestAgent.http: Request error: method=DELETE url=http://127.0.0.1:13961/v1/connect/intentions/ from=192.168.1.2:5555 error="Access is restricted" -41:57.714 [DEBUG] TestAgent.http: Request finished: method=DELETE url=http://127.0.0.1:13961/v1/connect/intentions/ from=192.168.1.2:5555 latency=15.26µs -41:57.714 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/agent/check/fail/ from=192.168.1.2:5555 error="Access is restricted" -41:57.714 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/agent/check/fail/ from=192.168.1.2:5555 latency=17.88µs -41:57.714 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/session/create from=192.168.1.2:5555 error="Access is restricted" -41:57.715 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/session/create from=192.168.1.2:5555 latency=15.5µs -41:57.715 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/acl/bootstrap from=192.168.1.2:5555 error="Access is restricted" -41:57.715 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/acl/bootstrap from=192.168.1.2:5555 latency=16.52µs -41:57.715 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/acl/policy/ from=192.168.1.2:5555 error="Access is restricted" -41:57.715 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/acl/policy/ from=192.168.1.2:5555 latency=19.44µs -41:57.715 [ERROR] TestAgent.http: Request error: method=DELETE url=http://127.0.0.1:13961/v1/acl/policy/ from=192.168.1.2:5555 error="Access is restricted" -41:57.715 [DEBUG] TestAgent.http: Request finished: method=DELETE url=http://127.0.0.1:13961/v1/acl/policy/ from=192.168.1.2:5555 latency=17.66µs -41:57.715 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/acl/role/ from=192.168.1.2:5555 error="Access is restricted" -41:57.715 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/acl/role/ from=192.168.1.2:5555 latency=16.62µs -41:57.715 [ERROR] TestAgent.http: Request error: method=DELETE url=http://127.0.0.1:13961/v1/acl/role/ from=192.168.1.2:5555 error="Access is restricted" -41:57.715 [DEBUG] TestAgent.http: Request finished: method=DELETE url=http://127.0.0.1:13961/v1/acl/role/ from=192.168.1.2:5555 latency=14.96µs -41:57.715 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/acl/auth-method from=192.168.1.2:5555 error="Access is restricted" -41:57.715 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/acl/auth-method from=192.168.1.2:5555 latency=17.28µs -41:57.715 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/agent/force-leave/ from=192.168.1.2:5555 error="Access is restricted" -41:57.715 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/agent/force-leave/ from=192.168.1.2:5555 latency=20.28µs -41:57.715 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/agent/leave from=192.168.1.2:5555 error="Access is restricted" -41:57.715 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/agent/leave from=192.168.1.2:5555 latency=16.34µs -41:57.716 [ERROR] TestAgent.http: Request error: method=POST url=http://127.0.0.1:13961/v1/discovery-chain/ from=192.168.1.2:5555 error="Access is restricted" -41:57.716 [DEBUG] TestAgent.http: Request finished: method=POST url=http://127.0.0.1:13961/v1/discovery-chain/ from=192.168.1.2:5555 latency=78.64µs -41:57.716 [ERROR] TestAgent.http: Request error: method=DELETE url=http://127.0.0.1:13961/v1/operator/raft/peer from=192.168.1.2:5555 error="Access is restricted" -41:57.716 [DEBUG] TestAgent.http: Request finished: method=DELETE url=http://127.0.0.1:13961/v1/operator/raft/peer from=192.168.1.2:5555 latency=24.22µs -41:57.716 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/config from=192.168.1.2:5555 error="Access is restricted" -41:57.716 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/config from=192.168.1.2:5555 latency=18.06µs -41:57.716 [ERROR] TestAgent.http: Request error: method=POST url=http://127.0.0.1:13961/v1/acl/login from=192.168.1.2:5555 error="Access is restricted" -41:57.716 [DEBUG] TestAgent.http: Request finished: method=POST url=http://127.0.0.1:13961/v1/acl/login from=192.168.1.2:5555 latency=20.32µs -41:57.716 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/acl/policy from=192.168.1.2:5555 error="Access is restricted" -41:57.716 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/acl/policy from=192.168.1.2:5555 latency=19.36µs -41:57.716 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/acl/binding-rule/ from=192.168.1.2:5555 error="Access is restricted" -41:57.716 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/acl/binding-rule/ from=192.168.1.2:5555 latency=21.72µs -41:57.716 [ERROR] TestAgent.http: Request error: method=DELETE url=http://127.0.0.1:13961/v1/acl/binding-rule/ from=192.168.1.2:5555 error="Access is restricted" -41:57.716 [DEBUG] TestAgent.http: Request finished: method=DELETE url=http://127.0.0.1:13961/v1/acl/binding-rule/ from=192.168.1.2:5555 latency=22.74µs -41:57.717 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/agent/service/maintenance/ from=192.168.1.2:5555 error="Access is restricted" -41:57.717 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/agent/service/maintenance/ from=192.168.1.2:5555 latency=23.42µs -41:57.717 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/catalog/deregister from=192.168.1.2:5555 error="Access is restricted" -41:57.717 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/catalog/deregister from=192.168.1.2:5555 latency=24.32µs -41:57.717 [ERROR] TestAgent.http: Request error: method=POST url=http://127.0.0.1:13961/v1/acl/logout from=192.168.1.2:5555 error="Access is restricted" -41:57.717 [DEBUG] TestAgent.http: Request finished: method=POST url=http://127.0.0.1:13961/v1/acl/logout from=192.168.1.2:5555 latency=22.82µs -41:57.717 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/agent/check/pass/ from=192.168.1.2:5555 error="Access is restricted" -41:57.717 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/agent/check/pass/ from=192.168.1.2:5555 latency=23.74µs -41:57.717 [ERROR] TestAgent.http: Request error: method=POST url=http://127.0.0.1:13961/v1/agent/connect/authorize from=192.168.1.2:5555 error="Access is restricted" -41:57.717 [DEBUG] TestAgent.http: Request finished: method=POST url=http://127.0.0.1:13961/v1/agent/connect/authorize from=192.168.1.2:5555 latency=26µs -41:57.717 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/agent/check/register from=192.168.1.2:5555 error="Access is restricted" -41:57.717 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/agent/check/register from=192.168.1.2:5555 latency=20.8µs -41:57.717 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/catalog/register from=192.168.1.2:5555 error="Access is restricted" -41:57.718 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/catalog/register from=192.168.1.2:5555 latency=25.22µs -41:57.722 [ERROR] TestAgent.http: Request error: method=POST url=http://127.0.0.1:13961/v1/acl/rules/translate from=192.168.1.2:5555 error="Access is restricted" -41:57.722 [DEBUG] TestAgent.http: Request finished: method=POST url=http://127.0.0.1:13961/v1/acl/rules/translate from=192.168.1.2:5555 latency=38.56µs -41:57.722 [ERROR] TestAgent.http: Request error: method=POST url=http://127.0.0.1:13961/v1/internal/acl/authorize from=192.168.1.2:5555 error="Access is restricted" -41:57.722 [DEBUG] TestAgent.http: Request finished: method=POST url=http://127.0.0.1:13961/v1/internal/acl/authorize from=192.168.1.2:5555 latency=17.54µs -41:57.722 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/agent/service/deregister/ from=192.168.1.2:5555 error="Access is restricted" -41:57.722 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/agent/service/deregister/ from=192.168.1.2:5555 latency=17.34µs -41:57.722 [ERROR] TestAgent.http: Request error: method=POST url=http://127.0.0.1:13961/v1/operator/keyring from=192.168.1.2:5555 error="Access is restricted" -41:57.722 [DEBUG] TestAgent.http: Request finished: method=POST url=http://127.0.0.1:13961/v1/operator/keyring from=192.168.1.2:5555 latency=17.62µs -41:57.722 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/operator/keyring from=192.168.1.2:5555 error="Access is restricted" -41:57.722 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/operator/keyring from=192.168.1.2:5555 latency=19.96µs -41:57.723 [ERROR] TestAgent.http: Request error: method=DELETE url=http://127.0.0.1:13961/v1/operator/keyring from=192.168.1.2:5555 error="Access is restricted" -41:57.723 [DEBUG] TestAgent.http: Request finished: method=DELETE url=http://127.0.0.1:13961/v1/operator/keyring from=192.168.1.2:5555 latency=17.14µs -41:57.723 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/txn from=192.168.1.2:5555 error="Access is restricted" -41:57.723 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/txn from=192.168.1.2:5555 latency=20.18µs -41:57.723 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/acl/auth-method/ from=192.168.1.2:5555 error="Access is restricted" -41:57.723 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/acl/auth-method/ from=192.168.1.2:5555 latency=14.88µs -41:57.723 [ERROR] TestAgent.http: Request error: method=DELETE url=http://127.0.0.1:13961/v1/acl/auth-method/ from=192.168.1.2:5555 error="Access is restricted" -41:57.723 [DEBUG] TestAgent.http: Request finished: method=DELETE url=http://127.0.0.1:13961/v1/acl/auth-method/ from=192.168.1.2:5555 latency=17.9µs -41:57.723 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/agent/service/register from=192.168.1.2:5555 error="Access is restricted" -41:57.723 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/agent/service/register from=192.168.1.2:5555 latency=18.96µs -41:57.723 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/operator/autopilot/configuration from=192.168.1.2:5555 error="Access is restricted" -41:57.723 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/operator/autopilot/configuration from=192.168.1.2:5555 latency=17.88µs -41:57.723 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/acl/destroy/ from=192.168.1.2:5555 error="Access is restricted" -41:57.723 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/acl/destroy/ from=192.168.1.2:5555 latency=16.22µs -41:57.723 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/agent/check/update/ from=192.168.1.2:5555 error="Access is restricted" -41:57.724 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/agent/check/update/ from=192.168.1.2:5555 latency=22.06µs -41:57.724 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:13961/v1/coordinate/update from=192.168.1.2:5555 error="Access is restricted" -41:57.724 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:13961/v1/coordinate/update from=192.168.1.2:5555 latency=22.52µs -41:57.724 [INFO] TestAgent: Requesting shutdown -41:57.724 [INFO] TestAgent.server: shutting down server -41:57.724 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:57.724 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -41:57.724 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -41:57.724 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:57.724 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:57.724 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:57.724 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:57.724 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:57.724 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:57.724 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:57.724 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -41:57.724 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -41:57.724 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:57.724 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:57.724 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:57.725Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:57.725Z [INFO] agent.router.manager: shutting down -41:57.725 [INFO] TestAgent: consul server down -41:57.725 [INFO] TestAgent: shutdown complete -41:57.725 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13960 network=tcp -41:57.725 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13960 network=udp -41:57.725 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13961 network=tcp -41:58.225 [INFO] TestAgent: Waiting for endpoints to shut down -41:58.225 [INFO] TestAgent: Endpoints down ---- PASS: TestHTTPAPI_AllowedNets_OSS (1.03s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/session/renew/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/deregister/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/create (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/token/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/token/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/acl/token/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/connect/ca/configuration (0.00s) +2021-08-13T15:33:38.654Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:33:38.654Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:38.682Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:33:38.682Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:38.685 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e65a2fff-2fdf-beec-6af7-5eb249fe9f21 Address:127.0.0.1:25196}]" +33:38.685 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25196 [Follower]" leader= +33:38.686 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e65a2fff-2fdf-beec-6af7-5eb249fe9f21.dc1 127.0.0.1 +33:38.687 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e65a2fff-2fdf-beec-6af7-5eb249fe9f21 127.0.0.1 +2021-08-13T15:33:38.687Z [INFO] agent.router: Initializing LAN area manager +33:38.687 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e65a2fff-2fdf-beec-6af7-5eb249fe9f21.dc1 area=wan +33:38.687 [INFO] TestAgent.server: Adding LAN server: server="Node-e65a2fff-2fdf-beec-6af7-5eb249fe9f21 (Addr: tcp/127.0.0.1:25196) (DC: dc1)" +33:38.687 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25191 network=tcp +33:38.687 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25191 network=udp +33:38.688 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25192 network=tcp +33:38.688 [INFO] TestAgent: started state syncer +33:38.688 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25197 network=tcp +33:38.737 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +33:38.754 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:38.754 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25196 [Candidate]" term=2 +33:38.755 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:38.755 [DEBUG] TestAgent.server.raft: vote granted: from=e65a2fff-2fdf-beec-6af7-5eb249fe9f21 term=2 tally=1 +33:38.755 [INFO] TestAgent.server.raft: election won: tally=1 +33:38.755 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25196 [Leader]" +33:38.755 [INFO] TestAgent.server: cluster leadership acquired +33:38.755 [INFO] TestAgent.server: New leader elected: payload=Node-e65a2fff-2fdf-beec-6af7-5eb249fe9f21 +33:38.755 [INFO] TestAgent.server: initializing acls +33:38.755 [INFO] TestAgent.server: Created ACL 'global-management' policy +33:38.756 [INFO] TestAgent.server: Created ACL anonymous token from configuration +33:38.756 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +33:38.756 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +33:38.756 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-e65a2fff-2fdf-beec-6af7-5eb249fe9f21 +33:38.756 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-e65a2fff-2fdf-beec-6af7-5eb249fe9f21.dc1 +33:38.756 [INFO] TestAgent.server: Updating LAN server: server="Node-e65a2fff-2fdf-beec-6af7-5eb249fe9f21 (Addr: tcp/127.0.0.1:25196) (DC: dc1)" +33:38.756 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-e65a2fff-2fdf-beec-6af7-5eb249fe9f21.dc1 area=wan +33:38.757 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:38.760 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:38.760 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:38.760 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:38.760 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:38.760 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:38.760 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e65a2fff-2fdf-beec-6af7-5eb249fe9f21 +33:38.760 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e65a2fff-2fdf-beec-6af7-5eb249fe9f21 +33:38.760 [INFO] TestAgent.server: federation state anti-entropy synced +33:38.761 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e65a2fff-2fdf-beec-6af7-5eb249fe9f21 +33:38.843 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/acl/update from=192.168.1.2:5555 error="Access is restricted" +33:38.844 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/acl/update from=192.168.1.2:5555 latency=59.4µs +33:38.844 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/agent/check/register from=192.168.1.2:5555 error="Access is restricted" +33:38.844 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/agent/check/register from=192.168.1.2:5555 latency=16.56µs +33:38.844 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/agent/service/maintenance/ from=192.168.1.2:5555 error="Access is restricted" +33:38.844 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/agent/service/maintenance/ from=192.168.1.2:5555 latency=15.36µs +33:38.844 [ERROR] TestAgent.http: Request error: method=DELETE url=http://127.0.0.1:25192/v1/config/ from=192.168.1.2:5555 error="Access is restricted" +33:38.844 [DEBUG] TestAgent.http: Request finished: method=DELETE url=http://127.0.0.1:25192/v1/config/ from=192.168.1.2:5555 latency=18.72µs +33:38.844 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/snapshot from=192.168.1.2:5555 error="Access is restricted" +33:38.844 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/snapshot from=192.168.1.2:5555 latency=16.18µs +33:38.844 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/agent/check/update/ from=192.168.1.2:5555 error="Access is restricted" +33:38.844 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/agent/check/update/ from=192.168.1.2:5555 latency=14.92µs +33:38.844 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/agent/leave from=192.168.1.2:5555 error="Access is restricted" +33:38.844 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/agent/leave from=192.168.1.2:5555 latency=15.18µs +33:38.844 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/acl/role from=192.168.1.2:5555 error="Access is restricted" +33:38.844 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/acl/role from=192.168.1.2:5555 latency=14.6µs +33:38.844 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/agent/service/register from=192.168.1.2:5555 error="Access is restricted" +33:38.845 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/agent/service/register from=192.168.1.2:5555 latency=15.02µs +33:38.845 [ERROR] TestAgent.http: Request error: method=POST url=http://127.0.0.1:25192/v1/connect/intentions from=192.168.1.2:5555 error="Access is restricted" +33:38.845 [DEBUG] TestAgent.http: Request finished: method=POST url=http://127.0.0.1:25192/v1/connect/intentions from=192.168.1.2:5555 latency=40.86µs +33:38.845 [ERROR] TestAgent.http: Request error: method=DELETE url=http://127.0.0.1:25192/v1/operator/raft/peer from=192.168.1.2:5555 error="Access is restricted" +33:38.845 [DEBUG] TestAgent.http: Request finished: method=DELETE url=http://127.0.0.1:25192/v1/operator/raft/peer from=192.168.1.2:5555 latency=20.58µs +33:38.845 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/agent/maintenance from=192.168.1.2:5555 error="Access is restricted" +33:38.845 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/agent/maintenance from=192.168.1.2:5555 latency=16.66µs +33:38.845 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/agent/force-leave/ from=192.168.1.2:5555 error="Access is restricted" +33:38.845 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/agent/force-leave/ from=192.168.1.2:5555 latency=15.72µs +33:38.845 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/acl/bootstrap from=192.168.1.2:5555 error="Access is restricted" +33:38.845 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/acl/bootstrap from=192.168.1.2:5555 latency=15.76µs +33:38.845 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/session/create from=192.168.1.2:5555 error="Access is restricted" +33:38.845 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/session/create from=192.168.1.2:5555 latency=15.24µs +33:38.845 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/coordinate/update from=192.168.1.2:5555 error="Access is restricted" +33:38.845 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/coordinate/update from=192.168.1.2:5555 latency=15.04µs +33:38.846 [ERROR] TestAgent.http: Request error: method=POST url=http://127.0.0.1:25192/v1/operator/keyring from=192.168.1.2:5555 error="Access is restricted" +33:38.846 [DEBUG] TestAgent.http: Request finished: method=POST url=http://127.0.0.1:25192/v1/operator/keyring from=192.168.1.2:5555 latency=14.521µs +33:38.846 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/operator/keyring from=192.168.1.2:5555 error="Access is restricted" +33:38.846 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/operator/keyring from=192.168.1.2:5555 latency=15.2µs +33:38.846 [ERROR] TestAgent.http: Request error: method=DELETE url=http://127.0.0.1:25192/v1/operator/keyring from=192.168.1.2:5555 error="Access is restricted" +33:38.846 [DEBUG] TestAgent.http: Request finished: method=DELETE url=http://127.0.0.1:25192/v1/operator/keyring from=192.168.1.2:5555 latency=14.42µs +33:38.846 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/acl/clone/ from=192.168.1.2:5555 error="Access is restricted" +33:38.846 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/acl/clone/ from=192.168.1.2:5555 latency=14.72µs +33:38.846 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/acl/binding-rule/ from=192.168.1.2:5555 error="Access is restricted" +33:38.846 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/acl/binding-rule/ from=192.168.1.2:5555 latency=16.98µs +33:38.846 [ERROR] TestAgent.http: Request error: method=DELETE url=http://127.0.0.1:25192/v1/acl/binding-rule/ from=192.168.1.2:5555 error="Access is restricted" +33:38.846 [DEBUG] TestAgent.http: Request finished: method=DELETE url=http://127.0.0.1:25192/v1/acl/binding-rule/ from=192.168.1.2:5555 latency=14.12µs +33:38.846 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/agent/token/ from=192.168.1.2:5555 error="Access is restricted" +33:38.846 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/agent/token/ from=192.168.1.2:5555 latency=14.96µs +33:38.846 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/operator/autopilot/configuration from=192.168.1.2:5555 error="Access is restricted" +33:38.846 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/operator/autopilot/configuration from=192.168.1.2:5555 latency=14.62µs +33:38.846 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/session/destroy/ from=192.168.1.2:5555 error="Access is restricted" +33:38.846 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/session/destroy/ from=192.168.1.2:5555 latency=14.32µs +33:38.847 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/agent/check/warn/ from=192.168.1.2:5555 error="Access is restricted" +33:38.847 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/agent/check/warn/ from=192.168.1.2:5555 latency=15.18µs +33:38.847 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/config from=192.168.1.2:5555 error="Access is restricted" +33:38.847 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/config from=192.168.1.2:5555 latency=14.7µs +33:38.847 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/agent/check/deregister/ from=192.168.1.2:5555 error="Access is restricted" +33:38.847 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/agent/check/deregister/ from=192.168.1.2:5555 latency=21.62µs +33:38.847 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/agent/check/pass/ from=192.168.1.2:5555 error="Access is restricted" +33:38.847 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/agent/check/pass/ from=192.168.1.2:5555 latency=17.62µs +33:38.847 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/agent/service/deregister/ from=192.168.1.2:5555 error="Access is restricted" +33:38.847 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/agent/service/deregister/ from=192.168.1.2:5555 latency=15.28µs +33:38.847 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/acl/role/ from=192.168.1.2:5555 error="Access is restricted" +33:38.847 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/acl/role/ from=192.168.1.2:5555 latency=18.04µs +33:38.847 [ERROR] TestAgent.http: Request error: method=DELETE url=http://127.0.0.1:25192/v1/acl/role/ from=192.168.1.2:5555 error="Access is restricted" +33:38.847 [DEBUG] TestAgent.http: Request finished: method=DELETE url=http://127.0.0.1:25192/v1/acl/role/ from=192.168.1.2:5555 latency=15.72µs +33:38.848 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/acl/binding-rule from=192.168.1.2:5555 error="Access is restricted" +33:38.848 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/acl/binding-rule from=192.168.1.2:5555 latency=17.48µs +33:38.848 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/acl/token/ from=192.168.1.2:5555 error="Access is restricted" +33:38.848 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/acl/token/ from=192.168.1.2:5555 latency=15.481µs +33:38.848 [ERROR] TestAgent.http: Request error: method=DELETE url=http://127.0.0.1:25192/v1/acl/token/ from=192.168.1.2:5555 error="Access is restricted" +33:38.848 [DEBUG] TestAgent.http: Request finished: method=DELETE url=http://127.0.0.1:25192/v1/acl/token/ from=192.168.1.2:5555 latency=18µs +33:38.848 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/acl/create from=192.168.1.2:5555 error="Access is restricted" +33:38.848 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/acl/create from=192.168.1.2:5555 latency=15.98µs +33:38.848 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/acl/destroy/ from=192.168.1.2:5555 error="Access is restricted" +33:38.848 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/acl/destroy/ from=192.168.1.2:5555 latency=15.701µs +33:38.848 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/session/renew/ from=192.168.1.2:5555 error="Access is restricted" +33:38.848 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/session/renew/ from=192.168.1.2:5555 latency=15.34µs +33:38.848 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/txn from=192.168.1.2:5555 error="Access is restricted" +33:38.848 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/txn from=192.168.1.2:5555 latency=16.88µs +33:38.849 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/acl/policy from=192.168.1.2:5555 error="Access is restricted" +33:38.849 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/acl/policy from=192.168.1.2:5555 latency=16.04µs +33:38.849 [ERROR] TestAgent.http: Request error: method=POST url=http://127.0.0.1:25192/v1/agent/connect/authorize from=192.168.1.2:5555 error="Access is restricted" +33:38.849 [DEBUG] TestAgent.http: Request finished: method=POST url=http://127.0.0.1:25192/v1/agent/connect/authorize from=192.168.1.2:5555 latency=17.36µs +33:38.849 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/agent/check/fail/ from=192.168.1.2:5555 error="Access is restricted" +33:38.849 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/agent/check/fail/ from=192.168.1.2:5555 latency=17.32µs +33:38.849 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/acl/auth-method/ from=192.168.1.2:5555 error="Access is restricted" +33:38.849 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/acl/auth-method/ from=192.168.1.2:5555 latency=15.78µs +33:38.849 [ERROR] TestAgent.http: Request error: method=DELETE url=http://127.0.0.1:25192/v1/acl/auth-method/ from=192.168.1.2:5555 error="Access is restricted" +33:38.849 [DEBUG] TestAgent.http: Request finished: method=DELETE url=http://127.0.0.1:25192/v1/acl/auth-method/ from=192.168.1.2:5555 latency=14.94µs +33:38.849 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/agent/join/ from=192.168.1.2:5555 error="Access is restricted" +33:38.849 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/agent/join/ from=192.168.1.2:5555 latency=18µs +33:38.849 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/event/fire/ from=192.168.1.2:5555 error="Access is restricted" +33:38.849 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/event/fire/ from=192.168.1.2:5555 latency=15.52µs +33:38.849 [ERROR] TestAgent.http: Request error: method=POST url=http://127.0.0.1:25192/v1/acl/rules/translate from=192.168.1.2:5555 error="Access is restricted" +33:38.849 [DEBUG] TestAgent.http: Request finished: method=POST url=http://127.0.0.1:25192/v1/acl/rules/translate from=192.168.1.2:5555 latency=15.52µs +33:38.850 [ERROR] TestAgent.http: Request error: method=POST url=http://127.0.0.1:25192/v1/acl/logout from=192.168.1.2:5555 error="Access is restricted" +33:38.850 [DEBUG] TestAgent.http: Request finished: method=POST url=http://127.0.0.1:25192/v1/acl/logout from=192.168.1.2:5555 latency=14.9µs +33:38.850 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/connect/intentions/ from=192.168.1.2:5555 error="Access is restricted" +33:38.850 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/connect/intentions/ from=192.168.1.2:5555 latency=17.62µs +33:38.850 [ERROR] TestAgent.http: Request error: method=DELETE url=http://127.0.0.1:25192/v1/connect/intentions/ from=192.168.1.2:5555 error="Access is restricted" +33:38.850 [DEBUG] TestAgent.http: Request finished: method=DELETE url=http://127.0.0.1:25192/v1/connect/intentions/ from=192.168.1.2:5555 latency=14.58µs +33:38.850 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/acl/policy/ from=192.168.1.2:5555 error="Access is restricted" +33:38.850 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/acl/policy/ from=192.168.1.2:5555 latency=14.26µs +33:38.850 [ERROR] TestAgent.http: Request error: method=DELETE url=http://127.0.0.1:25192/v1/acl/policy/ from=192.168.1.2:5555 error="Access is restricted" +33:38.850 [DEBUG] TestAgent.http: Request finished: method=DELETE url=http://127.0.0.1:25192/v1/acl/policy/ from=192.168.1.2:5555 latency=14.72µs +33:38.850 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/acl/token from=192.168.1.2:5555 error="Access is restricted" +33:38.850 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/acl/token from=192.168.1.2:5555 latency=15.7µs +33:38.850 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/connect/ca/configuration from=192.168.1.2:5555 error="Access is restricted" +33:38.850 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/connect/ca/configuration from=192.168.1.2:5555 latency=15.22µs +33:38.850 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/catalog/register from=192.168.1.2:5555 error="Access is restricted" +33:38.850 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/catalog/register from=192.168.1.2:5555 latency=15.84µs +33:38.851 [ERROR] TestAgent.http: Request error: method=POST url=http://127.0.0.1:25192/v1/discovery-chain/ from=192.168.1.2:5555 error="Access is restricted" +33:38.851 [DEBUG] TestAgent.http: Request finished: method=POST url=http://127.0.0.1:25192/v1/discovery-chain/ from=192.168.1.2:5555 latency=16.08µs +33:38.851 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/catalog/deregister from=192.168.1.2:5555 error="Access is restricted" +33:38.851 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/catalog/deregister from=192.168.1.2:5555 latency=16.4µs +33:38.851 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/kv/ from=192.168.1.2:5555 error="Access is restricted" +33:38.851 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/kv/ from=192.168.1.2:5555 latency=93.74µs +33:38.851 [ERROR] TestAgent.http: Request error: method=DELETE url=http://127.0.0.1:25192/v1/kv/ from=192.168.1.2:5555 error="Access is restricted" +33:38.851 [DEBUG] TestAgent.http: Request finished: method=DELETE url=http://127.0.0.1:25192/v1/kv/ from=192.168.1.2:5555 latency=19.1µs +33:38.851 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:25192/v1/acl/auth-method from=192.168.1.2:5555 error="Access is restricted" +33:38.851 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:25192/v1/acl/auth-method from=192.168.1.2:5555 latency=16.56µs +33:38.851 [ERROR] TestAgent.http: Request error: method=POST url=http://127.0.0.1:25192/v1/acl/login from=192.168.1.2:5555 error="Access is restricted" +33:38.851 [DEBUG] TestAgent.http: Request finished: method=POST url=http://127.0.0.1:25192/v1/acl/login from=192.168.1.2:5555 latency=16.3µs +33:38.851 [ERROR] TestAgent.http: Request error: method=POST url=http://127.0.0.1:25192/v1/internal/acl/authorize from=192.168.1.2:5555 error="Access is restricted" +33:38.851 [DEBUG] TestAgent.http: Request finished: method=POST url=http://127.0.0.1:25192/v1/internal/acl/authorize from=192.168.1.2:5555 latency=15.7µs +33:38.851 [INFO] TestAgent: Requesting shutdown +33:38.852 [INFO] TestAgent.server: shutting down server +33:38.852 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +33:38.852 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +33:38.852 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:38.852 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:38.852 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:38.852 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:38.852 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:38.852 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:38.852 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:38.852 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:38.852 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +33:38.852 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +33:38.852 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:38.852 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:38.852 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +33:38.852 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2021-08-13T15:33:38.852Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:38.852Z [INFO] agent.router.manager: shutting down +33:38.852 [INFO] TestAgent: consul server down +33:38.852 [INFO] TestAgent: shutdown complete +33:38.852 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25191 network=tcp +33:38.852 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25191 network=udp +33:38.852 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25192 network=tcp +33:39.353 [INFO] TestAgent: Waiting for endpoints to shut down +33:39.353 [INFO] TestAgent: Endpoints down +--- PASS: TestHTTPAPI_AllowedNets_OSS (0.72s) --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/update (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/binding-rule (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/token (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/register (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/service/maintenance/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/config/ (0.00s) --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/snapshot (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/update/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/leave (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/role (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/service/register (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/POST_/v1/connect/intentions (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/operator/raft/peer (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/maintenance (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/force-leave/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/bootstrap (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/session/create (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/coordinate/update (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/POST_/v1/operator/keyring (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/operator/keyring (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/operator/keyring (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/clone/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/binding-rule/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/acl/binding-rule/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/token/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/operator/autopilot/configuration (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/session/destroy/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/warn/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/config (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/deregister/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/pass/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/service/deregister/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/role/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/acl/role/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/binding-rule (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/token/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/acl/token/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/create (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/destroy/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/session/renew/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/txn (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/policy (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/POST_/v1/agent/connect/authorize (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/fail/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/auth-method/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/acl/auth-method/ (0.00s) --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/join/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/config/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/maintenance (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/session/destroy/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/clone/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/role (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/kv/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/kv/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/warn/ (0.00s) --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/event/fire/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/POST_/v1/connect/intentions (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/POST_/v1/acl/rules/translate (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/POST_/v1/acl/logout (0.00s) --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/connect/intentions/ (0.00s) --- PASS: TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/connect/intentions/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/fail/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/session/create (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/bootstrap (0.00s) --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/policy/ (0.00s) --- PASS: TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/acl/policy/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/role/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/acl/role/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/auth-method (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/force-leave/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/leave (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/token (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/connect/ca/configuration (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/catalog/register (0.00s) --- PASS: TestHTTPAPI_AllowedNets_OSS/POST_/v1/discovery-chain/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/operator/raft/peer (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/config (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/POST_/v1/acl/login (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/policy (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/binding-rule/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/acl/binding-rule/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/service/maintenance/ (0.00s) --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/catalog/deregister (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/POST_/v1/acl/logout (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/pass/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/POST_/v1/agent/connect/authorize (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/register (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/catalog/register (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/POST_/v1/acl/rules/translate (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/kv/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/kv/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/auth-method (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/POST_/v1/acl/login (0.00s) --- PASS: TestHTTPAPI_AllowedNets_OSS/POST_/v1/internal/acl/authorize (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/service/deregister/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/POST_/v1/operator/keyring (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/operator/keyring (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/operator/keyring (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/txn (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/auth-method/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/acl/auth-method/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/service/register (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/operator/autopilot/configuration (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/destroy/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/update/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/coordinate/update (0.00s) === RUN TestHTTPServer_UnixSocket === PAUSE TestHTTPServer_UnixSocket === RUN TestHTTPServer_UnixSocket_FileExists @@ -18446,124 +18450,123 @@ [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:58.251Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:58.268Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:58.271 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a68b52a5-1c8f-1a6c-0164-f6ad4438ebf9 Address:127.0.0.1:13972}]" -41:58.271 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13972 [Follower]" leader= -41:58.272 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a68b52a5-1c8f-1a6c-0164-f6ad4438ebf9.dc1 127.0.0.1 -41:58.273 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a68b52a5-1c8f-1a6c-0164-f6ad4438ebf9 127.0.0.1 -2022-09-15T21:41:58.273Z [INFO] agent.router: Initializing LAN area manager -41:58.273 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13967 network=udp -41:58.273 [INFO] TestAgent.server: Adding LAN server: server="Node-a68b52a5-1c8f-1a6c-0164-f6ad4438ebf9 (Addr: tcp/127.0.0.1:13972) (DC: dc1)" -41:58.273 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a68b52a5-1c8f-1a6c-0164-f6ad4438ebf9.dc1 area=wan -41:58.274 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13967 network=tcp -41:58.275 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13968 network=tcp -41:58.275 [INFO] TestAgent: started state syncer -41:58.275 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13973 network=tcp -41:58.319 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:58.319 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13972 [Candidate]" term=2 -41:58.320 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:58.320 [DEBUG] TestAgent.server.raft: vote granted: from=a68b52a5-1c8f-1a6c-0164-f6ad4438ebf9 term=2 tally=1 -41:58.320 [INFO] TestAgent.server.raft: election won: tally=1 -41:58.320 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13972 [Leader]" -41:58.320 [INFO] TestAgent.server: cluster leadership acquired -41:58.320 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13972 -41:58.322 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:58.323 [INFO] TestAgent.server: New leader elected: payload=Node-a68b52a5-1c8f-1a6c-0164-f6ad4438ebf9 -41:58.325 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:58.325 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:58.325 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:58.325 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:58.325 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:58.325 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a68b52a5-1c8f-1a6c-0164-f6ad4438ebf9 -41:58.325 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a68b52a5-1c8f-1a6c-0164-f6ad4438ebf9 -41:58.327 [INFO] TestAgent: Synced node info -41:58.327 [INFO] TestAgent.server: federation state anti-entropy synced -41:58.414 [INFO] TestAgent: Requesting shutdown -41:58.414 [INFO] TestAgent.server: shutting down server -41:58.414 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:58.414 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:58.414 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:58.414 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:58.414 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:58.414 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:58.414 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:58.414 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:58.414 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:58.414 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:58.414 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:58.414Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:58.414Z [INFO] agent.router.manager: shutting down -41:58.415 [INFO] TestAgent: consul server down -41:58.415 [INFO] TestAgent: shutdown complete -41:58.415 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13967 network=tcp -41:58.415 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13967 network=udp -41:58.415 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13968 network=tcp -41:58.917 [INFO] TestAgent: Waiting for endpoints to shut down -41:58.917 [INFO] TestAgent: Endpoints down ---- PASS: TestHTTPAPI_Ban_Nonprintable_Characters (0.69s) +2021-08-13T15:33:39.372Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:39.389Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:39.392 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:46f70603-007b-6073-5e7f-64a310a9dd5e Address:127.0.0.1:25203}]" +33:39.393 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-46f70603-007b-6073-5e7f-64a310a9dd5e.dc1 127.0.0.1 +33:39.394 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-46f70603-007b-6073-5e7f-64a310a9dd5e 127.0.0.1 +2021-08-13T15:33:39.394Z [INFO] agent.router: Initializing LAN area manager +33:39.394 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25198 network=udp +33:39.394 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25203 [Follower]" leader= +33:39.395 [INFO] TestAgent.server: Adding LAN server: server="Node-46f70603-007b-6073-5e7f-64a310a9dd5e (Addr: tcp/127.0.0.1:25203) (DC: dc1)" +33:39.395 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-46f70603-007b-6073-5e7f-64a310a9dd5e.dc1 area=wan +33:39.395 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25198 network=tcp +33:39.396 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25199 network=tcp +33:39.396 [INFO] TestAgent: started state syncer +33:39.396 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25204 network=tcp +33:39.450 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:39.450 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25203 [Candidate]" term=2 +33:39.450 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:39.450 [DEBUG] TestAgent.server.raft: vote granted: from=46f70603-007b-6073-5e7f-64a310a9dd5e term=2 tally=1 +33:39.450 [INFO] TestAgent.server.raft: election won: tally=1 +33:39.450 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25203 [Leader]" +33:39.450 [INFO] TestAgent.server: cluster leadership acquired +33:39.451 [INFO] TestAgent.server: New leader elected: payload=Node-46f70603-007b-6073-5e7f-64a310a9dd5e +33:39.451 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25203 +33:39.452 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:39.454 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:39.454 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:39.454 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:39.454 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:39.454 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:39.454 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-46f70603-007b-6073-5e7f-64a310a9dd5e +33:39.454 [INFO] TestAgent.server: member joined, marking health alive: member=Node-46f70603-007b-6073-5e7f-64a310a9dd5e +33:39.455 [INFO] TestAgent.server: federation state anti-entropy synced +33:39.662 [INFO] TestAgent: Requesting shutdown +33:39.662 [INFO] TestAgent.server: shutting down server +33:39.662 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:39.662 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:39.662 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:39.662 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:39.662 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:39.662 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:39.662 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:39.662 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:39.662 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:39.662 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:39.662 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:39.662 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:39.662Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:39.663Z [INFO] agent.router.manager: shutting down +33:39.663 [INFO] TestAgent: consul server down +33:39.663 [INFO] TestAgent: shutdown complete +33:39.663 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25198 network=tcp +33:39.663 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25198 network=udp +33:39.663 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25199 network=tcp +33:40.163 [INFO] TestAgent: Waiting for endpoints to shut down +33:40.163 [INFO] TestAgent: Endpoints down +--- PASS: TestHTTPAPI_Ban_Nonprintable_Characters (0.81s) === RUN TestHTTPAPI_Allow_Nonprintable_Characters_With_Flag [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:58.976Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:59.011Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:59.023 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f255727e-2f5f-a1e5-e3d5-e3953cfc56ef Address:127.0.0.1:13979}]" -41:59.024 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13979 [Follower]" leader= -41:59.029 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f255727e-2f5f-a1e5-e3d5-e3953cfc56ef.dc1 127.0.0.1 -41:59.038 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f255727e-2f5f-a1e5-e3d5-e3953cfc56ef 127.0.0.1 -2022-09-15T21:41:59.038Z [INFO] agent.router: Initializing LAN area manager -41:59.039 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13974 network=udp -41:59.039 [INFO] TestAgent.server: Adding LAN server: server="Node-f255727e-2f5f-a1e5-e3d5-e3953cfc56ef (Addr: tcp/127.0.0.1:13979) (DC: dc1)" -41:59.039 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f255727e-2f5f-a1e5-e3d5-e3953cfc56ef.dc1 area=wan -41:59.039 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13974 network=tcp -41:59.040 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13975 network=tcp -41:59.040 [INFO] TestAgent: started state syncer -41:59.041 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13980 network=tcp -41:59.062 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:59.062 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13979 [Candidate]" term=2 -41:59.062 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:59.062 [DEBUG] TestAgent.server.raft: vote granted: from=f255727e-2f5f-a1e5-e3d5-e3953cfc56ef term=2 tally=1 -41:59.062 [INFO] TestAgent.server.raft: election won: tally=1 -41:59.062 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13979 [Leader]" -41:59.062 [INFO] TestAgent.server: cluster leadership acquired -41:59.063 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13979 -41:59.063 [INFO] TestAgent.server: New leader elected: payload=Node-f255727e-2f5f-a1e5-e3d5-e3953cfc56ef -41:59.064 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:59.067 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:59.067 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:59.067 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:59.067 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:59.067 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:59.067 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f255727e-2f5f-a1e5-e3d5-e3953cfc56ef -41:59.067 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f255727e-2f5f-a1e5-e3d5-e3953cfc56ef -41:59.068 [INFO] TestAgent.server: federation state anti-entropy synced -41:59.310 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/bad%00ness from= latency=213.5µs -41:59.310 [INFO] TestAgent: Requesting shutdown -41:59.310 [INFO] TestAgent.server: shutting down server -41:59.310 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:59.310 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:59.310 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:59.310 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:59.310 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:59.310 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:59.310 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:59.311 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:59.311 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:59.311 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:59.311 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:59.311 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:59.311Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:59.311Z [INFO] agent.router.manager: shutting down -41:59.311 [INFO] TestAgent: consul server down -41:59.311 [INFO] TestAgent: shutdown complete -41:59.311 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13974 network=tcp -41:59.311 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13974 network=udp -41:59.311 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13975 network=tcp -41:59.811 [INFO] TestAgent: Waiting for endpoints to shut down -41:59.811 [INFO] TestAgent: Endpoints down ---- PASS: TestHTTPAPI_Allow_Nonprintable_Characters_With_Flag (0.89s) +2021-08-13T15:33:40.180Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:40.196Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:40.198 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:07d54344-4f80-73fb-ad01-11fda85a44c7 Address:127.0.0.1:25210}]" +33:40.198 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25210 [Follower]" leader= +33:40.199 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-07d54344-4f80-73fb-ad01-11fda85a44c7.dc1 127.0.0.1 +33:40.200 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-07d54344-4f80-73fb-ad01-11fda85a44c7 127.0.0.1 +2021-08-13T15:33:40.200Z [INFO] agent.router: Initializing LAN area manager +33:40.200 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25205 network=udp +33:40.201 [INFO] TestAgent.server: Adding LAN server: server="Node-07d54344-4f80-73fb-ad01-11fda85a44c7 (Addr: tcp/127.0.0.1:25210) (DC: dc1)" +33:40.201 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-07d54344-4f80-73fb-ad01-11fda85a44c7.dc1 area=wan +33:40.201 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25205 network=tcp +33:40.202 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25206 network=tcp +33:40.202 [INFO] TestAgent: started state syncer +33:40.202 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25211 network=tcp +33:40.250 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:40.250 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25210 [Candidate]" term=2 +33:40.250 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:40.250 [DEBUG] TestAgent.server.raft: vote granted: from=07d54344-4f80-73fb-ad01-11fda85a44c7 term=2 tally=1 +33:40.250 [INFO] TestAgent.server.raft: election won: tally=1 +33:40.250 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25210 [Leader]" +33:40.250 [INFO] TestAgent.server: cluster leadership acquired +33:40.251 [INFO] TestAgent.server: New leader elected: payload=Node-07d54344-4f80-73fb-ad01-11fda85a44c7 +33:40.251 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25210 +33:40.252 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:40.254 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:40.254 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:40.254 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:40.254 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:40.254 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:40.254 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-07d54344-4f80-73fb-ad01-11fda85a44c7 +33:40.254 [INFO] TestAgent.server: member joined, marking health alive: member=Node-07d54344-4f80-73fb-ad01-11fda85a44c7 +33:40.256 [INFO] TestAgent.server: federation state anti-entropy synced +33:40.399 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/bad%00ness from= latency=204.88µs +33:40.399 [INFO] TestAgent: Requesting shutdown +33:40.399 [INFO] TestAgent.server: shutting down server +33:40.399 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:40.399 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:40.399 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:40.399 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:40.399 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:40.399 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:40.399 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:40.399 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:40.399 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:40.399 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:40.399 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:40.399 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:40.399Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:40.399Z [INFO] agent.router.manager: shutting down +33:40.400 [INFO] TestAgent: consul server down +33:40.400 [INFO] TestAgent: shutdown complete +33:40.400 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25205 network=tcp +33:40.400 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25205 network=udp +33:40.400 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25206 network=tcp +33:40.900 [INFO] TestAgent: Waiting for endpoints to shut down +33:40.900 [INFO] TestAgent: Endpoints down +--- PASS: TestHTTPAPI_Allow_Nonprintable_Characters_With_Flag (0.74s) === RUN TestHTTPAPI_TranslateAddrHeader === PAUSE TestHTTPAPI_TranslateAddrHeader === RUN TestHTTPAPIResponseHeaders @@ -18627,62 +18630,62 @@ === RUN TestParseConsistencyAndMaxStale [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:41:59.831Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:59.848Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:59.851 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4b5f48b2-fbaa-7b95-8b35-de879b7bd635 Address:127.0.0.1:13986}]" -41:59.851 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13986 [Follower]" leader= -41:59.852 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4b5f48b2-fbaa-7b95-8b35-de879b7bd635.dc1 127.0.0.1 -41:59.853 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4b5f48b2-fbaa-7b95-8b35-de879b7bd635 127.0.0.1 -2022-09-15T21:41:59.853Z [INFO] agent.router: Initializing LAN area manager -41:59.853 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13981 network=udp -41:59.853 [INFO] TestAgent.server: Adding LAN server: server="Node-4b5f48b2-fbaa-7b95-8b35-de879b7bd635 (Addr: tcp/127.0.0.1:13986) (DC: dc1)" -41:59.853 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4b5f48b2-fbaa-7b95-8b35-de879b7bd635.dc1 area=wan -41:59.853 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13981 network=tcp -41:59.854 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13982 network=tcp -41:59.854 [INFO] TestAgent: started state syncer -41:59.855 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13987 network=tcp -41:59.921 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:59.922 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13986 [Candidate]" term=2 -41:59.922 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:59.922 [DEBUG] TestAgent.server.raft: vote granted: from=4b5f48b2-fbaa-7b95-8b35-de879b7bd635 term=2 tally=1 -41:59.922 [INFO] TestAgent.server.raft: election won: tally=1 -41:59.922 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13986 [Leader]" -41:59.922 [INFO] TestAgent.server: cluster leadership acquired -41:59.922 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13986 -41:59.924 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:59.926 [INFO] TestAgent.server: New leader elected: payload=Node-4b5f48b2-fbaa-7b95-8b35-de879b7bd635 -41:59.927 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:59.927 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:59.927 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:59.927 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:59.927 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:59.927 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4b5f48b2-fbaa-7b95-8b35-de879b7bd635 -41:59.927 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4b5f48b2-fbaa-7b95-8b35-de879b7bd635 -41:59.928 [INFO] TestAgent.server: federation state anti-entropy synced -42:00.089 [INFO] TestAgent: Requesting shutdown -42:00.089 [INFO] TestAgent.server: shutting down server -42:00.089 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:00.089 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:00.089 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:00.089 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:00.089 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:00.089 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:00.089 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:00.089 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:00.089 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:00.089 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:00.089 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:00.090Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:00.090Z [INFO] agent.router.manager: shutting down -42:00.090 [INFO] TestAgent: consul server down -42:00.090 [INFO] TestAgent: shutdown complete -42:00.090 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13981 network=tcp -42:00.090 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13981 network=udp -42:00.090 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13982 network=tcp -42:00.590 [INFO] TestAgent: Waiting for endpoints to shut down -42:00.590 [INFO] TestAgent: Endpoints down ---- PASS: TestParseConsistencyAndMaxStale (0.78s) +2021-08-13T15:33:40.918Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:40.933Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:40.936 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a18d6e73-82d0-476a-1171-aab4d6267405 Address:127.0.0.1:25217}]" +33:40.936 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25217 [Follower]" leader= +33:40.937 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a18d6e73-82d0-476a-1171-aab4d6267405.dc1 127.0.0.1 +33:40.937 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a18d6e73-82d0-476a-1171-aab4d6267405 127.0.0.1 +2021-08-13T15:33:40.937Z [INFO] agent.router: Initializing LAN area manager +33:40.938 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25212 network=udp +33:40.938 [INFO] TestAgent.server: Adding LAN server: server="Node-a18d6e73-82d0-476a-1171-aab4d6267405 (Addr: tcp/127.0.0.1:25217) (DC: dc1)" +33:40.938 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a18d6e73-82d0-476a-1171-aab4d6267405.dc1 area=wan +33:40.938 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25212 network=tcp +33:40.939 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25213 network=tcp +33:40.939 [INFO] TestAgent: started state syncer +33:40.939 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25218 network=tcp +33:40.979 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:40.979 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25217 [Candidate]" term=2 +33:40.979 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:40.979 [DEBUG] TestAgent.server.raft: vote granted: from=a18d6e73-82d0-476a-1171-aab4d6267405 term=2 tally=1 +33:40.979 [INFO] TestAgent.server.raft: election won: tally=1 +33:40.979 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25217 [Leader]" +33:40.979 [INFO] TestAgent.server: cluster leadership acquired +33:40.979 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25217 +33:40.981 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:40.983 [INFO] TestAgent.server: New leader elected: payload=Node-a18d6e73-82d0-476a-1171-aab4d6267405 +33:40.983 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:40.983 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:40.983 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:40.983 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:40.983 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:40.983 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a18d6e73-82d0-476a-1171-aab4d6267405 +33:40.983 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a18d6e73-82d0-476a-1171-aab4d6267405 +33:40.984 [INFO] TestAgent.server: federation state anti-entropy synced +33:41.019 [INFO] TestAgent: Requesting shutdown +33:41.019 [INFO] TestAgent.server: shutting down server +33:41.019 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:41.019 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:41.019 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:41.019 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:41.019 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:41.020 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:41.020 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:41.020 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:41.020 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:41.020 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:41.020 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:41.020 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:41.020Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:41.020Z [INFO] agent.router.manager: shutting down +33:41.020 [INFO] TestAgent: consul server down +33:41.020 [INFO] TestAgent: shutdown complete +33:41.020 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25212 network=tcp +33:41.020 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25212 network=udp +33:41.020 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25213 network=tcp +33:41.520 [INFO] TestAgent: Waiting for endpoints to shut down +33:41.520 [INFO] TestAgent: Endpoints down +--- PASS: TestParseConsistencyAndMaxStale (0.62s) === RUN TestParseConsistency_Invalid === PAUSE TestParseConsistency_Invalid === RUN TestACLResolution @@ -18764,6 +18767,7 @@ === PAUSE TestKVSEndpoint_PUT_ConflictingFlags === RUN TestKVSEndpoint_GET kvs_endpoint_test.go:477: too slow for testing.Short +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! --- SKIP: TestKVSEndpoint_GET (0.00s) === RUN TestKVSEndpoint_DELETE_ConflictingFlags === PAUSE TestKVSEndpoint_DELETE_ConflictingFlags @@ -18866,254 +18870,252 @@ service_checks_test.go:19: DM-Disabled --- SKIP: TestAgent_ServiceHTTPChecksNotification (0.00s) === RUN TestServiceManager_RegisterService -[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:00.629Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:00.679Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:00.685 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ed287349-4ab7-607f-f115-4980f262609a Address:127.0.0.1:13993}]" -42:00.686 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ed287349-4ab7-607f-f115-4980f262609a.dc1 127.0.0.1 -42:00.686 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13993 [Follower]" leader= -42:00.687 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ed287349-4ab7-607f-f115-4980f262609a 127.0.0.1 -2022-09-15T21:42:00.687Z [INFO] agent.router: Initializing LAN area manager -42:00.688 [INFO] TestAgent.server: Adding LAN server: server="Node-ed287349-4ab7-607f-f115-4980f262609a (Addr: tcp/127.0.0.1:13993) (DC: dc1)" -42:00.688 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ed287349-4ab7-607f-f115-4980f262609a.dc1 area=wan -42:00.688 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13988 network=udp -42:00.688 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13988 network=tcp -42:00.689 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13989 network=tcp -42:00.689 [INFO] TestAgent: started state syncer -42:00.689 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13994 network=tcp -42:00.730 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:00.730 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13993 [Candidate]" term=2 -42:00.731 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:00.731 [DEBUG] TestAgent.server.raft: vote granted: from=ed287349-4ab7-607f-f115-4980f262609a term=2 tally=1 -42:00.731 [INFO] TestAgent.server.raft: election won: tally=1 -42:00.731 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13993 [Leader]" -42:00.731 [INFO] TestAgent.server: cluster leadership acquired -42:00.731 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13993 -42:00.732 [INFO] TestAgent.server: New leader elected: payload=Node-ed287349-4ab7-607f-f115-4980f262609a -42:00.732 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:00.735 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:00.735 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:00.735 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:00.735 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:00.735 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:00.735 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ed287349-4ab7-607f-f115-4980f262609a -42:00.735 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ed287349-4ab7-607f-f115-4980f262609a -42:00.736 [INFO] TestAgent.server: federation state anti-entropy synced -42:00.754 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:00.755 [INFO] TestAgent: Synced node info -42:00.755 [DEBUG] TestAgent: Node info in sync -42:00.896 [INFO] TestAgent: Requesting shutdown -42:00.896 [INFO] TestAgent.server: shutting down server -42:00.896 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:00.896 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:00.896 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:00.896 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:00.896 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:00.896 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:00.896 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:00.896 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:00.896 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:00.896 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:00.896 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:00.896Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:00.896Z [INFO] agent.router.manager: shutting down -42:00.897 [INFO] TestAgent: consul server down -42:00.897 [INFO] TestAgent: shutdown complete -42:00.897 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13988 network=tcp -42:00.897 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13988 network=udp -42:00.897 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13989 network=tcp -42:01.397 [INFO] TestAgent: Waiting for endpoints to shut down -42:01.397 [INFO] TestAgent: Endpoints down ---- PASS: TestServiceManager_RegisterService (0.79s) +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:33:41.564Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:41.625Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:41.632 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b7ab2d1d-5f6f-4400-c824-49d3314012ed Address:127.0.0.1:25224}]" +33:41.633 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b7ab2d1d-5f6f-4400-c824-49d3314012ed.dc1 127.0.0.1 +33:41.634 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b7ab2d1d-5f6f-4400-c824-49d3314012ed 127.0.0.1 +2021-08-13T15:33:41.634Z [INFO] agent.router: Initializing LAN area manager +33:41.634 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25219 network=udp +33:41.634 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25224 [Follower]" leader= +33:41.635 [INFO] TestAgent.server: Adding LAN server: server="Node-b7ab2d1d-5f6f-4400-c824-49d3314012ed (Addr: tcp/127.0.0.1:25224) (DC: dc1)" +33:41.635 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b7ab2d1d-5f6f-4400-c824-49d3314012ed.dc1 area=wan +33:41.635 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25219 network=tcp +33:41.636 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25220 network=tcp +33:41.636 [INFO] TestAgent: started state syncer +33:41.636 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25225 network=tcp +33:41.692 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:41.692 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25224 [Candidate]" term=2 +33:41.693 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:41.693 [DEBUG] TestAgent.server.raft: vote granted: from=b7ab2d1d-5f6f-4400-c824-49d3314012ed term=2 tally=1 +33:41.693 [INFO] TestAgent.server.raft: election won: tally=1 +33:41.693 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25224 [Leader]" +33:41.693 [INFO] TestAgent.server: cluster leadership acquired +33:41.693 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25224 +33:41.694 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:41.695 [INFO] TestAgent.server: New leader elected: payload=Node-b7ab2d1d-5f6f-4400-c824-49d3314012ed +33:41.697 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:41.697 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:41.697 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:41.697 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:41.697 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:41.697 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b7ab2d1d-5f6f-4400-c824-49d3314012ed +33:41.697 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b7ab2d1d-5f6f-4400-c824-49d3314012ed +33:41.698 [INFO] TestAgent.server: federation state anti-entropy synced +33:41.740 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:41.740 [INFO] TestAgent: Synced node info +33:41.740 [DEBUG] TestAgent: Node info in sync +33:41.832 [INFO] TestAgent: Requesting shutdown +33:41.832 [INFO] TestAgent.server: shutting down server +33:41.832 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:41.832 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:41.832 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:41.832 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:41.832 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:41.832 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:41.832 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:41.832 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:41.832 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:41.832 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:41.832 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:41.832Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:41.832Z [INFO] agent.router.manager: shutting down +33:41.833 [INFO] TestAgent: consul server down +33:41.833 [INFO] TestAgent: shutdown complete +33:41.833 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25219 network=tcp +33:41.833 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25219 network=udp +33:41.833 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25220 network=tcp +33:42.335 [INFO] TestAgent: Waiting for endpoints to shut down +33:42.335 [INFO] TestAgent: Endpoints down +--- PASS: TestServiceManager_RegisterService (0.80s) === RUN TestServiceManager_RegisterSidecar [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:01.416Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:01.438Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:01.441 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:98526f27-5bc0-7761-dd59-d4cc4bd4a351 Address:127.0.0.1:14000}]" -42:01.441 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14000 [Follower]" leader= -42:01.442 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-98526f27-5bc0-7761-dd59-d4cc4bd4a351.dc1 127.0.0.1 -42:01.443 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-98526f27-5bc0-7761-dd59-d4cc4bd4a351 127.0.0.1 -2022-09-15T21:42:01.443Z [INFO] agent.router: Initializing LAN area manager -42:01.443 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13995 network=udp -42:01.443 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-98526f27-5bc0-7761-dd59-d4cc4bd4a351.dc1 area=wan -42:01.443 [INFO] TestAgent.server: Adding LAN server: server="Node-98526f27-5bc0-7761-dd59-d4cc4bd4a351 (Addr: tcp/127.0.0.1:14000) (DC: dc1)" -42:01.443 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13995 network=tcp -42:01.444 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13996 network=tcp -42:01.444 [INFO] TestAgent: started state syncer -42:01.444 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14001 network=tcp -42:01.484 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:01.484 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14000 [Candidate]" term=2 -42:01.485 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:01.485 [DEBUG] TestAgent.server.raft: vote granted: from=98526f27-5bc0-7761-dd59-d4cc4bd4a351 term=2 tally=1 -42:01.485 [INFO] TestAgent.server.raft: election won: tally=1 -42:01.485 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14000 [Leader]" -42:01.485 [INFO] TestAgent.server: cluster leadership acquired -42:01.485 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14000 -42:01.485 [INFO] TestAgent.server: New leader elected: payload=Node-98526f27-5bc0-7761-dd59-d4cc4bd4a351 -42:01.486 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:01.489 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:01.489 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:01.489 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:01.489 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:01.489 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:01.489 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-98526f27-5bc0-7761-dd59-d4cc4bd4a351 -42:01.489 [INFO] TestAgent.server: member joined, marking health alive: member=Node-98526f27-5bc0-7761-dd59-d4cc4bd4a351 -42:01.490 [INFO] TestAgent.server: federation state anti-entropy synced -42:01.637 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:01.638 [INFO] TestAgent: Synced node info -42:01.865 [DEBUG] TestAgent: added local registration for service: service=web-sidecar-proxy -42:01.865 [INFO] TestAgent: Requesting shutdown -42:01.866 [INFO] TestAgent.server: shutting down server -42:01.866 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:01.866 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:01.866 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:01.866 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:01.866 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:01.866 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:01.866 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:01.866 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:01.866 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:01.866 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:01.866 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:01.866Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:01.866Z [INFO] agent.router.manager: shutting down -42:01.866 [INFO] TestAgent: consul server down -42:01.866 [INFO] TestAgent: shutdown complete -42:01.866 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13995 network=tcp -42:01.866 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13995 network=udp -42:01.866 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13996 network=tcp -42:02.367 [INFO] TestAgent: Waiting for endpoints to shut down -42:02.367 [INFO] TestAgent: Endpoints down ---- PASS: TestServiceManager_RegisterSidecar (0.97s) +2021-08-13T15:33:42.353Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:42.369Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:42.372 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2df36acc-70f7-d7f7-e3dd-f9a697101953 Address:127.0.0.1:25231}]" +33:42.372 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25231 [Follower]" leader= +33:42.373 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2df36acc-70f7-d7f7-e3dd-f9a697101953.dc1 127.0.0.1 +33:42.374 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2df36acc-70f7-d7f7-e3dd-f9a697101953 127.0.0.1 +2021-08-13T15:33:42.374Z [INFO] agent.router: Initializing LAN area manager +33:42.374 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25226 network=udp +33:42.374 [INFO] TestAgent.server: Adding LAN server: server="Node-2df36acc-70f7-d7f7-e3dd-f9a697101953 (Addr: tcp/127.0.0.1:25231) (DC: dc1)" +33:42.375 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2df36acc-70f7-d7f7-e3dd-f9a697101953.dc1 area=wan +33:42.375 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25226 network=tcp +33:42.376 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25227 network=tcp +33:42.376 [INFO] TestAgent: started state syncer +33:42.376 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25232 network=tcp +33:42.411 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:42.411 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25231 [Candidate]" term=2 +33:42.411 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:42.411 [DEBUG] TestAgent.server.raft: vote granted: from=2df36acc-70f7-d7f7-e3dd-f9a697101953 term=2 tally=1 +33:42.411 [INFO] TestAgent.server.raft: election won: tally=1 +33:42.411 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25231 [Leader]" +33:42.412 [INFO] TestAgent.server: cluster leadership acquired +33:42.412 [INFO] TestAgent.server: New leader elected: payload=Node-2df36acc-70f7-d7f7-e3dd-f9a697101953 +33:42.412 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25231 +33:42.413 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:42.416 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:42.416 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:42.416 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:42.416 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:42.416 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:42.416 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2df36acc-70f7-d7f7-e3dd-f9a697101953 +33:42.416 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2df36acc-70f7-d7f7-e3dd-f9a697101953 +33:42.417 [INFO] TestAgent.server: federation state anti-entropy synced +33:42.522 [DEBUG] TestAgent: added local registration for service: service=web-sidecar-proxy +33:42.522 [INFO] TestAgent: Requesting shutdown +33:42.522 [INFO] TestAgent.server: shutting down server +33:42.522 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:42.522 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:42.522 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:42.522 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:42.522 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:42.522 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:42.522 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:42.522 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:42.522 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:42.522 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:42.522 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:42.522 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:42.522Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:42.523Z [INFO] agent.router.manager: shutting down +33:42.523 [INFO] TestAgent: consul server down +33:42.523 [INFO] TestAgent: shutdown complete +33:42.523 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25226 network=tcp +33:42.523 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25226 network=udp +33:42.523 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25227 network=tcp +33:43.023 [INFO] TestAgent: Waiting for endpoints to shut down +33:43.023 [INFO] TestAgent: Endpoints down +--- PASS: TestServiceManager_RegisterSidecar (0.69s) === RUN TestServiceManager_RegisterMeshGateway [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:02.390Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:02.411Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:02.414 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5752ec6e-266a-7214-4390-7e631098af3e Address:127.0.0.1:14007}]" -42:02.415 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14007 [Follower]" leader= -42:02.415 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5752ec6e-266a-7214-4390-7e631098af3e.dc1 127.0.0.1 -42:02.416 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5752ec6e-266a-7214-4390-7e631098af3e 127.0.0.1 -2022-09-15T21:42:02.416Z [INFO] agent.router: Initializing LAN area manager -42:02.416 [INFO] TestAgent.server: Adding LAN server: server="Node-5752ec6e-266a-7214-4390-7e631098af3e (Addr: tcp/127.0.0.1:14007) (DC: dc1)" -42:02.416 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5752ec6e-266a-7214-4390-7e631098af3e.dc1 area=wan -42:02.416 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14002 network=udp -42:02.417 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14002 network=tcp -42:02.417 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14003 network=tcp -42:02.418 [INFO] TestAgent: started state syncer -42:02.418 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14008 network=tcp -42:02.471 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:02.471 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14007 [Candidate]" term=2 -42:02.472 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:02.472 [DEBUG] TestAgent.server.raft: vote granted: from=5752ec6e-266a-7214-4390-7e631098af3e term=2 tally=1 -42:02.472 [INFO] TestAgent.server.raft: election won: tally=1 -42:02.472 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14007 [Leader]" -42:02.472 [INFO] TestAgent.server: cluster leadership acquired -42:02.472 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14007 -42:02.473 [INFO] TestAgent.server: New leader elected: payload=Node-5752ec6e-266a-7214-4390-7e631098af3e -42:02.473 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:02.476 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:02.476 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:02.476 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:02.476 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:02.476 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:02.476 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5752ec6e-266a-7214-4390-7e631098af3e -42:02.476 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5752ec6e-266a-7214-4390-7e631098af3e -42:02.477 [INFO] TestAgent.server: federation state anti-entropy synced -42:02.504 [DEBUG] TestAgent: added local registration for service: service=mesh-gateway -42:02.504 [INFO] TestAgent: Requesting shutdown -42:02.504 [INFO] TestAgent.server: shutting down server -42:02.504 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:02.504 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:02.504 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:02.504 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:02.504 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:02.504 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:02.504 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:02.504 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:02.504 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:02.504 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:02.504 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:02.504 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:02.504Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:02.504Z [INFO] agent.router.manager: shutting down -42:02.504 [INFO] TestAgent: consul server down -42:02.504 [INFO] TestAgent: shutdown complete -42:02.504 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14002 network=tcp -42:02.504 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14002 network=udp -42:02.505 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14003 network=tcp -42:03.005 [INFO] TestAgent: Waiting for endpoints to shut down -42:03.005 [INFO] TestAgent: Endpoints down ---- PASS: TestServiceManager_RegisterMeshGateway (0.64s) +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:33:43.040Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:43.056Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:43.059 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b2221f88-6654-7ff0-b0e3-767c6f50bf17 Address:127.0.0.1:25238}]" +33:43.059 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25238 [Follower]" leader= +33:43.060 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b2221f88-6654-7ff0-b0e3-767c6f50bf17.dc1 127.0.0.1 +33:43.061 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b2221f88-6654-7ff0-b0e3-767c6f50bf17 127.0.0.1 +2021-08-13T15:33:43.061Z [INFO] agent.router: Initializing LAN area manager +33:43.061 [INFO] TestAgent.server: Adding LAN server: server="Node-b2221f88-6654-7ff0-b0e3-767c6f50bf17 (Addr: tcp/127.0.0.1:25238) (DC: dc1)" +33:43.061 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b2221f88-6654-7ff0-b0e3-767c6f50bf17.dc1 area=wan +33:43.061 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25233 network=tcp +33:43.061 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25233 network=udp +33:43.062 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25234 network=tcp +33:43.062 [INFO] TestAgent: started state syncer +33:43.062 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25239 network=tcp +33:43.098 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:43.098 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25238 [Candidate]" term=2 +33:43.098 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:43.098 [DEBUG] TestAgent.server.raft: vote granted: from=b2221f88-6654-7ff0-b0e3-767c6f50bf17 term=2 tally=1 +33:43.098 [INFO] TestAgent.server.raft: election won: tally=1 +33:43.098 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25238 [Leader]" +33:43.098 [INFO] TestAgent.server: cluster leadership acquired +33:43.098 [INFO] TestAgent.server: New leader elected: payload=Node-b2221f88-6654-7ff0-b0e3-767c6f50bf17 +33:43.099 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25238 +33:43.100 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:43.102 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:43.102 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:43.102 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:43.102 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:43.102 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:43.102 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b2221f88-6654-7ff0-b0e3-767c6f50bf17 +33:43.102 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b2221f88-6654-7ff0-b0e3-767c6f50bf17 +33:43.103 [INFO] TestAgent.server: federation state anti-entropy synced +33:43.311 [DEBUG] TestAgent: added local registration for service: service=mesh-gateway +33:43.311 [INFO] TestAgent: Requesting shutdown +33:43.312 [INFO] TestAgent.server: shutting down server +33:43.312 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:43.312 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:43.312 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:43.312 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:43.312 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:43.312 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:43.312 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:43.312 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:43.312 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:43.312 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:43.312 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:43.312 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:43.312Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:43.312Z [INFO] agent.router.manager: shutting down +33:43.312 [INFO] TestAgent: consul server down +33:43.312 [INFO] TestAgent: shutdown complete +33:43.312 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25233 network=tcp +33:43.312 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25233 network=udp +33:43.312 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25234 network=tcp +33:43.815 [INFO] TestAgent: Waiting for endpoints to shut down +33:43.815 [INFO] TestAgent: Endpoints down +--- PASS: TestServiceManager_RegisterMeshGateway (0.79s) === RUN TestServiceManager_RegisterTerminatingGateway [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:03.025Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:03.043Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:03.053 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:97c74a02-22ca-f432-c53e-bb93093f36e8 Address:127.0.0.1:14014}]" -42:03.053 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14014 [Follower]" leader= -42:03.055 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-97c74a02-22ca-f432-c53e-bb93093f36e8.dc1 127.0.0.1 -42:03.058 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-97c74a02-22ca-f432-c53e-bb93093f36e8 127.0.0.1 -2022-09-15T21:42:03.058Z [INFO] agent.router: Initializing LAN area manager -42:03.061 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-97c74a02-22ca-f432-c53e-bb93093f36e8.dc1 area=wan -42:03.061 [INFO] TestAgent.server: Adding LAN server: server="Node-97c74a02-22ca-f432-c53e-bb93093f36e8 (Addr: tcp/127.0.0.1:14014) (DC: dc1)" -42:03.062 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14009 network=udp -42:03.062 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14009 network=tcp -42:03.063 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14010 network=tcp -42:03.063 [INFO] TestAgent: started state syncer -42:03.064 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14015 network=tcp -42:03.120 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:03.120 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14014 [Candidate]" term=2 -42:03.120 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:03.120 [DEBUG] TestAgent.server.raft: vote granted: from=97c74a02-22ca-f432-c53e-bb93093f36e8 term=2 tally=1 -42:03.120 [INFO] TestAgent.server.raft: election won: tally=1 -42:03.120 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14014 [Leader]" -42:03.120 [INFO] TestAgent.server: cluster leadership acquired -42:03.121 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14014 -42:03.122 [INFO] TestAgent.server: New leader elected: payload=Node-97c74a02-22ca-f432-c53e-bb93093f36e8 -42:03.122 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:03.125 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:03.125 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:03.125 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:03.125 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:03.125 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:03.125 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-97c74a02-22ca-f432-c53e-bb93093f36e8 -42:03.125 [INFO] TestAgent.server: member joined, marking health alive: member=Node-97c74a02-22ca-f432-c53e-bb93093f36e8 -42:03.126 [INFO] TestAgent.server: federation state anti-entropy synced -42:03.372 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:03.373 [INFO] TestAgent: Synced node info -42:03.373 [DEBUG] TestAgent: Node info in sync -42:03.381 [DEBUG] TestAgent: added local registration for service: service=terminating-gateway -42:03.381 [INFO] TestAgent: Requesting shutdown -42:03.381 [INFO] TestAgent.server: shutting down server -42:03.381 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:03.381 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:03.381 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:03.381 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:03.381 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:03.381 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:03.381 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:03.381 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:03.381 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:03.381 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:03.381 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -42:03.382 [INFO] TestAgent: consul server down -42:03.382 [INFO] TestAgent: shutdown complete -42:03.382 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14009 network=tcp -42:03.382 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14009 network=udp -42:03.382 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14010 network=tcp -2022-09-15T21:42:03.381Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:03.382Z [INFO] agent.router.manager: shutting down -42:03.884 [INFO] TestAgent: Waiting for endpoints to shut down -42:03.884 [INFO] TestAgent: Endpoints down ---- PASS: TestServiceManager_RegisterTerminatingGateway (0.88s) +2021-08-13T15:33:43.849Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:43.896Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:43.899 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:80c9908a-2c82-c702-989f-d4a2e25c4c78 Address:127.0.0.1:25245}]" +33:43.899 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25245 [Follower]" leader= +33:43.900 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-80c9908a-2c82-c702-989f-d4a2e25c4c78.dc1 127.0.0.1 +33:43.901 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-80c9908a-2c82-c702-989f-d4a2e25c4c78 127.0.0.1 +2021-08-13T15:33:43.901Z [INFO] agent.router: Initializing LAN area manager +33:43.901 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25240 network=udp +33:43.902 [INFO] TestAgent.server: Adding LAN server: server="Node-80c9908a-2c82-c702-989f-d4a2e25c4c78 (Addr: tcp/127.0.0.1:25245) (DC: dc1)" +33:43.902 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-80c9908a-2c82-c702-989f-d4a2e25c4c78.dc1 area=wan +33:43.902 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25240 network=tcp +33:43.903 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25241 network=tcp +33:43.903 [INFO] TestAgent: started state syncer +33:43.903 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25246 network=tcp +33:43.964 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:43.964 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25245 [Candidate]" term=2 +33:43.964 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:43.964 [DEBUG] TestAgent.server.raft: vote granted: from=80c9908a-2c82-c702-989f-d4a2e25c4c78 term=2 tally=1 +33:43.964 [INFO] TestAgent.server.raft: election won: tally=1 +33:43.964 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25245 [Leader]" +33:43.965 [INFO] TestAgent.server: cluster leadership acquired +33:43.965 [INFO] TestAgent.server: New leader elected: payload=Node-80c9908a-2c82-c702-989f-d4a2e25c4c78 +33:43.965 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25245 +33:43.966 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:43.969 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:43.969 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:43.969 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:43.969 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:43.969 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:43.969 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-80c9908a-2c82-c702-989f-d4a2e25c4c78 +33:43.969 [INFO] TestAgent.server: member joined, marking health alive: member=Node-80c9908a-2c82-c702-989f-d4a2e25c4c78 +33:43.970 [INFO] TestAgent.server: federation state anti-entropy synced +33:44.078 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:44.079 [INFO] TestAgent: Synced node info +33:44.343 [DEBUG] TestAgent: added local registration for service: service=terminating-gateway +33:44.343 [INFO] TestAgent: Requesting shutdown +33:44.343 [INFO] TestAgent.server: shutting down server +33:44.343 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:44.343 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:44.343 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:44.343 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:44.344 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:44.344 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:44.344 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:44.344 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:44.344 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:44.344 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:44.344 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:44.344Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:44.344Z [INFO] agent.router.manager: shutting down +33:44.344 [INFO] TestAgent: consul server down +33:44.344 [INFO] TestAgent: shutdown complete +33:44.344 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25240 network=tcp +33:44.344 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25240 network=udp +33:44.344 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25241 network=tcp +33:44.845 [INFO] TestAgent: Waiting for endpoints to shut down +33:44.845 [INFO] TestAgent: Endpoints down +--- PASS: TestServiceManager_RegisterTerminatingGateway (1.03s) === RUN TestServiceManager_PersistService_API service_manager_test.go:264: DM-disabled --- SKIP: TestServiceManager_PersistService_API (0.00s) @@ -19122,62 +19124,65 @@ === RUN TestServiceManager_Disabled [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:03.903Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:03.922Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:03.924 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c31d76aa-bfbc-1ba1-67cc-85b2ec2c03c1 Address:127.0.0.1:14021}]" -42:03.930 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14021 [Follower]" leader= -42:03.931 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c31d76aa-bfbc-1ba1-67cc-85b2ec2c03c1.dc1 127.0.0.1 -42:03.932 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c31d76aa-bfbc-1ba1-67cc-85b2ec2c03c1 127.0.0.1 -2022-09-15T21:42:03.932Z [INFO] agent.router: Initializing LAN area manager -42:03.932 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14016 network=udp -42:03.933 [INFO] TestAgent.server: Adding LAN server: server="Node-c31d76aa-bfbc-1ba1-67cc-85b2ec2c03c1 (Addr: tcp/127.0.0.1:14021) (DC: dc1)" -42:03.933 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c31d76aa-bfbc-1ba1-67cc-85b2ec2c03c1.dc1 area=wan -42:03.933 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14016 network=tcp -42:03.934 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14017 network=tcp -42:03.934 [INFO] TestAgent: started state syncer -42:03.934 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14022 network=tcp -42:03.971 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:03.971 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14021 [Candidate]" term=2 -42:03.971 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:03.971 [DEBUG] TestAgent.server.raft: vote granted: from=c31d76aa-bfbc-1ba1-67cc-85b2ec2c03c1 term=2 tally=1 -42:03.971 [INFO] TestAgent.server.raft: election won: tally=1 -42:03.971 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14021 [Leader]" -42:03.972 [INFO] TestAgent.server: cluster leadership acquired -42:03.972 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14021 -42:03.974 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:03.975 [INFO] TestAgent.server: New leader elected: payload=Node-c31d76aa-bfbc-1ba1-67cc-85b2ec2c03c1 -42:03.977 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:03.977 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:03.977 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:03.977 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:03.977 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:03.977 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c31d76aa-bfbc-1ba1-67cc-85b2ec2c03c1 -42:03.977 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c31d76aa-bfbc-1ba1-67cc-85b2ec2c03c1 -42:03.978 [INFO] TestAgent.server: federation state anti-entropy synced -42:04.012 [INFO] TestAgent: Requesting shutdown -42:04.012 [INFO] TestAgent.server: shutting down server -42:04.012 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:04.012 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:04.012 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:04.012 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:04.012 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:04.013 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:04.013 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:04.013 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:04.013 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:04.013 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:04.013 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:04.013 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:04.013Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:04.013Z [INFO] agent.router.manager: shutting down -42:04.013 [INFO] TestAgent: consul server down -42:04.013 [INFO] TestAgent: shutdown complete -42:04.013 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14016 network=tcp -42:04.013 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14016 network=udp -42:04.013 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14017 network=tcp -42:04.519 [INFO] TestAgent: Waiting for endpoints to shut down -42:04.519 [INFO] TestAgent: Endpoints down ---- PASS: TestServiceManager_Disabled (0.63s) +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:33:44.863Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:44.880Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:44.883 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0f3643ea-e702-f266-8cdc-37de7bd17c9b Address:127.0.0.1:25252}]" +33:44.883 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25252 [Follower]" leader= +33:44.884 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0f3643ea-e702-f266-8cdc-37de7bd17c9b.dc1 127.0.0.1 +33:44.884 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0f3643ea-e702-f266-8cdc-37de7bd17c9b 127.0.0.1 +2021-08-13T15:33:44.884Z [INFO] agent.router: Initializing LAN area manager +33:44.885 [INFO] TestAgent.server: Adding LAN server: server="Node-0f3643ea-e702-f266-8cdc-37de7bd17c9b (Addr: tcp/127.0.0.1:25252) (DC: dc1)" +33:44.885 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0f3643ea-e702-f266-8cdc-37de7bd17c9b.dc1 area=wan +33:44.885 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25247 network=udp +33:44.885 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25247 network=tcp +33:44.886 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25248 network=tcp +33:44.886 [INFO] TestAgent: started state syncer +33:44.886 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25253 network=tcp +33:44.935 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:44.935 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25252 [Candidate]" term=2 +33:44.935 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:44.935 [DEBUG] TestAgent.server.raft: vote granted: from=0f3643ea-e702-f266-8cdc-37de7bd17c9b term=2 tally=1 +33:44.935 [INFO] TestAgent.server.raft: election won: tally=1 +33:44.935 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25252 [Leader]" +33:44.935 [INFO] TestAgent.server: cluster leadership acquired +33:44.935 [INFO] TestAgent.server: New leader elected: payload=Node-0f3643ea-e702-f266-8cdc-37de7bd17c9b +33:44.935 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25252 +33:44.937 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:44.939 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:44.939 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:44.939 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:44.939 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:44.939 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:44.939 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0f3643ea-e702-f266-8cdc-37de7bd17c9b +33:44.939 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0f3643ea-e702-f266-8cdc-37de7bd17c9b +33:44.940 [INFO] TestAgent.server: federation state anti-entropy synced +33:45.081 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:45.082 [INFO] TestAgent: Synced node info +33:45.082 [DEBUG] TestAgent: Node info in sync +33:45.145 [INFO] TestAgent: Requesting shutdown +33:45.145 [INFO] TestAgent.server: shutting down server +33:45.145 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:45.145 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:45.145 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:45.145 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:45.145 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:45.145 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:45.145 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:45.145 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:45.145 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:45.145 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:45.146 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:45.146Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:45.146Z [INFO] agent.router.manager: shutting down +33:45.146 [INFO] TestAgent: consul server down +33:45.146 [INFO] TestAgent: shutdown complete +33:45.146 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25247 network=tcp +33:45.146 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25247 network=udp +33:45.146 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25248 network=tcp +33:45.646 [INFO] TestAgent: Waiting for endpoints to shut down +33:45.646 [INFO] TestAgent: Endpoints down +--- PASS: TestServiceManager_Disabled (0.80s) === RUN TestSessionCreate === PAUSE TestSessionCreate === RUN TestSessionCreate_NodeChecks @@ -19203,124 +19208,126 @@ [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:04.543Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:04.561Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:04.564 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:369c1ba1-7fbb-0f87-3396-7e44f2151d68 Address:127.0.0.1:14028}]" -42:04.565 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-369c1ba1-7fbb-0f87-3396-7e44f2151d68.dc1 127.0.0.1 -42:04.566 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14028 [Follower]" leader= -42:04.571 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-369c1ba1-7fbb-0f87-3396-7e44f2151d68 127.0.0.1 -2022-09-15T21:42:04.571Z [INFO] agent.router: Initializing LAN area manager -42:04.571 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14023 network=udp -42:04.571 [INFO] TestAgent.server: Adding LAN server: server="Node-369c1ba1-7fbb-0f87-3396-7e44f2151d68 (Addr: tcp/127.0.0.1:14028) (DC: dc1)" -42:04.572 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-369c1ba1-7fbb-0f87-3396-7e44f2151d68.dc1 area=wan -42:04.572 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14023 network=tcp -42:04.573 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14024 network=tcp -42:04.573 [INFO] TestAgent: started state syncer -42:04.573 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14029 network=tcp -42:04.622 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:04.622 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14028 [Candidate]" term=2 -42:04.622 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:04.622 [DEBUG] TestAgent.server.raft: vote granted: from=369c1ba1-7fbb-0f87-3396-7e44f2151d68 term=2 tally=1 -42:04.622 [INFO] TestAgent.server.raft: election won: tally=1 -42:04.622 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14028 [Leader]" -42:04.622 [INFO] TestAgent.server: cluster leadership acquired -42:04.623 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14028 -42:04.623 [INFO] TestAgent.server: New leader elected: payload=Node-369c1ba1-7fbb-0f87-3396-7e44f2151d68 -42:04.624 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:04.626 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:04.626 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:04.626 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:04.626 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:04.626 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:04.626 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-369c1ba1-7fbb-0f87-3396-7e44f2151d68 -42:04.626 [INFO] TestAgent.server: member joined, marking health alive: member=Node-369c1ba1-7fbb-0f87-3396-7e44f2151d68 -42:04.628 [INFO] TestAgent.server: federation state anti-entropy synced -42:04.654 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:04.654 [INFO] TestAgent: Synced node info -42:04.925 [INFO] TestAgent: Requesting shutdown -42:04.925 [INFO] TestAgent.server: shutting down server -42:04.925 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:04.925 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:04.925 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:04.925 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:04.925 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:04.925 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:04.925 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:04.925 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:04.925 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:04.925 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:04.925 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:04.926Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:04.926Z [INFO] agent.router.manager: shutting down -42:04.926 [INFO] TestAgent: consul server down -42:04.926 [INFO] TestAgent: shutdown complete -42:04.926 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14023 network=tcp -42:04.926 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14023 network=udp -42:04.926 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14024 network=tcp -42:05.426 [INFO] TestAgent: Waiting for endpoints to shut down -42:05.426 [INFO] TestAgent: Endpoints down +2021-08-13T15:33:45.664Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:45.679Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:45.682 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:df31729c-c2b3-92fc-bff6-ff208c624319 Address:127.0.0.1:25259}]" +33:45.682 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25259 [Follower]" leader= +33:45.683 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-df31729c-c2b3-92fc-bff6-ff208c624319.dc1 127.0.0.1 +33:45.684 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-df31729c-c2b3-92fc-bff6-ff208c624319 127.0.0.1 +2021-08-13T15:33:45.684Z [INFO] agent.router: Initializing LAN area manager +33:45.684 [INFO] TestAgent.server: Adding LAN server: server="Node-df31729c-c2b3-92fc-bff6-ff208c624319 (Addr: tcp/127.0.0.1:25259) (DC: dc1)" +33:45.684 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-df31729c-c2b3-92fc-bff6-ff208c624319.dc1 area=wan +33:45.684 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25254 network=tcp +33:45.684 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25254 network=udp +33:45.685 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25255 network=tcp +33:45.685 [INFO] TestAgent: started state syncer +33:45.685 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25260 network=tcp +33:45.737 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:45.737 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25259 [Candidate]" term=2 +33:45.737 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:45.737 [DEBUG] TestAgent.server.raft: vote granted: from=df31729c-c2b3-92fc-bff6-ff208c624319 term=2 tally=1 +33:45.737 [INFO] TestAgent.server.raft: election won: tally=1 +33:45.737 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25259 [Leader]" +33:45.737 [INFO] TestAgent.server: cluster leadership acquired +33:45.737 [INFO] TestAgent.server: New leader elected: payload=Node-df31729c-c2b3-92fc-bff6-ff208c624319 +33:45.738 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25259 +33:45.739 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:45.741 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:45.741 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:45.741 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:45.741 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:45.741 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:45.742 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-df31729c-c2b3-92fc-bff6-ff208c624319 +33:45.742 [INFO] TestAgent.server: member joined, marking health alive: member=Node-df31729c-c2b3-92fc-bff6-ff208c624319 +33:45.742 [INFO] TestAgent.server: federation state anti-entropy synced +33:45.887 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:45.888 [INFO] TestAgent: Synced node info +33:46.044 [INFO] TestAgent: Requesting shutdown +33:46.044 [INFO] TestAgent.server: shutting down server +33:46.044 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:46.044 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:46.044 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:46.044 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:46.044 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:46.044 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:46.044 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:46.044 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:46.044 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:46.044 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:46.044 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:46.044Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:46.044Z [INFO] agent.router.manager: shutting down +33:46.044 [INFO] TestAgent: consul server down +33:46.044 [INFO] TestAgent: shutdown complete +33:46.044 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25254 network=tcp +33:46.044 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25254 network=udp +33:46.044 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25255 network=tcp +33:46.544 [INFO] TestAgent: Waiting for endpoints to shut down +33:46.545 [INFO] TestAgent: Endpoints down === RUN TestSessionList/#01 [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:05.447Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:05.466Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:05.469 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f21d912e-8857-8fae-c685-3f154b6d9ade Address:127.0.0.1:14035}]" -42:05.469 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14035 [Follower]" leader= -42:05.470 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f21d912e-8857-8fae-c685-3f154b6d9ade.dc1 127.0.0.1 -42:05.471 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f21d912e-8857-8fae-c685-3f154b6d9ade 127.0.0.1 -2022-09-15T21:42:05.471Z [INFO] agent.router: Initializing LAN area manager -42:05.472 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14030 network=udp -42:05.472 [INFO] TestAgent.server: Adding LAN server: server="Node-f21d912e-8857-8fae-c685-3f154b6d9ade (Addr: tcp/127.0.0.1:14035) (DC: dc1)" -42:05.472 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f21d912e-8857-8fae-c685-3f154b6d9ade.dc1 area=wan -42:05.472 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14030 network=tcp -42:05.473 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14031 network=tcp -42:05.473 [INFO] TestAgent: started state syncer -42:05.474 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14036 network=tcp -42:05.521 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:05.521 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14035 [Candidate]" term=2 -42:05.522 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:05.522 [DEBUG] TestAgent.server.raft: vote granted: from=f21d912e-8857-8fae-c685-3f154b6d9ade term=2 tally=1 -42:05.522 [INFO] TestAgent.server.raft: election won: tally=1 -42:05.522 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14035 [Leader]" -42:05.522 [INFO] TestAgent.server: cluster leadership acquired -42:05.522 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14035 -42:05.526 [INFO] TestAgent.server: New leader elected: payload=Node-f21d912e-8857-8fae-c685-3f154b6d9ade -42:05.528 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:05.532 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:05.532 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:05.532 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:05.532 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:05.532 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:05.532 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f21d912e-8857-8fae-c685-3f154b6d9ade -42:05.532 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f21d912e-8857-8fae-c685-3f154b6d9ade -42:05.533 [INFO] TestAgent.server: federation state anti-entropy synced -42:05.678 [INFO] TestAgent: Requesting shutdown -42:05.678 [INFO] TestAgent.server: shutting down server -42:05.678 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:05.678 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:05.678 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:05.678 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:05.678 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:05.678 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:05.678 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:05.678 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:05.678 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:05.678 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:05.678 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:05.679 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:05.679Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:05.679Z [INFO] agent.router.manager: shutting down -42:05.679 [INFO] TestAgent: consul server down -42:05.679 [INFO] TestAgent: shutdown complete -42:05.679 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14030 network=tcp -42:05.679 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14030 network=udp -42:05.679 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14031 network=tcp -42:06.180 [INFO] TestAgent: Waiting for endpoints to shut down -42:06.180 [INFO] TestAgent: Endpoints down ---- PASS: TestSessionList (1.66s) - --- PASS: TestSessionList/#00 (0.91s) - --- PASS: TestSessionList/#01 (0.75s) +2021-08-13T15:33:46.564Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:46.620Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:46.625 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8a797c19-1dff-9d75-8195-68753e1b5ac7 Address:127.0.0.1:25266}]" +33:46.626 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8a797c19-1dff-9d75-8195-68753e1b5ac7.dc1 127.0.0.1 +33:46.627 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25266 [Follower]" leader= +33:46.627 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8a797c19-1dff-9d75-8195-68753e1b5ac7 127.0.0.1 +2021-08-13T15:33:46.627Z [INFO] agent.router: Initializing LAN area manager +33:46.627 [INFO] TestAgent.server: Adding LAN server: server="Node-8a797c19-1dff-9d75-8195-68753e1b5ac7 (Addr: tcp/127.0.0.1:25266) (DC: dc1)" +33:46.627 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25261 network=udp +33:46.628 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8a797c19-1dff-9d75-8195-68753e1b5ac7.dc1 area=wan +33:46.628 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25261 network=tcp +33:46.629 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25262 network=tcp +33:46.629 [INFO] TestAgent: started state syncer +33:46.629 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25267 network=tcp +33:46.667 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:46.667 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25266 [Candidate]" term=2 +33:46.667 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:46.667 [DEBUG] TestAgent.server.raft: vote granted: from=8a797c19-1dff-9d75-8195-68753e1b5ac7 term=2 tally=1 +33:46.667 [INFO] TestAgent.server.raft: election won: tally=1 +33:46.667 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25266 [Leader]" +33:46.667 [INFO] TestAgent.server: cluster leadership acquired +33:46.668 [INFO] TestAgent.server: New leader elected: payload=Node-8a797c19-1dff-9d75-8195-68753e1b5ac7 +33:46.668 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25266 +33:46.669 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:46.672 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:46.672 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:46.672 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:46.672 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:46.672 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:46.672 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8a797c19-1dff-9d75-8195-68753e1b5ac7 +33:46.672 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8a797c19-1dff-9d75-8195-68753e1b5ac7 +33:46.673 [INFO] TestAgent.server: federation state anti-entropy synced +33:46.822 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:46.823 [INFO] TestAgent: Synced node info +33:46.823 [DEBUG] TestAgent: Node info in sync +33:46.847 [INFO] TestAgent: Requesting shutdown +33:46.847 [INFO] TestAgent.server: shutting down server +33:46.847 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:46.847 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:46.847 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:46.847 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:46.847 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:46.847 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:46.847 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:46.847 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:46.847 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:46.847 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:46.847 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:46.847Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:46.847Z [INFO] agent.router.manager: shutting down +33:46.847 [INFO] TestAgent: consul server down +33:46.847 [INFO] TestAgent: shutdown complete +33:46.847 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25261 network=tcp +33:46.847 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25261 network=udp +33:46.847 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25262 network=tcp +33:47.348 [INFO] TestAgent: Waiting for endpoints to shut down +33:47.348 [INFO] TestAgent: Endpoints down +--- PASS: TestSessionList (1.70s) + --- PASS: TestSessionList/#00 (0.90s) + --- PASS: TestSessionList/#01 (0.80s) === RUN TestSessionsForNode session_endpoint_test.go:677: DM-skipped --- SKIP: TestSessionsForNode (0.00s) @@ -19331,551 +19338,550 @@ [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:06.209Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:06.229Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:06.232 [INFO] jones.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:59f65b77-194f-185c-1548-1b1cd6d4641d Address:127.0.0.1:14042}]" -42:06.233 [INFO] jones.server.serf.wan: serf: EventMemberJoin: Node-59f65b77-194f-185c-1548-1b1cd6d4641d.dc1 127.0.0.1 -42:06.234 [INFO] jones.server.raft: entering follower state: follower="Node at 127.0.0.1:14042 [Follower]" leader= -42:06.234 [INFO] jones.server.serf.lan: serf: EventMemberJoin: Node-59f65b77-194f-185c-1548-1b1cd6d4641d 127.0.0.1 -2022-09-15T21:42:06.235Z [INFO] agent.router: Initializing LAN area manager -42:06.235 [INFO] jones: Started DNS server: address=127.0.0.1:14037 network=udp -42:06.235 [INFO] jones.server: Adding LAN server: server="Node-59f65b77-194f-185c-1548-1b1cd6d4641d (Addr: tcp/127.0.0.1:14042) (DC: dc1)" -42:06.235 [INFO] jones.server: Handled event for server in area: event=member-join server=Node-59f65b77-194f-185c-1548-1b1cd6d4641d.dc1 area=wan -42:06.235 [INFO] jones: Started DNS server: address=127.0.0.1:14037 network=tcp -42:06.236 [INFO] jones: Started HTTP server: address=127.0.0.1:14038 network=tcp -42:06.236 [INFO] jones: started state syncer -42:06.237 [INFO] jones: Started gRPC server: address=127.0.0.1:14043 network=tcp -42:06.287 [WARN] jones.server.raft: heartbeat timeout reached, starting election: last-leader= -42:06.287 [INFO] jones.server.raft: entering candidate state: node="Node at 127.0.0.1:14042 [Candidate]" term=2 -42:06.287 [DEBUG] jones.server.raft: votes: needed=1 -42:06.287 [DEBUG] jones.server.raft: vote granted: from=59f65b77-194f-185c-1548-1b1cd6d4641d term=2 tally=1 -42:06.287 [INFO] jones.server.raft: election won: tally=1 -42:06.287 [INFO] jones.server.raft: entering leader state: leader="Node at 127.0.0.1:14042 [Leader]" -42:06.287 [INFO] jones.server: cluster leadership acquired -42:06.288 [INFO] jones.server: New leader elected: payload=Node-59f65b77-194f-185c-1548-1b1cd6d4641d -42:06.288 [DEBUG] jones.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14042 -42:06.289 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:06.292 [INFO] jones.server.connect: initialized primary datacenter CA with provider: provider=consul -42:06.292 [INFO] jones.leader: started routine: routine="federation state anti-entropy" -42:06.292 [INFO] jones.leader: started routine: routine="federation state pruning" -42:06.292 [INFO] jones.leader: started routine: routine="intermediate cert renew watch" -42:06.292 [INFO] jones.leader: started routine: routine="CA root pruning" -42:06.292 [DEBUG] jones.server: Skipping self join check for node since the cluster is too small: node=Node-59f65b77-194f-185c-1548-1b1cd6d4641d -42:06.292 [INFO] jones.server: member joined, marking health alive: member=Node-59f65b77-194f-185c-1548-1b1cd6d4641d -42:06.293 [INFO] jones.server: federation state anti-entropy synced -42:06.308 [DEBUG] jones: Skipping remote check since it is managed automatically: check=serfHealth -42:06.309 [INFO] jones: Synced node info -42:06.395 [INFO] jones: Requesting shutdown -42:06.395 [INFO] jones.server: shutting down server -42:06.395 [DEBUG] jones.leader: stopping routine: routine="federation state anti-entropy" -42:06.395 [DEBUG] jones.leader: stopping routine: routine="federation state pruning" -42:06.395 [DEBUG] jones.leader: stopping routine: routine="intermediate cert renew watch" -42:06.395 [DEBUG] jones.leader: stopping routine: routine="CA root pruning" -42:06.395 [WARN] jones.server.serf.lan: serf: Shutdown without a Leave -42:06.395 [ERROR] jones.server: error performing anti-entropy sync of federation state: error="context canceled" -42:06.395 [DEBUG] jones.leader: stopped routine: routine="federation state anti-entropy" -42:06.396 [DEBUG] jones.leader: stopped routine: routine="federation state pruning" -42:06.396 [DEBUG] jones.leader: stopped routine: routine="intermediate cert renew watch" -42:06.396 [DEBUG] jones.leader: stopped routine: routine="CA root pruning" -42:06.396 [WARN] jones.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:06.396Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:06.396Z [INFO] agent.router.manager: shutting down -42:06.396 [INFO] jones: consul server down -42:06.396 [INFO] jones: shutdown complete -42:06.396 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:14037 network=tcp -42:06.396 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:14037 network=udp -42:06.396 [INFO] jones: Stopping server: protocol=HTTP address=127.0.0.1:14038 network=tcp -42:06.896 [INFO] jones: Waiting for endpoints to shut down -42:06.896 [INFO] jones: Endpoints down +2021-08-13T15:33:47.365Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:47.381Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:47.384 [INFO] jones.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4bc2f38a-70b4-17fd-9b13-f80cfa607d43 Address:127.0.0.1:25273}]" +33:47.384 [INFO] jones.server.raft: entering follower state: follower="Node at 127.0.0.1:25273 [Follower]" leader= +33:47.385 [INFO] jones.server.serf.wan: serf: EventMemberJoin: Node-4bc2f38a-70b4-17fd-9b13-f80cfa607d43.dc1 127.0.0.1 +33:47.386 [INFO] jones.server.serf.lan: serf: EventMemberJoin: Node-4bc2f38a-70b4-17fd-9b13-f80cfa607d43 127.0.0.1 +2021-08-13T15:33:47.386Z [INFO] agent.router: Initializing LAN area manager +33:47.386 [INFO] jones.server: Adding LAN server: server="Node-4bc2f38a-70b4-17fd-9b13-f80cfa607d43 (Addr: tcp/127.0.0.1:25273) (DC: dc1)" +33:47.386 [INFO] jones.server: Handled event for server in area: event=member-join server=Node-4bc2f38a-70b4-17fd-9b13-f80cfa607d43.dc1 area=wan +33:47.386 [INFO] jones: Started DNS server: address=127.0.0.1:25268 network=udp +33:47.386 [INFO] jones: Started DNS server: address=127.0.0.1:25268 network=tcp +33:47.387 [INFO] jones: Started HTTP server: address=127.0.0.1:25269 network=tcp +33:47.387 [INFO] jones: started state syncer +33:47.387 [INFO] jones: Started gRPC server: address=127.0.0.1:25274 network=tcp +33:47.437 [WARN] jones.server.raft: heartbeat timeout reached, starting election: last-leader= +33:47.437 [INFO] jones.server.raft: entering candidate state: node="Node at 127.0.0.1:25273 [Candidate]" term=2 +33:47.437 [DEBUG] jones.server.raft: votes: needed=1 +33:47.437 [DEBUG] jones.server.raft: vote granted: from=4bc2f38a-70b4-17fd-9b13-f80cfa607d43 term=2 tally=1 +33:47.437 [INFO] jones.server.raft: election won: tally=1 +33:47.437 [INFO] jones.server.raft: entering leader state: leader="Node at 127.0.0.1:25273 [Leader]" +33:47.437 [INFO] jones.server: cluster leadership acquired +33:47.438 [INFO] jones.server: New leader elected: payload=Node-4bc2f38a-70b4-17fd-9b13-f80cfa607d43 +33:47.438 [DEBUG] jones.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25273 +33:47.439 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:47.441 [INFO] jones.server.connect: initialized primary datacenter CA with provider: provider=consul +33:47.441 [INFO] jones.leader: started routine: routine="federation state anti-entropy" +33:47.441 [INFO] jones.leader: started routine: routine="federation state pruning" +33:47.442 [INFO] jones.leader: started routine: routine="intermediate cert renew watch" +33:47.442 [INFO] jones.leader: started routine: routine="CA root pruning" +33:47.442 [DEBUG] jones.server: Skipping self join check for node since the cluster is too small: node=Node-4bc2f38a-70b4-17fd-9b13-f80cfa607d43 +33:47.442 [INFO] jones.server: member joined, marking health alive: member=Node-4bc2f38a-70b4-17fd-9b13-f80cfa607d43 +33:47.443 [INFO] jones.server: federation state anti-entropy synced +33:47.634 [DEBUG] jones: Skipping remote check since it is managed automatically: check=serfHealth +33:47.634 [INFO] jones: Synced node info +33:47.760 [INFO] jones: Requesting shutdown +33:47.760 [INFO] jones.server: shutting down server +33:47.760 [DEBUG] jones.leader: stopping routine: routine="federation state anti-entropy" +33:47.760 [DEBUG] jones.leader: stopping routine: routine="federation state pruning" +33:47.760 [DEBUG] jones.leader: stopping routine: routine="intermediate cert renew watch" +33:47.760 [DEBUG] jones.leader: stopping routine: routine="CA root pruning" +33:47.760 [WARN] jones.server.serf.lan: serf: Shutdown without a Leave +33:47.761 [DEBUG] jones.leader: stopped routine: routine="intermediate cert renew watch" +33:47.761 [DEBUG] jones.leader: stopped routine: routine="federation state pruning" +33:47.761 [ERROR] jones.server: error performing anti-entropy sync of federation state: error="context canceled" +33:47.761 [DEBUG] jones.leader: stopped routine: routine="federation state anti-entropy" +33:47.761 [DEBUG] jones.leader: stopped routine: routine="CA root pruning" +33:47.761 [WARN] jones.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:47.761Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:47.761Z [INFO] agent.router.manager: shutting down +33:47.761 [INFO] jones: consul server down +33:47.761 [INFO] jones: shutdown complete +33:47.761 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:25268 network=tcp +33:47.761 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:25268 network=udp +33:47.761 [INFO] jones: Stopping server: protocol=HTTP address=127.0.0.1:25269 network=tcp +33:48.261 [INFO] jones: Waiting for endpoints to shut down +33:48.261 [INFO] jones: Endpoints down === RUN TestAgent_sidecarServiceFromNodeService/all_the_defaults [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:06.917Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:06.937Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:06.940 [INFO] jones.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4f543da1-0136-411b-9898-4e601014309c Address:127.0.0.1:14049}]" -42:06.941 [INFO] jones.server.serf.wan: serf: EventMemberJoin: Node-4f543da1-0136-411b-9898-4e601014309c.dc1 127.0.0.1 -42:06.942 [INFO] jones.server.raft: entering follower state: follower="Node at 127.0.0.1:14049 [Follower]" leader= -42:06.943 [INFO] jones.server.serf.lan: serf: EventMemberJoin: Node-4f543da1-0136-411b-9898-4e601014309c 127.0.0.1 -2022-09-15T21:42:06.943Z [INFO] agent.router: Initializing LAN area manager -42:06.943 [INFO] jones: Started DNS server: address=127.0.0.1:14044 network=udp -42:06.943 [INFO] jones.server: Adding LAN server: server="Node-4f543da1-0136-411b-9898-4e601014309c (Addr: tcp/127.0.0.1:14049) (DC: dc1)" -42:06.944 [INFO] jones.server: Handled event for server in area: event=member-join server=Node-4f543da1-0136-411b-9898-4e601014309c.dc1 area=wan -42:06.944 [INFO] jones: Started DNS server: address=127.0.0.1:14044 network=tcp -42:06.945 [INFO] jones: Started HTTP server: address=127.0.0.1:14045 network=tcp -42:06.945 [INFO] jones: started state syncer -42:06.945 [INFO] jones: Started gRPC server: address=127.0.0.1:14050 network=tcp -42:06.994 [WARN] jones.server.raft: heartbeat timeout reached, starting election: last-leader= -42:06.994 [INFO] jones.server.raft: entering candidate state: node="Node at 127.0.0.1:14049 [Candidate]" term=2 -42:06.995 [DEBUG] jones.server.raft: votes: needed=1 -42:06.995 [DEBUG] jones.server.raft: vote granted: from=4f543da1-0136-411b-9898-4e601014309c term=2 tally=1 -42:06.995 [INFO] jones.server.raft: election won: tally=1 -42:06.995 [INFO] jones.server.raft: entering leader state: leader="Node at 127.0.0.1:14049 [Leader]" -42:06.995 [INFO] jones.server: cluster leadership acquired -42:06.995 [DEBUG] jones.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14049 -42:06.995 [INFO] jones.server: New leader elected: payload=Node-4f543da1-0136-411b-9898-4e601014309c -42:06.996 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:06.999 [INFO] jones.server.connect: initialized primary datacenter CA with provider: provider=consul -42:06.999 [INFO] jones.leader: started routine: routine="federation state anti-entropy" -42:06.999 [INFO] jones.leader: started routine: routine="federation state pruning" -42:06.999 [INFO] jones.leader: started routine: routine="intermediate cert renew watch" -42:06.999 [INFO] jones.leader: started routine: routine="CA root pruning" -42:06.999 [DEBUG] jones.server: Skipping self join check for node since the cluster is too small: node=Node-4f543da1-0136-411b-9898-4e601014309c -42:06.999 [INFO] jones.server: member joined, marking health alive: member=Node-4f543da1-0136-411b-9898-4e601014309c -42:07.000 [INFO] jones.server: federation state anti-entropy synced -42:07.083 [DEBUG] jones: Skipping remote check since it is managed automatically: check=serfHealth -42:07.084 [INFO] jones: Synced node info -42:07.084 [DEBUG] jones: Node info in sync -42:07.210 [INFO] jones: Requesting shutdown -42:07.210 [INFO] jones.server: shutting down server -42:07.210 [DEBUG] jones.leader: stopping routine: routine="federation state anti-entropy" -42:07.210 [DEBUG] jones.leader: stopping routine: routine="federation state pruning" -42:07.210 [DEBUG] jones.leader: stopping routine: routine="intermediate cert renew watch" -42:07.210 [DEBUG] jones.leader: stopping routine: routine="CA root pruning" -42:07.210 [WARN] jones.server.serf.lan: serf: Shutdown without a Leave -42:07.210 [ERROR] jones.server: error performing anti-entropy sync of federation state: error="context canceled" -42:07.210 [DEBUG] jones.leader: stopped routine: routine="federation state anti-entropy" -42:07.211 [DEBUG] jones.leader: stopped routine: routine="federation state pruning" -42:07.211 [DEBUG] jones.leader: stopped routine: routine="intermediate cert renew watch" -42:07.211 [DEBUG] jones.leader: stopped routine: routine="CA root pruning" -42:07.211 [WARN] jones.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:07.211Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:07.211Z [INFO] agent.router.manager: shutting down -42:07.211 [INFO] jones: consul server down -42:07.211 [INFO] jones: shutdown complete -42:07.211 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:14044 network=tcp -42:07.211 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:14044 network=udp -42:07.211 [INFO] jones: Stopping server: protocol=HTTP address=127.0.0.1:14045 network=tcp -42:07.713 [INFO] jones: Waiting for endpoints to shut down -42:07.713 [INFO] jones: Endpoints down +2021-08-13T15:33:48.281Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:48.299Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:48.302 [INFO] jones.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ff8eaf8d-57bf-d578-0adb-df269b2e5e14 Address:127.0.0.1:25280}]" +33:48.302 [INFO] jones.server.raft: entering follower state: follower="Node at 127.0.0.1:25280 [Follower]" leader= +33:48.303 [INFO] jones.server.serf.wan: serf: EventMemberJoin: Node-ff8eaf8d-57bf-d578-0adb-df269b2e5e14.dc1 127.0.0.1 +33:48.304 [INFO] jones.server.serf.lan: serf: EventMemberJoin: Node-ff8eaf8d-57bf-d578-0adb-df269b2e5e14 127.0.0.1 +2021-08-13T15:33:48.304Z [INFO] agent.router: Initializing LAN area manager +33:48.304 [INFO] jones: Started DNS server: address=127.0.0.1:25275 network=udp +33:48.304 [INFO] jones.server: Adding LAN server: server="Node-ff8eaf8d-57bf-d578-0adb-df269b2e5e14 (Addr: tcp/127.0.0.1:25280) (DC: dc1)" +33:48.304 [INFO] jones.server: Handled event for server in area: event=member-join server=Node-ff8eaf8d-57bf-d578-0adb-df269b2e5e14.dc1 area=wan +33:48.305 [INFO] jones: Started DNS server: address=127.0.0.1:25275 network=tcp +33:48.306 [INFO] jones: Started HTTP server: address=127.0.0.1:25276 network=tcp +33:48.306 [INFO] jones: started state syncer +33:48.306 [INFO] jones: Started gRPC server: address=127.0.0.1:25281 network=tcp +33:48.348 [WARN] jones.server.raft: heartbeat timeout reached, starting election: last-leader= +33:48.348 [INFO] jones.server.raft: entering candidate state: node="Node at 127.0.0.1:25280 [Candidate]" term=2 +33:48.348 [DEBUG] jones.server.raft: votes: needed=1 +33:48.348 [DEBUG] jones.server.raft: vote granted: from=ff8eaf8d-57bf-d578-0adb-df269b2e5e14 term=2 tally=1 +33:48.348 [INFO] jones.server.raft: election won: tally=1 +33:48.348 [INFO] jones.server.raft: entering leader state: leader="Node at 127.0.0.1:25280 [Leader]" +33:48.349 [INFO] jones.server: cluster leadership acquired +33:48.349 [INFO] jones.server: New leader elected: payload=Node-ff8eaf8d-57bf-d578-0adb-df269b2e5e14 +33:48.349 [DEBUG] jones.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25280 +33:48.350 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:48.353 [INFO] jones.server.connect: initialized primary datacenter CA with provider: provider=consul +33:48.353 [INFO] jones.leader: started routine: routine="federation state anti-entropy" +33:48.353 [INFO] jones.leader: started routine: routine="federation state pruning" +33:48.353 [INFO] jones.leader: started routine: routine="intermediate cert renew watch" +33:48.353 [INFO] jones.leader: started routine: routine="CA root pruning" +33:48.353 [DEBUG] jones.server: Skipping self join check for node since the cluster is too small: node=Node-ff8eaf8d-57bf-d578-0adb-df269b2e5e14 +33:48.353 [INFO] jones.server: member joined, marking health alive: member=Node-ff8eaf8d-57bf-d578-0adb-df269b2e5e14 +33:48.354 [INFO] jones.server: federation state anti-entropy synced +33:48.455 [INFO] jones: Requesting shutdown +33:48.455 [INFO] jones.server: shutting down server +33:48.455 [DEBUG] jones.leader: stopping routine: routine="federation state anti-entropy" +33:48.455 [DEBUG] jones.leader: stopping routine: routine="federation state pruning" +33:48.455 [DEBUG] jones.leader: stopping routine: routine="intermediate cert renew watch" +33:48.455 [DEBUG] jones.leader: stopping routine: routine="CA root pruning" +33:48.455 [WARN] jones.server.serf.lan: serf: Shutdown without a Leave +33:48.455 [ERROR] jones.server: error performing anti-entropy sync of federation state: error="context canceled" +33:48.455 [DEBUG] jones.leader: stopped routine: routine="federation state anti-entropy" +33:48.455 [ERROR] jones.anti_entropy: failed to sync remote state: error="No cluster leader" +33:48.455 [DEBUG] jones.leader: stopped routine: routine="federation state pruning" +33:48.455 [DEBUG] jones.leader: stopped routine: routine="CA root pruning" +33:48.455 [DEBUG] jones.leader: stopped routine: routine="intermediate cert renew watch" +33:48.455 [WARN] jones.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:48.456Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:48.456Z [INFO] agent.router.manager: shutting down +33:48.456 [INFO] jones: consul server down +33:48.456 [INFO] jones: shutdown complete +33:48.456 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:25275 network=tcp +33:48.456 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:25275 network=udp +33:48.456 [INFO] jones: Stopping server: protocol=HTTP address=127.0.0.1:25276 network=tcp +33:48.956 [INFO] jones: Waiting for endpoints to shut down +33:48.956 [INFO] jones: Endpoints down === RUN TestAgent_sidecarServiceFromNodeService/all_the_allowed_overrides [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:07.734Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:07.759Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:07.762 [INFO] jones.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bde97f09-800a-21f6-fdab-8555ed480808 Address:127.0.0.1:14056}]" -42:07.763 [INFO] jones.server.serf.wan: serf: EventMemberJoin: Node-bde97f09-800a-21f6-fdab-8555ed480808.dc1 127.0.0.1 -42:07.764 [INFO] jones.server.serf.lan: serf: EventMemberJoin: Node-bde97f09-800a-21f6-fdab-8555ed480808 127.0.0.1 -2022-09-15T21:42:07.764Z [INFO] agent.router: Initializing LAN area manager -42:07.765 [INFO] jones: Started DNS server: address=127.0.0.1:14051 network=udp -42:07.765 [INFO] jones.server.raft: entering follower state: follower="Node at 127.0.0.1:14056 [Follower]" leader= -42:07.765 [INFO] jones.server: Adding LAN server: server="Node-bde97f09-800a-21f6-fdab-8555ed480808 (Addr: tcp/127.0.0.1:14056) (DC: dc1)" -42:07.765 [INFO] jones.server: Handled event for server in area: event=member-join server=Node-bde97f09-800a-21f6-fdab-8555ed480808.dc1 area=wan -42:07.765 [INFO] jones: Started DNS server: address=127.0.0.1:14051 network=tcp -42:07.766 [INFO] jones: Started HTTP server: address=127.0.0.1:14052 network=tcp -42:07.766 [INFO] jones: started state syncer -42:07.767 [INFO] jones: Started gRPC server: address=127.0.0.1:14057 network=tcp -42:07.806 [WARN] jones.server.raft: heartbeat timeout reached, starting election: last-leader= -42:07.806 [INFO] jones.server.raft: entering candidate state: node="Node at 127.0.0.1:14056 [Candidate]" term=2 -42:07.806 [DEBUG] jones.server.raft: votes: needed=1 -42:07.806 [DEBUG] jones.server.raft: vote granted: from=bde97f09-800a-21f6-fdab-8555ed480808 term=2 tally=1 -42:07.806 [INFO] jones.server.raft: election won: tally=1 -42:07.806 [INFO] jones.server.raft: entering leader state: leader="Node at 127.0.0.1:14056 [Leader]" -42:07.806 [INFO] jones.server: cluster leadership acquired -42:07.807 [DEBUG] jones.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14056 -42:07.808 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:07.810 [INFO] jones.server: New leader elected: payload=Node-bde97f09-800a-21f6-fdab-8555ed480808 -42:07.811 [INFO] jones.server.connect: initialized primary datacenter CA with provider: provider=consul -42:07.811 [INFO] jones.leader: started routine: routine="federation state anti-entropy" -42:07.811 [INFO] jones.leader: started routine: routine="federation state pruning" -42:07.811 [INFO] jones.leader: started routine: routine="intermediate cert renew watch" -42:07.811 [INFO] jones.leader: started routine: routine="CA root pruning" -42:07.811 [DEBUG] jones.server: Skipping self join check for node since the cluster is too small: node=Node-bde97f09-800a-21f6-fdab-8555ed480808 -42:07.811 [INFO] jones.server: member joined, marking health alive: member=Node-bde97f09-800a-21f6-fdab-8555ed480808 -42:07.813 [INFO] jones.server: federation state anti-entropy synced -42:07.921 [DEBUG] jones: Skipping remote check since it is managed automatically: check=serfHealth -42:07.922 [INFO] jones: Synced node info -42:08.019 [INFO] jones: Requesting shutdown -42:08.019 [INFO] jones.server: shutting down server -42:08.019 [DEBUG] jones.leader: stopping routine: routine="federation state pruning" -42:08.019 [DEBUG] jones.leader: stopping routine: routine="intermediate cert renew watch" -42:08.019 [DEBUG] jones.leader: stopping routine: routine="CA root pruning" -42:08.019 [DEBUG] jones.leader: stopping routine: routine="federation state anti-entropy" -42:08.019 [WARN] jones.server.serf.lan: serf: Shutdown without a Leave -42:08.020 [ERROR] jones.server: error performing anti-entropy sync of federation state: error="context canceled" -42:08.020 [DEBUG] jones.leader: stopped routine: routine="federation state anti-entropy" -42:08.020 [DEBUG] jones.leader: stopped routine: routine="federation state pruning" -42:08.020 [DEBUG] jones.leader: stopped routine: routine="intermediate cert renew watch" -42:08.020 [DEBUG] jones.leader: stopped routine: routine="CA root pruning" -42:08.020 [WARN] jones.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:08.020Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:08.020Z [INFO] agent.router.manager: shutting down -42:08.020 [INFO] jones: consul server down -42:08.020 [INFO] jones: shutdown complete -42:08.020 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:14051 network=tcp -42:08.020 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:14051 network=udp -42:08.020 [INFO] jones: Stopping server: protocol=HTTP address=127.0.0.1:14052 network=tcp -42:08.521 [INFO] jones: Waiting for endpoints to shut down -42:08.521 [INFO] jones: Endpoints down +2021-08-13T15:33:48.973Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:48.989Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:48.992 [INFO] jones.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a7d87944-f83e-c765-54b3-09e5bbae89b7 Address:127.0.0.1:25287}]" +33:48.992 [INFO] jones.server.raft: entering follower state: follower="Node at 127.0.0.1:25287 [Follower]" leader= +33:48.993 [INFO] jones.server.serf.wan: serf: EventMemberJoin: Node-a7d87944-f83e-c765-54b3-09e5bbae89b7.dc1 127.0.0.1 +33:48.993 [INFO] jones.server.serf.lan: serf: EventMemberJoin: Node-a7d87944-f83e-c765-54b3-09e5bbae89b7 127.0.0.1 +2021-08-13T15:33:48.993Z [INFO] agent.router: Initializing LAN area manager +33:48.994 [INFO] jones.server: Handled event for server in area: event=member-join server=Node-a7d87944-f83e-c765-54b3-09e5bbae89b7.dc1 area=wan +33:48.994 [INFO] jones.server: Adding LAN server: server="Node-a7d87944-f83e-c765-54b3-09e5bbae89b7 (Addr: tcp/127.0.0.1:25287) (DC: dc1)" +33:48.994 [INFO] jones: Started DNS server: address=127.0.0.1:25282 network=tcp +33:48.994 [INFO] jones: Started DNS server: address=127.0.0.1:25282 network=udp +33:48.995 [INFO] jones: Started HTTP server: address=127.0.0.1:25283 network=tcp +33:48.995 [INFO] jones: started state syncer +33:48.995 [INFO] jones: Started gRPC server: address=127.0.0.1:25288 network=tcp +33:49.036 [WARN] jones.server.raft: heartbeat timeout reached, starting election: last-leader= +33:49.036 [INFO] jones.server.raft: entering candidate state: node="Node at 127.0.0.1:25287 [Candidate]" term=2 +33:49.037 [DEBUG] jones.server.raft: votes: needed=1 +33:49.037 [DEBUG] jones.server.raft: vote granted: from=a7d87944-f83e-c765-54b3-09e5bbae89b7 term=2 tally=1 +33:49.037 [INFO] jones.server.raft: election won: tally=1 +33:49.037 [INFO] jones.server.raft: entering leader state: leader="Node at 127.0.0.1:25287 [Leader]" +33:49.037 [INFO] jones.server: cluster leadership acquired +33:49.037 [INFO] jones.server: New leader elected: payload=Node-a7d87944-f83e-c765-54b3-09e5bbae89b7 +33:49.037 [DEBUG] jones.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25287 +33:49.041 [INFO] jones: Synced node info +33:49.041 [DEBUG] jones: Node info in sync +33:49.041 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:49.045 [INFO] jones.server.connect: initialized primary datacenter CA with provider: provider=consul +33:49.045 [INFO] jones.leader: started routine: routine="federation state anti-entropy" +33:49.045 [INFO] jones.leader: started routine: routine="federation state pruning" +33:49.045 [INFO] jones.leader: started routine: routine="intermediate cert renew watch" +33:49.045 [INFO] jones.leader: started routine: routine="CA root pruning" +33:49.045 [DEBUG] jones.server: Skipping self join check for node since the cluster is too small: node=Node-a7d87944-f83e-c765-54b3-09e5bbae89b7 +33:49.045 [INFO] jones.server: member joined, marking health alive: member=Node-a7d87944-f83e-c765-54b3-09e5bbae89b7 +33:49.047 [INFO] jones.server: federation state anti-entropy synced +33:49.173 [INFO] jones: Requesting shutdown +33:49.173 [INFO] jones.server: shutting down server +33:49.173 [DEBUG] jones.leader: stopping routine: routine="federation state anti-entropy" +33:49.173 [DEBUG] jones.leader: stopping routine: routine="federation state pruning" +33:49.173 [DEBUG] jones.leader: stopping routine: routine="intermediate cert renew watch" +33:49.173 [DEBUG] jones.leader: stopping routine: routine="CA root pruning" +33:49.173 [WARN] jones.server.serf.lan: serf: Shutdown without a Leave +33:49.173 [DEBUG] jones.leader: stopped routine: routine="intermediate cert renew watch" +33:49.173 [ERROR] jones.server: error performing anti-entropy sync of federation state: error="context canceled" +33:49.174 [DEBUG] jones.leader: stopped routine: routine="federation state anti-entropy" +33:49.173 [DEBUG] jones.leader: stopped routine: routine="federation state pruning" +33:49.173 [DEBUG] jones.leader: stopped routine: routine="CA root pruning" +33:49.174 [WARN] jones.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:49.174Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:49.174Z [INFO] agent.router.manager: shutting down +33:49.174 [INFO] jones: consul server down +33:49.174 [INFO] jones: shutdown complete +33:49.174 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:25282 network=tcp +33:49.174 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:25282 network=udp +33:49.174 [INFO] jones: Stopping server: protocol=HTTP address=127.0.0.1:25283 network=tcp +33:49.674 [INFO] jones: Waiting for endpoints to shut down +33:49.674 [INFO] jones: Endpoints down === RUN TestAgent_sidecarServiceFromNodeService/no_auto_ports_available [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:08.573Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:08.593Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:08.596 [INFO] jones.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:337ad586-9599-4d58-f5e8-dbd5ce23fcdd Address:127.0.0.1:14063}]" -42:08.597 [INFO] jones.server.serf.wan: serf: EventMemberJoin: Node-337ad586-9599-4d58-f5e8-dbd5ce23fcdd.dc1 127.0.0.1 -42:08.598 [INFO] jones.server.raft: entering follower state: follower="Node at 127.0.0.1:14063 [Follower]" leader= -42:08.598 [INFO] jones.server.serf.lan: serf: EventMemberJoin: Node-337ad586-9599-4d58-f5e8-dbd5ce23fcdd 127.0.0.1 -2022-09-15T21:42:08.598Z [INFO] agent.router: Initializing LAN area manager -42:08.599 [INFO] jones: Started DNS server: address=127.0.0.1:14058 network=udp -42:08.599 [INFO] jones.server: Adding LAN server: server="Node-337ad586-9599-4d58-f5e8-dbd5ce23fcdd (Addr: tcp/127.0.0.1:14063) (DC: dc1)" -42:08.599 [INFO] jones.server: Handled event for server in area: event=member-join server=Node-337ad586-9599-4d58-f5e8-dbd5ce23fcdd.dc1 area=wan -42:08.599 [INFO] jones: Started DNS server: address=127.0.0.1:14058 network=tcp -42:08.600 [INFO] jones: Started HTTP server: address=127.0.0.1:14059 network=tcp -42:08.600 [INFO] jones: started state syncer -42:08.600 [INFO] jones: Started gRPC server: address=127.0.0.1:14064 network=tcp -42:08.664 [WARN] jones.server.raft: heartbeat timeout reached, starting election: last-leader= -42:08.664 [INFO] jones.server.raft: entering candidate state: node="Node at 127.0.0.1:14063 [Candidate]" term=2 -42:08.664 [DEBUG] jones.server.raft: votes: needed=1 -42:08.664 [DEBUG] jones.server.raft: vote granted: from=337ad586-9599-4d58-f5e8-dbd5ce23fcdd term=2 tally=1 -42:08.664 [INFO] jones.server.raft: election won: tally=1 -42:08.664 [INFO] jones.server.raft: entering leader state: leader="Node at 127.0.0.1:14063 [Leader]" -42:08.664 [INFO] jones.server: cluster leadership acquired -42:08.665 [DEBUG] jones.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14063 -42:08.666 [INFO] jones.server: New leader elected: payload=Node-337ad586-9599-4d58-f5e8-dbd5ce23fcdd -42:08.666 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:08.669 [INFO] jones.server.connect: initialized primary datacenter CA with provider: provider=consul -42:08.669 [INFO] jones.leader: started routine: routine="federation state anti-entropy" -42:08.669 [INFO] jones.leader: started routine: routine="federation state pruning" -42:08.669 [INFO] jones.leader: started routine: routine="intermediate cert renew watch" -42:08.669 [INFO] jones.leader: started routine: routine="CA root pruning" -42:08.669 [DEBUG] jones.server: Skipping self join check for node since the cluster is too small: node=Node-337ad586-9599-4d58-f5e8-dbd5ce23fcdd -42:08.669 [INFO] jones.server: member joined, marking health alive: member=Node-337ad586-9599-4d58-f5e8-dbd5ce23fcdd -42:08.675 [INFO] jones.server: federation state anti-entropy synced -42:08.723 [DEBUG] jones: Skipping remote check since it is managed automatically: check=serfHealth -42:08.723 [INFO] jones: Synced node info -42:08.952 [INFO] jones: Requesting shutdown -42:08.952 [INFO] jones.server: shutting down server -42:08.952 [DEBUG] jones.leader: stopping routine: routine="federation state anti-entropy" -42:08.952 [DEBUG] jones.leader: stopping routine: routine="federation state pruning" -42:08.952 [DEBUG] jones.leader: stopping routine: routine="intermediate cert renew watch" -42:08.952 [DEBUG] jones.leader: stopping routine: routine="CA root pruning" -42:08.952 [WARN] jones.server.serf.lan: serf: Shutdown without a Leave -42:08.953 [ERROR] jones.server: error performing anti-entropy sync of federation state: error="context canceled" -42:08.953 [DEBUG] jones.leader: stopped routine: routine="federation state anti-entropy" -42:08.953 [DEBUG] jones.leader: stopped routine: routine="federation state pruning" -42:08.953 [DEBUG] jones.leader: stopped routine: routine="intermediate cert renew watch" -42:08.953 [DEBUG] jones.leader: stopped routine: routine="CA root pruning" -42:08.953 [WARN] jones.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:08.953Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:08.953Z [INFO] agent.router.manager: shutting down -42:08.953 [INFO] jones: consul server down -42:08.953 [INFO] jones: shutdown complete -42:08.953 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:14058 network=tcp -42:08.953 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:14058 network=udp -42:08.953 [INFO] jones: Stopping server: protocol=HTTP address=127.0.0.1:14059 network=tcp -42:09.454 [INFO] jones: Waiting for endpoints to shut down -42:09.454 [INFO] jones: Endpoints down +2021-08-13T15:33:49.694Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:49.711Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:49.714 [INFO] jones.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:39ca5f77-15db-c26e-ce5c-bcc04fafa996 Address:127.0.0.1:25294}]" +33:49.714 [INFO] jones.server.raft: entering follower state: follower="Node at 127.0.0.1:25294 [Follower]" leader= +33:49.715 [INFO] jones.server.serf.wan: serf: EventMemberJoin: Node-39ca5f77-15db-c26e-ce5c-bcc04fafa996.dc1 127.0.0.1 +33:49.716 [INFO] jones.server.serf.lan: serf: EventMemberJoin: Node-39ca5f77-15db-c26e-ce5c-bcc04fafa996 127.0.0.1 +2021-08-13T15:33:49.716Z [INFO] agent.router: Initializing LAN area manager +33:49.716 [INFO] jones.server: Adding LAN server: server="Node-39ca5f77-15db-c26e-ce5c-bcc04fafa996 (Addr: tcp/127.0.0.1:25294) (DC: dc1)" +33:49.716 [INFO] jones.server: Handled event for server in area: event=member-join server=Node-39ca5f77-15db-c26e-ce5c-bcc04fafa996.dc1 area=wan +33:49.716 [INFO] jones: Started DNS server: address=127.0.0.1:25289 network=udp +33:49.716 [INFO] jones: Started DNS server: address=127.0.0.1:25289 network=tcp +33:49.717 [INFO] jones: Started HTTP server: address=127.0.0.1:25290 network=tcp +33:49.717 [INFO] jones: started state syncer +33:49.717 [INFO] jones: Started gRPC server: address=127.0.0.1:25295 network=tcp +33:49.769 [WARN] jones.server.raft: heartbeat timeout reached, starting election: last-leader= +33:49.769 [INFO] jones.server.raft: entering candidate state: node="Node at 127.0.0.1:25294 [Candidate]" term=2 +33:49.769 [DEBUG] jones.server.raft: votes: needed=1 +33:49.769 [DEBUG] jones.server.raft: vote granted: from=39ca5f77-15db-c26e-ce5c-bcc04fafa996 term=2 tally=1 +33:49.769 [INFO] jones.server.raft: election won: tally=1 +33:49.769 [INFO] jones.server.raft: entering leader state: leader="Node at 127.0.0.1:25294 [Leader]" +33:49.769 [INFO] jones.server: cluster leadership acquired +33:49.769 [INFO] jones.server: New leader elected: payload=Node-39ca5f77-15db-c26e-ce5c-bcc04fafa996 +33:49.770 [DEBUG] jones.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25294 +33:49.771 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:49.773 [INFO] jones.server.connect: initialized primary datacenter CA with provider: provider=consul +33:49.774 [INFO] jones.leader: started routine: routine="federation state anti-entropy" +33:49.774 [INFO] jones.leader: started routine: routine="federation state pruning" +33:49.774 [INFO] jones.leader: started routine: routine="intermediate cert renew watch" +33:49.774 [INFO] jones.leader: started routine: routine="CA root pruning" +33:49.774 [DEBUG] jones.server: Skipping self join check for node since the cluster is too small: node=Node-39ca5f77-15db-c26e-ce5c-bcc04fafa996 +33:49.774 [INFO] jones.server: member joined, marking health alive: member=Node-39ca5f77-15db-c26e-ce5c-bcc04fafa996 +33:49.774 [INFO] jones.server: federation state anti-entropy synced +33:50.032 [INFO] jones: Requesting shutdown +33:50.032 [INFO] jones.server: shutting down server +33:50.032 [DEBUG] jones.leader: stopping routine: routine="intermediate cert renew watch" +33:50.032 [DEBUG] jones.leader: stopping routine: routine="CA root pruning" +33:50.032 [DEBUG] jones.leader: stopping routine: routine="federation state anti-entropy" +33:50.032 [DEBUG] jones.leader: stopping routine: routine="federation state pruning" +33:50.032 [WARN] jones.server.serf.lan: serf: Shutdown without a Leave +33:50.032 [ERROR] jones.server: error performing anti-entropy sync of federation state: error="context canceled" +33:50.032 [DEBUG] jones.leader: stopped routine: routine="federation state anti-entropy" +33:50.032 [ERROR] jones.anti_entropy: failed to sync remote state: error="No cluster leader" +33:50.032 [DEBUG] jones.leader: stopped routine: routine="federation state pruning" +33:50.032 [DEBUG] jones.leader: stopped routine: routine="CA root pruning" +33:50.032 [DEBUG] jones.leader: stopped routine: routine="intermediate cert renew watch" +33:50.032 [WARN] jones.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:50.033Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:50.033Z [INFO] agent.router.manager: shutting down +33:50.033 [INFO] jones: consul server down +33:50.033 [INFO] jones: shutdown complete +33:50.033 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:25289 network=tcp +33:50.033 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:25289 network=udp +33:50.033 [INFO] jones: Stopping server: protocol=HTTP address=127.0.0.1:25290 network=tcp +33:50.533 [INFO] jones: Waiting for endpoints to shut down +33:50.533 [INFO] jones: Endpoints down === RUN TestAgent_sidecarServiceFromNodeService/auto_ports_disabled [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:09.510Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:09.560Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:09.567 [INFO] jones.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e0b220cd-913f-7090-f4ce-3d7c2e1a2609 Address:127.0.0.1:14070}]" -42:09.568 [INFO] jones.server.serf.wan: serf: EventMemberJoin: Node-e0b220cd-913f-7090-f4ce-3d7c2e1a2609.dc1 127.0.0.1 -42:09.569 [INFO] jones.server.serf.lan: serf: EventMemberJoin: Node-e0b220cd-913f-7090-f4ce-3d7c2e1a2609 127.0.0.1 -2022-09-15T21:42:09.569Z [INFO] agent.router: Initializing LAN area manager -42:09.572 [INFO] jones.server.raft: entering follower state: follower="Node at 127.0.0.1:14070 [Follower]" leader= -42:09.573 [INFO] jones.server: Handled event for server in area: event=member-join server=Node-e0b220cd-913f-7090-f4ce-3d7c2e1a2609.dc1 area=wan -42:09.573 [INFO] jones.server: Adding LAN server: server="Node-e0b220cd-913f-7090-f4ce-3d7c2e1a2609 (Addr: tcp/127.0.0.1:14070) (DC: dc1)" -42:09.574 [INFO] jones: Started DNS server: address=127.0.0.1:14065 network=tcp -42:09.574 [INFO] jones: Started DNS server: address=127.0.0.1:14065 network=udp -42:09.575 [INFO] jones: Started HTTP server: address=127.0.0.1:14066 network=tcp -42:09.575 [INFO] jones: started state syncer -42:09.575 [INFO] jones: Started gRPC server: address=127.0.0.1:14071 network=tcp -42:09.614 [WARN] jones.server.raft: heartbeat timeout reached, starting election: last-leader= -42:09.614 [INFO] jones.server.raft: entering candidate state: node="Node at 127.0.0.1:14070 [Candidate]" term=2 -42:09.614 [DEBUG] jones.server.raft: votes: needed=1 -42:09.614 [DEBUG] jones.server.raft: vote granted: from=e0b220cd-913f-7090-f4ce-3d7c2e1a2609 term=2 tally=1 -42:09.614 [INFO] jones.server.raft: election won: tally=1 -42:09.614 [INFO] jones.server.raft: entering leader state: leader="Node at 127.0.0.1:14070 [Leader]" -42:09.614 [INFO] jones.server: cluster leadership acquired -42:09.615 [DEBUG] jones.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14070 -42:09.616 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:09.618 [INFO] jones.server: New leader elected: payload=Node-e0b220cd-913f-7090-f4ce-3d7c2e1a2609 -42:09.623 [INFO] jones.server.connect: initialized primary datacenter CA with provider: provider=consul -42:09.623 [INFO] jones.leader: started routine: routine="federation state anti-entropy" -42:09.623 [INFO] jones.leader: started routine: routine="federation state pruning" -42:09.623 [INFO] jones.leader: started routine: routine="intermediate cert renew watch" -42:09.623 [INFO] jones.leader: started routine: routine="CA root pruning" -42:09.623 [DEBUG] jones.server: Skipping self join check for node since the cluster is too small: node=Node-e0b220cd-913f-7090-f4ce-3d7c2e1a2609 -42:09.623 [INFO] jones.server: member joined, marking health alive: member=Node-e0b220cd-913f-7090-f4ce-3d7c2e1a2609 -42:09.624 [INFO] jones.server: federation state anti-entropy synced -42:09.680 [DEBUG] jones: Skipping remote check since it is managed automatically: check=serfHealth -42:09.681 [INFO] jones: Synced node info -42:09.800 [INFO] jones: Requesting shutdown -42:09.801 [INFO] jones.server: shutting down server -42:09.801 [DEBUG] jones.leader: stopping routine: routine="federation state anti-entropy" -42:09.801 [DEBUG] jones.leader: stopping routine: routine="federation state pruning" -42:09.801 [DEBUG] jones.leader: stopping routine: routine="intermediate cert renew watch" -42:09.801 [DEBUG] jones.leader: stopping routine: routine="CA root pruning" -42:09.801 [WARN] jones.server.serf.lan: serf: Shutdown without a Leave -42:09.801 [ERROR] jones.server: error performing anti-entropy sync of federation state: error="context canceled" -42:09.801 [DEBUG] jones.leader: stopped routine: routine="federation state anti-entropy" -42:09.801 [DEBUG] jones.leader: stopped routine: routine="federation state pruning" -42:09.801 [DEBUG] jones.leader: stopped routine: routine="intermediate cert renew watch" -42:09.801 [DEBUG] jones.leader: stopped routine: routine="CA root pruning" -42:09.801 [WARN] jones.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:09.801Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:09.801Z [INFO] agent.router.manager: shutting down -42:09.801 [INFO] jones: consul server down -42:09.801 [INFO] jones: shutdown complete -42:09.801 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:14065 network=tcp -42:09.801 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:14065 network=udp -42:09.802 [INFO] jones: Stopping server: protocol=HTTP address=127.0.0.1:14066 network=tcp -42:10.302 [INFO] jones: Waiting for endpoints to shut down -42:10.302 [INFO] jones: Endpoints down +2021-08-13T15:33:50.552Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:50.569Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:50.572 [INFO] jones.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2f6859ca-c3c1-9615-509b-39dd21e5ad87 Address:127.0.0.1:25301}]" +33:50.572 [INFO] jones.server.raft: entering follower state: follower="Node at 127.0.0.1:25301 [Follower]" leader= +33:50.573 [INFO] jones.server.serf.wan: serf: EventMemberJoin: Node-2f6859ca-c3c1-9615-509b-39dd21e5ad87.dc1 127.0.0.1 +33:50.573 [INFO] jones.server.serf.lan: serf: EventMemberJoin: Node-2f6859ca-c3c1-9615-509b-39dd21e5ad87 127.0.0.1 +2021-08-13T15:33:50.574Z [INFO] agent.router: Initializing LAN area manager +33:50.574 [INFO] jones.server: Adding LAN server: server="Node-2f6859ca-c3c1-9615-509b-39dd21e5ad87 (Addr: tcp/127.0.0.1:25301) (DC: dc1)" +33:50.574 [INFO] jones.server: Handled event for server in area: event=member-join server=Node-2f6859ca-c3c1-9615-509b-39dd21e5ad87.dc1 area=wan +33:50.574 [INFO] jones: Started DNS server: address=127.0.0.1:25296 network=tcp +33:50.574 [INFO] jones: Started DNS server: address=127.0.0.1:25296 network=udp +33:50.575 [INFO] jones: Started HTTP server: address=127.0.0.1:25297 network=tcp +33:50.575 [INFO] jones: started state syncer +33:50.575 [INFO] jones: Started gRPC server: address=127.0.0.1:25302 network=tcp +33:50.612 [WARN] jones.server.raft: heartbeat timeout reached, starting election: last-leader= +33:50.612 [INFO] jones.server.raft: entering candidate state: node="Node at 127.0.0.1:25301 [Candidate]" term=2 +33:50.612 [DEBUG] jones.server.raft: votes: needed=1 +33:50.612 [DEBUG] jones.server.raft: vote granted: from=2f6859ca-c3c1-9615-509b-39dd21e5ad87 term=2 tally=1 +33:50.612 [INFO] jones.server.raft: election won: tally=1 +33:50.612 [INFO] jones.server.raft: entering leader state: leader="Node at 127.0.0.1:25301 [Leader]" +33:50.612 [INFO] jones.server: cluster leadership acquired +33:50.613 [INFO] jones.server: New leader elected: payload=Node-2f6859ca-c3c1-9615-509b-39dd21e5ad87 +33:50.613 [DEBUG] jones.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25301 +33:50.614 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:50.617 [INFO] jones.server.connect: initialized primary datacenter CA with provider: provider=consul +33:50.617 [INFO] jones.leader: started routine: routine="federation state anti-entropy" +33:50.617 [INFO] jones.leader: started routine: routine="federation state pruning" +33:50.617 [INFO] jones.leader: started routine: routine="intermediate cert renew watch" +33:50.617 [INFO] jones.leader: started routine: routine="CA root pruning" +33:50.617 [DEBUG] jones.server: Skipping self join check for node since the cluster is too small: node=Node-2f6859ca-c3c1-9615-509b-39dd21e5ad87 +33:50.617 [INFO] jones.server: member joined, marking health alive: member=Node-2f6859ca-c3c1-9615-509b-39dd21e5ad87 +33:50.618 [INFO] jones.server: federation state anti-entropy synced +33:50.655 [DEBUG] jones: Skipping remote check since it is managed automatically: check=serfHealth +33:50.655 [INFO] jones: Synced node info +33:50.655 [DEBUG] jones: Node info in sync +33:50.720 [INFO] jones: Requesting shutdown +33:50.720 [INFO] jones.server: shutting down server +33:50.720 [DEBUG] jones.leader: stopping routine: routine="CA root pruning" +33:50.721 [DEBUG] jones.leader: stopping routine: routine="federation state anti-entropy" +33:50.721 [DEBUG] jones.leader: stopping routine: routine="federation state pruning" +33:50.721 [DEBUG] jones.leader: stopping routine: routine="intermediate cert renew watch" +33:50.721 [WARN] jones.server.serf.lan: serf: Shutdown without a Leave +33:50.721 [ERROR] jones.server: error performing anti-entropy sync of federation state: error="context canceled" +33:50.721 [DEBUG] jones.leader: stopped routine: routine="federation state anti-entropy" +33:50.721 [DEBUG] jones.leader: stopped routine: routine="CA root pruning" +33:50.721 [DEBUG] jones.leader: stopped routine: routine="federation state pruning" +33:50.721 [DEBUG] jones.leader: stopped routine: routine="intermediate cert renew watch" +33:50.721 [WARN] jones.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:50.721Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:50.721Z [INFO] agent.router.manager: shutting down +33:50.721 [INFO] jones: consul server down +33:50.721 [INFO] jones: shutdown complete +33:50.721 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:25296 network=tcp +33:50.721 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:25296 network=udp +33:50.721 [INFO] jones: Stopping server: protocol=HTTP address=127.0.0.1:25297 network=tcp +33:51.221 [INFO] jones: Waiting for endpoints to shut down +33:51.221 [INFO] jones: Endpoints down === RUN TestAgent_sidecarServiceFromNodeService/inherit_tags_and_meta [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:10.322Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:10.342Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:10.345 [INFO] jones.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d5ace381-41e6-8b1d-d85e-8c751b1aed6f Address:127.0.0.1:14077}]" -42:10.346 [INFO] jones.server.raft: entering follower state: follower="Node at 127.0.0.1:14077 [Follower]" leader= -42:10.347 [INFO] jones.server.serf.wan: serf: EventMemberJoin: Node-d5ace381-41e6-8b1d-d85e-8c751b1aed6f.dc1 127.0.0.1 -42:10.348 [INFO] jones.server.serf.lan: serf: EventMemberJoin: Node-d5ace381-41e6-8b1d-d85e-8c751b1aed6f 127.0.0.1 -2022-09-15T21:42:10.348Z [INFO] agent.router: Initializing LAN area manager -42:10.348 [INFO] jones: Started DNS server: address=127.0.0.1:14072 network=udp -42:10.348 [INFO] jones.server: Adding LAN server: server="Node-d5ace381-41e6-8b1d-d85e-8c751b1aed6f (Addr: tcp/127.0.0.1:14077) (DC: dc1)" -42:10.349 [INFO] jones.server: Handled event for server in area: event=member-join server=Node-d5ace381-41e6-8b1d-d85e-8c751b1aed6f.dc1 area=wan -42:10.349 [INFO] jones: Started DNS server: address=127.0.0.1:14072 network=tcp -42:10.350 [INFO] jones: Started HTTP server: address=127.0.0.1:14073 network=tcp -42:10.350 [INFO] jones: started state syncer -42:10.350 [INFO] jones: Started gRPC server: address=127.0.0.1:14078 network=tcp -42:10.399 [WARN] jones.server.raft: heartbeat timeout reached, starting election: last-leader= -42:10.399 [INFO] jones.server.raft: entering candidate state: node="Node at 127.0.0.1:14077 [Candidate]" term=2 -42:10.400 [DEBUG] jones.server.raft: votes: needed=1 -42:10.400 [DEBUG] jones.server.raft: vote granted: from=d5ace381-41e6-8b1d-d85e-8c751b1aed6f term=2 tally=1 -42:10.400 [INFO] jones.server.raft: election won: tally=1 -42:10.400 [INFO] jones.server.raft: entering leader state: leader="Node at 127.0.0.1:14077 [Leader]" -42:10.400 [INFO] jones.server: cluster leadership acquired -42:10.400 [DEBUG] jones.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14077 -42:10.401 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:10.402 [INFO] jones.server: New leader elected: payload=Node-d5ace381-41e6-8b1d-d85e-8c751b1aed6f -42:10.404 [INFO] jones.server.connect: initialized primary datacenter CA with provider: provider=consul -42:10.404 [INFO] jones.leader: started routine: routine="federation state anti-entropy" -42:10.404 [INFO] jones.leader: started routine: routine="federation state pruning" -42:10.404 [INFO] jones.leader: started routine: routine="intermediate cert renew watch" -42:10.404 [INFO] jones.leader: started routine: routine="CA root pruning" -42:10.404 [DEBUG] jones.server: Skipping self join check for node since the cluster is too small: node=Node-d5ace381-41e6-8b1d-d85e-8c751b1aed6f -42:10.404 [INFO] jones.server: member joined, marking health alive: member=Node-d5ace381-41e6-8b1d-d85e-8c751b1aed6f -42:10.405 [INFO] jones.server: federation state anti-entropy synced -42:10.513 [INFO] jones: Requesting shutdown -42:10.513 [INFO] jones.server: shutting down server -42:10.513 [DEBUG] jones.leader: stopping routine: routine="intermediate cert renew watch" -42:10.513 [DEBUG] jones.leader: stopping routine: routine="CA root pruning" -42:10.513 [DEBUG] jones.leader: stopping routine: routine="federation state anti-entropy" -42:10.513 [DEBUG] jones.leader: stopping routine: routine="federation state pruning" -42:10.513 [WARN] jones.server.serf.lan: serf: Shutdown without a Leave -42:10.514 [ERROR] jones.server: error performing anti-entropy sync of federation state: error="context canceled" -42:10.514 [DEBUG] jones.leader: stopped routine: routine="federation state anti-entropy" -42:10.514 [ERROR] jones.anti_entropy: failed to sync remote state: error="No cluster leader" -42:10.514 [DEBUG] jones.leader: stopped routine: routine="CA root pruning" -42:10.514 [DEBUG] jones.leader: stopped routine: routine="intermediate cert renew watch" -42:10.514 [DEBUG] jones.leader: stopped routine: routine="federation state pruning" -42:10.514 [WARN] jones.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:10.514Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:10.515Z [INFO] agent.router.manager: shutting down -42:10.515 [INFO] jones: consul server down -42:10.515 [INFO] jones: shutdown complete -42:10.515 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:14072 network=tcp -42:10.515 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:14072 network=udp -42:10.515 [INFO] jones: Stopping server: protocol=HTTP address=127.0.0.1:14073 network=tcp -42:11.015 [INFO] jones: Waiting for endpoints to shut down -42:11.015 [INFO] jones: Endpoints down +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:33:51.238Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:51.254Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:51.257 [INFO] jones.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:118608b2-60e1-213d-8d37-cb060537b4bf Address:127.0.0.1:25308}]" +33:51.257 [INFO] jones.server.raft: entering follower state: follower="Node at 127.0.0.1:25308 [Follower]" leader= +33:51.258 [INFO] jones.server.serf.wan: serf: EventMemberJoin: Node-118608b2-60e1-213d-8d37-cb060537b4bf.dc1 127.0.0.1 +33:51.258 [INFO] jones.server.serf.lan: serf: EventMemberJoin: Node-118608b2-60e1-213d-8d37-cb060537b4bf 127.0.0.1 +2021-08-13T15:33:51.258Z [INFO] agent.router: Initializing LAN area manager +33:51.259 [INFO] jones.server: Adding LAN server: server="Node-118608b2-60e1-213d-8d37-cb060537b4bf (Addr: tcp/127.0.0.1:25308) (DC: dc1)" +33:51.259 [INFO] jones.server: Handled event for server in area: event=member-join server=Node-118608b2-60e1-213d-8d37-cb060537b4bf.dc1 area=wan +33:51.259 [INFO] jones: Started DNS server: address=127.0.0.1:25303 network=udp +33:51.259 [INFO] jones: Started DNS server: address=127.0.0.1:25303 network=tcp +33:51.260 [INFO] jones: Started HTTP server: address=127.0.0.1:25304 network=tcp +33:51.260 [INFO] jones: started state syncer +33:51.260 [INFO] jones: Started gRPC server: address=127.0.0.1:25309 network=tcp +33:51.309 [WARN] jones.server.raft: heartbeat timeout reached, starting election: last-leader= +33:51.309 [INFO] jones.server.raft: entering candidate state: node="Node at 127.0.0.1:25308 [Candidate]" term=2 +33:51.310 [DEBUG] jones.server.raft: votes: needed=1 +33:51.310 [DEBUG] jones.server.raft: vote granted: from=118608b2-60e1-213d-8d37-cb060537b4bf term=2 tally=1 +33:51.310 [INFO] jones.server.raft: election won: tally=1 +33:51.310 [INFO] jones.server.raft: entering leader state: leader="Node at 127.0.0.1:25308 [Leader]" +33:51.310 [INFO] jones.server: cluster leadership acquired +33:51.310 [INFO] jones.server: New leader elected: payload=Node-118608b2-60e1-213d-8d37-cb060537b4bf +33:51.310 [DEBUG] jones.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25308 +33:51.311 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:51.314 [INFO] jones: Requesting shutdown +33:51.314 [INFO] jones.server: shutting down server +33:51.314 [WARN] jones.server.serf.lan: serf: Shutdown without a Leave +33:51.314 [INFO] jones.server.connect: initialized primary datacenter CA with provider: provider=consul +33:51.314 [INFO] jones.leader: started routine: routine="federation state anti-entropy" +33:51.314 [INFO] jones.leader: started routine: routine="federation state pruning" +33:51.314 [INFO] jones.leader: started routine: routine="intermediate cert renew watch" +33:51.314 [INFO] jones.leader: started routine: routine="CA root pruning" +33:51.314 [ERROR] jones.anti_entropy: failed to sync remote state: error="No cluster leader" +33:51.314 [DEBUG] jones.server: Skipping self join check for node since the cluster is too small: node=Node-118608b2-60e1-213d-8d37-cb060537b4bf +33:51.314 [INFO] jones.server: member joined, marking health alive: member=Node-118608b2-60e1-213d-8d37-cb060537b4bf +33:51.314 [WARN] jones.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:51.314Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:51.314Z [INFO] agent.router.manager: shutting down +33:51.314 [ERROR] jones.server: failed to reconcile member: member="{Node-118608b2-60e1-213d-8d37-cb060537b4bf 127.0.0.1 25306 map[acls:0 bootstrap:1 build:1.8.7: dc:dc1 ft_fs:1 id:118608b2-60e1-213d-8d37-cb060537b4bf port:25308 raft_vsn:3 role:consul segment: vsn:2 vsn_max:3 vsn_min:2 wan_join_port:25307] alive 1 5 2 2 5 4}" error="leadership lost while committing log" +33:51.315 [INFO] jones: consul server down +33:51.315 [DEBUG] jones.leader: stopping routine: routine="federation state anti-entropy" +33:51.315 [DEBUG] jones.leader: stopping routine: routine="federation state pruning" +33:51.315 [DEBUG] jones.leader: stopping routine: routine="intermediate cert renew watch" +33:51.315 [DEBUG] jones.leader: stopped routine: routine="federation state anti-entropy" +33:51.315 [DEBUG] jones.leader: stopping routine: routine="CA root pruning" +33:51.315 [INFO] jones: shutdown complete +33:51.315 [DEBUG] jones.leader: stopped routine: routine="CA root pruning" +33:51.315 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:25303 network=tcp +33:51.315 [DEBUG] jones.leader: stopped routine: routine="intermediate cert renew watch" +33:51.315 [DEBUG] jones.leader: stopped routine: routine="federation state pruning" +33:51.315 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:25303 network=udp +33:51.315 [INFO] jones: Stopping server: protocol=HTTP address=127.0.0.1:25304 network=tcp +33:51.815 [INFO] jones: Waiting for endpoints to shut down +33:51.815 [INFO] jones: Endpoints down === RUN TestAgent_sidecarServiceFromNodeService/invalid_check_type [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:11.034Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:11.091Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:11.098 [INFO] jones.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e99ebac0-9410-a5a4-16c9-c386dfc5cf87 Address:127.0.0.1:13064}]" -42:11.099 [INFO] jones.server.raft: entering follower state: follower="Node at 127.0.0.1:13064 [Follower]" leader= -42:11.099 [INFO] jones.server.serf.wan: serf: EventMemberJoin: Node-e99ebac0-9410-a5a4-16c9-c386dfc5cf87.dc1 127.0.0.1 -42:11.106 [INFO] jones.server.serf.lan: serf: EventMemberJoin: Node-e99ebac0-9410-a5a4-16c9-c386dfc5cf87 127.0.0.1 -2022-09-15T21:42:11.106Z [INFO] agent.router: Initializing LAN area manager -42:11.106 [INFO] jones: Started DNS server: address=127.0.0.1:14079 network=udp -42:11.107 [INFO] jones.server: Handled event for server in area: event=member-join server=Node-e99ebac0-9410-a5a4-16c9-c386dfc5cf87.dc1 area=wan -42:11.107 [INFO] jones.server: Adding LAN server: server="Node-e99ebac0-9410-a5a4-16c9-c386dfc5cf87 (Addr: tcp/127.0.0.1:13064) (DC: dc1)" -42:11.107 [INFO] jones: Started DNS server: address=127.0.0.1:14079 network=tcp -42:11.108 [INFO] jones: Started HTTP server: address=127.0.0.1:14080 network=tcp -42:11.108 [INFO] jones: started state syncer -42:11.108 [INFO] jones: Started gRPC server: address=127.0.0.1:13065 network=tcp -42:11.152 [WARN] jones.server.raft: heartbeat timeout reached, starting election: last-leader= -42:11.152 [INFO] jones.server.raft: entering candidate state: node="Node at 127.0.0.1:13064 [Candidate]" term=2 -42:11.153 [DEBUG] jones.server.raft: votes: needed=1 -42:11.153 [DEBUG] jones.server.raft: vote granted: from=e99ebac0-9410-a5a4-16c9-c386dfc5cf87 term=2 tally=1 -42:11.153 [INFO] jones.server.raft: election won: tally=1 -42:11.153 [INFO] jones.server.raft: entering leader state: leader="Node at 127.0.0.1:13064 [Leader]" -42:11.153 [INFO] jones.server: cluster leadership acquired -42:11.153 [DEBUG] jones.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13064 -42:11.154 [INFO] jones.server: New leader elected: payload=Node-e99ebac0-9410-a5a4-16c9-c386dfc5cf87 -42:11.154 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:11.157 [INFO] jones.server.connect: initialized primary datacenter CA with provider: provider=consul -42:11.157 [INFO] jones.leader: started routine: routine="federation state anti-entropy" -42:11.157 [INFO] jones.leader: started routine: routine="federation state pruning" -42:11.157 [INFO] jones.leader: started routine: routine="intermediate cert renew watch" -42:11.157 [INFO] jones.leader: started routine: routine="CA root pruning" -42:11.157 [DEBUG] jones.server: Skipping self join check for node since the cluster is too small: node=Node-e99ebac0-9410-a5a4-16c9-c386dfc5cf87 -42:11.157 [INFO] jones.server: member joined, marking health alive: member=Node-e99ebac0-9410-a5a4-16c9-c386dfc5cf87 -42:11.158 [INFO] jones.server: federation state anti-entropy synced -42:11.176 [INFO] jones: Requesting shutdown -42:11.176 [INFO] jones.server: shutting down server -42:11.176 [DEBUG] jones.leader: stopping routine: routine="federation state anti-entropy" -42:11.176 [DEBUG] jones.leader: stopping routine: routine="federation state pruning" -42:11.176 [DEBUG] jones.leader: stopping routine: routine="intermediate cert renew watch" -42:11.176 [DEBUG] jones.leader: stopping routine: routine="CA root pruning" -42:11.176 [WARN] jones.server.serf.lan: serf: Shutdown without a Leave -42:11.176 [ERROR] jones.server: error performing anti-entropy sync of federation state: error="context canceled" -42:11.176 [DEBUG] jones.leader: stopped routine: routine="federation state anti-entropy" -42:11.176 [ERROR] jones.anti_entropy: failed to sync remote state: error="No cluster leader" -42:11.176 [DEBUG] jones.leader: stopped routine: routine="federation state pruning" -42:11.177 [DEBUG] jones.leader: stopped routine: routine="intermediate cert renew watch" -42:11.177 [DEBUG] jones.leader: stopped routine: routine="CA root pruning" -42:11.177 [WARN] jones.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:11.177Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:11.177Z [INFO] agent.router.manager: shutting down -42:11.177 [INFO] jones: consul server down -42:11.177 [INFO] jones: shutdown complete -42:11.177 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:14079 network=tcp -42:11.177 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:14079 network=udp -42:11.177 [INFO] jones: Stopping server: protocol=HTTP address=127.0.0.1:14080 network=tcp -42:11.677 [INFO] jones: Waiting for endpoints to shut down -42:11.677 [INFO] jones: Endpoints down +2021-08-13T15:33:51.838Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:51.864Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:51.866 [INFO] jones.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a992f0b0-a299-1f96-f052-ba62316765ff Address:127.0.0.1:24295}]" +33:51.866 [INFO] jones.server.raft: entering follower state: follower="Node at 127.0.0.1:24295 [Follower]" leader= +33:51.867 [INFO] jones.server.serf.wan: serf: EventMemberJoin: Node-a992f0b0-a299-1f96-f052-ba62316765ff.dc1 127.0.0.1 +33:51.868 [INFO] jones.server.serf.lan: serf: EventMemberJoin: Node-a992f0b0-a299-1f96-f052-ba62316765ff 127.0.0.1 +2021-08-13T15:33:51.868Z [INFO] agent.router: Initializing LAN area manager +33:51.869 [INFO] jones.server: Adding LAN server: server="Node-a992f0b0-a299-1f96-f052-ba62316765ff (Addr: tcp/127.0.0.1:24295) (DC: dc1)" +33:51.869 [INFO] jones.server: Handled event for server in area: event=member-join server=Node-a992f0b0-a299-1f96-f052-ba62316765ff.dc1 area=wan +33:51.869 [INFO] jones: Started DNS server: address=127.0.0.1:25310 network=udp +33:51.869 [INFO] jones: Started DNS server: address=127.0.0.1:25310 network=tcp +33:51.870 [INFO] jones: Started HTTP server: address=127.0.0.1:25311 network=tcp +33:51.870 [INFO] jones: started state syncer +33:51.870 [INFO] jones: Started gRPC server: address=127.0.0.1:24296 network=tcp +33:51.907 [WARN] jones.server.raft: heartbeat timeout reached, starting election: last-leader= +33:51.907 [INFO] jones.server.raft: entering candidate state: node="Node at 127.0.0.1:24295 [Candidate]" term=2 +33:51.908 [DEBUG] jones.server.raft: votes: needed=1 +33:51.908 [DEBUG] jones.server.raft: vote granted: from=a992f0b0-a299-1f96-f052-ba62316765ff term=2 tally=1 +33:51.908 [INFO] jones.server.raft: election won: tally=1 +33:51.908 [INFO] jones.server.raft: entering leader state: leader="Node at 127.0.0.1:24295 [Leader]" +33:51.908 [INFO] jones.server: cluster leadership acquired +33:51.908 [DEBUG] jones.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24295 +33:51.909 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:51.911 [INFO] jones.server: New leader elected: payload=Node-a992f0b0-a299-1f96-f052-ba62316765ff +33:51.912 [INFO] jones.server.connect: initialized primary datacenter CA with provider: provider=consul +33:51.912 [INFO] jones.leader: started routine: routine="federation state anti-entropy" +33:51.912 [INFO] jones.leader: started routine: routine="federation state pruning" +33:51.912 [INFO] jones.leader: started routine: routine="intermediate cert renew watch" +33:51.912 [INFO] jones.leader: started routine: routine="CA root pruning" +33:51.912 [DEBUG] jones.server: Skipping self join check for node since the cluster is too small: node=Node-a992f0b0-a299-1f96-f052-ba62316765ff +33:51.912 [INFO] jones.server: member joined, marking health alive: member=Node-a992f0b0-a299-1f96-f052-ba62316765ff +33:51.913 [INFO] jones.server: federation state anti-entropy synced +33:52.258 [DEBUG] jones: Skipping remote check since it is managed automatically: check=serfHealth +33:52.259 [INFO] jones: Synced node info +33:52.259 [DEBUG] jones: Node info in sync +33:52.265 [INFO] jones: Requesting shutdown +33:52.266 [INFO] jones.server: shutting down server +33:52.266 [DEBUG] jones.leader: stopping routine: routine="intermediate cert renew watch" +33:52.266 [DEBUG] jones.leader: stopping routine: routine="CA root pruning" +33:52.266 [DEBUG] jones.leader: stopping routine: routine="federation state anti-entropy" +33:52.266 [DEBUG] jones.leader: stopping routine: routine="federation state pruning" +33:52.266 [WARN] jones.server.serf.lan: serf: Shutdown without a Leave +33:52.266 [ERROR] jones.server: error performing anti-entropy sync of federation state: error="context canceled" +33:52.266 [DEBUG] jones.leader: stopped routine: routine="federation state anti-entropy" +33:52.266 [DEBUG] jones.leader: stopped routine: routine="CA root pruning" +33:52.266 [DEBUG] jones.leader: stopped routine: routine="federation state pruning" +33:52.266 [DEBUG] jones.leader: stopped routine: routine="intermediate cert renew watch" +33:52.266 [WARN] jones.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:52.266Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:52.266Z [INFO] agent.router.manager: shutting down +33:52.266 [INFO] jones: consul server down +33:52.266 [INFO] jones: shutdown complete +33:52.266 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:25310 network=tcp +33:52.266 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:25310 network=udp +33:52.266 [INFO] jones: Stopping server: protocol=HTTP address=127.0.0.1:25311 network=tcp +33:52.766 [INFO] jones: Waiting for endpoints to shut down +33:52.766 [INFO] jones: Endpoints down === RUN TestAgent_sidecarServiceFromNodeService/invalid_meta [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:11.697Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:11.715Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:11.718 [INFO] jones.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c7391d03-611d-91a4-c663-b2953dd75fc7 Address:127.0.0.1:13078}]" -42:11.718 [INFO] jones.server.raft: entering follower state: follower="Node at 127.0.0.1:13078 [Follower]" leader= -42:11.719 [INFO] jones.server.serf.wan: serf: EventMemberJoin: Node-c7391d03-611d-91a4-c663-b2953dd75fc7.dc1 127.0.0.1 -42:11.720 [INFO] jones.server.serf.lan: serf: EventMemberJoin: Node-c7391d03-611d-91a4-c663-b2953dd75fc7 127.0.0.1 -2022-09-15T21:42:11.720Z [INFO] agent.router: Initializing LAN area manager -42:11.720 [INFO] jones: Started DNS server: address=127.0.0.1:13066 network=udp -42:11.720 [INFO] jones.server: Adding LAN server: server="Node-c7391d03-611d-91a4-c663-b2953dd75fc7 (Addr: tcp/127.0.0.1:13078) (DC: dc1)" -42:11.720 [INFO] jones.server: Handled event for server in area: event=member-join server=Node-c7391d03-611d-91a4-c663-b2953dd75fc7.dc1 area=wan -42:11.720 [INFO] jones: Started DNS server: address=127.0.0.1:13066 network=tcp -42:11.721 [INFO] jones: Started HTTP server: address=127.0.0.1:13067 network=tcp -42:11.721 [INFO] jones: started state syncer -42:11.721 [INFO] jones: Started gRPC server: address=127.0.0.1:13079 network=tcp -42:11.767 [WARN] jones.server.raft: heartbeat timeout reached, starting election: last-leader= -42:11.767 [INFO] jones.server.raft: entering candidate state: node="Node at 127.0.0.1:13078 [Candidate]" term=2 -42:11.768 [DEBUG] jones.server.raft: votes: needed=1 -42:11.768 [DEBUG] jones.server.raft: vote granted: from=c7391d03-611d-91a4-c663-b2953dd75fc7 term=2 tally=1 -42:11.768 [INFO] jones.server.raft: election won: tally=1 -42:11.768 [INFO] jones.server.raft: entering leader state: leader="Node at 127.0.0.1:13078 [Leader]" -42:11.768 [INFO] jones.server: cluster leadership acquired -42:11.768 [DEBUG] jones.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13078 -42:11.769 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:11.770 [INFO] jones.server: New leader elected: payload=Node-c7391d03-611d-91a4-c663-b2953dd75fc7 -42:11.772 [INFO] jones.server.connect: initialized primary datacenter CA with provider: provider=consul -42:11.772 [INFO] jones.leader: started routine: routine="federation state anti-entropy" -42:11.772 [INFO] jones.leader: started routine: routine="federation state pruning" -42:11.772 [INFO] jones.leader: started routine: routine="intermediate cert renew watch" -42:11.772 [INFO] jones.leader: started routine: routine="CA root pruning" -42:11.772 [DEBUG] jones.server: Skipping self join check for node since the cluster is too small: node=Node-c7391d03-611d-91a4-c663-b2953dd75fc7 -42:11.772 [INFO] jones.server: member joined, marking health alive: member=Node-c7391d03-611d-91a4-c663-b2953dd75fc7 -42:11.773 [INFO] jones.server: federation state anti-entropy synced -42:11.950 [DEBUG] jones: Skipping remote check since it is managed automatically: check=serfHealth -42:11.951 [INFO] jones: Synced node info -42:12.058 [INFO] jones: Requesting shutdown -42:12.058 [INFO] jones.server: shutting down server -42:12.058 [DEBUG] jones.leader: stopping routine: routine="federation state anti-entropy" -42:12.058 [DEBUG] jones.leader: stopping routine: routine="federation state pruning" -42:12.058 [DEBUG] jones.leader: stopping routine: routine="intermediate cert renew watch" -42:12.058 [DEBUG] jones.leader: stopping routine: routine="CA root pruning" -42:12.058 [WARN] jones.server.serf.lan: serf: Shutdown without a Leave -42:12.058 [ERROR] jones.server: error performing anti-entropy sync of federation state: error="context canceled" -42:12.058 [DEBUG] jones.leader: stopped routine: routine="federation state anti-entropy" -42:12.058 [DEBUG] jones.leader: stopped routine: routine="federation state pruning" -42:12.058 [DEBUG] jones.leader: stopped routine: routine="intermediate cert renew watch" -42:12.058 [DEBUG] jones.leader: stopped routine: routine="CA root pruning" -42:12.059 [WARN] jones.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:12.059Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:12.059Z [INFO] agent.router.manager: shutting down -42:12.059 [INFO] jones: consul server down -42:12.059 [INFO] jones: shutdown complete -42:12.059 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:13066 network=tcp -42:12.059 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:13066 network=udp -42:12.059 [INFO] jones: Stopping server: protocol=HTTP address=127.0.0.1:13067 network=tcp -42:12.559 [INFO] jones: Waiting for endpoints to shut down -42:12.559 [INFO] jones: Endpoints down +2021-08-13T15:33:52.783Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:52.799Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:52.802 [INFO] jones.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fec59dc8-9789-7398-a3db-2e1847e72b70 Address:127.0.0.1:24302}]" +33:52.802 [INFO] jones.server.raft: entering follower state: follower="Node at 127.0.0.1:24302 [Follower]" leader= +33:52.803 [INFO] jones.server.serf.wan: serf: EventMemberJoin: Node-fec59dc8-9789-7398-a3db-2e1847e72b70.dc1 127.0.0.1 +33:52.804 [INFO] jones.server.serf.lan: serf: EventMemberJoin: Node-fec59dc8-9789-7398-a3db-2e1847e72b70 127.0.0.1 +2021-08-13T15:33:52.804Z [INFO] agent.router: Initializing LAN area manager +33:52.804 [INFO] jones.server: Adding LAN server: server="Node-fec59dc8-9789-7398-a3db-2e1847e72b70 (Addr: tcp/127.0.0.1:24302) (DC: dc1)" +33:52.804 [INFO] jones.server: Handled event for server in area: event=member-join server=Node-fec59dc8-9789-7398-a3db-2e1847e72b70.dc1 area=wan +33:52.804 [INFO] jones: Started DNS server: address=127.0.0.1:24297 network=tcp +33:52.804 [INFO] jones: Started DNS server: address=127.0.0.1:24297 network=udp +33:52.805 [INFO] jones: Started HTTP server: address=127.0.0.1:24298 network=tcp +33:52.805 [INFO] jones: started state syncer +33:52.805 [INFO] jones: Started gRPC server: address=127.0.0.1:24303 network=tcp +33:52.847 [WARN] jones.server.raft: heartbeat timeout reached, starting election: last-leader= +33:52.847 [INFO] jones.server.raft: entering candidate state: node="Node at 127.0.0.1:24302 [Candidate]" term=2 +33:52.847 [DEBUG] jones.server.raft: votes: needed=1 +33:52.847 [DEBUG] jones.server.raft: vote granted: from=fec59dc8-9789-7398-a3db-2e1847e72b70 term=2 tally=1 +33:52.847 [INFO] jones.server.raft: election won: tally=1 +33:52.847 [INFO] jones.server.raft: entering leader state: leader="Node at 127.0.0.1:24302 [Leader]" +33:52.847 [INFO] jones.server: cluster leadership acquired +33:52.847 [INFO] jones.server: New leader elected: payload=Node-fec59dc8-9789-7398-a3db-2e1847e72b70 +33:52.847 [DEBUG] jones.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24302 +33:52.849 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:52.851 [INFO] jones.server.connect: initialized primary datacenter CA with provider: provider=consul +33:52.851 [INFO] jones.leader: started routine: routine="federation state anti-entropy" +33:52.851 [INFO] jones.leader: started routine: routine="federation state pruning" +33:52.851 [INFO] jones.leader: started routine: routine="intermediate cert renew watch" +33:52.851 [INFO] jones.leader: started routine: routine="CA root pruning" +33:52.852 [DEBUG] jones.server: Skipping self join check for node since the cluster is too small: node=Node-fec59dc8-9789-7398-a3db-2e1847e72b70 +33:52.852 [INFO] jones.server: member joined, marking health alive: member=Node-fec59dc8-9789-7398-a3db-2e1847e72b70 +33:52.852 [INFO] jones.server: federation state anti-entropy synced +33:53.086 [INFO] jones: Requesting shutdown +33:53.086 [INFO] jones.server: shutting down server +33:53.086 [DEBUG] jones.leader: stopping routine: routine="federation state anti-entropy" +33:53.086 [DEBUG] jones.leader: stopping routine: routine="federation state pruning" +33:53.086 [DEBUG] jones.leader: stopping routine: routine="intermediate cert renew watch" +33:53.086 [DEBUG] jones.leader: stopping routine: routine="CA root pruning" +33:53.086 [WARN] jones.server.serf.lan: serf: Shutdown without a Leave +33:53.086 [ERROR] jones.server: error performing anti-entropy sync of federation state: error="context canceled" +33:53.086 [DEBUG] jones.leader: stopped routine: routine="federation state anti-entropy" +33:53.086 [DEBUG] jones.leader: stopped routine: routine="federation state pruning" +33:53.086 [ERROR] jones.anti_entropy: failed to sync remote state: error="No cluster leader" +33:53.086 [DEBUG] jones.leader: stopped routine: routine="intermediate cert renew watch" +33:53.086 [DEBUG] jones.leader: stopped routine: routine="CA root pruning" +33:53.086 [WARN] jones.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:53.086Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:53.086Z [INFO] agent.router.manager: shutting down +33:53.087 [INFO] jones: consul server down +33:53.087 [INFO] jones: shutdown complete +33:53.087 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:24297 network=tcp +33:53.087 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:24297 network=udp +33:53.087 [INFO] jones: Stopping server: protocol=HTTP address=127.0.0.1:24298 network=tcp +33:53.587 [INFO] jones: Waiting for endpoints to shut down +33:53.587 [INFO] jones: Endpoints down === RUN TestAgent_sidecarServiceFromNodeService/re-registering_same_sidecar_with_no_port_should_pick_same_one [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:12.586Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:12.604Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:12.607 [INFO] jones.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3ea03d64-5fe9-978d-c1b7-cd122ce7dce9 Address:127.0.0.1:13071}]" -42:12.608 [INFO] jones.server.serf.wan: serf: EventMemberJoin: Node-3ea03d64-5fe9-978d-c1b7-cd122ce7dce9.dc1 127.0.0.1 -42:12.609 [INFO] jones.server.serf.lan: serf: EventMemberJoin: Node-3ea03d64-5fe9-978d-c1b7-cd122ce7dce9 127.0.0.1 -2022-09-15T21:42:12.609Z [INFO] agent.router: Initializing LAN area manager -42:12.610 [INFO] jones.server.raft: entering follower state: follower="Node at 127.0.0.1:13071 [Follower]" leader= -42:12.610 [INFO] jones.server: Adding LAN server: server="Node-3ea03d64-5fe9-978d-c1b7-cd122ce7dce9 (Addr: tcp/127.0.0.1:13071) (DC: dc1)" -42:12.610 [INFO] jones.server: Handled event for server in area: event=member-join server=Node-3ea03d64-5fe9-978d-c1b7-cd122ce7dce9.dc1 area=wan -42:12.610 [INFO] jones: Started DNS server: address=127.0.0.1:13080 network=udp -42:12.610 [INFO] jones: Started DNS server: address=127.0.0.1:13080 network=tcp -42:12.611 [INFO] jones: Started HTTP server: address=127.0.0.1:13081 network=tcp -42:12.611 [INFO] jones: started state syncer -42:12.611 [INFO] jones: Started gRPC server: address=127.0.0.1:13072 network=tcp -42:12.654 [WARN] jones.server.raft: heartbeat timeout reached, starting election: last-leader= -42:12.654 [INFO] jones.server.raft: entering candidate state: node="Node at 127.0.0.1:13071 [Candidate]" term=2 -42:12.655 [DEBUG] jones.server.raft: votes: needed=1 -42:12.655 [DEBUG] jones.server.raft: vote granted: from=3ea03d64-5fe9-978d-c1b7-cd122ce7dce9 term=2 tally=1 -42:12.655 [INFO] jones.server.raft: election won: tally=1 -42:12.655 [INFO] jones.server.raft: entering leader state: leader="Node at 127.0.0.1:13071 [Leader]" -42:12.655 [INFO] jones.server: cluster leadership acquired -42:12.655 [INFO] jones.server: New leader elected: payload=Node-3ea03d64-5fe9-978d-c1b7-cd122ce7dce9 -42:12.655 [DEBUG] jones.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13071 -42:12.657 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:12.659 [INFO] jones.server.connect: initialized primary datacenter CA with provider: provider=consul -42:12.659 [INFO] jones.leader: started routine: routine="federation state anti-entropy" -42:12.659 [INFO] jones.leader: started routine: routine="federation state pruning" -42:12.659 [INFO] jones.leader: started routine: routine="intermediate cert renew watch" -42:12.659 [INFO] jones.leader: started routine: routine="CA root pruning" -42:12.659 [DEBUG] jones.server: Skipping self join check for node since the cluster is too small: node=Node-3ea03d64-5fe9-978d-c1b7-cd122ce7dce9 -42:12.659 [INFO] jones.server: member joined, marking health alive: member=Node-3ea03d64-5fe9-978d-c1b7-cd122ce7dce9 -42:12.661 [INFO] jones.server: federation state anti-entropy synced -42:12.665 [INFO] jones: Requesting shutdown -42:12.665 [INFO] jones.server: shutting down server -42:12.665 [DEBUG] jones.leader: stopping routine: routine="CA root pruning" -42:12.665 [DEBUG] jones.leader: stopping routine: routine="federation state anti-entropy" -42:12.665 [DEBUG] jones.leader: stopping routine: routine="federation state pruning" -42:12.665 [DEBUG] jones.leader: stopping routine: routine="intermediate cert renew watch" -42:12.665 [WARN] jones.server.serf.lan: serf: Shutdown without a Leave -42:12.665 [ERROR] jones.server: error performing anti-entropy sync of federation state: error="context canceled" -42:12.665 [DEBUG] jones.leader: stopped routine: routine="federation state anti-entropy" -42:12.665 [ERROR] jones.anti_entropy: failed to sync remote state: error="No cluster leader" -42:12.665 [DEBUG] jones.leader: stopped routine: routine="CA root pruning" -42:12.666 [DEBUG] jones.leader: stopped routine: routine="federation state pruning" -42:12.666 [WARN] jones.server.serf.wan: serf: Shutdown without a Leave -42:12.666 [DEBUG] jones.leader: stopped routine: routine="intermediate cert renew watch" -2022-09-15T21:42:12.666Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:12.666Z [INFO] agent.router.manager: shutting down -42:12.666 [INFO] jones: consul server down -42:12.666 [INFO] jones: shutdown complete -42:12.666 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:13080 network=tcp -42:12.666 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:13080 network=udp -42:12.666 [INFO] jones: Stopping server: protocol=HTTP address=127.0.0.1:13081 network=tcp -42:13.166 [INFO] jones: Waiting for endpoints to shut down -42:13.166 [INFO] jones: Endpoints down +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:33:53.604Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:53.620Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:53.623 [INFO] jones.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:66c54b11-5572-4063-5bd9-91d261a20d41 Address:127.0.0.1:24309}]" +33:53.623 [INFO] jones.server.raft: entering follower state: follower="Node at 127.0.0.1:24309 [Follower]" leader= +33:53.624 [INFO] jones.server.serf.wan: serf: EventMemberJoin: Node-66c54b11-5572-4063-5bd9-91d261a20d41.dc1 127.0.0.1 +33:53.624 [INFO] jones.server.serf.lan: serf: EventMemberJoin: Node-66c54b11-5572-4063-5bd9-91d261a20d41 127.0.0.1 +2021-08-13T15:33:53.624Z [INFO] agent.router: Initializing LAN area manager +33:53.625 [INFO] jones: Started DNS server: address=127.0.0.1:24304 network=udp +33:53.625 [INFO] jones.server: Adding LAN server: server="Node-66c54b11-5572-4063-5bd9-91d261a20d41 (Addr: tcp/127.0.0.1:24309) (DC: dc1)" +33:53.625 [INFO] jones.server: Handled event for server in area: event=member-join server=Node-66c54b11-5572-4063-5bd9-91d261a20d41.dc1 area=wan +33:53.625 [INFO] jones: Started DNS server: address=127.0.0.1:24304 network=tcp +33:53.626 [INFO] jones: Started HTTP server: address=127.0.0.1:24305 network=tcp +33:53.626 [INFO] jones: started state syncer +33:53.626 [INFO] jones: Started gRPC server: address=127.0.0.1:24310 network=tcp +33:53.664 [WARN] jones.server.raft: heartbeat timeout reached, starting election: last-leader= +33:53.664 [INFO] jones.server.raft: entering candidate state: node="Node at 127.0.0.1:24309 [Candidate]" term=2 +33:53.664 [DEBUG] jones.server.raft: votes: needed=1 +33:53.664 [DEBUG] jones.server.raft: vote granted: from=66c54b11-5572-4063-5bd9-91d261a20d41 term=2 tally=1 +33:53.664 [INFO] jones.server.raft: election won: tally=1 +33:53.665 [INFO] jones.server.raft: entering leader state: leader="Node at 127.0.0.1:24309 [Leader]" +33:53.665 [INFO] jones.server: cluster leadership acquired +33:53.665 [INFO] jones.server: New leader elected: payload=Node-66c54b11-5572-4063-5bd9-91d261a20d41 +33:53.665 [DEBUG] jones.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24309 +33:53.666 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:53.669 [INFO] jones.server.connect: initialized primary datacenter CA with provider: provider=consul +33:53.669 [INFO] jones.leader: started routine: routine="federation state anti-entropy" +33:53.669 [INFO] jones.leader: started routine: routine="federation state pruning" +33:53.669 [INFO] jones.leader: started routine: routine="intermediate cert renew watch" +33:53.669 [INFO] jones.leader: started routine: routine="CA root pruning" +33:53.669 [DEBUG] jones.server: Skipping self join check for node since the cluster is too small: node=Node-66c54b11-5572-4063-5bd9-91d261a20d41 +33:53.669 [INFO] jones.server: member joined, marking health alive: member=Node-66c54b11-5572-4063-5bd9-91d261a20d41 +33:53.670 [INFO] jones.server: federation state anti-entropy synced +33:53.806 [DEBUG] jones: Skipping remote check since it is managed automatically: check=serfHealth +33:53.807 [INFO] jones: Synced node info +33:53.838 [INFO] jones: Requesting shutdown +33:53.838 [INFO] jones.server: shutting down server +33:53.838 [DEBUG] jones.leader: stopping routine: routine="federation state pruning" +33:53.838 [DEBUG] jones.leader: stopping routine: routine="intermediate cert renew watch" +33:53.838 [DEBUG] jones.leader: stopping routine: routine="CA root pruning" +33:53.838 [DEBUG] jones.leader: stopping routine: routine="federation state anti-entropy" +33:53.838 [WARN] jones.server.serf.lan: serf: Shutdown without a Leave +33:53.838 [ERROR] jones.server: error performing anti-entropy sync of federation state: error="context canceled" +33:53.838 [DEBUG] jones.leader: stopped routine: routine="federation state anti-entropy" +33:53.838 [DEBUG] jones.leader: stopped routine: routine="federation state pruning" +33:53.838 [DEBUG] jones.leader: stopped routine: routine="intermediate cert renew watch" +33:53.838 [DEBUG] jones.leader: stopped routine: routine="CA root pruning" +33:53.838 [WARN] jones.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:53.839Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:53.839Z [INFO] agent.router.manager: shutting down +33:53.839 [INFO] jones: consul server down +33:53.839 [INFO] jones: shutdown complete +33:53.839 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:24304 network=tcp +33:53.839 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:24304 network=udp +33:53.839 [INFO] jones: Stopping server: protocol=HTTP address=127.0.0.1:24305 network=tcp +33:54.339 [INFO] jones: Waiting for endpoints to shut down +33:54.339 [INFO] jones: Endpoints down --- PASS: TestAgent_sidecarServiceFromNodeService (6.99s) - --- PASS: TestAgent_sidecarServiceFromNodeService/no_sidecar (0.72s) - --- PASS: TestAgent_sidecarServiceFromNodeService/all_the_defaults (0.82s) - --- PASS: TestAgent_sidecarServiceFromNodeService/all_the_allowed_overrides (0.81s) - --- PASS: TestAgent_sidecarServiceFromNodeService/no_auto_ports_available (0.93s) - --- PASS: TestAgent_sidecarServiceFromNodeService/auto_ports_disabled (0.85s) - --- PASS: TestAgent_sidecarServiceFromNodeService/inherit_tags_and_meta (0.71s) - --- PASS: TestAgent_sidecarServiceFromNodeService/invalid_check_type (0.66s) - --- PASS: TestAgent_sidecarServiceFromNodeService/invalid_meta (0.88s) - --- PASS: TestAgent_sidecarServiceFromNodeService/re-registering_same_sidecar_with_no_port_should_pick_same_one (0.61s) + --- PASS: TestAgent_sidecarServiceFromNodeService/no_sidecar (0.91s) + --- PASS: TestAgent_sidecarServiceFromNodeService/all_the_defaults (0.69s) + --- PASS: TestAgent_sidecarServiceFromNodeService/all_the_allowed_overrides (0.72s) + --- PASS: TestAgent_sidecarServiceFromNodeService/no_auto_ports_available (0.86s) + --- PASS: TestAgent_sidecarServiceFromNodeService/auto_ports_disabled (0.69s) + --- PASS: TestAgent_sidecarServiceFromNodeService/inherit_tags_and_meta (0.59s) + --- PASS: TestAgent_sidecarServiceFromNodeService/invalid_check_type (0.95s) + --- PASS: TestAgent_sidecarServiceFromNodeService/invalid_meta (0.82s) + --- PASS: TestAgent_sidecarServiceFromNodeService/re-registering_same_sidecar_with_no_port_should_pick_same_one (0.75s) === RUN TestSnapshot snapshot_endpoint_test.go:16: DM-skipped --- SKIP: TestSnapshot (0.00s) @@ -20891,32 +20897,19 @@ === RUN TestDefaultConfig/#499 === PAUSE TestDefaultConfig/#499 === CONT TestDefaultConfig/#00 -=== CONT TestDefaultConfig/#242 === CONT TestDefaultConfig/#499 === CONT TestDefaultConfig/#466 === CONT TestDefaultConfig/#437 === CONT TestDefaultConfig/#412 -=== CONT TestDefaultConfig/#389 === CONT TestDefaultConfig/#390 === CONT TestDefaultConfig/#243 === CONT TestDefaultConfig/#371 -=== CONT TestDefaultConfig/#370 -=== CONT TestDefaultConfig/#369 -[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== CONT TestDefaultConfig/#368 -=== CONT TestDefaultConfig/#367 +=== CONT TestDefaultConfig/#411 +=== CONT TestDefaultConfig/#410 +=== CONT TestDefaultConfig/#409 +=== CONT TestDefaultConfig/#408 === CONT TestDefaultConfig/#355 -=== CONT TestDefaultConfig/#366 -=== CONT TestDefaultConfig/#365 -=== CONT TestDefaultConfig/#364 -=== CONT TestDefaultConfig/#363 -=== CONT TestDefaultConfig/#362 -=== CONT TestDefaultConfig/#361 -=== CONT TestDefaultConfig/#360 -=== CONT TestDefaultConfig/#359 -=== CONT TestDefaultConfig/#358 -=== CONT TestDefaultConfig/#357 -=== CONT TestDefaultConfig/#356 +=== CONT TestDefaultConfig/#407 === CONT TestDefaultConfig/#354 === CONT TestDefaultConfig/#353 === CONT TestDefaultConfig/#352 @@ -20926,6 +20919,7 @@ === CONT TestDefaultConfig/#348 === CONT TestDefaultConfig/#347 === CONT TestDefaultConfig/#346 +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! === CONT TestDefaultConfig/#345 === CONT TestDefaultConfig/#344 === CONT TestDefaultConfig/#343 @@ -20937,7 +20931,6 @@ === CONT TestDefaultConfig/#337 === CONT TestDefaultConfig/#336 === CONT TestDefaultConfig/#335 -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! === CONT TestDefaultConfig/#334 === CONT TestDefaultConfig/#333 === CONT TestDefaultConfig/#332 @@ -20945,10 +20938,10 @@ === CONT TestDefaultConfig/#330 === CONT TestDefaultConfig/#329 === CONT TestDefaultConfig/#328 +=== CONT TestDefaultConfig/#327 === CONT TestDefaultConfig/#326 === CONT TestDefaultConfig/#325 === CONT TestDefaultConfig/#324 -=== CONT TestDefaultConfig/#327 === CONT TestDefaultConfig/#323 === CONT TestDefaultConfig/#322 === CONT TestDefaultConfig/#321 @@ -20958,8 +20951,9 @@ === CONT TestDefaultConfig/#317 === CONT TestDefaultConfig/#316 === CONT TestDefaultConfig/#315 -=== CONT TestDefaultConfig/#313 === CONT TestDefaultConfig/#314 +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== CONT TestDefaultConfig/#313 === CONT TestDefaultConfig/#312 === CONT TestDefaultConfig/#311 === CONT TestDefaultConfig/#310 @@ -20970,7 +20964,6 @@ === CONT TestDefaultConfig/#305 === CONT TestDefaultConfig/#304 === CONT TestDefaultConfig/#303 -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! === CONT TestDefaultConfig/#302 === CONT TestDefaultConfig/#301 === CONT TestDefaultConfig/#300 @@ -20978,34 +20971,33 @@ === CONT TestDefaultConfig/#298 === CONT TestDefaultConfig/#297 === CONT TestDefaultConfig/#296 +=== CONT TestDefaultConfig/#288 === CONT TestDefaultConfig/#295 === CONT TestDefaultConfig/#294 === CONT TestDefaultConfig/#293 === CONT TestDefaultConfig/#292 === CONT TestDefaultConfig/#291 -=== CONT TestDefaultConfig/#290 === CONT TestDefaultConfig/#289 -=== CONT TestDefaultConfig/#288 +=== CONT TestDefaultConfig/#290 === CONT TestDefaultConfig/#287 -=== CONT TestDefaultConfig/#286 === CONT TestDefaultConfig/#285 +=== CONT TestDefaultConfig/#286 +=== CONT TestDefaultConfig/#284 === CONT TestDefaultConfig/#283 === CONT TestDefaultConfig/#282 === CONT TestDefaultConfig/#281 === CONT TestDefaultConfig/#280 -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== CONT TestDefaultConfig/#284 === CONT TestDefaultConfig/#279 === CONT TestDefaultConfig/#278 === CONT TestDefaultConfig/#277 === CONT TestDefaultConfig/#276 +=== CONT TestDefaultConfig/#275 +=== CONT TestDefaultConfig/#274 +=== CONT TestDefaultConfig/#273 === CONT TestDefaultConfig/#272 === CONT TestDefaultConfig/#271 -=== CONT TestDefaultConfig/#275 === CONT TestDefaultConfig/#270 === CONT TestDefaultConfig/#269 -=== CONT TestDefaultConfig/#274 -=== CONT TestDefaultConfig/#273 === CONT TestDefaultConfig/#268 === CONT TestDefaultConfig/#267 === CONT TestDefaultConfig/#266 @@ -21013,6 +21005,7 @@ === CONT TestDefaultConfig/#264 === CONT TestDefaultConfig/#263 === CONT TestDefaultConfig/#262 +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! === CONT TestDefaultConfig/#261 === CONT TestDefaultConfig/#260 === CONT TestDefaultConfig/#259 @@ -21020,7 +21013,6 @@ === CONT TestDefaultConfig/#257 === CONT TestDefaultConfig/#256 === CONT TestDefaultConfig/#255 -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! === CONT TestDefaultConfig/#254 === CONT TestDefaultConfig/#253 === CONT TestDefaultConfig/#252 @@ -21031,47 +21023,81 @@ === CONT TestDefaultConfig/#247 === CONT TestDefaultConfig/#246 === CONT TestDefaultConfig/#245 -=== CONT TestDefaultConfig/#244 === CONT TestDefaultConfig/#114 -=== CONT TestDefaultConfig/#465 -=== CONT TestDefaultConfig/#464 === CONT TestDefaultConfig/#498 -=== CONT TestDefaultConfig/#452 -=== CONT TestDefaultConfig/#496 === CONT TestDefaultConfig/#497 -=== CONT TestDefaultConfig/#494 +=== CONT TestDefaultConfig/#244 +=== CONT TestDefaultConfig/#496 === CONT TestDefaultConfig/#495 +=== CONT TestDefaultConfig/#494 === CONT TestDefaultConfig/#493 -=== CONT TestDefaultConfig/#492 === CONT TestDefaultConfig/#491 -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! === CONT TestDefaultConfig/#490 +=== CONT TestDefaultConfig/#492 +=== CONT TestDefaultConfig/#489 +=== CONT TestDefaultConfig/#488 +=== CONT TestDefaultConfig/#487 +=== CONT TestDefaultConfig/#486 +=== CONT TestDefaultConfig/#485 +=== CONT TestDefaultConfig/#484 +=== CONT TestDefaultConfig/#483 +=== CONT TestDefaultConfig/#482 +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== CONT TestDefaultConfig/#481 +=== CONT TestDefaultConfig/#480 +=== CONT TestDefaultConfig/#478 +=== CONT TestDefaultConfig/#477 +=== CONT TestDefaultConfig/#476 +=== CONT TestDefaultConfig/#479 +=== CONT TestDefaultConfig/#474 +=== CONT TestDefaultConfig/#475 +=== CONT TestDefaultConfig/#473 +=== CONT TestDefaultConfig/#472 +=== CONT TestDefaultConfig/#471 +=== CONT TestDefaultConfig/#470 +=== CONT TestDefaultConfig/#469 +=== CONT TestDefaultConfig/#468 +=== CONT TestDefaultConfig/#467 +=== CONT TestDefaultConfig/#242 +=== CONT TestDefaultConfig/#370 +=== CONT TestDefaultConfig/#389 +=== CONT TestDefaultConfig/#369 +=== CONT TestDefaultConfig/#368 +=== CONT TestDefaultConfig/#388 +=== CONT TestDefaultConfig/#387 +=== CONT TestDefaultConfig/#386 +=== CONT TestDefaultConfig/#385 === CONT TestDefaultConfig/#436 +=== CONT TestDefaultConfig/#384 === CONT TestDefaultConfig/#435 === CONT TestDefaultConfig/#434 +=== CONT TestDefaultConfig/#383 === CONT TestDefaultConfig/#433 === CONT TestDefaultConfig/#432 +=== CONT TestDefaultConfig/#382 === CONT TestDefaultConfig/#431 +=== CONT TestDefaultConfig/#381 === CONT TestDefaultConfig/#430 +=== CONT TestDefaultConfig/#380 === CONT TestDefaultConfig/#429 +=== CONT TestDefaultConfig/#379 +=== CONT TestDefaultConfig/#378 === CONT TestDefaultConfig/#428 === CONT TestDefaultConfig/#427 -=== CONT TestDefaultConfig/#426 -=== CONT TestDefaultConfig/#425 -=== CONT TestDefaultConfig/#424 -=== CONT TestDefaultConfig/#423 -=== CONT TestDefaultConfig/#422 -=== CONT TestDefaultConfig/#421 -=== CONT TestDefaultConfig/#420 -=== CONT TestDefaultConfig/#419 -=== CONT TestDefaultConfig/#418 -=== CONT TestDefaultConfig/#417 -=== CONT TestDefaultConfig/#415 -=== CONT TestDefaultConfig/#414 -=== CONT TestDefaultConfig/#416 -=== CONT TestDefaultConfig/#413 -=== CONT TestDefaultConfig/#451 +=== CONT TestDefaultConfig/#463 +=== CONT TestDefaultConfig/#462 [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== CONT TestDefaultConfig/#460 +=== CONT TestDefaultConfig/#461 +=== CONT TestDefaultConfig/#459 +=== CONT TestDefaultConfig/#458 +=== CONT TestDefaultConfig/#457 +=== CONT TestDefaultConfig/#456 +=== CONT TestDefaultConfig/#455 +=== CONT TestDefaultConfig/#454 +=== CONT TestDefaultConfig/#453 +=== CONT TestDefaultConfig/#452 +=== CONT TestDefaultConfig/#451 === CONT TestDefaultConfig/#450 === CONT TestDefaultConfig/#449 === CONT TestDefaultConfig/#448 @@ -21083,189 +21109,131 @@ === CONT TestDefaultConfig/#442 === CONT TestDefaultConfig/#441 === CONT TestDefaultConfig/#440 -=== CONT TestDefaultConfig/#438 === CONT TestDefaultConfig/#439 -=== CONT TestDefaultConfig/#241 -=== CONT TestDefaultConfig/#240 -=== CONT TestDefaultConfig/#239 -=== CONT TestDefaultConfig/#238 -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== CONT TestDefaultConfig/#236 -=== CONT TestDefaultConfig/#237 -=== CONT TestDefaultConfig/#235 -=== CONT TestDefaultConfig/#234 -=== CONT TestDefaultConfig/#233 -=== CONT TestDefaultConfig/#232 -=== CONT TestDefaultConfig/#231 -=== CONT TestDefaultConfig/#230 -=== CONT TestDefaultConfig/#229 -=== CONT TestDefaultConfig/#228 -=== CONT TestDefaultConfig/#227 -=== CONT TestDefaultConfig/#226 -=== CONT TestDefaultConfig/#225 -=== CONT TestDefaultConfig/#224 -=== CONT TestDefaultConfig/#223 -=== CONT TestDefaultConfig/#222 -=== CONT TestDefaultConfig/#221 -=== CONT TestDefaultConfig/#220 -=== CONT TestDefaultConfig/#219 -=== CONT TestDefaultConfig/#218 -=== CONT TestDefaultConfig/#217 -=== CONT TestDefaultConfig/#216 -=== CONT TestDefaultConfig/#215 -=== CONT TestDefaultConfig/#214 -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== CONT TestDefaultConfig/#213 -=== CONT TestDefaultConfig/#212 -=== CONT TestDefaultConfig/#211 -=== CONT TestDefaultConfig/#210 -=== CONT TestDefaultConfig/#209 -=== CONT TestDefaultConfig/#388 -=== CONT TestDefaultConfig/#387 -=== CONT TestDefaultConfig/#463 -=== CONT TestDefaultConfig/#386 -=== CONT TestDefaultConfig/#385 -=== CONT TestDefaultConfig/#384 -=== CONT TestDefaultConfig/#383 -=== CONT TestDefaultConfig/#382 -=== CONT TestDefaultConfig/#381 -=== CONT TestDefaultConfig/#380 -=== CONT TestDefaultConfig/#379 -=== CONT TestDefaultConfig/#378 +=== CONT TestDefaultConfig/#438 === CONT TestDefaultConfig/#377 === CONT TestDefaultConfig/#376 === CONT TestDefaultConfig/#375 -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! === CONT TestDefaultConfig/#374 === CONT TestDefaultConfig/#373 === CONT TestDefaultConfig/#372 -=== CONT TestDefaultConfig/#410 -=== CONT TestDefaultConfig/#409 -=== CONT TestDefaultConfig/#408 -=== CONT TestDefaultConfig/#407 -=== CONT TestDefaultConfig/#406 -=== CONT TestDefaultConfig/#405 -=== CONT TestDefaultConfig/#404 === CONT TestDefaultConfig/#403 +=== CONT TestDefaultConfig/#404 +=== CONT TestDefaultConfig/#406 +=== CONT TestDefaultConfig/#426 === CONT TestDefaultConfig/#402 -=== CONT TestDefaultConfig/#401 +=== CONT TestDefaultConfig/#405 +=== CONT TestDefaultConfig/#425 === CONT TestDefaultConfig/#400 -=== CONT TestDefaultConfig/#399 +=== CONT TestDefaultConfig/#424 +=== CONT TestDefaultConfig/#401 +=== CONT TestDefaultConfig/#423 === CONT TestDefaultConfig/#398 === CONT TestDefaultConfig/#397 === CONT TestDefaultConfig/#396 === CONT TestDefaultConfig/#395 +=== CONT TestDefaultConfig/#422 +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== CONT TestDefaultConfig/#394 === CONT TestDefaultConfig/#393 -=== CONT TestDefaultConfig/#488 === CONT TestDefaultConfig/#392 === CONT TestDefaultConfig/#391 -=== CONT TestDefaultConfig/#487 -=== CONT TestDefaultConfig/#394 -=== CONT TestDefaultConfig/#486 -=== CONT TestDefaultConfig/#489 -=== CONT TestDefaultConfig/#485 -=== CONT TestDefaultConfig/#484 -=== CONT TestDefaultConfig/#483 -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== CONT TestDefaultConfig/#482 -=== CONT TestDefaultConfig/#481 -=== CONT TestDefaultConfig/#480 -=== CONT TestDefaultConfig/#479 -=== CONT TestDefaultConfig/#478 -=== CONT TestDefaultConfig/#476 -=== CONT TestDefaultConfig/#477 -=== CONT TestDefaultConfig/#475 -=== CONT TestDefaultConfig/#474 -=== CONT TestDefaultConfig/#473 -=== CONT TestDefaultConfig/#472 -=== CONT TestDefaultConfig/#471 -=== CONT TestDefaultConfig/#469 -=== CONT TestDefaultConfig/#470 -=== CONT TestDefaultConfig/#468 -=== CONT TestDefaultConfig/#467 -=== CONT TestDefaultConfig/#462 -=== CONT TestDefaultConfig/#461 -=== CONT TestDefaultConfig/#460 -=== CONT TestDefaultConfig/#459 -=== CONT TestDefaultConfig/#458 -=== CONT TestDefaultConfig/#457 -=== CONT TestDefaultConfig/#456 -=== CONT TestDefaultConfig/#454 -=== CONT TestDefaultConfig/#453 -=== CONT TestDefaultConfig/#208 -=== CONT TestDefaultConfig/#207 +=== CONT TestDefaultConfig/#417 +=== CONT TestDefaultConfig/#420 +=== CONT TestDefaultConfig/#419 +=== CONT TestDefaultConfig/#418 +=== CONT TestDefaultConfig/#465 +=== CONT TestDefaultConfig/#367 +=== CONT TestDefaultConfig/#366 +=== CONT TestDefaultConfig/#365 +=== CONT TestDefaultConfig/#364 +=== CONT TestDefaultConfig/#363 +=== CONT TestDefaultConfig/#362 +=== CONT TestDefaultConfig/#361 +=== CONT TestDefaultConfig/#360 +=== CONT TestDefaultConfig/#359 +=== CONT TestDefaultConfig/#358 +=== CONT TestDefaultConfig/#357 +=== CONT TestDefaultConfig/#356 +=== CONT TestDefaultConfig/#421 +=== CONT TestDefaultConfig/#416 +=== CONT TestDefaultConfig/#415 +=== CONT TestDefaultConfig/#414 +=== CONT TestDefaultConfig/#413 +=== CONT TestDefaultConfig/#464 +=== CONT TestDefaultConfig/#121 +=== CONT TestDefaultConfig/#240 +=== CONT TestDefaultConfig/#239 +=== CONT TestDefaultConfig/#238 +=== CONT TestDefaultConfig/#237 +=== CONT TestDefaultConfig/#235 +=== CONT TestDefaultConfig/#236 +=== CONT TestDefaultConfig/#234 +=== CONT TestDefaultConfig/#233 [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== CONT TestDefaultConfig/#206 -=== CONT TestDefaultConfig/#455 -=== CONT TestDefaultConfig/#205 -=== CONT TestDefaultConfig/#204 -=== CONT TestDefaultConfig/#203 -=== CONT TestDefaultConfig/#202 -=== CONT TestDefaultConfig/#200 -=== CONT TestDefaultConfig/#199 -=== CONT TestDefaultConfig/#201 -=== CONT TestDefaultConfig/#198 -=== CONT TestDefaultConfig/#197 -=== CONT TestDefaultConfig/#196 -=== CONT TestDefaultConfig/#195 -=== CONT TestDefaultConfig/#194 -=== CONT TestDefaultConfig/#193 -=== CONT TestDefaultConfig/#192 -=== CONT TestDefaultConfig/#191 -=== CONT TestDefaultConfig/#189 -=== CONT TestDefaultConfig/#188 -=== CONT TestDefaultConfig/#187 -=== CONT TestDefaultConfig/#190 -=== CONT TestDefaultConfig/#186 -=== CONT TestDefaultConfig/#185 -=== CONT TestDefaultConfig/#184 -=== CONT TestDefaultConfig/#182 -=== CONT TestDefaultConfig/#183 -=== CONT TestDefaultConfig/#181 -=== CONT TestDefaultConfig/#180 -=== CONT TestDefaultConfig/#179 -=== CONT TestDefaultConfig/#177 +=== CONT TestDefaultConfig/#232 +=== CONT TestDefaultConfig/#231 === CONT TestDefaultConfig/#176 -=== CONT TestDefaultConfig/#178 -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== CONT TestDefaultConfig/#230 +=== CONT TestDefaultConfig/#229 +=== CONT TestDefaultConfig/#228 === CONT TestDefaultConfig/#175 +=== CONT TestDefaultConfig/#227 === CONT TestDefaultConfig/#174 +=== CONT TestDefaultConfig/#226 === CONT TestDefaultConfig/#173 +=== CONT TestDefaultConfig/#225 +=== CONT TestDefaultConfig/#224 === CONT TestDefaultConfig/#172 === CONT TestDefaultConfig/#171 +=== CONT TestDefaultConfig/#223 +=== CONT TestDefaultConfig/#170 +=== CONT TestDefaultConfig/#222 === CONT TestDefaultConfig/#169 +=== CONT TestDefaultConfig/#221 === CONT TestDefaultConfig/#168 -=== CONT TestDefaultConfig/#170 +=== CONT TestDefaultConfig/#220 === CONT TestDefaultConfig/#167 === CONT TestDefaultConfig/#166 === CONT TestDefaultConfig/#165 === CONT TestDefaultConfig/#164 +=== CONT TestDefaultConfig/#219 +=== CONT TestDefaultConfig/#218 +=== CONT TestDefaultConfig/#217 === CONT TestDefaultConfig/#163 +=== CONT TestDefaultConfig/#216 === CONT TestDefaultConfig/#162 +=== CONT TestDefaultConfig/#215 === CONT TestDefaultConfig/#161 +=== CONT TestDefaultConfig/#214 === CONT TestDefaultConfig/#160 +=== CONT TestDefaultConfig/#213 === CONT TestDefaultConfig/#159 +=== CONT TestDefaultConfig/#212 === CONT TestDefaultConfig/#158 +=== CONT TestDefaultConfig/#211 === CONT TestDefaultConfig/#157 === CONT TestDefaultConfig/#156 +=== CONT TestDefaultConfig/#210 === CONT TestDefaultConfig/#155 +=== CONT TestDefaultConfig/#209 === CONT TestDefaultConfig/#154 +=== CONT TestDefaultConfig/#208 === CONT TestDefaultConfig/#153 === CONT TestDefaultConfig/#152 +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! === CONT TestDefaultConfig/#151 === CONT TestDefaultConfig/#150 === CONT TestDefaultConfig/#149 -=== CONT TestDefaultConfig/#148 === CONT TestDefaultConfig/#147 === CONT TestDefaultConfig/#146 +=== CONT TestDefaultConfig/#148 === CONT TestDefaultConfig/#145 === CONT TestDefaultConfig/#144 === CONT TestDefaultConfig/#143 === CONT TestDefaultConfig/#142 === CONT TestDefaultConfig/#141 === CONT TestDefaultConfig/#140 -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== CONT TestDefaultConfig/#139 === CONT TestDefaultConfig/#138 === CONT TestDefaultConfig/#137 === CONT TestDefaultConfig/#136 @@ -21273,6 +21241,7 @@ === CONT TestDefaultConfig/#134 === CONT TestDefaultConfig/#133 === CONT TestDefaultConfig/#132 +=== CONT TestDefaultConfig/#139 === CONT TestDefaultConfig/#131 === CONT TestDefaultConfig/#130 === CONT TestDefaultConfig/#129 @@ -21283,19 +21252,59 @@ === CONT TestDefaultConfig/#124 === CONT TestDefaultConfig/#123 === CONT TestDefaultConfig/#122 -=== CONT TestDefaultConfig/#121 +=== CONT TestDefaultConfig/#206 +=== CONT TestDefaultConfig/#192 +=== CONT TestDefaultConfig/#205 +=== CONT TestDefaultConfig/#204 +=== CONT TestDefaultConfig/#203 +=== CONT TestDefaultConfig/#202 +=== CONT TestDefaultConfig/#201 +=== CONT TestDefaultConfig/#200 +=== CONT TestDefaultConfig/#199 +=== CONT TestDefaultConfig/#198 +=== CONT TestDefaultConfig/#197 +=== CONT TestDefaultConfig/#196 +=== CONT TestDefaultConfig/#195 +=== CONT TestDefaultConfig/#194 +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== CONT TestDefaultConfig/#193 +=== CONT TestDefaultConfig/#184 +=== CONT TestDefaultConfig/#191 +=== CONT TestDefaultConfig/#190 +=== CONT TestDefaultConfig/#189 +=== CONT TestDefaultConfig/#188 +=== CONT TestDefaultConfig/#187 +=== CONT TestDefaultConfig/#186 +=== CONT TestDefaultConfig/#185 +=== CONT TestDefaultConfig/#183 +=== CONT TestDefaultConfig/#182 +=== CONT TestDefaultConfig/#181 +=== CONT TestDefaultConfig/#180 +=== CONT TestDefaultConfig/#60 === CONT TestDefaultConfig/#120 === CONT TestDefaultConfig/#119 === CONT TestDefaultConfig/#118 === CONT TestDefaultConfig/#117 === CONT TestDefaultConfig/#116 -=== CONT TestDefaultConfig/#115 +=== CONT TestDefaultConfig/#178 === CONT TestDefaultConfig/#113 +=== CONT TestDefaultConfig/#179 === CONT TestDefaultConfig/#112 +=== CONT TestDefaultConfig/#115 === CONT TestDefaultConfig/#111 +=== CONT TestDefaultConfig/#30 === CONT TestDefaultConfig/#110 +=== CONT TestDefaultConfig/#59 === CONT TestDefaultConfig/#109 +=== CONT TestDefaultConfig/#58 +=== CONT TestDefaultConfig/#57 +=== CONT TestDefaultConfig/#56 +=== CONT TestDefaultConfig/#55 +=== CONT TestDefaultConfig/#53 +=== CONT TestDefaultConfig/#54 +=== CONT TestDefaultConfig/#52 === CONT TestDefaultConfig/#108 +=== CONT TestDefaultConfig/#51 === CONT TestDefaultConfig/#107 === CONT TestDefaultConfig/#106 === CONT TestDefaultConfig/#105 @@ -21303,611 +21312,603 @@ === CONT TestDefaultConfig/#103 === CONT TestDefaultConfig/#102 === CONT TestDefaultConfig/#101 +=== CONT TestDefaultConfig/#50 +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! === CONT TestDefaultConfig/#100 +=== CONT TestDefaultConfig/#49 === CONT TestDefaultConfig/#99 +=== CONT TestDefaultConfig/#48 === CONT TestDefaultConfig/#98 -=== CONT TestDefaultConfig/#97 -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== CONT TestDefaultConfig/#96 -=== CONT TestDefaultConfig/#411 -=== CONT TestDefaultConfig/#95 -=== CONT TestDefaultConfig/#94 === CONT TestDefaultConfig/#47 +=== CONT TestDefaultConfig/#97 === CONT TestDefaultConfig/#46 -=== CONT TestDefaultConfig/#93 === CONT TestDefaultConfig/#45 -=== CONT TestDefaultConfig/#92 === CONT TestDefaultConfig/#44 -=== CONT TestDefaultConfig/#91 === CONT TestDefaultConfig/#43 -=== CONT TestDefaultConfig/#90 === CONT TestDefaultConfig/#42 -=== CONT TestDefaultConfig/#89 === CONT TestDefaultConfig/#41 -=== CONT TestDefaultConfig/#88 +=== CONT TestDefaultConfig/#96 === CONT TestDefaultConfig/#40 +=== CONT TestDefaultConfig/#95 === CONT TestDefaultConfig/#39 +=== CONT TestDefaultConfig/#94 +=== CONT TestDefaultConfig/#93 +=== CONT TestDefaultConfig/#92 +=== CONT TestDefaultConfig/#91 +=== CONT TestDefaultConfig/#88 === CONT TestDefaultConfig/#38 -=== CONT TestDefaultConfig/#37 -=== CONT TestDefaultConfig/#36 -=== CONT TestDefaultConfig/#35 -=== CONT TestDefaultConfig/#33 -=== CONT TestDefaultConfig/#34 -=== CONT TestDefaultConfig/#32 -=== CONT TestDefaultConfig/#31 -=== CONT TestDefaultConfig/#30 -=== CONT TestDefaultConfig/#29 -=== CONT TestDefaultConfig/#28 -=== CONT TestDefaultConfig/#27 -=== CONT TestDefaultConfig/#26 === CONT TestDefaultConfig/#87 +=== CONT TestDefaultConfig/#37 +=== CONT TestDefaultConfig/#90 === CONT TestDefaultConfig/#86 +=== CONT TestDefaultConfig/#36 +=== CONT TestDefaultConfig/#89 === CONT TestDefaultConfig/#85 +=== CONT TestDefaultConfig/#35 === CONT TestDefaultConfig/#84 -=== CONT TestDefaultConfig/#66 -=== CONT TestDefaultConfig/#65 === CONT TestDefaultConfig/#83 -=== CONT TestDefaultConfig/#64 === CONT TestDefaultConfig/#82 -=== CONT TestDefaultConfig/#63 === CONT TestDefaultConfig/#81 -=== CONT TestDefaultConfig/#62 -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! === CONT TestDefaultConfig/#80 -=== CONT TestDefaultConfig/#61 === CONT TestDefaultConfig/#79 -=== CONT TestDefaultConfig/#60 === CONT TestDefaultConfig/#78 -=== CONT TestDefaultConfig/#59 === CONT TestDefaultConfig/#77 -=== CONT TestDefaultConfig/#58 === CONT TestDefaultConfig/#76 -=== CONT TestDefaultConfig/#57 === CONT TestDefaultConfig/#75 -=== CONT TestDefaultConfig/#56 === CONT TestDefaultConfig/#74 -=== CONT TestDefaultConfig/#55 === CONT TestDefaultConfig/#73 -=== CONT TestDefaultConfig/#54 -=== CONT TestDefaultConfig/#72 -=== CONT TestDefaultConfig/#53 +=== CONT TestDefaultConfig/#34 +=== CONT TestDefaultConfig/#33 +=== CONT TestDefaultConfig/#32 +=== CONT TestDefaultConfig/#31 +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== CONT TestDefaultConfig/#66 === CONT TestDefaultConfig/#71 -=== CONT TestDefaultConfig/#52 === CONT TestDefaultConfig/#70 -=== CONT TestDefaultConfig/#51 === CONT TestDefaultConfig/#69 -=== CONT TestDefaultConfig/#50 -=== CONT TestDefaultConfig/#49 === CONT TestDefaultConfig/#68 -=== CONT TestDefaultConfig/#48 === CONT TestDefaultConfig/#67 -=== CONT TestDefaultConfig/#13 -=== CONT TestDefaultConfig/#07 +=== CONT TestDefaultConfig/#177 +=== CONT TestDefaultConfig/#15 +=== CONT TestDefaultConfig/#29 +=== CONT TestDefaultConfig/#28 +=== CONT TestDefaultConfig/#27 +=== CONT TestDefaultConfig/#26 +=== CONT TestDefaultConfig/#25 === CONT TestDefaultConfig/#24 -=== CONT TestDefaultConfig/#12 === CONT TestDefaultConfig/#23 -=== CONT TestDefaultConfig/#11 === CONT TestDefaultConfig/#22 === CONT TestDefaultConfig/#21 -=== CONT TestDefaultConfig/#10 -=== CONT TestDefaultConfig/#09 === CONT TestDefaultConfig/#20 === CONT TestDefaultConfig/#19 === CONT TestDefaultConfig/#18 === CONT TestDefaultConfig/#17 === CONT TestDefaultConfig/#16 -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== CONT TestDefaultConfig/#15 -=== CONT TestDefaultConfig/#14 +=== CONT TestDefaultConfig/#63 +=== CONT TestDefaultConfig/#65 +=== CONT TestDefaultConfig/#64 === CONT TestDefaultConfig/#08 +=== CONT TestDefaultConfig/#14 +=== CONT TestDefaultConfig/#13 +=== CONT TestDefaultConfig/#12 +=== CONT TestDefaultConfig/#11 +=== CONT TestDefaultConfig/#10 +=== CONT TestDefaultConfig/#09 === CONT TestDefaultConfig/#04 +=== CONT TestDefaultConfig/#07 === CONT TestDefaultConfig/#06 === CONT TestDefaultConfig/#05 +=== CONT TestDefaultConfig/#62 === CONT TestDefaultConfig/#02 === CONT TestDefaultConfig/#03 +=== CONT TestDefaultConfig/#61 === CONT TestDefaultConfig/#01 -=== CONT TestDefaultConfig/#25 +=== CONT TestDefaultConfig/#399 +=== CONT TestDefaultConfig/#207 +=== CONT TestDefaultConfig/#241 +=== CONT TestDefaultConfig/#72 +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! --- PASS: TestDefaultConfig (0.02s) + --- PASS: TestDefaultConfig/#412 (0.05s) --- PASS: TestDefaultConfig/#499 (0.06s) - --- PASS: TestDefaultConfig/#389 (0.06s) - --- PASS: TestDefaultConfig/#412 (0.07s) - --- PASS: TestDefaultConfig/#371 (0.01s) - --- PASS: TestDefaultConfig/#242 (0.10s) - --- PASS: TestDefaultConfig/#370 (0.02s) - --- PASS: TestDefaultConfig/#466 (0.10s) - --- PASS: TestDefaultConfig/#367 (0.01s) - --- PASS: TestDefaultConfig/#390 (0.08s) - --- PASS: TestDefaultConfig/#243 (0.05s) - --- PASS: TestDefaultConfig/#437 (0.12s) - --- PASS: TestDefaultConfig/#00 (0.16s) - --- PASS: TestDefaultConfig/#369 (0.08s) - --- PASS: TestDefaultConfig/#368 (0.07s) - --- PASS: TestDefaultConfig/#366 (0.06s) - --- PASS: TestDefaultConfig/#365 (0.07s) - --- PASS: TestDefaultConfig/#355 (0.07s) - --- PASS: TestDefaultConfig/#364 (0.07s) - --- PASS: TestDefaultConfig/#363 (0.06s) - --- PASS: TestDefaultConfig/#361 (0.03s) - --- PASS: TestDefaultConfig/#358 (0.02s) - --- PASS: TestDefaultConfig/#362 (0.07s) - --- PASS: TestDefaultConfig/#359 (0.07s) - --- PASS: TestDefaultConfig/#360 (0.08s) - --- PASS: TestDefaultConfig/#353 (0.05s) - --- PASS: TestDefaultConfig/#351 (0.02s) - --- PASS: TestDefaultConfig/#356 (0.07s) - --- PASS: TestDefaultConfig/#350 (0.02s) - --- PASS: TestDefaultConfig/#347 (0.01s) - --- PASS: TestDefaultConfig/#348 (0.03s) - --- PASS: TestDefaultConfig/#352 (0.07s) - --- PASS: TestDefaultConfig/#349 (0.03s) - --- PASS: TestDefaultConfig/#354 (0.11s) - --- PASS: TestDefaultConfig/#344 (0.04s) - --- PASS: TestDefaultConfig/#357 (0.13s) - --- PASS: TestDefaultConfig/#346 (0.07s) - --- PASS: TestDefaultConfig/#341 (0.06s) + --- PASS: TestDefaultConfig/#243 (0.07s) + --- PASS: TestDefaultConfig/#466 (0.08s) + --- PASS: TestDefaultConfig/#437 (0.08s) + --- PASS: TestDefaultConfig/#410 (0.02s) + --- PASS: TestDefaultConfig/#411 (0.04s) + --- PASS: TestDefaultConfig/#390 (0.09s) + --- PASS: TestDefaultConfig/#355 (0.02s) + --- PASS: TestDefaultConfig/#00 (0.13s) + --- PASS: TestDefaultConfig/#408 (0.05s) + --- PASS: TestDefaultConfig/#371 (0.12s) + --- PASS: TestDefaultConfig/#354 (0.06s) + --- PASS: TestDefaultConfig/#409 (0.08s) + --- PASS: TestDefaultConfig/#407 (0.07s) + --- PASS: TestDefaultConfig/#349 (0.04s) + --- PASS: TestDefaultConfig/#353 (0.08s) + --- PASS: TestDefaultConfig/#351 (0.05s) + --- PASS: TestDefaultConfig/#348 (0.04s) + --- PASS: TestDefaultConfig/#350 (0.07s) + --- PASS: TestDefaultConfig/#347 (0.04s) + --- PASS: TestDefaultConfig/#352 (0.09s) + --- PASS: TestDefaultConfig/#346 (0.08s) + --- PASS: TestDefaultConfig/#344 (0.06s) + --- PASS: TestDefaultConfig/#342 (0.06s) + --- PASS: TestDefaultConfig/#345 (0.07s) --- PASS: TestDefaultConfig/#343 (0.07s) - --- PASS: TestDefaultConfig/#342 (0.07s) - --- PASS: TestDefaultConfig/#337 (0.02s) - --- PASS: TestDefaultConfig/#345 (0.09s) - --- PASS: TestDefaultConfig/#339 (0.05s) - --- PASS: TestDefaultConfig/#333 (0.01s) - --- PASS: TestDefaultConfig/#334 (0.04s) - --- PASS: TestDefaultConfig/#336 (0.05s) - --- PASS: TestDefaultConfig/#335 (0.05s) - --- PASS: TestDefaultConfig/#332 (0.04s) - --- PASS: TestDefaultConfig/#338 (0.08s) - --- PASS: TestDefaultConfig/#340 (0.09s) - --- PASS: TestDefaultConfig/#331 (0.06s) + --- PASS: TestDefaultConfig/#339 (0.03s) + --- PASS: TestDefaultConfig/#338 (0.02s) + --- PASS: TestDefaultConfig/#336 (0.01s) + --- PASS: TestDefaultConfig/#335 (0.01s) + --- PASS: TestDefaultConfig/#341 (0.09s) + --- PASS: TestDefaultConfig/#334 (0.05s) + --- PASS: TestDefaultConfig/#333 (0.06s) + --- PASS: TestDefaultConfig/#340 (0.13s) --- PASS: TestDefaultConfig/#328 (0.03s) - --- PASS: TestDefaultConfig/#326 (0.03s) + --- PASS: TestDefaultConfig/#337 (0.09s) --- PASS: TestDefaultConfig/#330 (0.07s) - --- PASS: TestDefaultConfig/#325 (0.04s) - --- PASS: TestDefaultConfig/#327 (0.04s) - --- PASS: TestDefaultConfig/#323 (0.02s) - --- PASS: TestDefaultConfig/#317 (0.02s) - --- PASS: TestDefaultConfig/#319 (0.03s) - --- PASS: TestDefaultConfig/#329 (0.10s) - --- PASS: TestDefaultConfig/#324 (0.09s) - --- PASS: TestDefaultConfig/#318 (0.07s) - --- PASS: TestDefaultConfig/#321 (0.12s) - --- PASS: TestDefaultConfig/#322 (0.12s) - --- PASS: TestDefaultConfig/#313 (0.06s) - --- PASS: TestDefaultConfig/#320 (0.13s) + --- PASS: TestDefaultConfig/#331 (0.07s) + --- PASS: TestDefaultConfig/#329 (0.05s) + --- PASS: TestDefaultConfig/#332 (0.08s) + --- PASS: TestDefaultConfig/#327 (0.03s) + --- PASS: TestDefaultConfig/#326 (0.02s) + --- PASS: TestDefaultConfig/#321 (0.01s) + --- PASS: TestDefaultConfig/#325 (0.03s) + --- PASS: TestDefaultConfig/#324 (0.04s) + --- PASS: TestDefaultConfig/#317 (0.06s) + --- PASS: TestDefaultConfig/#320 (0.07s) + --- PASS: TestDefaultConfig/#319 (0.08s) + --- PASS: TestDefaultConfig/#315 (0.05s) + --- PASS: TestDefaultConfig/#318 (0.08s) + --- PASS: TestDefaultConfig/#322 (0.09s) + --- PASS: TestDefaultConfig/#314 (0.02s) + --- PASS: TestDefaultConfig/#323 (0.10s) + --- PASS: TestDefaultConfig/#311 (0.01s) + --- PASS: TestDefaultConfig/#310 (0.01s) + --- PASS: TestDefaultConfig/#313 (0.04s) --- PASS: TestDefaultConfig/#312 (0.05s) --- PASS: TestDefaultConfig/#316 (0.12s) - --- PASS: TestDefaultConfig/#314 (0.08s) - --- PASS: TestDefaultConfig/#315 (0.12s) - --- PASS: TestDefaultConfig/#309 (0.03s) - --- PASS: TestDefaultConfig/#310 (0.08s) - --- PASS: TestDefaultConfig/#311 (0.10s) - --- PASS: TestDefaultConfig/#308 (0.09s) - --- PASS: TestDefaultConfig/#307 (0.09s) - --- PASS: TestDefaultConfig/#305 (0.09s) - --- PASS: TestDefaultConfig/#301 (0.03s) - --- PASS: TestDefaultConfig/#306 (0.10s) - --- PASS: TestDefaultConfig/#303 (0.10s) - --- PASS: TestDefaultConfig/#302 (0.06s) - --- PASS: TestDefaultConfig/#298 (0.03s) - --- PASS: TestDefaultConfig/#304 (0.11s) - --- PASS: TestDefaultConfig/#296 (0.03s) - --- PASS: TestDefaultConfig/#297 (0.03s) - --- PASS: TestDefaultConfig/#299 (0.07s) - --- PASS: TestDefaultConfig/#300 (0.09s) - --- PASS: TestDefaultConfig/#295 (0.07s) - --- PASS: TestDefaultConfig/#291 (0.09s) - --- PASS: TestDefaultConfig/#292 (0.10s) - --- PASS: TestDefaultConfig/#290 (0.10s) - --- PASS: TestDefaultConfig/#287 (0.07s) - --- PASS: TestDefaultConfig/#286 (0.03s) - --- PASS: TestDefaultConfig/#293 (0.13s) - --- PASS: TestDefaultConfig/#283 (0.01s) - --- PASS: TestDefaultConfig/#288 (0.09s) - --- PASS: TestDefaultConfig/#289 (0.12s) - --- PASS: TestDefaultConfig/#294 (0.15s) + --- PASS: TestDefaultConfig/#309 (0.07s) + --- PASS: TestDefaultConfig/#306 (0.06s) + --- PASS: TestDefaultConfig/#308 (0.06s) + --- PASS: TestDefaultConfig/#307 (0.07s) + --- PASS: TestDefaultConfig/#301 (0.01s) + --- PASS: TestDefaultConfig/#305 (0.07s) + --- PASS: TestDefaultConfig/#299 (0.01s) + --- PASS: TestDefaultConfig/#304 (0.06s) + --- PASS: TestDefaultConfig/#298 (0.01s) + --- PASS: TestDefaultConfig/#297 (0.01s) + --- PASS: TestDefaultConfig/#300 (0.03s) + --- PASS: TestDefaultConfig/#303 (0.07s) + --- PASS: TestDefaultConfig/#294 (0.03s) + --- PASS: TestDefaultConfig/#288 (0.05s) + --- PASS: TestDefaultConfig/#295 (0.05s) + --- PASS: TestDefaultConfig/#293 (0.04s) + --- PASS: TestDefaultConfig/#296 (0.05s) + --- PASS: TestDefaultConfig/#302 (0.08s) + --- PASS: TestDefaultConfig/#290 (0.02s) --- PASS: TestDefaultConfig/#284 (0.01s) - --- PASS: TestDefaultConfig/#281 (0.03s) - --- PASS: TestDefaultConfig/#285 (0.06s) - --- PASS: TestDefaultConfig/#282 (0.04s) - --- PASS: TestDefaultConfig/#280 (0.05s) - --- PASS: TestDefaultConfig/#279 (0.05s) - --- PASS: TestDefaultConfig/#277 (0.06s) - --- PASS: TestDefaultConfig/#278 (0.08s) - --- PASS: TestDefaultConfig/#276 (0.09s) - --- PASS: TestDefaultConfig/#272 (0.11s) - --- PASS: TestDefaultConfig/#271 (0.10s) - --- PASS: TestDefaultConfig/#273 (0.06s) - --- PASS: TestDefaultConfig/#275 (0.12s) + --- PASS: TestDefaultConfig/#287 (0.02s) + --- PASS: TestDefaultConfig/#285 (0.02s) + --- PASS: TestDefaultConfig/#283 (0.01s) + --- PASS: TestDefaultConfig/#280 (0.02s) + --- PASS: TestDefaultConfig/#291 (0.06s) + --- PASS: TestDefaultConfig/#286 (0.05s) + --- PASS: TestDefaultConfig/#279 (0.03s) + --- PASS: TestDefaultConfig/#289 (0.06s) + --- PASS: TestDefaultConfig/#292 (0.11s) + --- PASS: TestDefaultConfig/#282 (0.06s) + --- PASS: TestDefaultConfig/#276 (0.03s) + --- PASS: TestDefaultConfig/#281 (0.06s) + --- PASS: TestDefaultConfig/#277 (0.04s) + --- PASS: TestDefaultConfig/#274 (0.03s) + --- PASS: TestDefaultConfig/#275 (0.03s) + --- PASS: TestDefaultConfig/#271 (0.01s) + --- PASS: TestDefaultConfig/#272 (0.02s) + --- PASS: TestDefaultConfig/#270 (0.03s) + --- PASS: TestDefaultConfig/#273 (0.04s) --- PASS: TestDefaultConfig/#266 (0.02s) - --- PASS: TestDefaultConfig/#269 (0.11s) - --- PASS: TestDefaultConfig/#270 (0.13s) - --- PASS: TestDefaultConfig/#264 (0.02s) - --- PASS: TestDefaultConfig/#274 (0.11s) - --- PASS: TestDefaultConfig/#267 (0.06s) - --- PASS: TestDefaultConfig/#268 (0.09s) - --- PASS: TestDefaultConfig/#263 (0.08s) - --- PASS: TestDefaultConfig/#260 (0.06s) - --- PASS: TestDefaultConfig/#265 (0.10s) - --- PASS: TestDefaultConfig/#262 (0.09s) - --- PASS: TestDefaultConfig/#259 (0.12s) - --- PASS: TestDefaultConfig/#261 (0.14s) - --- PASS: TestDefaultConfig/#255 (0.07s) - --- PASS: TestDefaultConfig/#252 (0.02s) - --- PASS: TestDefaultConfig/#257 (0.13s) - --- PASS: TestDefaultConfig/#258 (0.14s) - --- PASS: TestDefaultConfig/#251 (0.01s) - --- PASS: TestDefaultConfig/#253 (0.07s) - --- PASS: TestDefaultConfig/#249 (0.02s) - --- PASS: TestDefaultConfig/#250 (0.03s) - --- PASS: TestDefaultConfig/#254 (0.11s) - --- PASS: TestDefaultConfig/#256 (0.13s) - --- PASS: TestDefaultConfig/#247 (0.07s) - --- PASS: TestDefaultConfig/#246 (0.07s) - --- PASS: TestDefaultConfig/#114 (0.07s) - --- PASS: TestDefaultConfig/#245 (0.09s) - --- PASS: TestDefaultConfig/#244 (0.09s) - --- PASS: TestDefaultConfig/#248 (0.10s) - --- PASS: TestDefaultConfig/#452 (0.06s) - --- PASS: TestDefaultConfig/#464 (0.07s) - --- PASS: TestDefaultConfig/#465 (0.11s) - --- PASS: TestDefaultConfig/#495 (0.05s) - --- PASS: TestDefaultConfig/#493 (0.03s) + --- PASS: TestDefaultConfig/#268 (0.04s) + --- PASS: TestDefaultConfig/#263 (0.04s) + --- PASS: TestDefaultConfig/#265 (0.06s) + --- PASS: TestDefaultConfig/#262 (0.03s) + --- PASS: TestDefaultConfig/#278 (0.12s) + --- PASS: TestDefaultConfig/#261 (0.03s) + --- PASS: TestDefaultConfig/#264 (0.05s) + --- PASS: TestDefaultConfig/#267 (0.08s) + --- PASS: TestDefaultConfig/#259 (0.02s) + --- PASS: TestDefaultConfig/#269 (0.09s) + --- PASS: TestDefaultConfig/#260 (0.03s) + --- PASS: TestDefaultConfig/#257 (0.02s) + --- PASS: TestDefaultConfig/#256 (0.03s) + --- PASS: TestDefaultConfig/#258 (0.03s) + --- PASS: TestDefaultConfig/#253 (0.04s) + --- PASS: TestDefaultConfig/#255 (0.06s) + --- PASS: TestDefaultConfig/#252 (0.05s) + --- PASS: TestDefaultConfig/#254 (0.07s) + --- PASS: TestDefaultConfig/#251 (0.08s) + --- PASS: TestDefaultConfig/#248 (0.08s) + --- PASS: TestDefaultConfig/#247 (0.05s) + --- PASS: TestDefaultConfig/#114 (0.01s) + --- PASS: TestDefaultConfig/#498 (0.01s) + --- PASS: TestDefaultConfig/#250 (0.10s) + --- PASS: TestDefaultConfig/#244 (0.02s) + --- PASS: TestDefaultConfig/#246 (0.06s) + --- PASS: TestDefaultConfig/#495 (0.01s) + --- PASS: TestDefaultConfig/#245 (0.06s) + --- PASS: TestDefaultConfig/#249 (0.11s) --- PASS: TestDefaultConfig/#497 (0.06s) - --- PASS: TestDefaultConfig/#492 (0.03s) - --- PASS: TestDefaultConfig/#494 (0.07s) - --- PASS: TestDefaultConfig/#498 (0.10s) - --- PASS: TestDefaultConfig/#434 (0.04s) - --- PASS: TestDefaultConfig/#496 (0.13s) - --- PASS: TestDefaultConfig/#490 (0.07s) - --- PASS: TestDefaultConfig/#491 (0.09s) - --- PASS: TestDefaultConfig/#435 (0.09s) - --- PASS: TestDefaultConfig/#432 (0.11s) - --- PASS: TestDefaultConfig/#433 (0.12s) - --- PASS: TestDefaultConfig/#431 (0.08s) - --- PASS: TestDefaultConfig/#427 (0.03s) - --- PASS: TestDefaultConfig/#436 (0.14s) - --- PASS: TestDefaultConfig/#430 (0.08s) - --- PASS: TestDefaultConfig/#424 (0.02s) - --- PASS: TestDefaultConfig/#429 (0.08s) - --- PASS: TestDefaultConfig/#428 (0.08s) - --- PASS: TestDefaultConfig/#422 (0.02s) - --- PASS: TestDefaultConfig/#426 (0.07s) - --- PASS: TestDefaultConfig/#421 (0.05s) - --- PASS: TestDefaultConfig/#425 (0.08s) - --- PASS: TestDefaultConfig/#423 (0.09s) - --- PASS: TestDefaultConfig/#419 (0.09s) - --- PASS: TestDefaultConfig/#420 (0.12s) - --- PASS: TestDefaultConfig/#418 (0.14s) - --- PASS: TestDefaultConfig/#413 (0.08s) - --- PASS: TestDefaultConfig/#415 (0.11s) - --- PASS: TestDefaultConfig/#414 (0.11s) - --- PASS: TestDefaultConfig/#416 (0.11s) - --- PASS: TestDefaultConfig/#446 (0.02s) - --- PASS: TestDefaultConfig/#417 (0.17s) - --- PASS: TestDefaultConfig/#444 (0.02s) - --- PASS: TestDefaultConfig/#451 (0.11s) - --- PASS: TestDefaultConfig/#449 (0.09s) - --- PASS: TestDefaultConfig/#443 (0.07s) - --- PASS: TestDefaultConfig/#450 (0.15s) - --- PASS: TestDefaultConfig/#442 (0.07s) - --- PASS: TestDefaultConfig/#447 (0.10s) - --- PASS: TestDefaultConfig/#448 (0.14s) - --- PASS: TestDefaultConfig/#440 (0.07s) - --- PASS: TestDefaultConfig/#445 (0.14s) - --- PASS: TestDefaultConfig/#438 (0.08s) - --- PASS: TestDefaultConfig/#439 (0.08s) - --- PASS: TestDefaultConfig/#241 (0.12s) - --- PASS: TestDefaultConfig/#240 (0.12s) - --- PASS: TestDefaultConfig/#441 (0.19s) - --- PASS: TestDefaultConfig/#239 (0.12s) - --- PASS: TestDefaultConfig/#236 (0.07s) - --- PASS: TestDefaultConfig/#237 (0.07s) - --- PASS: TestDefaultConfig/#238 (0.12s) - --- PASS: TestDefaultConfig/#235 (0.08s) - --- PASS: TestDefaultConfig/#232 (0.07s) - --- PASS: TestDefaultConfig/#233 (0.09s) - --- PASS: TestDefaultConfig/#226 (0.04s) - --- PASS: TestDefaultConfig/#229 (0.10s) - --- PASS: TestDefaultConfig/#234 (0.14s) - --- PASS: TestDefaultConfig/#230 (0.12s) - --- PASS: TestDefaultConfig/#228 (0.11s) - --- PASS: TestDefaultConfig/#231 (0.14s) - --- PASS: TestDefaultConfig/#224 (0.07s) - --- PASS: TestDefaultConfig/#227 (0.15s) - --- PASS: TestDefaultConfig/#225 (0.10s) - --- PASS: TestDefaultConfig/#222 (0.06s) - --- PASS: TestDefaultConfig/#220 (0.06s) - --- PASS: TestDefaultConfig/#221 (0.07s) - --- PASS: TestDefaultConfig/#223 (0.09s) - --- PASS: TestDefaultConfig/#219 (0.07s) - --- PASS: TestDefaultConfig/#216 (0.05s) - --- PASS: TestDefaultConfig/#215 (0.07s) - --- PASS: TestDefaultConfig/#217 (0.09s) - --- PASS: TestDefaultConfig/#218 (0.13s) - --- PASS: TestDefaultConfig/#213 (0.11s) - --- PASS: TestDefaultConfig/#212 (0.13s) - --- PASS: TestDefaultConfig/#210 (0.11s) - --- PASS: TestDefaultConfig/#209 (0.10s) - --- PASS: TestDefaultConfig/#385 (0.01s) - --- PASS: TestDefaultConfig/#214 (0.18s) - --- PASS: TestDefaultConfig/#384 (0.01s) - --- PASS: TestDefaultConfig/#387 (0.07s) - --- PASS: TestDefaultConfig/#383 (0.01s) - --- PASS: TestDefaultConfig/#211 (0.17s) - --- PASS: TestDefaultConfig/#388 (0.11s) - --- PASS: TestDefaultConfig/#379 (0.01s) - --- PASS: TestDefaultConfig/#463 (0.13s) - --- PASS: TestDefaultConfig/#381 (0.08s) - --- PASS: TestDefaultConfig/#380 (0.09s) - --- PASS: TestDefaultConfig/#378 (0.10s) - --- PASS: TestDefaultConfig/#377 (0.10s) - --- PASS: TestDefaultConfig/#375 (0.05s) - --- PASS: TestDefaultConfig/#382 (0.13s) - --- PASS: TestDefaultConfig/#408 (0.02s) - --- PASS: TestDefaultConfig/#376 (0.13s) - --- PASS: TestDefaultConfig/#386 (0.19s) - --- PASS: TestDefaultConfig/#372 (0.05s) - --- PASS: TestDefaultConfig/#373 (0.06s) - --- PASS: TestDefaultConfig/#374 (0.07s) - --- PASS: TestDefaultConfig/#410 (0.06s) - --- PASS: TestDefaultConfig/#406 (0.02s) - --- PASS: TestDefaultConfig/#403 (0.05s) - --- PASS: TestDefaultConfig/#409 (0.09s) - --- PASS: TestDefaultConfig/#405 (0.06s) - --- PASS: TestDefaultConfig/#407 (0.09s) - --- PASS: TestDefaultConfig/#402 (0.08s) - --- PASS: TestDefaultConfig/#401 (0.09s) - --- PASS: TestDefaultConfig/#398 (0.07s) - --- PASS: TestDefaultConfig/#399 (0.08s) - --- PASS: TestDefaultConfig/#404 (0.13s) - --- PASS: TestDefaultConfig/#396 (0.05s) - --- PASS: TestDefaultConfig/#397 (0.08s) - --- PASS: TestDefaultConfig/#400 (0.13s) - --- PASS: TestDefaultConfig/#392 (0.02s) - --- PASS: TestDefaultConfig/#395 (0.08s) - --- PASS: TestDefaultConfig/#393 (0.03s) - --- PASS: TestDefaultConfig/#487 (0.03s) - --- PASS: TestDefaultConfig/#488 (0.08s) - --- PASS: TestDefaultConfig/#391 (0.09s) - --- PASS: TestDefaultConfig/#394 (0.09s) - --- PASS: TestDefaultConfig/#489 (0.12s) - --- PASS: TestDefaultConfig/#484 (0.12s) - --- PASS: TestDefaultConfig/#483 (0.12s) - --- PASS: TestDefaultConfig/#481 (0.05s) - --- PASS: TestDefaultConfig/#485 (0.13s) - --- PASS: TestDefaultConfig/#486 (0.14s) - --- PASS: TestDefaultConfig/#482 (0.08s) + --- PASS: TestDefaultConfig/#494 (0.05s) + --- PASS: TestDefaultConfig/#496 (0.06s) + --- PASS: TestDefaultConfig/#490 (0.05s) + --- PASS: TestDefaultConfig/#489 (0.05s) + --- PASS: TestDefaultConfig/#486 (0.02s) + --- PASS: TestDefaultConfig/#493 (0.07s) + --- PASS: TestDefaultConfig/#492 (0.07s) + --- PASS: TestDefaultConfig/#491 (0.08s) + --- PASS: TestDefaultConfig/#484 (0.02s) + --- PASS: TestDefaultConfig/#483 (0.01s) + --- PASS: TestDefaultConfig/#488 (0.05s) + --- PASS: TestDefaultConfig/#487 (0.05s) + --- PASS: TestDefaultConfig/#482 (0.02s) + --- PASS: TestDefaultConfig/#481 (0.02s) --- PASS: TestDefaultConfig/#477 (0.01s) - --- PASS: TestDefaultConfig/#480 (0.07s) --- PASS: TestDefaultConfig/#478 (0.04s) - --- PASS: TestDefaultConfig/#473 (0.02s) --- PASS: TestDefaultConfig/#479 (0.05s) - --- PASS: TestDefaultConfig/#476 (0.05s) - --- PASS: TestDefaultConfig/#472 (0.04s) + --- PASS: TestDefaultConfig/#480 (0.07s) + --- PASS: TestDefaultConfig/#485 (0.10s) + --- PASS: TestDefaultConfig/#476 (0.07s) + --- PASS: TestDefaultConfig/#471 (0.03s) --- PASS: TestDefaultConfig/#475 (0.07s) - --- PASS: TestDefaultConfig/#471 (0.07s) - --- PASS: TestDefaultConfig/#462 (0.04s) - --- PASS: TestDefaultConfig/#470 (0.07s) - --- PASS: TestDefaultConfig/#468 (0.06s) - --- PASS: TestDefaultConfig/#474 (0.11s) - --- PASS: TestDefaultConfig/#461 (0.04s) - --- PASS: TestDefaultConfig/#460 (0.02s) - --- PASS: TestDefaultConfig/#469 (0.09s) - --- PASS: TestDefaultConfig/#457 (0.02s) - --- PASS: TestDefaultConfig/#467 (0.07s) - --- PASS: TestDefaultConfig/#454 (0.01s) - --- PASS: TestDefaultConfig/#456 (0.05s) - --- PASS: TestDefaultConfig/#459 (0.07s) - --- PASS: TestDefaultConfig/#458 (0.07s) - --- PASS: TestDefaultConfig/#206 (0.04s) - --- PASS: TestDefaultConfig/#453 (0.06s) - --- PASS: TestDefaultConfig/#208 (0.06s) - --- PASS: TestDefaultConfig/#455 (0.06s) - --- PASS: TestDefaultConfig/#207 (0.08s) - --- PASS: TestDefaultConfig/#202 (0.02s) - --- PASS: TestDefaultConfig/#198 (0.01s) - --- PASS: TestDefaultConfig/#205 (0.06s) - --- PASS: TestDefaultConfig/#197 (0.01s) - --- PASS: TestDefaultConfig/#203 (0.04s) - --- PASS: TestDefaultConfig/#199 (0.04s) - --- PASS: TestDefaultConfig/#200 (0.04s) - --- PASS: TestDefaultConfig/#204 (0.08s) - --- PASS: TestDefaultConfig/#196 (0.04s) - --- PASS: TestDefaultConfig/#201 (0.07s) - --- PASS: TestDefaultConfig/#194 (0.04s) - --- PASS: TestDefaultConfig/#195 (0.08s) - --- PASS: TestDefaultConfig/#193 (0.08s) - --- PASS: TestDefaultConfig/#192 (0.08s) - --- PASS: TestDefaultConfig/#189 (0.07s) - --- PASS: TestDefaultConfig/#190 (0.06s) - --- PASS: TestDefaultConfig/#187 (0.07s) - --- PASS: TestDefaultConfig/#186 (0.04s) - --- PASS: TestDefaultConfig/#184 (0.02s) - --- PASS: TestDefaultConfig/#191 (0.11s) - --- PASS: TestDefaultConfig/#188 (0.09s) - --- PASS: TestDefaultConfig/#183 (0.03s) - --- PASS: TestDefaultConfig/#185 (0.07s) - --- PASS: TestDefaultConfig/#180 (0.05s) - --- PASS: TestDefaultConfig/#181 (0.06s) - --- PASS: TestDefaultConfig/#177 (0.05s) - --- PASS: TestDefaultConfig/#182 (0.08s) - --- PASS: TestDefaultConfig/#178 (0.05s) - --- PASS: TestDefaultConfig/#179 (0.08s) - --- PASS: TestDefaultConfig/#173 (0.04s) - --- PASS: TestDefaultConfig/#174 (0.05s) - --- PASS: TestDefaultConfig/#176 (0.09s) - --- PASS: TestDefaultConfig/#170 (0.01s) - --- PASS: TestDefaultConfig/#172 (0.06s) - --- PASS: TestDefaultConfig/#168 (0.02s) - --- PASS: TestDefaultConfig/#175 (0.07s) - --- PASS: TestDefaultConfig/#171 (0.06s) - --- PASS: TestDefaultConfig/#169 (0.03s) - --- PASS: TestDefaultConfig/#163 (0.03s) - --- PASS: TestDefaultConfig/#165 (0.06s) - --- PASS: TestDefaultConfig/#167 (0.07s) - --- PASS: TestDefaultConfig/#166 (0.09s) - --- PASS: TestDefaultConfig/#160 (0.08s) - --- PASS: TestDefaultConfig/#162 (0.08s) - --- PASS: TestDefaultConfig/#164 (0.09s) - --- PASS: TestDefaultConfig/#157 (0.03s) + --- PASS: TestDefaultConfig/#470 (0.03s) + --- PASS: TestDefaultConfig/#473 (0.07s) + --- PASS: TestDefaultConfig/#469 (0.03s) + --- PASS: TestDefaultConfig/#472 (0.06s) + --- PASS: TestDefaultConfig/#370 (0.01s) + --- PASS: TestDefaultConfig/#467 (0.02s) + --- PASS: TestDefaultConfig/#474 (0.10s) + --- PASS: TestDefaultConfig/#368 (0.01s) + --- PASS: TestDefaultConfig/#468 (0.05s) + --- PASS: TestDefaultConfig/#242 (0.05s) + --- PASS: TestDefaultConfig/#389 (0.04s) + --- PASS: TestDefaultConfig/#388 (0.03s) + --- PASS: TestDefaultConfig/#369 (0.05s) + --- PASS: TestDefaultConfig/#387 (0.05s) + --- PASS: TestDefaultConfig/#386 (0.05s) + --- PASS: TestDefaultConfig/#435 (0.03s) + --- PASS: TestDefaultConfig/#432 (0.01s) + --- PASS: TestDefaultConfig/#385 (0.07s) + --- PASS: TestDefaultConfig/#433 (0.03s) + --- PASS: TestDefaultConfig/#382 (0.03s) + --- PASS: TestDefaultConfig/#431 (0.01s) + --- PASS: TestDefaultConfig/#383 (0.04s) + --- PASS: TestDefaultConfig/#436 (0.07s) + --- PASS: TestDefaultConfig/#381 (0.01s) + --- PASS: TestDefaultConfig/#384 (0.07s) + --- PASS: TestDefaultConfig/#378 (0.04s) + --- PASS: TestDefaultConfig/#380 (0.05s) + --- PASS: TestDefaultConfig/#434 (0.10s) + --- PASS: TestDefaultConfig/#430 (0.06s) + --- PASS: TestDefaultConfig/#428 (0.05s) + --- PASS: TestDefaultConfig/#427 (0.06s) + --- PASS: TestDefaultConfig/#462 (0.03s) + --- PASS: TestDefaultConfig/#459 (0.02s) + --- PASS: TestDefaultConfig/#458 (0.01s) + --- PASS: TestDefaultConfig/#463 (0.05s) + --- PASS: TestDefaultConfig/#429 (0.09s) + --- PASS: TestDefaultConfig/#457 (0.01s) + --- PASS: TestDefaultConfig/#379 (0.09s) + --- PASS: TestDefaultConfig/#456 (0.01s) + --- PASS: TestDefaultConfig/#461 (0.03s) + --- PASS: TestDefaultConfig/#455 (0.01s) + --- PASS: TestDefaultConfig/#450 (0.03s) + --- PASS: TestDefaultConfig/#451 (0.03s) + --- PASS: TestDefaultConfig/#460 (0.08s) + --- PASS: TestDefaultConfig/#454 (0.05s) + --- PASS: TestDefaultConfig/#453 (0.05s) + --- PASS: TestDefaultConfig/#448 (0.05s) + --- PASS: TestDefaultConfig/#443 (0.02s) + --- PASS: TestDefaultConfig/#444 (0.02s) + --- PASS: TestDefaultConfig/#446 (0.03s) + --- PASS: TestDefaultConfig/#452 (0.07s) + --- PASS: TestDefaultConfig/#449 (0.07s) + --- PASS: TestDefaultConfig/#442 (0.03s) + --- PASS: TestDefaultConfig/#441 (0.01s) + --- PASS: TestDefaultConfig/#440 (0.02s) + --- PASS: TestDefaultConfig/#447 (0.06s) + --- PASS: TestDefaultConfig/#375 (0.02s) + --- PASS: TestDefaultConfig/#438 (0.05s) + --- PASS: TestDefaultConfig/#377 (0.05s) + --- PASS: TestDefaultConfig/#439 (0.06s) + --- PASS: TestDefaultConfig/#445 (0.09s) + --- PASS: TestDefaultConfig/#373 (0.05s) + --- PASS: TestDefaultConfig/#374 (0.05s) + --- PASS: TestDefaultConfig/#376 (0.07s) + --- PASS: TestDefaultConfig/#372 (0.05s) + --- PASS: TestDefaultConfig/#406 (0.03s) + --- PASS: TestDefaultConfig/#403 (0.05s) + --- PASS: TestDefaultConfig/#426 (0.04s) + --- PASS: TestDefaultConfig/#400 (0.03s) + --- PASS: TestDefaultConfig/#404 (0.05s) + --- PASS: TestDefaultConfig/#425 (0.04s) + --- PASS: TestDefaultConfig/#405 (0.05s) + --- PASS: TestDefaultConfig/#402 (0.05s) + --- PASS: TestDefaultConfig/#424 (0.07s) + --- PASS: TestDefaultConfig/#401 (0.08s) + --- PASS: TestDefaultConfig/#396 (0.06s) + --- PASS: TestDefaultConfig/#423 (0.08s) + --- PASS: TestDefaultConfig/#398 (0.08s) + --- PASS: TestDefaultConfig/#395 (0.08s) + --- PASS: TestDefaultConfig/#397 (0.09s) + --- PASS: TestDefaultConfig/#417 (0.02s) + --- PASS: TestDefaultConfig/#392 (0.03s) + --- PASS: TestDefaultConfig/#422 (0.09s) + --- PASS: TestDefaultConfig/#394 (0.05s) + --- PASS: TestDefaultConfig/#391 (0.04s) + --- PASS: TestDefaultConfig/#420 (0.02s) + --- PASS: TestDefaultConfig/#393 (0.05s) + --- PASS: TestDefaultConfig/#366 (0.01s) + --- PASS: TestDefaultConfig/#465 (0.02s) + --- PASS: TestDefaultConfig/#419 (0.05s) + --- PASS: TestDefaultConfig/#418 (0.07s) + --- PASS: TestDefaultConfig/#364 (0.06s) + --- PASS: TestDefaultConfig/#367 (0.07s) + --- PASS: TestDefaultConfig/#365 (0.07s) + --- PASS: TestDefaultConfig/#362 (0.07s) + --- PASS: TestDefaultConfig/#361 (0.08s) + --- PASS: TestDefaultConfig/#356 (0.02s) + --- PASS: TestDefaultConfig/#359 (0.04s) + --- PASS: TestDefaultConfig/#358 (0.04s) + --- PASS: TestDefaultConfig/#360 (0.05s) + --- PASS: TestDefaultConfig/#357 (0.03s) + --- PASS: TestDefaultConfig/#416 (0.01s) + --- PASS: TestDefaultConfig/#415 (0.01s) + --- PASS: TestDefaultConfig/#421 (0.03s) + --- PASS: TestDefaultConfig/#363 (0.11s) + --- PASS: TestDefaultConfig/#240 (0.03s) + --- PASS: TestDefaultConfig/#414 (0.04s) + --- PASS: TestDefaultConfig/#121 (0.04s) + --- PASS: TestDefaultConfig/#464 (0.05s) + --- PASS: TestDefaultConfig/#239 (0.06s) + --- PASS: TestDefaultConfig/#413 (0.07s) + --- PASS: TestDefaultConfig/#237 (0.06s) + --- PASS: TestDefaultConfig/#236 (0.04s) + --- PASS: TestDefaultConfig/#235 (0.04s) + --- PASS: TestDefaultConfig/#233 (0.03s) + --- PASS: TestDefaultConfig/#234 (0.05s) + --- PASS: TestDefaultConfig/#230 (0.01s) + --- PASS: TestDefaultConfig/#238 (0.08s) + --- PASS: TestDefaultConfig/#228 (0.01s) + --- PASS: TestDefaultConfig/#232 (0.03s) + --- PASS: TestDefaultConfig/#231 (0.04s) + --- PASS: TestDefaultConfig/#229 (0.04s) + --- PASS: TestDefaultConfig/#176 (0.05s) + --- PASS: TestDefaultConfig/#173 (0.05s) + --- PASS: TestDefaultConfig/#174 (0.06s) + --- PASS: TestDefaultConfig/#227 (0.06s) + --- PASS: TestDefaultConfig/#226 (0.06s) + --- PASS: TestDefaultConfig/#172 (0.03s) + --- PASS: TestDefaultConfig/#223 (0.01s) + --- PASS: TestDefaultConfig/#225 (0.05s) + --- PASS: TestDefaultConfig/#222 (0.01s) + --- PASS: TestDefaultConfig/#171 (0.02s) + --- PASS: TestDefaultConfig/#224 (0.05s) + --- PASS: TestDefaultConfig/#175 (0.09s) + --- PASS: TestDefaultConfig/#221 (0.01s) + --- PASS: TestDefaultConfig/#170 (0.03s) + --- PASS: TestDefaultConfig/#169 (0.05s) + --- PASS: TestDefaultConfig/#166 (0.05s) + --- PASS: TestDefaultConfig/#165 (0.05s) + --- PASS: TestDefaultConfig/#219 (0.04s) + --- PASS: TestDefaultConfig/#167 (0.06s) + --- PASS: TestDefaultConfig/#220 (0.07s) + --- PASS: TestDefaultConfig/#217 (0.02s) + --- PASS: TestDefaultConfig/#168 (0.07s) + --- PASS: TestDefaultConfig/#162 (0.01s) + --- PASS: TestDefaultConfig/#164 (0.07s) + --- PASS: TestDefaultConfig/#215 (0.01s) + --- PASS: TestDefaultConfig/#160 (0.01s) + --- PASS: TestDefaultConfig/#163 (0.04s) + --- PASS: TestDefaultConfig/#159 (0.02s) + --- PASS: TestDefaultConfig/#218 (0.06s) + --- PASS: TestDefaultConfig/#214 (0.05s) + --- PASS: TestDefaultConfig/#212 (0.03s) + --- PASS: TestDefaultConfig/#161 (0.05s) + --- PASS: TestDefaultConfig/#213 (0.05s) + --- PASS: TestDefaultConfig/#157 (0.06s) + --- PASS: TestDefaultConfig/#216 (0.12s) + --- PASS: TestDefaultConfig/#158 (0.08s) --- PASS: TestDefaultConfig/#154 (0.01s) - --- PASS: TestDefaultConfig/#153 (0.01s) - --- PASS: TestDefaultConfig/#161 (0.10s) - --- PASS: TestDefaultConfig/#152 (0.01s) - --- PASS: TestDefaultConfig/#159 (0.08s) + --- PASS: TestDefaultConfig/#208 (0.01s) + --- PASS: TestDefaultConfig/#210 (0.07s) + --- PASS: TestDefaultConfig/#156 (0.07s) + --- PASS: TestDefaultConfig/#211 (0.09s) + --- PASS: TestDefaultConfig/#209 (0.06s) --- PASS: TestDefaultConfig/#151 (0.02s) - --- PASS: TestDefaultConfig/#158 (0.06s) - --- PASS: TestDefaultConfig/#156 (0.05s) - --- PASS: TestDefaultConfig/#155 (0.05s) + --- PASS: TestDefaultConfig/#152 (0.03s) + --- PASS: TestDefaultConfig/#155 (0.11s) + --- PASS: TestDefaultConfig/#153 (0.05s) + --- PASS: TestDefaultConfig/#147 (0.04s) + --- PASS: TestDefaultConfig/#149 (0.05s) --- PASS: TestDefaultConfig/#150 (0.05s) - --- PASS: TestDefaultConfig/#148 (0.05s) - --- PASS: TestDefaultConfig/#145 (0.04s) --- PASS: TestDefaultConfig/#146 (0.05s) - --- PASS: TestDefaultConfig/#147 (0.06s) - --- PASS: TestDefaultConfig/#149 (0.07s) - --- PASS: TestDefaultConfig/#141 (0.03s) - --- PASS: TestDefaultConfig/#142 (0.04s) + --- PASS: TestDefaultConfig/#148 (0.04s) + --- PASS: TestDefaultConfig/#145 (0.04s) + --- PASS: TestDefaultConfig/#144 (0.03s) + --- PASS: TestDefaultConfig/#142 (0.03s) + --- PASS: TestDefaultConfig/#138 (0.01s) --- PASS: TestDefaultConfig/#137 (0.01s) - --- PASS: TestDefaultConfig/#144 (0.06s) - --- PASS: TestDefaultConfig/#139 (0.03s) - --- PASS: TestDefaultConfig/#143 (0.06s) - --- PASS: TestDefaultConfig/#136 (0.01s) - --- PASS: TestDefaultConfig/#140 (0.05s) - --- PASS: TestDefaultConfig/#134 (0.02s) - --- PASS: TestDefaultConfig/#130 (0.03s) + --- PASS: TestDefaultConfig/#143 (0.04s) + --- PASS: TestDefaultConfig/#134 (0.01s) + --- PASS: TestDefaultConfig/#136 (0.02s) + --- PASS: TestDefaultConfig/#139 (0.01s) + --- PASS: TestDefaultConfig/#133 (0.03s) + --- PASS: TestDefaultConfig/#140 (0.06s) --- PASS: TestDefaultConfig/#135 (0.05s) - --- PASS: TestDefaultConfig/#138 (0.07s) - --- PASS: TestDefaultConfig/#132 (0.05s) - --- PASS: TestDefaultConfig/#133 (0.06s) - --- PASS: TestDefaultConfig/#129 (0.07s) - --- PASS: TestDefaultConfig/#131 (0.08s) - --- PASS: TestDefaultConfig/#128 (0.07s) - --- PASS: TestDefaultConfig/#123 (0.04s) - --- PASS: TestDefaultConfig/#120 (0.01s) + --- PASS: TestDefaultConfig/#132 (0.04s) + --- PASS: TestDefaultConfig/#130 (0.05s) + --- PASS: TestDefaultConfig/#131 (0.05s) + --- PASS: TestDefaultConfig/#129 (0.05s) + --- PASS: TestDefaultConfig/#141 (0.10s) + --- PASS: TestDefaultConfig/#128 (0.04s) + --- PASS: TestDefaultConfig/#123 (0.03s) + --- PASS: TestDefaultConfig/#192 (0.02s) + --- PASS: TestDefaultConfig/#125 (0.04s) + --- PASS: TestDefaultConfig/#127 (0.05s) --- PASS: TestDefaultConfig/#126 (0.06s) - --- PASS: TestDefaultConfig/#124 (0.05s) - --- PASS: TestDefaultConfig/#121 (0.02s) - --- PASS: TestDefaultConfig/#122 (0.03s) - --- PASS: TestDefaultConfig/#118 (0.02s) - --- PASS: TestDefaultConfig/#127 (0.08s) - --- PASS: TestDefaultConfig/#119 (0.02s) - --- PASS: TestDefaultConfig/#116 (0.02s) - --- PASS: TestDefaultConfig/#112 (0.04s) + --- PASS: TestDefaultConfig/#124 (0.04s) + --- PASS: TestDefaultConfig/#122 (0.04s) + --- PASS: TestDefaultConfig/#204 (0.01s) + --- PASS: TestDefaultConfig/#206 (0.03s) + --- PASS: TestDefaultConfig/#205 (0.04s) + --- PASS: TestDefaultConfig/#203 (0.04s) + --- PASS: TestDefaultConfig/#198 (0.04s) + --- PASS: TestDefaultConfig/#199 (0.05s) + --- PASS: TestDefaultConfig/#201 (0.05s) + --- PASS: TestDefaultConfig/#202 (0.06s) + --- PASS: TestDefaultConfig/#196 (0.03s) + --- PASS: TestDefaultConfig/#197 (0.06s) + --- PASS: TestDefaultConfig/#194 (0.04s) + --- PASS: TestDefaultConfig/#200 (0.08s) + --- PASS: TestDefaultConfig/#195 (0.04s) + --- PASS: TestDefaultConfig/#191 (0.02s) + --- PASS: TestDefaultConfig/#190 (0.01s) + --- PASS: TestDefaultConfig/#193 (0.03s) + --- PASS: TestDefaultConfig/#185 (0.01s) + --- PASS: TestDefaultConfig/#184 (0.04s) + --- PASS: TestDefaultConfig/#187 (0.03s) + --- PASS: TestDefaultConfig/#189 (0.05s) + --- PASS: TestDefaultConfig/#181 (0.03s) + --- PASS: TestDefaultConfig/#186 (0.05s) + --- PASS: TestDefaultConfig/#188 (0.05s) + --- PASS: TestDefaultConfig/#60 (0.04s) + --- PASS: TestDefaultConfig/#183 (0.06s) + --- PASS: TestDefaultConfig/#180 (0.06s) + --- PASS: TestDefaultConfig/#120 (0.04s) + --- PASS: TestDefaultConfig/#119 (0.04s) + --- PASS: TestDefaultConfig/#182 (0.07s) + --- PASS: TestDefaultConfig/#178 (0.02s) + --- PASS: TestDefaultConfig/#112 (0.01s) + --- PASS: TestDefaultConfig/#115 (0.02s) + --- PASS: TestDefaultConfig/#30 (0.01s) --- PASS: TestDefaultConfig/#117 (0.06s) + --- PASS: TestDefaultConfig/#118 (0.06s) + --- PASS: TestDefaultConfig/#111 (0.03s) --- PASS: TestDefaultConfig/#113 (0.05s) - --- PASS: TestDefaultConfig/#109 (0.06s) - --- PASS: TestDefaultConfig/#125 (0.14s) - --- PASS: TestDefaultConfig/#115 (0.09s) - --- PASS: TestDefaultConfig/#111 (0.08s) - --- PASS: TestDefaultConfig/#110 (0.08s) - --- PASS: TestDefaultConfig/#107 (0.05s) - --- PASS: TestDefaultConfig/#108 (0.06s) + --- PASS: TestDefaultConfig/#58 (0.03s) + --- PASS: TestDefaultConfig/#179 (0.07s) + --- PASS: TestDefaultConfig/#116 (0.09s) + --- PASS: TestDefaultConfig/#110 (0.06s) + --- PASS: TestDefaultConfig/#56 (0.04s) + --- PASS: TestDefaultConfig/#55 (0.03s) + --- PASS: TestDefaultConfig/#59 (0.06s) + --- PASS: TestDefaultConfig/#57 (0.05s) + --- PASS: TestDefaultConfig/#109 (0.07s) + --- PASS: TestDefaultConfig/#51 (0.01s) + --- PASS: TestDefaultConfig/#52 (0.02s) + --- PASS: TestDefaultConfig/#53 (0.04s) + --- PASS: TestDefaultConfig/#107 (0.02s) + --- PASS: TestDefaultConfig/#104 (0.01s) --- PASS: TestDefaultConfig/#103 (0.02s) - --- PASS: TestDefaultConfig/#105 (0.03s) - --- PASS: TestDefaultConfig/#101 (0.03s) - --- PASS: TestDefaultConfig/#106 (0.07s) - --- PASS: TestDefaultConfig/#104 (0.05s) - --- PASS: TestDefaultConfig/#102 (0.05s) - --- PASS: TestDefaultConfig/#99 (0.04s) - --- PASS: TestDefaultConfig/#411 (0.02s) - --- PASS: TestDefaultConfig/#98 (0.06s) - --- PASS: TestDefaultConfig/#100 (0.06s) - --- PASS: TestDefaultConfig/#95 (0.04s) - --- PASS: TestDefaultConfig/#47 (0.03s) - --- PASS: TestDefaultConfig/#93 (0.01s) - --- PASS: TestDefaultConfig/#92 (0.01s) - --- PASS: TestDefaultConfig/#96 (0.05s) + --- PASS: TestDefaultConfig/#102 (0.03s) + --- PASS: TestDefaultConfig/#54 (0.06s) + --- PASS: TestDefaultConfig/#108 (0.07s) + --- PASS: TestDefaultConfig/#50 (0.04s) + --- PASS: TestDefaultConfig/#105 (0.06s) + --- PASS: TestDefaultConfig/#49 (0.03s) + --- PASS: TestDefaultConfig/#100 (0.04s) + --- PASS: TestDefaultConfig/#99 (0.03s) + --- PASS: TestDefaultConfig/#106 (0.08s) + --- PASS: TestDefaultConfig/#97 (0.02s) + --- PASS: TestDefaultConfig/#101 (0.07s) + --- PASS: TestDefaultConfig/#45 (0.01s) + --- PASS: TestDefaultConfig/#98 (0.03s) + --- PASS: TestDefaultConfig/#43 (0.02s) + --- PASS: TestDefaultConfig/#46 (0.05s) + --- PASS: TestDefaultConfig/#44 (0.04s) + --- PASS: TestDefaultConfig/#96 (0.03s) + --- PASS: TestDefaultConfig/#47 (0.07s) + --- PASS: TestDefaultConfig/#39 (0.03s) + --- PASS: TestDefaultConfig/#42 (0.07s) + --- PASS: TestDefaultConfig/#93 (0.03s) + --- PASS: TestDefaultConfig/#48 (0.10s) + --- PASS: TestDefaultConfig/#40 (0.05s) --- PASS: TestDefaultConfig/#94 (0.04s) - --- PASS: TestDefaultConfig/#97 (0.07s) - --- PASS: TestDefaultConfig/#45 (0.03s) - --- PASS: TestDefaultConfig/#44 (0.02s) - --- PASS: TestDefaultConfig/#90 (0.03s) - --- PASS: TestDefaultConfig/#42 (0.03s) - --- PASS: TestDefaultConfig/#43 (0.05s) - --- PASS: TestDefaultConfig/#46 (0.08s) - --- PASS: TestDefaultConfig/#88 (0.05s) - --- PASS: TestDefaultConfig/#89 (0.06s) - --- PASS: TestDefaultConfig/#40 (0.04s) - --- PASS: TestDefaultConfig/#37 (0.02s) - --- PASS: TestDefaultConfig/#35 (0.01s) - --- PASS: TestDefaultConfig/#36 (0.03s) - --- PASS: TestDefaultConfig/#91 (0.09s) + --- PASS: TestDefaultConfig/#92 (0.01s) --- PASS: TestDefaultConfig/#41 (0.08s) - --- PASS: TestDefaultConfig/#39 (0.06s) - --- PASS: TestDefaultConfig/#34 (0.02s) - --- PASS: TestDefaultConfig/#33 (0.03s) - --- PASS: TestDefaultConfig/#30 (0.02s) - --- PASS: TestDefaultConfig/#38 (0.07s) - --- PASS: TestDefaultConfig/#32 (0.05s) - --- PASS: TestDefaultConfig/#31 (0.05s) - --- PASS: TestDefaultConfig/#87 (0.05s) - --- PASS: TestDefaultConfig/#26 (0.06s) - --- PASS: TestDefaultConfig/#86 (0.05s) - --- PASS: TestDefaultConfig/#29 (0.08s) - --- PASS: TestDefaultConfig/#28 (0.08s) + --- PASS: TestDefaultConfig/#95 (0.05s) + --- PASS: TestDefaultConfig/#88 (0.01s) + --- PASS: TestDefaultConfig/#90 (0.02s) + --- PASS: TestDefaultConfig/#87 (0.03s) + --- PASS: TestDefaultConfig/#91 (0.03s) + --- PASS: TestDefaultConfig/#89 (0.02s) + --- PASS: TestDefaultConfig/#38 (0.05s) + --- PASS: TestDefaultConfig/#35 (0.03s) + --- PASS: TestDefaultConfig/#86 (0.06s) --- PASS: TestDefaultConfig/#84 (0.04s) - --- PASS: TestDefaultConfig/#85 (0.04s) - --- PASS: TestDefaultConfig/#66 (0.02s) - --- PASS: TestDefaultConfig/#27 (0.09s) - --- PASS: TestDefaultConfig/#65 (0.03s) - --- PASS: TestDefaultConfig/#83 (0.05s) - --- PASS: TestDefaultConfig/#81 (0.04s) - --- PASS: TestDefaultConfig/#64 (0.05s) - --- PASS: TestDefaultConfig/#63 (0.05s) - --- PASS: TestDefaultConfig/#62 (0.05s) - --- PASS: TestDefaultConfig/#61 (0.05s) - --- PASS: TestDefaultConfig/#60 (0.03s) - --- PASS: TestDefaultConfig/#79 (0.03s) - --- PASS: TestDefaultConfig/#77 (0.02s) - --- PASS: TestDefaultConfig/#80 (0.07s) - --- PASS: TestDefaultConfig/#78 (0.04s) - --- PASS: TestDefaultConfig/#76 (0.01s) - --- PASS: TestDefaultConfig/#58 (0.02s) - --- PASS: TestDefaultConfig/#74 (0.02s) - --- PASS: TestDefaultConfig/#59 (0.05s) - --- PASS: TestDefaultConfig/#57 (0.03s) - --- PASS: TestDefaultConfig/#55 (0.02s) - --- PASS: TestDefaultConfig/#82 (0.12s) - --- PASS: TestDefaultConfig/#56 (0.04s) - --- PASS: TestDefaultConfig/#72 (0.03s) - --- PASS: TestDefaultConfig/#71 (0.03s) - --- PASS: TestDefaultConfig/#51 (0.02s) + --- PASS: TestDefaultConfig/#37 (0.06s) + --- PASS: TestDefaultConfig/#81 (0.02s) + --- PASS: TestDefaultConfig/#80 (0.02s) + --- PASS: TestDefaultConfig/#36 (0.08s) + --- PASS: TestDefaultConfig/#79 (0.02s) + --- PASS: TestDefaultConfig/#78 (0.02s) + --- PASS: TestDefaultConfig/#83 (0.06s) + --- PASS: TestDefaultConfig/#82 (0.06s) + --- PASS: TestDefaultConfig/#76 (0.02s) + --- PASS: TestDefaultConfig/#74 (0.03s) + --- PASS: TestDefaultConfig/#77 (0.04s) + --- PASS: TestDefaultConfig/#75 (0.04s) + --- PASS: TestDefaultConfig/#85 (0.12s) + --- PASS: TestDefaultConfig/#33 (0.04s) + --- PASS: TestDefaultConfig/#32 (0.05s) + --- PASS: TestDefaultConfig/#31 (0.04s) + --- PASS: TestDefaultConfig/#66 (0.03s) + --- PASS: TestDefaultConfig/#177 (0.01s) --- PASS: TestDefaultConfig/#73 (0.07s) - --- PASS: TestDefaultConfig/#52 (0.04s) - --- PASS: TestDefaultConfig/#70 (0.04s) - --- PASS: TestDefaultConfig/#75 (0.09s) - --- PASS: TestDefaultConfig/#53 (0.06s) - --- PASS: TestDefaultConfig/#48 (0.01s) + --- PASS: TestDefaultConfig/#71 (0.04s) + --- PASS: TestDefaultConfig/#67 (0.02s) + --- PASS: TestDefaultConfig/#70 (0.03s) + --- PASS: TestDefaultConfig/#34 (0.08s) + --- PASS: TestDefaultConfig/#15 (0.01s) + --- PASS: TestDefaultConfig/#68 (0.03s) --- PASS: TestDefaultConfig/#69 (0.04s) - --- PASS: TestDefaultConfig/#50 (0.02s) - --- PASS: TestDefaultConfig/#67 (0.01s) - --- PASS: TestDefaultConfig/#49 (0.05s) - --- PASS: TestDefaultConfig/#54 (0.11s) - --- PASS: TestDefaultConfig/#12 (0.03s) - --- PASS: TestDefaultConfig/#68 (0.05s) - --- PASS: TestDefaultConfig/#24 (0.04s) - --- PASS: TestDefaultConfig/#13 (0.06s) - --- PASS: TestDefaultConfig/#07 (0.06s) - --- PASS: TestDefaultConfig/#11 (0.04s) - --- PASS: TestDefaultConfig/#09 (0.03s) - --- PASS: TestDefaultConfig/#20 (0.02s) - --- PASS: TestDefaultConfig/#10 (0.04s) - --- PASS: TestDefaultConfig/#23 (0.08s) - --- PASS: TestDefaultConfig/#19 (0.01s) + --- PASS: TestDefaultConfig/#28 (0.03s) + --- PASS: TestDefaultConfig/#29 (0.04s) + --- PASS: TestDefaultConfig/#26 (0.04s) + --- PASS: TestDefaultConfig/#25 (0.05s) + --- PASS: TestDefaultConfig/#22 (0.04s) + --- PASS: TestDefaultConfig/#23 (0.06s) + --- PASS: TestDefaultConfig/#20 (0.03s) + --- PASS: TestDefaultConfig/#21 (0.05s) --- PASS: TestDefaultConfig/#18 (0.02s) - --- PASS: TestDefaultConfig/#17 (0.01s) - --- PASS: TestDefaultConfig/#16 (0.02s) - --- PASS: TestDefaultConfig/#22 (0.06s) + --- PASS: TestDefaultConfig/#16 (0.01s) + --- PASS: TestDefaultConfig/#27 (0.08s) + --- PASS: TestDefaultConfig/#19 (0.04s) + --- PASS: TestDefaultConfig/#24 (0.08s) + --- PASS: TestDefaultConfig/#17 (0.03s) + --- PASS: TestDefaultConfig/#64 (0.01s) + --- PASS: TestDefaultConfig/#63 (0.02s) + --- PASS: TestDefaultConfig/#65 (0.03s) --- PASS: TestDefaultConfig/#08 (0.03s) - --- PASS: TestDefaultConfig/#21 (0.08s) - --- PASS: TestDefaultConfig/#06 (0.04s) - --- PASS: TestDefaultConfig/#15 (0.06s) - --- PASS: TestDefaultConfig/#04 (0.06s) - --- PASS: TestDefaultConfig/#05 (0.06s) - --- PASS: TestDefaultConfig/#14 (0.07s) - --- PASS: TestDefaultConfig/#02 (0.06s) - --- PASS: TestDefaultConfig/#25 (0.03s) - --- PASS: TestDefaultConfig/#01 (0.05s) - --- PASS: TestDefaultConfig/#03 (0.05s) + --- PASS: TestDefaultConfig/#13 (0.04s) + --- PASS: TestDefaultConfig/#14 (0.05s) + --- PASS: TestDefaultConfig/#10 (0.05s) + --- PASS: TestDefaultConfig/#07 (0.03s) + --- PASS: TestDefaultConfig/#11 (0.05s) + --- PASS: TestDefaultConfig/#04 (0.04s) + --- PASS: TestDefaultConfig/#06 (0.03s) + --- PASS: TestDefaultConfig/#05 (0.02s) + --- PASS: TestDefaultConfig/#61 (0.02s) + --- PASS: TestDefaultConfig/#09 (0.07s) + --- PASS: TestDefaultConfig/#01 (0.01s) + --- PASS: TestDefaultConfig/#12 (0.08s) + --- PASS: TestDefaultConfig/#03 (0.04s) + --- PASS: TestDefaultConfig/#02 (0.04s) + --- PASS: TestDefaultConfig/#72 (0.02s) + --- PASS: TestDefaultConfig/#399 (0.03s) + --- PASS: TestDefaultConfig/#62 (0.06s) + --- PASS: TestDefaultConfig/#207 (0.04s) + --- PASS: TestDefaultConfig/#241 (0.05s) === RUN TestTxnEndpoint_Bad_JSON === PAUSE TestTxnEndpoint_Bad_JSON === RUN TestTxnEndpoint_Bad_Size_Item @@ -21958,20 +21959,20 @@ util_test.go:230: DM-skipped --- SKIP: TestForwardSignals (0.00s) === RUN TestMakeWatchHandler -2022-09-15T21:42:17.398Z [DEBUG] TestMakeWatchHandler: watch handler output: watch_handler="bash -c 'echo $CONSUL_INDEX >> handler_index_out && cat >> handler_out'" output= ---- PASS: TestMakeWatchHandler (0.03s) +2021-08-13T15:33:57.307Z [DEBUG] TestMakeWatchHandler: watch handler output: watch_handler="bash -c 'echo $CONSUL_INDEX >> handler_index_out && cat >> handler_out'" output= +--- PASS: TestMakeWatchHandler (0.01s) === RUN TestMakeHTTPWatchHandler -2022-09-15T21:42:17.400Z [TRACE] TestMakeHTTPWatchHandler: http watch handler output: watch=http://127.0.0.1:34843 output="Ok, i see" +2021-08-13T15:33:57.308Z [TRACE] TestMakeHTTPWatchHandler: http watch handler output: watch=http://127.0.0.1:45631 output="Ok, i see" --- PASS: TestMakeHTTPWatchHandler (0.00s) === RUN TestMakeWatchPlan === RUN TestMakeWatchPlan/handler_type_script,_with_deprecated_handler_field -2022-09-15T21:42:17.400Z [WARN] The 'handler' field in watches has been deprecated and replaced with the 'args' field. See https://www.consul.io/docs/agent/watches.html +2021-08-13T15:33:57.309Z [WARN] The 'handler' field in watches has been deprecated and replaced with the 'args' field. See https://www.consul.io/docs/agent/watches.html === RUN TestMakeWatchPlan/handler_type_script,_with_single_arg === RUN TestMakeWatchPlan/handler_type_script,_with_multiple_args_from_slice_of_interface === RUN TestMakeWatchPlan/handler_type_script,_with_multiple_args_from_slice_of_strings === RUN TestMakeWatchPlan/handler_type_script,_with_not_string_args === RUN TestMakeWatchPlan/conflicting_handler -2022-09-15T21:42:17.400Z [WARN] The 'handler' field in watches has been deprecated and replaced with the 'args' field. See https://www.consul.io/docs/agent/watches.html +2021-08-13T15:33:57.309Z [WARN] The 'handler' field in watches has been deprecated and replaced with the 'args' field. See https://www.consul.io/docs/agent/watches.html === RUN TestMakeWatchPlan/no_handler_type --- PASS: TestMakeWatchPlan (0.00s) --- PASS: TestMakeWatchPlan/handler_type_script,_with_deprecated_handler_field (0.00s) @@ -21981,19 +21982,18 @@ --- PASS: TestMakeWatchPlan/handler_type_script,_with_not_string_args (0.00s) --- PASS: TestMakeWatchPlan/conflicting_handler (0.00s) --- PASS: TestMakeWatchPlan/no_handler_type (0.00s) -=== CONT TestACL_Legacy_Disabled_Response -=== CONT TestSetFilePermissions ---- PASS: TestSetFilePermissions (0.00s) +=== CONT TestUiNodes_Filter === CONT TestStringHash --- PASS: TestStringHash (0.00s) +=== CONT TestUiNodes +=== CONT TestACL_Legacy_Disabled_Response === CONT TestUserEventToken -=== CONT TestSessionCreate_NodeChecks -=== CONT TestOperator_Keyring_InvalidRelayFactor -=== CONT TestOperator_KeyringUse -=== CONT TestOperator_KeyringRemove -=== CONT TestOperator_KeyringListLocalOnly -=== CONT TestIntentionsCreate_noBody -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== CONT TestCatalogServiceNodes_DistanceSort +=== CONT TestFireReceiveEvent +=== CONT TestShouldProcessUserEvent +=== CONT TestSetFilePermissions +--- PASS: TestSetFilePermissions (0.00s) +=== CONT TestCatalogServiceNodes_WanTranslation [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! === RUN TestACL_Legacy_Disabled_Response/0 === RUN TestACL_Legacy_Disabled_Response/1 @@ -22002,12857 +22002,1815 @@ === RUN TestACL_Legacy_Disabled_Response/4 === RUN TestACL_Legacy_Disabled_Response/5 [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:17.434Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:17.467Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:17.473 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6cbf65f2-3031-b870-9fc7-24b209669220 Address:127.0.0.1:13127}]" -42:17.474 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13127 [Follower]" leader= -42:17.475 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6cbf65f2-3031-b870-9fc7-24b209669220.dc1 127.0.0.1 -42:17.476 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6cbf65f2-3031-b870-9fc7-24b209669220 127.0.0.1 -2022-09-15T21:42:17.476Z [INFO] agent.router: Initializing LAN area manager -42:17.477 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13115 network=udp -42:17.487 [INFO] TestAgent.server: Adding LAN server: server="Node-6cbf65f2-3031-b870-9fc7-24b209669220 (Addr: tcp/127.0.0.1:13127) (DC: dc1)" -42:17.487 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6cbf65f2-3031-b870-9fc7-24b209669220.dc1 area=wan -42:17.487 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13115 network=tcp -42:17.488 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13116 network=tcp -42:17.488 [INFO] TestAgent: started state syncer -42:17.488 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13128 network=tcp -42:17.522 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:17.522 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13127 [Candidate]" term=2 -42:17.522 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:17.522 [DEBUG] TestAgent.server.raft: vote granted: from=6cbf65f2-3031-b870-9fc7-24b209669220 term=2 tally=1 -42:17.522 [INFO] TestAgent.server.raft: election won: tally=1 -42:17.522 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13127 [Leader]" -42:17.522 [INFO] TestAgent.server: cluster leadership acquired -42:17.522 [INFO] TestAgent.server: New leader elected: payload=Node-6cbf65f2-3031-b870-9fc7-24b209669220 -42:17.522 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13127 -42:17.524 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:17.526 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:17.526 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:17.526 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:17.526 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:17.526 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:17.527 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6cbf65f2-3031-b870-9fc7-24b209669220 -42:17.527 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6cbf65f2-3031-b870-9fc7-24b209669220 -42:17.528 [INFO] TestAgent.server: federation state anti-entropy synced -42:17.588 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:17.589 [INFO] TestAgent: Synced node info -42:17.589 [DEBUG] TestAgent: Node info in sync -42:17.609 [INFO] TestAgent.server.serf.wan: serf: Received install-key query -42:17.610 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-6cbf65f2-3031-b870-9fc7-24b209669220.dc1 -42:17.610 [INFO] TestAgent.server.serf.lan: serf: Received install-key query -42:17.611 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-6cbf65f2-3031-b870-9fc7-24b209669220 -42:17.611 [INFO] TestAgent.server.serf.wan: serf: Received use-key query -42:17.612 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-6cbf65f2-3031-b870-9fc7-24b209669220.dc1 -42:17.612 [INFO] TestAgent.server.serf.lan: serf: Received use-key query -42:17.612 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-6cbf65f2-3031-b870-9fc7-24b209669220 -42:17.613 [INFO] TestAgent.server.serf.wan: serf: Received remove-key query -42:17.613 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-6cbf65f2-3031-b870-9fc7-24b209669220.dc1 -42:17.613 [INFO] TestAgent.server.serf.lan: serf: Received remove-key query -42:17.614 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-6cbf65f2-3031-b870-9fc7-24b209669220 -42:17.614 [INFO] TestAgent.server.serf.wan: serf: Received list-keys query -42:17.614 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-6cbf65f2-3031-b870-9fc7-24b209669220.dc1 -42:17.614 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query -42:17.615 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-6cbf65f2-3031-b870-9fc7-24b209669220 -42:17.615 [INFO] TestAgent: Requesting shutdown -42:17.615 [INFO] TestAgent.server: shutting down server -42:17.615 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:17.615 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:17.615 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:17.615 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:17.615 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:17.615 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:17.615 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:17.615 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:17.615 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:17.615 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:17.615 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:17.615Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:17.615Z [INFO] agent.router.manager: shutting down -42:17.615 [INFO] TestAgent: consul server down -42:17.615 [INFO] TestAgent: shutdown complete -42:17.616 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13115 network=tcp -42:17.616 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13115 network=udp -42:17.616 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13116 network=tcp -42:18.116 [INFO] TestAgent: Waiting for endpoints to shut down -42:18.116 [INFO] TestAgent: Endpoints down ---- PASS: TestOperator_KeyringUse (0.71s) -=== CONT TestParseWait ---- PASS: TestParseWait (0.00s) -=== CONT TestParseSource -2022-09-15T21:42:17.467Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:17.505Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:17.508 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:17ed706e-cc1a-a2c6-0bcb-2abe8f8c030e Address:127.0.0.1:13113}]" -42:17.509 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-17ed706e-cc1a-a2c6-0bcb-2abe8f8c030e.dc1 127.0.0.1 -42:17.509 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13113 [Follower]" leader= -42:17.510 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-17ed706e-cc1a-a2c6-0bcb-2abe8f8c030e 127.0.0.1 -2022-09-15T21:42:17.510Z [INFO] agent.router: Initializing LAN area manager -42:17.510 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13122 network=udp -42:17.511 [INFO] TestAgent.server: Adding LAN server: server="Node-17ed706e-cc1a-a2c6-0bcb-2abe8f8c030e (Addr: tcp/127.0.0.1:13113) (DC: dc1)" -42:17.511 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-17ed706e-cc1a-a2c6-0bcb-2abe8f8c030e.dc1 area=wan -42:17.511 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13122 network=tcp -42:17.512 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13123 network=tcp -42:17.512 [INFO] TestAgent: started state syncer -42:17.512 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13114 network=tcp -42:17.572 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:17.572 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13113 [Candidate]" term=2 -42:17.572 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:17.572 [DEBUG] TestAgent.server.raft: vote granted: from=17ed706e-cc1a-a2c6-0bcb-2abe8f8c030e term=2 tally=1 -42:17.572 [INFO] TestAgent.server.raft: election won: tally=1 -42:17.572 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13113 [Leader]" -42:17.572 [INFO] TestAgent.server: cluster leadership acquired -42:17.572 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13113 -42:17.574 [INFO] TestAgent.server: New leader elected: payload=Node-17ed706e-cc1a-a2c6-0bcb-2abe8f8c030e -42:17.574 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:17.577 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:17.577 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:17.577 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:17.577 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:17.577 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:17.577 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-17ed706e-cc1a-a2c6-0bcb-2abe8f8c030e -42:17.577 [INFO] TestAgent.server: member joined, marking health alive: member=Node-17ed706e-cc1a-a2c6-0bcb-2abe8f8c030e -42:17.579 [INFO] TestAgent.server: federation state anti-entropy synced -42:17.638 [INFO] TestAgent: Requesting shutdown -42:17.638 [INFO] TestAgent.server: shutting down server -42:17.638 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:17.638 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:17.638 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:17.638 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:17.638 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:17.638 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:17.638 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:17.638 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:17.638 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:17.638 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:17.638 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:17.638 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:17.639Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:17.639Z [INFO] agent.router.manager: shutting down -42:17.639 [INFO] TestAgent: consul server down -42:17.639 [INFO] TestAgent: shutdown complete -42:17.639 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13122 network=tcp -42:17.639 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13122 network=udp -42:17.639 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13123 network=tcp -42:18.141 [INFO] TestAgent: Waiting for endpoints to shut down -42:18.141 [INFO] TestAgent: Endpoints down ---- PASS: TestOperator_Keyring_InvalidRelayFactor (0.74s) -=== CONT TestPrettyPrintBare -2022-09-15T21:42:17.441Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:17.501Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:17.504 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:681de776-6856-3b5d-6ee8-1d0275c962ad Address:127.0.0.1:13099}]" -42:17.505 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13099 [Follower]" leader= -42:17.505 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-681de776-6856-3b5d-6ee8-1d0275c962ad.dc1 127.0.0.1 -42:17.506 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-681de776-6856-3b5d-6ee8-1d0275c962ad 127.0.0.1 -2022-09-15T21:42:17.506Z [INFO] agent.router: Initializing LAN area manager -42:17.506 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13094 network=udp -42:17.506 [INFO] TestAgent.server: Adding LAN server: server="Node-681de776-6856-3b5d-6ee8-1d0275c962ad (Addr: tcp/127.0.0.1:13099) (DC: dc1)" -42:17.507 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-681de776-6856-3b5d-6ee8-1d0275c962ad.dc1 area=wan -42:17.507 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13094 network=tcp -42:17.508 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13095 network=tcp -42:17.508 [INFO] TestAgent: started state syncer -42:17.508 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13100 network=tcp -42:17.575 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:17.575 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13099 [Candidate]" term=2 -42:17.575 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:17.575 [DEBUG] TestAgent.server.raft: vote granted: from=681de776-6856-3b5d-6ee8-1d0275c962ad term=2 tally=1 -42:17.575 [INFO] TestAgent.server.raft: election won: tally=1 -42:17.575 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13099 [Leader]" -42:17.576 [INFO] TestAgent.server: cluster leadership acquired -42:17.576 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13099 -42:17.577 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:17.579 [INFO] TestAgent.server: New leader elected: payload=Node-681de776-6856-3b5d-6ee8-1d0275c962ad -42:17.580 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:17.580 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:17.580 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:17.580 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:17.580 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:17.581 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-681de776-6856-3b5d-6ee8-1d0275c962ad -42:17.581 [INFO] TestAgent.server: member joined, marking health alive: member=Node-681de776-6856-3b5d-6ee8-1d0275c962ad -42:17.582 [INFO] TestAgent.server: federation state anti-entropy synced -42:17.654 [INFO] TestAgent: Requesting shutdown -42:17.654 [INFO] TestAgent.server: shutting down server -42:17.654 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:17.654 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:17.654 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:17.654 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:17.654 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:17.654 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:17.654 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:17.654 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:17.654 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:17.654 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:17.654 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:17.654 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:17.655Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:17.655Z [INFO] agent.router.manager: shutting down -42:17.655 [INFO] TestAgent: consul server down -42:17.655 [INFO] TestAgent: shutdown complete -42:17.655 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13094 network=tcp -42:17.655 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13094 network=udp -42:17.655 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13095 network=tcp -42:18.156 [INFO] TestAgent: Waiting for endpoints to shut down -42:18.156 [INFO] TestAgent: Endpoints down ---- PASS: TestSessionCreate_NodeChecks (0.75s) -=== CONT TestPrettyPrint -[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:17.460Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:17.513Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:17.516 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:65eed8a8-270b-be45-7867-4902aff9014e Address:127.0.0.1:13134}]" -42:17.516 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13134 [Follower]" leader= -42:17.517 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-65eed8a8-270b-be45-7867-4902aff9014e.dc1 127.0.0.1 -42:17.518 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-65eed8a8-270b-be45-7867-4902aff9014e 127.0.0.1 -2022-09-15T21:42:17.518Z [INFO] agent.router: Initializing LAN area manager -42:17.518 [INFO] TestAgent.server: Adding LAN server: server="Node-65eed8a8-270b-be45-7867-4902aff9014e (Addr: tcp/127.0.0.1:13134) (DC: dc1)" -42:17.518 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13129 network=udp -42:17.518 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-65eed8a8-270b-be45-7867-4902aff9014e.dc1 area=wan -42:17.518 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13129 network=tcp -42:17.519 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13130 network=tcp -42:17.519 [INFO] TestAgent: started state syncer -42:17.519 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13135 network=tcp -42:17.580 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:17.580 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13134 [Candidate]" term=2 -42:17.580 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:17.580 [DEBUG] TestAgent.server.raft: vote granted: from=65eed8a8-270b-be45-7867-4902aff9014e term=2 tally=1 -42:17.580 [INFO] TestAgent.server.raft: election won: tally=1 -42:17.580 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13134 [Leader]" -42:17.580 [INFO] TestAgent.server: cluster leadership acquired -42:17.580 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13134 -42:17.581 [INFO] TestAgent.server: New leader elected: payload=Node-65eed8a8-270b-be45-7867-4902aff9014e -42:17.582 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:17.585 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:17.585 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:17.585 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:17.585 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:17.585 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:17.585 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-65eed8a8-270b-be45-7867-4902aff9014e -42:17.585 [INFO] TestAgent.server: member joined, marking health alive: member=Node-65eed8a8-270b-be45-7867-4902aff9014e -42:17.586 [INFO] TestAgent.server: federation state anti-entropy synced -42:17.597 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:17.598 [INFO] TestAgent: Synced node info -42:17.598 [DEBUG] TestAgent: Node info in sync -42:17.791 [INFO] TestAgent.server.serf.wan: serf: Received install-key query -42:17.792 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-65eed8a8-270b-be45-7867-4902aff9014e.dc1 -42:17.792 [INFO] TestAgent.server.serf.lan: serf: Received install-key query -42:17.792 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-65eed8a8-270b-be45-7867-4902aff9014e -42:17.792 [INFO] TestAgent.server.serf.wan: serf: Received list-keys query -42:17.793 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-65eed8a8-270b-be45-7867-4902aff9014e.dc1 -42:17.793 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query -42:17.793 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-65eed8a8-270b-be45-7867-4902aff9014e -42:17.793 [INFO] TestAgent.server.serf.wan: serf: Received remove-key query -42:17.794 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-65eed8a8-270b-be45-7867-4902aff9014e.dc1 -42:17.794 [INFO] TestAgent.server.serf.lan: serf: Received remove-key query -42:17.795 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-65eed8a8-270b-be45-7867-4902aff9014e -42:17.795 [INFO] TestAgent.server.serf.wan: serf: Received list-keys query -42:17.795 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-65eed8a8-270b-be45-7867-4902aff9014e.dc1 -42:17.796 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query -42:17.796 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-65eed8a8-270b-be45-7867-4902aff9014e -42:17.796 [INFO] TestAgent: Requesting shutdown -42:17.796 [INFO] TestAgent.server: shutting down server -42:17.796 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:17.796 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:17.796 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:17.796 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:17.796 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:17.796 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:17.796 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:17.796 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:17.796 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:17.796 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:17.796 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:17.797Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:17.797Z [INFO] agent.router.manager: shutting down -42:17.797 [INFO] TestAgent: consul server down -42:17.797 [INFO] TestAgent: shutdown complete -42:17.797 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13129 network=tcp -42:17.797 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13129 network=udp -42:17.797 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13130 network=tcp -42:18.298 [INFO] TestAgent: Waiting for endpoints to shut down -42:18.298 [INFO] TestAgent: Endpoints down ---- PASS: TestOperator_KeyringRemove (0.89s) -=== CONT TestHTTP_wrap_obfuscateLog -2022-09-15T21:42:17.433Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:17.477Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:17.501 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1befb46c-a2af-02fd-11a6-f3cabf4424bc Address:127.0.0.1:13120}]" -42:17.501 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1befb46c-a2af-02fd-11a6-f3cabf4424bc.dc1 127.0.0.1 -42:17.502 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1befb46c-a2af-02fd-11a6-f3cabf4424bc 127.0.0.1 -2022-09-15T21:42:17.502Z [INFO] agent.router: Initializing LAN area manager -42:17.503 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13108 network=udp -42:17.503 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13120 [Follower]" leader= -42:17.503 [INFO] TestAgent.server: Adding LAN server: server="Node-1befb46c-a2af-02fd-11a6-f3cabf4424bc (Addr: tcp/127.0.0.1:13120) (DC: dc1)" -42:17.503 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1befb46c-a2af-02fd-11a6-f3cabf4424bc.dc1 area=wan -42:17.504 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13108 network=tcp -42:17.504 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13109 network=tcp -42:17.505 [INFO] TestAgent: started state syncer -42:17.505 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13121 network=tcp -42:17.549 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:17.549 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13120 [Candidate]" term=2 -42:17.549 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:17.549 [DEBUG] TestAgent.server.raft: vote granted: from=1befb46c-a2af-02fd-11a6-f3cabf4424bc term=2 tally=1 -42:17.549 [INFO] TestAgent.server.raft: election won: tally=1 -42:17.549 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13120 [Leader]" -42:17.549 [INFO] TestAgent.server: cluster leadership acquired -42:17.550 [INFO] TestAgent.server: New leader elected: payload=Node-1befb46c-a2af-02fd-11a6-f3cabf4424bc -42:17.550 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13120 -42:17.551 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:17.553 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:17.553 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:17.553 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:17.553 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:17.553 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:17.553 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1befb46c-a2af-02fd-11a6-f3cabf4424bc -42:17.553 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1befb46c-a2af-02fd-11a6-f3cabf4424bc -42:17.554 [INFO] TestAgent.server: federation state anti-entropy synced -42:17.823 [INFO] TestAgent: Requesting shutdown -42:17.823 [INFO] TestAgent.server: shutting down server -42:17.823 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:17.823 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:17.823 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:17.823 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:17.823 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:17.823 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:17.823 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:17.823 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:17.823 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:17.823 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:17.823 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:17.823 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:17.824Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:17.824Z [INFO] agent.router.manager: shutting down -42:17.824 [INFO] TestAgent: consul server down -42:17.824 [INFO] TestAgent: shutdown complete -42:17.824 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13108 network=tcp -42:17.824 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13108 network=udp -42:17.824 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13109 network=tcp -42:18.325 [INFO] TestAgent: Waiting for endpoints to shut down -42:18.325 [INFO] TestAgent: Endpoints down ---- PASS: TestIntentionsCreate_noBody (0.92s) -=== CONT TestContentTypeIsJSON -2022-09-15T21:42:17.455Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:17.455Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:17.499Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:17.499Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:17.514 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4ed4ebdf-de78-138b-9529-3237b7f6bc79 Address:127.0.0.1:13092}]" -42:17.514 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13092 [Follower]" leader= -42:17.515 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4ed4ebdf-de78-138b-9529-3237b7f6bc79.dc1 127.0.0.1 -42:17.516 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4ed4ebdf-de78-138b-9529-3237b7f6bc79 127.0.0.1 -2022-09-15T21:42:17.516Z [INFO] agent.router: Initializing LAN area manager -42:17.516 [INFO] TestAgent.server: Adding LAN server: server="Node-4ed4ebdf-de78-138b-9529-3237b7f6bc79 (Addr: tcp/127.0.0.1:13092) (DC: dc1)" -42:17.516 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4ed4ebdf-de78-138b-9529-3237b7f6bc79.dc1 area=wan -42:17.517 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13087 network=udp -42:17.517 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13087 network=tcp -42:17.517 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13088 network=tcp -42:17.518 [INFO] TestAgent: started state syncer -42:17.518 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13093 network=tcp -42:17.563 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:17.563 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13092 [Candidate]" term=2 -42:17.564 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:17.564 [DEBUG] TestAgent.server.raft: vote granted: from=4ed4ebdf-de78-138b-9529-3237b7f6bc79 term=2 tally=1 -42:17.564 [INFO] TestAgent.server.raft: election won: tally=1 -42:17.564 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13092 [Leader]" -42:17.564 [INFO] TestAgent.server: cluster leadership acquired -42:17.564 [INFO] TestAgent.server: initializing acls -42:17.565 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:17.565 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:17.565 [INFO] TestAgent.server: New leader elected: payload=Node-4ed4ebdf-de78-138b-9529-3237b7f6bc79 -42:17.565 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:17.566 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:17.566 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:17.566 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:17.566 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-4ed4ebdf-de78-138b-9529-3237b7f6bc79 -42:17.568 [INFO] TestAgent.server: Updating LAN server: server="Node-4ed4ebdf-de78-138b-9529-3237b7f6bc79 (Addr: tcp/127.0.0.1:13092) (DC: dc1)" -42:17.568 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-4ed4ebdf-de78-138b-9529-3237b7f6bc79.dc1 -42:17.570 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:17.573 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:17.573 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:17.573 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:17.573 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:17.573 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:17.573 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4ed4ebdf-de78-138b-9529-3237b7f6bc79 -42:17.573 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4ed4ebdf-de78-138b-9529-3237b7f6bc79 -42:17.575 [INFO] TestAgent.server: federation state anti-entropy synced -42:17.575 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4ed4ebdf-de78-138b-9529-3237b7f6bc79 -42:17.581 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-4ed4ebdf-de78-138b-9529-3237b7f6bc79.dc1 area=wan -42:17.834 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-4ed4ebdf-de78-138b-9529-3237b7f6bc79 -42:17.836 [WARN] TestAgent.server.internal: user event blocked by ACLs: event=foo accessorID= -42:17.836 [WARN] TestAgent.server.internal: user event blocked by ACLs: event=bar accessorID= -42:17.836 [WARN] TestAgent.server.internal: user event blocked by ACLs: event=zip accessorID= -42:17.836 [INFO] TestAgent: Requesting shutdown -42:17.836 [INFO] TestAgent.server: shutting down server -42:17.836 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:17.836 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:17.836 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:17.836 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:17.836 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:17.836 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:17.836 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:17.836 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:17.836 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:17.836 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:17.836 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:17.837 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:17.837 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:17.837 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:17.837 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:17.837 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:17.837Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:17.837Z [INFO] agent.router.manager: shutting down -42:17.837 [INFO] TestAgent: consul server down -42:17.837 [INFO] TestAgent: shutdown complete -42:17.837 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13087 network=tcp -42:17.837 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13087 network=udp -42:17.837 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13088 network=tcp -42:18.338 [INFO] TestAgent: Waiting for endpoints to shut down -42:18.338 [INFO] TestAgent: Endpoints down ---- PASS: TestUserEventToken (0.94s) -=== CONT TestUIResponseHeaders -2022-09-15T21:42:17.439Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:17.497Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:17.500 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4dc79a34-4f31-c8c1-0953-1f0007ae70dd Address:127.0.0.1:13106}]" -42:17.500 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13106 [Follower]" leader= -42:17.501 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4dc79a34-4f31-c8c1-0953-1f0007ae70dd.dc1 127.0.0.1 -42:17.507 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4dc79a34-4f31-c8c1-0953-1f0007ae70dd 127.0.0.1 -2022-09-15T21:42:17.507Z [INFO] agent.router: Initializing LAN area manager -42:17.508 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13101 network=udp -42:17.508 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4dc79a34-4f31-c8c1-0953-1f0007ae70dd.dc1 area=wan -42:17.508 [INFO] TestAgent.server: Adding LAN server: server="Node-4dc79a34-4f31-c8c1-0953-1f0007ae70dd (Addr: tcp/127.0.0.1:13106) (DC: dc1)" -42:17.508 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13101 network=tcp -42:17.509 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13102 network=tcp -42:17.509 [INFO] TestAgent: started state syncer -42:17.509 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13107 network=tcp -42:17.565 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:17.565 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13106 [Candidate]" term=2 -42:17.565 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:17.565 [DEBUG] TestAgent.server.raft: vote granted: from=4dc79a34-4f31-c8c1-0953-1f0007ae70dd term=2 tally=1 -42:17.565 [INFO] TestAgent.server.raft: election won: tally=1 -42:17.565 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13106 [Leader]" -42:17.565 [INFO] TestAgent.server: cluster leadership acquired -42:17.566 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13106 -42:17.566 [INFO] TestAgent.server: New leader elected: payload=Node-4dc79a34-4f31-c8c1-0953-1f0007ae70dd -42:17.568 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:17.572 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:17.572 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:17.572 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:17.572 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:17.572 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:17.572 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4dc79a34-4f31-c8c1-0953-1f0007ae70dd -42:17.572 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4dc79a34-4f31-c8c1-0953-1f0007ae70dd -42:17.575 [INFO] TestAgent.server: federation state anti-entropy synced -42:17.834 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query -42:17.837 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-4dc79a34-4f31-c8c1-0953-1f0007ae70dd -42:17.838 [INFO] TestAgent: Requesting shutdown -42:17.838 [INFO] TestAgent.server: shutting down server -42:17.838 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:17.838 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:17.838 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:17.838 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:17.838 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:17.838 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:17.838 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:17.838 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:17.838 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:17.838 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:17.838 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:17.838 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:17.838Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:17.838Z [INFO] agent.router.manager: shutting down -42:17.839 [INFO] TestAgent: consul server down -42:17.839 [INFO] TestAgent: shutdown complete -42:17.839 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13101 network=tcp -42:17.839 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13101 network=udp -42:17.839 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13102 network=tcp -42:18.342 [INFO] TestAgent: Waiting for endpoints to shut down -42:18.342 [INFO] TestAgent: Endpoints down ---- PASS: TestOperator_KeyringListLocalOnly (0.94s) -=== CONT TestHTTPAPIResponseHeaders -2022-09-15T21:42:17.433Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:17.505Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:17.509 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ea86a45f-3b33-654f-756f-8cf19ae32adf Address:127.0.0.1:13085}]" -42:17.509 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13085 [Follower]" leader= -42:17.510 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ea86a45f-3b33-654f-756f-8cf19ae32adf.dc1 127.0.0.1 -42:17.510 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ea86a45f-3b33-654f-756f-8cf19ae32adf 127.0.0.1 -2022-09-15T21:42:17.510Z [INFO] agent.router: Initializing LAN area manager -42:17.511 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13073 network=udp -42:17.511 [INFO] TestAgent.server: Adding LAN server: server="Node-ea86a45f-3b33-654f-756f-8cf19ae32adf (Addr: tcp/127.0.0.1:13085) (DC: dc1)" -42:17.511 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ea86a45f-3b33-654f-756f-8cf19ae32adf.dc1 area=wan -42:17.511 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13073 network=tcp -42:17.512 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13074 network=tcp -42:17.512 [INFO] TestAgent: started state syncer -42:17.512 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13086 network=tcp -42:17.549 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:17.549 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13085 [Candidate]" term=2 -42:17.550 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:17.550 [DEBUG] TestAgent.server.raft: vote granted: from=ea86a45f-3b33-654f-756f-8cf19ae32adf term=2 tally=1 -42:17.550 [INFO] TestAgent.server.raft: election won: tally=1 -42:17.550 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13085 [Leader]" -42:17.550 [INFO] TestAgent.server: cluster leadership acquired -42:17.550 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13085 -42:17.550 [INFO] TestAgent.server: New leader elected: payload=Node-ea86a45f-3b33-654f-756f-8cf19ae32adf -42:17.551 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:17.553 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:17.553 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:17.553 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:17.553 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:17.553 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:17.553 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ea86a45f-3b33-654f-756f-8cf19ae32adf -42:17.553 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ea86a45f-3b33-654f-756f-8cf19ae32adf -42:17.554 [INFO] TestAgent.server: federation state anti-entropy synced -42:17.787 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:17.787 [INFO] TestAgent: Synced node info -42:17.870 [INFO] TestAgent: Requesting shutdown -42:17.870 [INFO] TestAgent.server: shutting down server -42:17.870 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:17.870 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:17.870 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:17.870 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:17.870 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:17.870 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:17.870 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:17.870 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:17.870 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:17.870 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -42:17.870 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2022-09-15T21:42:17.870Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:17.871Z [INFO] agent.router.manager: shutting down -42:17.871 [INFO] TestAgent: consul server down -42:17.871 [INFO] TestAgent: shutdown complete -42:17.871 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13073 network=tcp -42:17.871 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13073 network=udp -42:17.871 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13074 network=tcp -42:18.374 [INFO] TestAgent: Waiting for endpoints to shut down -42:18.374 [INFO] TestAgent: Endpoints down ---- PASS: TestACL_Legacy_Disabled_Response (0.97s) +2021-08-13T15:33:57.375Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:33:57.375Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:57.401Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:33:57.401Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:57.404 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:36168d99-4265-741d-b5f9-a5ad22e3f4ad Address:127.0.0.1:24351}]" +33:57.404 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24351 [Follower]" leader= +33:57.405 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-36168d99-4265-741d-b5f9-a5ad22e3f4ad.dc1 127.0.0.1 +33:57.406 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-36168d99-4265-741d-b5f9-a5ad22e3f4ad 127.0.0.1 +2021-08-13T15:33:57.406Z [INFO] agent.router: Initializing LAN area manager +33:57.406 [INFO] TestAgent.server: Adding LAN server: server="Node-36168d99-4265-741d-b5f9-a5ad22e3f4ad (Addr: tcp/127.0.0.1:24351) (DC: dc1)" +33:57.406 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-36168d99-4265-741d-b5f9-a5ad22e3f4ad.dc1 area=wan +33:57.406 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24339 network=tcp +33:57.406 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24339 network=udp +33:57.407 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24340 network=tcp +33:57.407 [INFO] TestAgent: started state syncer +33:57.407 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24352 network=tcp +33:57.456 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +33:57.458 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:57.458 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24351 [Candidate]" term=2 +33:57.458 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:57.458 [DEBUG] TestAgent.server.raft: vote granted: from=36168d99-4265-741d-b5f9-a5ad22e3f4ad term=2 tally=1 +33:57.458 [INFO] TestAgent.server.raft: election won: tally=1 +33:57.458 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24351 [Leader]" +33:57.458 [INFO] TestAgent.server: cluster leadership acquired +33:57.458 [INFO] TestAgent.server: New leader elected: payload=Node-36168d99-4265-741d-b5f9-a5ad22e3f4ad +33:57.458 [INFO] TestAgent.server: initializing acls +33:57.459 [INFO] TestAgent.server: Created ACL 'global-management' policy +33:57.459 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +33:57.459 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +33:57.460 [INFO] TestAgent.server: Created ACL anonymous token from configuration +33:57.460 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +33:57.460 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +33:57.460 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-36168d99-4265-741d-b5f9-a5ad22e3f4ad +33:57.460 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-36168d99-4265-741d-b5f9-a5ad22e3f4ad.dc1 +33:57.461 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-36168d99-4265-741d-b5f9-a5ad22e3f4ad.dc1 area=wan +33:57.461 [INFO] TestAgent.server: Updating LAN server: server="Node-36168d99-4265-741d-b5f9-a5ad22e3f4ad (Addr: tcp/127.0.0.1:24351) (DC: dc1)" +33:57.461 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:57.463 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:57.463 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:57.463 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:57.463 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:57.463 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:57.463 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-36168d99-4265-741d-b5f9-a5ad22e3f4ad +33:57.463 [INFO] TestAgent.server: member joined, marking health alive: member=Node-36168d99-4265-741d-b5f9-a5ad22e3f4ad +33:57.464 [INFO] TestAgent.server: federation state anti-entropy synced +33:57.464 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-36168d99-4265-741d-b5f9-a5ad22e3f4ad +33:57.464 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-36168d99-4265-741d-b5f9-a5ad22e3f4ad +33:57.466 [WARN] TestAgent.server.internal: user event blocked by ACLs: event=foo accessorID= +33:57.466 [WARN] TestAgent.server.internal: user event blocked by ACLs: event=bar accessorID= +33:57.466 [WARN] TestAgent.server.internal: user event blocked by ACLs: event=zip accessorID= +33:57.466 [INFO] TestAgent: Requesting shutdown +33:57.466 [INFO] TestAgent.server: shutting down server +33:57.466 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +33:57.466 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +33:57.466 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:57.466 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:57.466 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:57.466 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:57.466 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:57.466 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +33:57.466 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:57.466 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:57.466 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:57.466 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +33:57.466 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:57.466 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:57.466 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:57.466 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:57.467Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:57.467Z [INFO] agent.router.manager: shutting down +33:57.467 [INFO] TestAgent: consul server down +33:57.467 [INFO] TestAgent: shutdown complete +33:57.467 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24339 network=tcp +33:57.467 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24339 network=udp +33:57.467 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24340 network=tcp +33:57.967 [INFO] TestAgent: Waiting for endpoints to shut down +33:57.967 [INFO] TestAgent: Endpoints down +--- PASS: TestUserEventToken (0.66s) +=== CONT TestCatalogServiceNodes_Filter +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:33:57.331Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:57.380Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:57.382 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3ef86ad5-83d1-e5c8-6664-3cb23d866dbc Address:127.0.0.1:24337}]" +33:57.389 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24337 [Follower]" leader= +33:57.390 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3ef86ad5-83d1-e5c8-6664-3cb23d866dbc.dc1 127.0.0.1 +33:57.395 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3ef86ad5-83d1-e5c8-6664-3cb23d866dbc 127.0.0.1 +2021-08-13T15:33:57.395Z [INFO] agent.router: Initializing LAN area manager +33:57.395 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24332 network=udp +33:57.395 [INFO] TestAgent.server: Adding LAN server: server="Node-3ef86ad5-83d1-e5c8-6664-3cb23d866dbc (Addr: tcp/127.0.0.1:24337) (DC: dc1)" +33:57.395 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3ef86ad5-83d1-e5c8-6664-3cb23d866dbc.dc1 area=wan +33:57.396 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24332 network=tcp +33:57.399 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24333 network=tcp +33:57.400 [INFO] TestAgent: started state syncer +33:57.400 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24338 network=tcp +33:57.453 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:57.453 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24337 [Candidate]" term=2 +33:57.453 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:57.453 [DEBUG] TestAgent.server.raft: vote granted: from=3ef86ad5-83d1-e5c8-6664-3cb23d866dbc term=2 tally=1 +33:57.453 [INFO] TestAgent.server.raft: election won: tally=1 +33:57.453 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24337 [Leader]" +33:57.453 [INFO] TestAgent.server: cluster leadership acquired +33:57.454 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24337 +33:57.454 [INFO] TestAgent.server: New leader elected: payload=Node-3ef86ad5-83d1-e5c8-6664-3cb23d866dbc +33:57.455 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:57.457 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:57.457 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:57.457 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:57.457 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:57.457 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:57.457 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3ef86ad5-83d1-e5c8-6664-3cb23d866dbc +33:57.457 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3ef86ad5-83d1-e5c8-6664-3cb23d866dbc +33:57.458 [INFO] TestAgent.server: federation state anti-entropy synced +33:57.632 [INFO] TestAgent: Requesting shutdown +33:57.632 [INFO] TestAgent.server: shutting down server +33:57.632 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:57.632 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:57.632 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:57.632 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:57.632 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:57.632 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:57.632 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:57.632 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:57.632 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:57.632 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:57.632 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:57.632 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:57.633Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:57.633Z [INFO] agent.router.manager: shutting down +33:57.633 [INFO] TestAgent: consul server down +33:57.633 [INFO] TestAgent: shutdown complete +33:57.633 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24332 network=tcp +33:57.633 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24332 network=udp +33:57.633 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24333 network=tcp +33:58.133 [INFO] TestAgent: Waiting for endpoints to shut down +33:58.133 [INFO] TestAgent: Endpoints down +--- PASS: TestACL_Legacy_Disabled_Response (0.82s) --- PASS: TestACL_Legacy_Disabled_Response/0 (0.00s) --- PASS: TestACL_Legacy_Disabled_Response/1 (0.00s) --- PASS: TestACL_Legacy_Disabled_Response/2 (0.00s) --- PASS: TestACL_Legacy_Disabled_Response/3 (0.00s) --- PASS: TestACL_Legacy_Disabled_Response/4 (0.00s) --- PASS: TestACL_Legacy_Disabled_Response/5 (0.00s) -=== CONT TestHTTPAPI_TranslateAddrHeader -[WARN] freeport: 10 out of 45 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestHTTP_wrap_obfuscateLog//some/url?token=secret1&token=secret2 -=== RUN TestHTTP_wrap_obfuscateLog//v1/acl/clone/secret1 -=== RUN TestHTTP_wrap_obfuscateLog//v1/acl/clone/secret1?token=secret2 -=== RUN TestHTTP_wrap_obfuscateLog//v1/acl/destroy/secret1 -=== RUN TestHTTP_wrap_obfuscateLog//v1/acl/destroy/secret1?token=secret2 -=== RUN TestHTTP_wrap_obfuscateLog//v1/acl/info/secret1 -=== RUN TestHTTP_wrap_obfuscateLog//v1/acl/info/secret1?token=secret2 -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:18.142Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:18.163Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:18.167 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e597e12b-0a17-d9c0-70be-67a533c1e73f Address:127.0.0.1:13148}]" -42:18.168 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e597e12b-0a17-d9c0-70be-67a533c1e73f.dc1 127.0.0.1 -42:18.168 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e597e12b-0a17-d9c0-70be-67a533c1e73f 127.0.0.1 -2022-09-15T21:42:18.169Z [INFO] agent.router: Initializing LAN area manager -42:18.169 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13136 network=udp -42:18.169 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13148 [Follower]" leader= -42:18.169 [INFO] TestAgent.server: Adding LAN server: server="Node-e597e12b-0a17-d9c0-70be-67a533c1e73f (Addr: tcp/127.0.0.1:13148) (DC: dc1)" -42:18.170 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e597e12b-0a17-d9c0-70be-67a533c1e73f.dc1 area=wan -42:18.170 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13136 network=tcp -42:18.171 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13137 network=tcp -42:18.171 [INFO] TestAgent: started state syncer -42:18.171 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13149 network=tcp -42:18.222 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:18.222 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13148 [Candidate]" term=2 -42:18.223 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:18.223 [DEBUG] TestAgent.server.raft: vote granted: from=e597e12b-0a17-d9c0-70be-67a533c1e73f term=2 tally=1 -42:18.223 [INFO] TestAgent.server.raft: election won: tally=1 -42:18.223 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13148 [Leader]" -42:18.223 [INFO] TestAgent.server: cluster leadership acquired -42:18.223 [INFO] TestAgent.server: New leader elected: payload=Node-e597e12b-0a17-d9c0-70be-67a533c1e73f -42:18.223 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13148 -42:18.224 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:18.227 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:18.227 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:18.227 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:18.227 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:18.227 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:18.227 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e597e12b-0a17-d9c0-70be-67a533c1e73f -42:18.227 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e597e12b-0a17-d9c0-70be-67a533c1e73f -42:18.228 [INFO] TestAgent.server: federation state anti-entropy synced -42:18.383 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:18.384 [INFO] TestAgent: Synced node info -42:18.425 [INFO] TestAgent: Requesting shutdown -42:18.426 [INFO] TestAgent.server: shutting down server -42:18.426 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:18.426 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:18.426 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:18.426 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:18.426 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:18.426 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:18.426 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:18.426 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:18.426 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:18.426 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:18.427 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:18.427Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:18.427Z [INFO] agent.router.manager: shutting down -42:18.427 [INFO] TestAgent: consul server down -42:18.427 [INFO] TestAgent: shutdown complete -42:18.427 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13136 network=tcp -42:18.427 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13136 network=udp -42:18.427 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13137 network=tcp -42:18.927 [INFO] TestAgent: Waiting for endpoints to shut down -42:18.927 [INFO] TestAgent: Endpoints down ---- PASS: TestParseSource (0.81s) -=== CONT TestHTTPAPI_BlockEndpoints -2022-09-15T21:42:18.342Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:18.359Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:18.364 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cedfab61-1be3-b677-db41-8b42195e078a Address:127.0.0.1:13176}]" -42:18.365 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-cedfab61-1be3-b677-db41-8b42195e078a.dc1 127.0.0.1 -42:18.366 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13176 [Follower]" leader= -42:18.366 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-cedfab61-1be3-b677-db41-8b42195e078a 127.0.0.1 -2022-09-15T21:42:18.366Z [INFO] agent.router: Initializing LAN area manager -42:18.367 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13157 network=udp -42:18.367 [INFO] TestAgent.server: Adding LAN server: server="Node-cedfab61-1be3-b677-db41-8b42195e078a (Addr: tcp/127.0.0.1:13176) (DC: dc1)" -42:18.367 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-cedfab61-1be3-b677-db41-8b42195e078a.dc1 area=wan -42:18.367 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13157 network=tcp -42:18.368 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13158 network=tcp -42:18.368 [INFO] TestAgent: started state syncer -42:18.368 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13177 network=tcp -42:18.430 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:18.430 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13176 [Candidate]" term=2 -42:18.430 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:18.430 [DEBUG] TestAgent.server.raft: vote granted: from=cedfab61-1be3-b677-db41-8b42195e078a term=2 tally=1 -42:18.430 [INFO] TestAgent.server.raft: election won: tally=1 -42:18.430 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13176 [Leader]" -42:18.430 [INFO] TestAgent.server: cluster leadership acquired -42:18.431 [INFO] TestAgent.server: New leader elected: payload=Node-cedfab61-1be3-b677-db41-8b42195e078a -42:18.431 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13176 -42:18.433 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:18.436 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:18.436 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:18.436 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:18.436 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:18.436 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:18.436 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cedfab61-1be3-b677-db41-8b42195e078a -42:18.436 [INFO] TestAgent.server: member joined, marking health alive: member=Node-cedfab61-1be3-b677-db41-8b42195e078a -42:18.437 [INFO] TestAgent.server: federation state anti-entropy synced -42:18.485 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:18.486 [INFO] TestAgent: Synced node info -42:18.486 [DEBUG] TestAgent: Node info in sync -42:18.494 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/key from= latency=212.6µs -42:18.494 [INFO] TestAgent: Requesting shutdown -42:18.494 [INFO] TestAgent.server: shutting down server -42:18.494 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:18.494 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:18.494 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:18.494 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:18.494 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:18.494 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:18.495 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:18.495 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:18.495 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:18.495 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:18.495 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:18.495Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:18.495Z [INFO] agent.router.manager: shutting down -42:18.495 [INFO] TestAgent: consul server down -42:18.495 [INFO] TestAgent: shutdown complete -42:18.495 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13157 network=tcp -42:18.495 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13157 network=udp -42:18.495 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13158 network=tcp -42:18.995 [INFO] TestAgent: Waiting for endpoints to shut down -42:18.995 [INFO] TestAgent: Endpoints down ---- PASS: TestContentTypeIsJSON (0.67s) -=== CONT TestSetMeta ---- PASS: TestSetMeta (0.00s) -=== CONT TestSetLastContact -=== RUN TestSetLastContact/neg -=== RUN TestSetLastContact/zero -=== RUN TestSetLastContact/pos -=== RUN TestSetLastContact/pos_ms_only ---- PASS: TestSetLastContact (0.00s) - --- PASS: TestSetLastContact/neg (0.00s) - --- PASS: TestSetLastContact/zero (0.00s) - --- PASS: TestSetLastContact/pos (0.00s) - --- PASS: TestSetLastContact/pos_ms_only (0.00s) -=== CONT TestSetKnownLeader ---- PASS: TestSetKnownLeader (0.00s) -=== CONT TestSetIndex ---- PASS: TestSetIndex (0.00s) -=== CONT TestHTTPServer_UnixSocket_FileExists -[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:18.367Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:18.398Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:18.401 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3b42d413-60dc-8fb8-2687-c86ac6c1106a Address:127.0.0.1:13190}]" -42:18.402 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13190 [Follower]" leader= -42:18.402 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3b42d413-60dc-8fb8-2687-c86ac6c1106a.dc1 127.0.0.1 -42:18.406 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3b42d413-60dc-8fb8-2687-c86ac6c1106a 127.0.0.1 -2022-09-15T21:42:18.406Z [INFO] agent.router: Initializing LAN area manager -42:18.408 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13171 network=udp -42:18.410 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3b42d413-60dc-8fb8-2687-c86ac6c1106a.dc1 area=wan -42:18.411 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13171 network=tcp -42:18.414 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13172 network=tcp -42:18.414 [INFO] TestAgent: started state syncer -42:18.414 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13191 network=tcp -42:18.415 [INFO] TestAgent.server: Adding LAN server: server="Node-3b42d413-60dc-8fb8-2687-c86ac6c1106a (Addr: tcp/127.0.0.1:13190) (DC: dc1)" -42:18.452 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:18.452 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13190 [Candidate]" term=2 -42:18.452 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:18.452 [DEBUG] TestAgent.server.raft: vote granted: from=3b42d413-60dc-8fb8-2687-c86ac6c1106a term=2 tally=1 -42:18.452 [INFO] TestAgent.server.raft: election won: tally=1 -42:18.452 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13190 [Leader]" -42:18.453 [INFO] TestAgent.server: cluster leadership acquired -42:18.453 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13190 -42:18.453 [INFO] TestAgent.server: New leader elected: payload=Node-3b42d413-60dc-8fb8-2687-c86ac6c1106a -42:18.455 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:18.457 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:18.457 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:18.457 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:18.457 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:18.457 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:18.457 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3b42d413-60dc-8fb8-2687-c86ac6c1106a -42:18.457 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3b42d413-60dc-8fb8-2687-c86ac6c1106a -42:18.458 [INFO] TestAgent.server: federation state anti-entropy synced -42:18.525 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from= latency=3.78µs -42:18.525 [INFO] TestAgent: Requesting shutdown -42:18.525 [INFO] TestAgent.server: shutting down server -42:18.526 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:18.526 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:18.526 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:18.526 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:18.526 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:18.526 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:18.526 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:18.526 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:18.526 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:18.526 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:18.526 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:18.526 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:18.526Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:18.526Z [INFO] agent.router.manager: shutting down -42:18.527 [INFO] TestAgent: consul server down -42:18.527 [INFO] TestAgent: shutdown complete -42:18.527 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13171 network=tcp -42:18.527 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13171 network=udp -42:18.527 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13172 network=tcp -42:19.030 [INFO] TestAgent: Waiting for endpoints to shut down -42:19.030 [INFO] TestAgent: Endpoints down ---- PASS: TestHTTPAPIResponseHeaders (0.69s) -=== CONT TestHTTPServer_UnixSocket -2022-09-15T21:42:18.367Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:18.395Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:18.402 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a1bd6b88-e748-fec6-5140-b5915aa871e8 Address:127.0.0.1:13169}]" -42:18.407 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a1bd6b88-e748-fec6-5140-b5915aa871e8.dc1 127.0.0.1 -42:18.415 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13169 [Follower]" leader= -42:18.418 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a1bd6b88-e748-fec6-5140-b5915aa871e8 127.0.0.1 -2022-09-15T21:42:18.418Z [INFO] agent.router: Initializing LAN area manager -42:18.419 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13178 network=udp -42:18.420 [INFO] TestAgent.server: Adding LAN server: server="Node-a1bd6b88-e748-fec6-5140-b5915aa871e8 (Addr: tcp/127.0.0.1:13169) (DC: dc1)" -42:18.420 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a1bd6b88-e748-fec6-5140-b5915aa871e8.dc1 area=wan -42:18.421 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13178 network=tcp -42:18.425 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13179 network=tcp -42:18.425 [INFO] TestAgent: started state syncer -42:18.425 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13170 network=tcp -42:18.457 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:18.457 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13169 [Candidate]" term=2 -42:18.457 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:18.457 [DEBUG] TestAgent.server.raft: vote granted: from=a1bd6b88-e748-fec6-5140-b5915aa871e8 term=2 tally=1 -42:18.457 [INFO] TestAgent.server.raft: election won: tally=1 -42:18.457 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13169 [Leader]" -42:18.457 [INFO] TestAgent.server: cluster leadership acquired -42:18.458 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13169 -42:18.458 [INFO] TestAgent.server: New leader elected: payload=Node-a1bd6b88-e748-fec6-5140-b5915aa871e8 -42:18.462 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:18.468 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:18.468 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:18.468 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:18.468 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:18.468 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:18.468 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a1bd6b88-e748-fec6-5140-b5915aa871e8 -42:18.468 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a1bd6b88-e748-fec6-5140-b5915aa871e8 -42:18.469 [INFO] TestAgent.server: federation state anti-entropy synced -42:18.536 [DEBUG] TestAgent.http: Request finished: method=GET url=/ui from= latency=3µs -42:18.536 [INFO] TestAgent: Requesting shutdown -42:18.536 [INFO] TestAgent.server: shutting down server -42:18.536 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:18.536 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:18.536 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:18.536 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:18.536 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:18.536 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:18.536 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:18.536 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:18.536 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:18.536 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:18.536 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:18.536 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:18.536Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:18.536Z [INFO] agent.router.manager: shutting down -42:18.537 [INFO] TestAgent: consul server down -42:18.537 [INFO] TestAgent: shutdown complete -42:18.537 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13178 network=tcp -42:18.537 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13178 network=udp -42:18.537 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13179 network=tcp -42:19.037 [INFO] TestAgent: Waiting for endpoints to shut down -42:19.037 [INFO] TestAgent: Endpoints down ---- PASS: TestUIResponseHeaders (0.70s) -=== CONT TestFilterNonPassing ---- PASS: TestFilterNonPassing (0.00s) -=== CONT TestHealthConnectServiceNodes_PassingFilter -2022-09-15T21:42:18.174Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:18.191Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:18.193 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:52893b48-0a51-cc89-0f16-cf3ac931f1af Address:127.0.0.1:13162}]" -42:18.194 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-52893b48-0a51-cc89-0f16-cf3ac931f1af.dc1 127.0.0.1 -42:18.195 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-52893b48-0a51-cc89-0f16-cf3ac931f1af 127.0.0.1 -2022-09-15T21:42:18.195Z [INFO] agent.router: Initializing LAN area manager -42:18.196 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13143 network=udp -42:18.196 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13162 [Follower]" leader= -42:18.196 [INFO] TestAgent.server: Adding LAN server: server="Node-52893b48-0a51-cc89-0f16-cf3ac931f1af (Addr: tcp/127.0.0.1:13162) (DC: dc1)" -42:18.196 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-52893b48-0a51-cc89-0f16-cf3ac931f1af.dc1 area=wan -42:18.196 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13143 network=tcp -42:18.197 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13144 network=tcp -42:18.197 [INFO] TestAgent: started state syncer -42:18.197 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13163 network=tcp -42:18.254 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:18.254 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13162 [Candidate]" term=2 -42:18.254 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:18.254 [DEBUG] TestAgent.server.raft: vote granted: from=52893b48-0a51-cc89-0f16-cf3ac931f1af term=2 tally=1 -42:18.254 [INFO] TestAgent.server.raft: election won: tally=1 -42:18.254 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13162 [Leader]" -42:18.254 [INFO] TestAgent.server: cluster leadership acquired -42:18.255 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13162 -42:18.256 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:18.258 [INFO] TestAgent.server: New leader elected: payload=Node-52893b48-0a51-cc89-0f16-cf3ac931f1af -42:18.273 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:18.273 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:18.273 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:18.273 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:18.273 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:18.273 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-52893b48-0a51-cc89-0f16-cf3ac931f1af -42:18.273 [INFO] TestAgent.server: member joined, marking health alive: member=Node-52893b48-0a51-cc89-0f16-cf3ac931f1af -42:18.279 [INFO] TestAgent.server: federation state anti-entropy synced -42:18.409 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:18.411 [INFO] TestAgent: Synced node info -42:18.567 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/key?pretty=1 from= latency=84.72µs -42:18.567 [INFO] TestAgent: Requesting shutdown -42:18.567 [INFO] TestAgent.server: shutting down server -42:18.567 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:18.567 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:18.567 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:18.567 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:18.567 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:18.567 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:18.567 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:18.567 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:18.567 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:18.567 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:18.567 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:18.567Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:18.568Z [INFO] agent.router.manager: shutting down -42:18.568 [INFO] TestAgent: consul server down -42:18.568 [INFO] TestAgent: shutdown complete -42:18.568 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13143 network=tcp -42:18.568 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13143 network=udp -42:18.568 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13144 network=tcp -42:19.070 [INFO] TestAgent: Waiting for endpoints to shut down -42:19.070 [INFO] TestAgent: Endpoints down ---- PASS: TestPrettyPrint (0.91s) -=== CONT TestHealthConnectServiceNodes_Filter ---- PASS: TestHTTP_wrap_obfuscateLog (0.78s) - --- PASS: TestHTTP_wrap_obfuscateLog//some/url?token=secret1&token=secret2 (0.00s) - --- PASS: TestHTTP_wrap_obfuscateLog//v1/acl/clone/secret1 (0.00s) - --- PASS: TestHTTP_wrap_obfuscateLog//v1/acl/clone/secret1?token=secret2 (0.00s) - --- PASS: TestHTTP_wrap_obfuscateLog//v1/acl/destroy/secret1 (0.00s) - --- PASS: TestHTTP_wrap_obfuscateLog//v1/acl/destroy/secret1?token=secret2 (0.00s) - --- PASS: TestHTTP_wrap_obfuscateLog//v1/acl/info/secret1 (0.00s) - --- PASS: TestHTTP_wrap_obfuscateLog//v1/acl/info/secret1?token=secret2 (0.00s) -=== CONT TestConfig_Apply_IngressGateway -2022-09-15T21:42:18.172Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:18.191Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:18.193 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c3804a71-4e70-c6bc-88ba-af703e6b6234 Address:127.0.0.1:13141}]" -42:18.194 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c3804a71-4e70-c6bc-88ba-af703e6b6234.dc1 127.0.0.1 -42:18.195 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c3804a71-4e70-c6bc-88ba-af703e6b6234 127.0.0.1 -2022-09-15T21:42:18.195Z [INFO] agent.router: Initializing LAN area manager -42:18.196 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13150 network=udp -42:18.196 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13141 [Follower]" leader= -42:18.196 [INFO] TestAgent.server: Adding LAN server: server="Node-c3804a71-4e70-c6bc-88ba-af703e6b6234 (Addr: tcp/127.0.0.1:13141) (DC: dc1)" -42:18.196 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c3804a71-4e70-c6bc-88ba-af703e6b6234.dc1 area=wan -42:18.196 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13150 network=tcp -42:18.197 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13151 network=tcp -42:18.197 [INFO] TestAgent: started state syncer -42:18.197 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13142 network=tcp -42:18.236 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:18.236 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13141 [Candidate]" term=2 -42:18.237 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:18.237 [DEBUG] TestAgent.server.raft: vote granted: from=c3804a71-4e70-c6bc-88ba-af703e6b6234 term=2 tally=1 -42:18.237 [INFO] TestAgent.server.raft: election won: tally=1 -42:18.237 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13141 [Leader]" -42:18.237 [INFO] TestAgent.server: cluster leadership acquired -42:18.237 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13141 -42:18.237 [INFO] TestAgent.server: New leader elected: payload=Node-c3804a71-4e70-c6bc-88ba-af703e6b6234 -42:18.239 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:18.243 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:18.243 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:18.243 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:18.243 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:18.243 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:18.243 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c3804a71-4e70-c6bc-88ba-af703e6b6234 -42:18.243 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c3804a71-4e70-c6bc-88ba-af703e6b6234 -42:18.244 [INFO] TestAgent.server: federation state anti-entropy synced -42:18.586 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/key?pretty from= latency=67.16µs -42:18.586 [INFO] TestAgent: Requesting shutdown -42:18.586 [INFO] TestAgent.server: shutting down server -42:18.586 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:18.586 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:18.586 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:18.586 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:18.586 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:18.586 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:18.586 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:18.586 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:18.586 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:18.587 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:18.587 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:18.587 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:18.587Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:18.587Z [INFO] agent.router.manager: shutting down -42:18.587 [INFO] TestAgent: consul server down -42:18.587 [INFO] TestAgent: shutdown complete -42:18.587 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13150 network=tcp -42:18.587 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13150 network=udp -42:18.587 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13151 network=tcp -42:19.092 [INFO] TestAgent: Waiting for endpoints to shut down -42:19.092 [INFO] TestAgent: Endpoints down ---- PASS: TestPrettyPrintBare (0.95s) -=== CONT TestHealthIngressServiceNodes -[WARN] freeport: 10 out of 45 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestHealthConnectServiceNodes_PassingFilter/bc_no_query_value -=== RUN TestHealthConnectServiceNodes_PassingFilter/passing_true -=== RUN TestHealthConnectServiceNodes_PassingFilter/passing_false -=== RUN TestHealthConnectServiceNodes_PassingFilter/passing_bad -=== RUN TestHealthIngressServiceNodes/associated_service -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestHealthIngressServiceNodes/non-associated_service -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:19.136Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:19.161Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:19.164 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0393fb38-f19f-32b0-9b54-a209d8cfe761 Address:127.0.0.1:13232}]" -42:19.164 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13232 [Follower]" leader= -42:19.165 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0393fb38-f19f-32b0-9b54-a209d8cfe761.dc1 127.0.0.1 -42:19.166 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0393fb38-f19f-32b0-9b54-a209d8cfe761 127.0.0.1 -2022-09-15T21:42:19.166Z [INFO] agent.router: Initializing LAN area manager -42:19.166 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13227 network=udp -42:19.166 [INFO] TestAgent.server: Adding LAN server: server="Node-0393fb38-f19f-32b0-9b54-a209d8cfe761 (Addr: tcp/127.0.0.1:13232) (DC: dc1)" -42:19.166 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13227 network=tcp -42:19.166 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0393fb38-f19f-32b0-9b54-a209d8cfe761.dc1 area=wan -42:19.167 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13228 network=tcp -42:19.167 [INFO] TestAgent: started state syncer -42:19.167 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13233 network=tcp -42:19.204 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:19.205 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13232 [Candidate]" term=2 -42:19.205 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:19.205 [DEBUG] TestAgent.server.raft: vote granted: from=0393fb38-f19f-32b0-9b54-a209d8cfe761 term=2 tally=1 -42:19.205 [INFO] TestAgent.server.raft: election won: tally=1 -42:19.205 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13232 [Leader]" -42:19.205 [INFO] TestAgent.server: cluster leadership acquired -42:19.205 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13232 -42:19.206 [INFO] TestAgent.server: New leader elected: payload=Node-0393fb38-f19f-32b0-9b54-a209d8cfe761 -42:19.207 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:19.209 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:19.209 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:19.209 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:19.209 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:19.209 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:19.209 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0393fb38-f19f-32b0-9b54-a209d8cfe761 -42:19.209 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0393fb38-f19f-32b0-9b54-a209d8cfe761 -42:19.210 [INFO] TestAgent.server: federation state anti-entropy synced -42:19.290 [INFO] TestAgent: Requesting shutdown -42:19.290 [INFO] TestAgent.server: shutting down server -42:19.290 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:19.290 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:19.290 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:19.290 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:19.290 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:19.290 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:19.290 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:19.291 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:19.291 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:19.291 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:19.291 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:19.291 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:19.291Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:19.291Z [INFO] agent.router.manager: shutting down -42:19.291 [INFO] TestAgent: consul server down -42:19.291 [INFO] TestAgent: shutdown complete -42:19.291 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13227 network=tcp -42:19.291 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13227 network=udp -42:19.291 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13228 network=tcp -42:19.791 [INFO] TestAgent: Waiting for endpoints to shut down -42:19.791 [INFO] TestAgent: Endpoints down ---- PASS: TestHealthConnectServiceNodes_Filter (0.72s) -=== CONT TestHealthConnectServiceNodes -2022-09-15T21:42:18.945Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:18.967Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:18.975 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c59fec8c-fafb-beea-e0c9-9d833fb9706a Address:127.0.0.1:13197}]" -42:18.976 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c59fec8c-fafb-beea-e0c9-9d833fb9706a.dc1 127.0.0.1 -42:18.977 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c59fec8c-fafb-beea-e0c9-9d833fb9706a 127.0.0.1 -2022-09-15T21:42:18.977Z [INFO] agent.router: Initializing LAN area manager -42:18.977 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13206 network=udp -42:18.977 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13197 [Follower]" leader= -42:18.977 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c59fec8c-fafb-beea-e0c9-9d833fb9706a.dc1 area=wan -42:18.977 [INFO] TestAgent.server: Adding LAN server: server="Node-c59fec8c-fafb-beea-e0c9-9d833fb9706a (Addr: tcp/127.0.0.1:13197) (DC: dc1)" -42:18.977 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13206 network=tcp -42:18.980 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13207 network=tcp -42:18.980 [INFO] TestAgent: started state syncer -42:18.980 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13198 network=tcp -42:19.018 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:19.018 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13197 [Candidate]" term=2 -42:19.018 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:19.018 [DEBUG] TestAgent.server.raft: vote granted: from=c59fec8c-fafb-beea-e0c9-9d833fb9706a term=2 tally=1 -42:19.018 [INFO] TestAgent.server.raft: election won: tally=1 -42:19.018 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13197 [Leader]" -42:19.018 [INFO] TestAgent.server: cluster leadership acquired -42:19.018 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13197 -42:19.019 [INFO] TestAgent.server: New leader elected: payload=Node-c59fec8c-fafb-beea-e0c9-9d833fb9706a -42:19.020 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:19.023 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:19.023 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:19.023 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:19.023 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:19.023 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:19.023 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c59fec8c-fafb-beea-e0c9-9d833fb9706a -42:19.023 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c59fec8c-fafb-beea-e0c9-9d833fb9706a -42:19.024 [INFO] TestAgent.server: federation state anti-entropy synced -42:19.262 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:19.262 [INFO] TestAgent: Synced node info -42:19.370 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/self from= error="Endpoint is blocked by agent configuration" -42:19.370 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/checks from= latency=3.02µs -42:19.370 [INFO] TestAgent: Requesting shutdown -42:19.370 [INFO] TestAgent.server: shutting down server -42:19.370 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:19.370 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:19.370 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:19.370 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:19.370 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:19.370 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:19.370 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:19.370 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:19.370 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:19.370 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:19.370 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:19.370Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:19.370Z [INFO] agent.router.manager: shutting down -42:19.371 [INFO] TestAgent: consul server down -42:19.371 [INFO] TestAgent: shutdown complete -42:19.371 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13206 network=tcp -42:19.371 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13206 network=udp -42:19.371 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13207 network=tcp -42:19.871 [INFO] TestAgent: Waiting for endpoints to shut down -42:19.871 [INFO] TestAgent: Endpoints down ---- PASS: TestHTTPAPI_BlockEndpoints (0.94s) -=== CONT TestHealthServiceNodes_WanTranslation -2022-09-15T21:42:19.033Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:19.065Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:19.072 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9b7141ed-facb-6405-7913-3ff8f69b802e Address:127.0.0.1:13218}]" -42:19.074 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13218 [Follower]" leader= -42:19.076 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9b7141ed-facb-6405-7913-3ff8f69b802e.dc1 127.0.0.1 -42:19.083 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9b7141ed-facb-6405-7913-3ff8f69b802e 127.0.0.1 -2022-09-15T21:42:19.083Z [INFO] agent.router: Initializing LAN area manager -42:19.086 [INFO] TestAgent.server: Adding LAN server: server="Node-9b7141ed-facb-6405-7913-3ff8f69b802e (Addr: tcp/127.0.0.1:13218) (DC: dc1)" -42:19.086 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9b7141ed-facb-6405-7913-3ff8f69b802e.dc1 area=wan -42:19.092 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13199 network=udp -42:19.092 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13199 network=tcp -42:19.092 [WARN] TestAgent: Replacing socket: path=/tmp/consul-test/TestHTTPServer_UnixSocket_FileExists-consul246958013/test.sock -42:19.103 [INFO] TestAgent: Started HTTP server: address=/tmp/consul-test/TestHTTPServer_UnixSocket_FileExists-consul246958013/test.sock network=unix -42:19.103 [INFO] TestAgent: started state syncer -42:19.106 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13219 network=tcp -42:19.122 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:19.122 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13218 [Candidate]" term=2 -42:19.122 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:19.122 [DEBUG] TestAgent.server.raft: vote granted: from=9b7141ed-facb-6405-7913-3ff8f69b802e term=2 tally=1 -42:19.122 [INFO] TestAgent.server.raft: election won: tally=1 -42:19.122 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13218 [Leader]" -42:19.122 [INFO] TestAgent.server: cluster leadership acquired -42:19.122 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13218 -42:19.122 [INFO] TestAgent.server: New leader elected: payload=Node-9b7141ed-facb-6405-7913-3ff8f69b802e -42:19.124 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:19.126 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:19.126 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:19.126 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:19.126 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:19.126 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:19.126 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9b7141ed-facb-6405-7913-3ff8f69b802e -42:19.126 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9b7141ed-facb-6405-7913-3ff8f69b802e -42:19.127 [INFO] TestAgent.server: federation state anti-entropy synced -42:19.407 [INFO] TestAgent: Requesting shutdown -42:19.407 [INFO] TestAgent.server: shutting down server -42:19.407 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:19.407 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:19.407 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:19.407 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:19.407 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:19.407 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:19.407 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:19.408 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:19.408 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:19.408 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:19.408 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:19.408 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:19.408Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:19.408Z [INFO] agent.router.manager: shutting down -42:19.408 [INFO] TestAgent: consul server down -42:19.408 [INFO] TestAgent: shutdown complete -42:19.408 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13199 network=tcp -42:19.408 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13199 network=udp -42:19.408 [INFO] TestAgent: Stopping server: protocol=HTTP address=/tmp/consul-test/TestHTTPServer_UnixSocket_FileExists-consul246958013/test.sock network=unix -42:19.910 [INFO] TestAgent: Waiting for endpoints to shut down -42:19.910 [INFO] TestAgent: Endpoints down ---- PASS: TestHTTPServer_UnixSocket_FileExists (0.91s) -=== CONT TestHealthServiceNodes_CheckType -2022-09-15T21:42:19.118Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:19.140Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:19.143 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3d67d0b5-21e4-1b9b-f5da-90ab59960b54 Address:127.0.0.1:13239}]" -42:19.144 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3d67d0b5-21e4-1b9b-f5da-90ab59960b54.dc1 127.0.0.1 -42:19.145 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13239 [Follower]" leader= -42:19.145 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3d67d0b5-21e4-1b9b-f5da-90ab59960b54 127.0.0.1 -2022-09-15T21:42:19.145Z [INFO] agent.router: Initializing LAN area manager -42:19.145 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13234 network=udp -42:19.145 [INFO] TestAgent.server: Adding LAN server: server="Node-3d67d0b5-21e4-1b9b-f5da-90ab59960b54 (Addr: tcp/127.0.0.1:13239) (DC: dc1)" -42:19.145 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3d67d0b5-21e4-1b9b-f5da-90ab59960b54.dc1 area=wan -42:19.145 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13234 network=tcp -42:19.146 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13235 network=tcp -42:19.146 [INFO] TestAgent: started state syncer -42:19.146 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13240 network=tcp -42:19.205 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:19.205 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13239 [Candidate]" term=2 -42:19.205 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:19.205 [DEBUG] TestAgent.server.raft: vote granted: from=3d67d0b5-21e4-1b9b-f5da-90ab59960b54 term=2 tally=1 -42:19.205 [INFO] TestAgent.server.raft: election won: tally=1 -42:19.205 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13239 [Leader]" -42:19.205 [INFO] TestAgent.server: cluster leadership acquired -42:19.205 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13239 -42:19.206 [INFO] TestAgent.server: New leader elected: payload=Node-3d67d0b5-21e4-1b9b-f5da-90ab59960b54 -42:19.207 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:19.209 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:19.209 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:19.209 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:19.209 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:19.209 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:19.209 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3d67d0b5-21e4-1b9b-f5da-90ab59960b54 -42:19.209 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3d67d0b5-21e4-1b9b-f5da-90ab59960b54 -42:19.211 [INFO] TestAgent.server: federation state anti-entropy synced -42:19.264 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:19.265 [INFO] TestAgent: Synced node info -42:19.265 [DEBUG] TestAgent: Node info in sync -42:19.431 [INFO] TestAgent: Requesting shutdown -42:19.432 [INFO] TestAgent.server: shutting down server -42:19.432 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:19.432 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:19.432 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:19.432 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:19.432 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:19.432 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:19.432 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:19.432 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:19.432 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:19.432 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:19.432 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:19.432Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:19.432Z [INFO] agent.router.manager: shutting down -42:19.432 [INFO] TestAgent: consul server down -42:19.432 [INFO] TestAgent: shutdown complete -42:19.432 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13234 network=tcp -42:19.432 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13234 network=udp -42:19.432 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13235 network=tcp -42:19.934 [INFO] TestAgent: Waiting for endpoints to shut down -42:19.934 [INFO] TestAgent: Endpoints down ---- PASS: TestConfig_Apply_IngressGateway (0.86s) -=== CONT TestHealthServiceNodes_PassingFilter -2022-09-15T21:42:19.099Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:19.115Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:19.118 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:523951b7-5109-bf5b-3276-9791730a2317 Address:127.0.0.1:13225}]" -42:19.119 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-523951b7-5109-bf5b-3276-9791730a2317.dc1 127.0.0.1 -42:19.120 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-523951b7-5109-bf5b-3276-9791730a2317 127.0.0.1 -2022-09-15T21:42:19.120Z [INFO] agent.router: Initializing LAN area manager -42:19.120 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13213 network=udp -42:19.120 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13225 [Follower]" leader= -42:19.121 [INFO] TestAgent.server: Adding LAN server: server="Node-523951b7-5109-bf5b-3276-9791730a2317 (Addr: tcp/127.0.0.1:13225) (DC: dc1)" -42:19.121 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-523951b7-5109-bf5b-3276-9791730a2317.dc1 area=wan -42:19.121 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13213 network=tcp -42:19.122 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13214 network=tcp -42:19.122 [INFO] TestAgent: started state syncer -42:19.122 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13226 network=tcp -42:19.186 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:19.186 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13225 [Candidate]" term=2 -42:19.186 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:19.186 [DEBUG] TestAgent.server.raft: vote granted: from=523951b7-5109-bf5b-3276-9791730a2317 term=2 tally=1 -42:19.186 [INFO] TestAgent.server.raft: election won: tally=1 -42:19.186 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13225 [Leader]" -42:19.187 [INFO] TestAgent.server: cluster leadership acquired -42:19.187 [INFO] TestAgent.server: New leader elected: payload=Node-523951b7-5109-bf5b-3276-9791730a2317 -42:19.187 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13225 -42:19.188 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:19.191 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:19.191 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:19.191 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:19.191 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:19.191 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:19.191 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-523951b7-5109-bf5b-3276-9791730a2317 -42:19.191 [INFO] TestAgent.server: member joined, marking health alive: member=Node-523951b7-5109-bf5b-3276-9791730a2317 -42:19.192 [INFO] TestAgent.server: federation state anti-entropy synced -42:19.466 [INFO] TestAgent: Requesting shutdown -42:19.466 [INFO] TestAgent.server: shutting down server -42:19.466 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:19.466 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:19.466 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:19.466 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:19.466 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:19.467 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:19.467 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:19.467 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:19.467 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:19.467 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:19.467 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:19.467 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:19.467Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:19.467Z [INFO] agent.router.manager: shutting down -42:19.467 [INFO] TestAgent: consul server down -42:19.467 [INFO] TestAgent: shutdown complete -42:19.467 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13213 network=tcp -42:19.467 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13213 network=udp -42:19.467 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13214 network=tcp -42:19.967 [INFO] TestAgent: Waiting for endpoints to shut down -42:19.967 [INFO] TestAgent: Endpoints down ---- PASS: TestHealthConnectServiceNodes_PassingFilter (0.93s) - --- PASS: TestHealthConnectServiceNodes_PassingFilter/bc_no_query_value (0.00s) - --- PASS: TestHealthConnectServiceNodes_PassingFilter/passing_true (0.00s) - --- PASS: TestHealthConnectServiceNodes_PassingFilter/passing_false (0.00s) - --- PASS: TestHealthConnectServiceNodes_PassingFilter/passing_bad (0.00s) -=== CONT TestHealthServiceNodes_DistanceSort -2022-09-15T21:42:19.086Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:19.126Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:19.132 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:994c6715-da29-a508-a160-657b877bda9a Address:127.0.0.1:13211}]" -42:19.133 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-994c6715-da29-a508-a160-657b877bda9a.dc1 127.0.0.1 -42:19.134 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-994c6715-da29-a508-a160-657b877bda9a 127.0.0.1 -42:19.134 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13211 [Follower]" leader= -2022-09-15T21:42:19.134Z [INFO] agent.router: Initializing LAN area manager -42:19.138 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13220 network=udp -42:19.138 [INFO] TestAgent.server: Adding LAN server: server="Node-994c6715-da29-a508-a160-657b877bda9a (Addr: tcp/127.0.0.1:13211) (DC: dc1)" -42:19.138 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-994c6715-da29-a508-a160-657b877bda9a.dc1 area=wan -42:19.138 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13220 network=tcp -42:19.139 [INFO] TestAgent: Started HTTP server: address=/tmp/consul-test/TestHTTPServer_UnixSocket-consul843518711/test.sock network=unix -42:19.140 [INFO] TestAgent: started state syncer -42:19.140 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13212 network=tcp -42:19.179 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:19.179 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13211 [Candidate]" term=2 -42:19.179 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:19.179 [DEBUG] TestAgent.server.raft: vote granted: from=994c6715-da29-a508-a160-657b877bda9a term=2 tally=1 -42:19.179 [INFO] TestAgent.server.raft: election won: tally=1 -42:19.179 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13211 [Leader]" -42:19.179 [INFO] TestAgent.server: cluster leadership acquired -42:19.180 [INFO] TestAgent.server: New leader elected: payload=Node-994c6715-da29-a508-a160-657b877bda9a -42:19.180 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13211 -42:19.181 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:19.183 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:19.184 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:19.184 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:19.184 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:19.184 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:19.184 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-994c6715-da29-a508-a160-657b877bda9a -42:19.184 [INFO] TestAgent.server: member joined, marking health alive: member=Node-994c6715-da29-a508-a160-657b877bda9a -42:19.185 [INFO] TestAgent.server: federation state anti-entropy synced -42:19.411 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:19.412 [INFO] TestAgent: Synced node info -42:19.412 [DEBUG] TestAgent: Node info in sync -42:19.510 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=@ latency=22.46414ms -42:19.512 [INFO] TestAgent: Requesting shutdown -42:19.512 [INFO] TestAgent.server: shutting down server -42:19.512 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:19.512 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:19.512 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:19.512 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:19.512 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:19.512 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:19.512 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:19.512 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:19.512 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:19.512 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:19.512 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:19.513Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:19.513Z [INFO] agent.router.manager: shutting down -42:19.513 [INFO] TestAgent: consul server down -42:19.513 [INFO] TestAgent: shutdown complete -42:19.513 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13220 network=tcp -42:19.513 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13220 network=udp -42:19.513 [INFO] TestAgent: Stopping server: protocol=HTTP address=/tmp/consul-test/TestHTTPServer_UnixSocket-consul843518711/test.sock network=unix -42:20.014 [INFO] TestAgent: Waiting for endpoints to shut down -42:20.014 [INFO] TestAgent: Endpoints down ---- PASS: TestHTTPServer_UnixSocket (0.98s) -=== CONT TestHealthServiceNodes_Filter -[WARN] freeport: 10 out of 52 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:19.145Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:19.166Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:19.170 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:706ca695-c87a-4c7a-68fc-2ab09e177f6c Address:127.0.0.1:13246}]" -42:19.170 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13246 [Follower]" leader= -42:19.171 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-706ca695-c87a-4c7a-68fc-2ab09e177f6c.dc1 127.0.0.1 -42:19.171 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-706ca695-c87a-4c7a-68fc-2ab09e177f6c 127.0.0.1 -2022-09-15T21:42:19.172Z [INFO] agent.router: Initializing LAN area manager -42:19.172 [INFO] TestAgent.server: Adding LAN server: server="Node-706ca695-c87a-4c7a-68fc-2ab09e177f6c (Addr: tcp/127.0.0.1:13246) (DC: dc1)" -42:19.172 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-706ca695-c87a-4c7a-68fc-2ab09e177f6c.dc1 area=wan -42:19.172 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13241 network=udp -42:19.172 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13241 network=tcp -42:19.173 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13242 network=tcp -42:19.173 [INFO] TestAgent: started state syncer -42:19.173 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13247 network=tcp -42:19.225 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:19.225 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13246 [Candidate]" term=2 -42:19.225 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:19.225 [DEBUG] TestAgent.server.raft: vote granted: from=706ca695-c87a-4c7a-68fc-2ab09e177f6c term=2 tally=1 -42:19.225 [INFO] TestAgent.server.raft: election won: tally=1 -42:19.225 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13246 [Leader]" -42:19.225 [INFO] TestAgent.server: cluster leadership acquired -42:19.225 [INFO] TestAgent.server: New leader elected: payload=Node-706ca695-c87a-4c7a-68fc-2ab09e177f6c -42:19.226 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13246 -42:19.227 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:19.231 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:19.231 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:19.231 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:19.231 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:19.231 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:19.231 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-706ca695-c87a-4c7a-68fc-2ab09e177f6c -42:19.231 [INFO] TestAgent.server: member joined, marking health alive: member=Node-706ca695-c87a-4c7a-68fc-2ab09e177f6c -42:19.232 [INFO] TestAgent.server: federation state anti-entropy synced -42:19.421 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:19.422 [INFO] TestAgent: Synced node info -42:19.525 [INFO] TestAgent: Requesting shutdown -42:19.525 [INFO] TestAgent.server: shutting down server -42:19.525 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:19.525 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:19.525 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:19.525 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:19.525 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:19.527 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:19.527 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:19.527 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:19.527Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:19.527Z [INFO] agent.router.manager: shutting down -42:19.527 [INFO] TestAgent: consul server down -42:19.527 [INFO] TestAgent: shutdown complete -42:19.527 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13241 network=tcp -42:19.527 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13241 network=udp -42:19.527 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13242 network=tcp -42:19.529 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:19.529 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:19.530 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:20.037 [INFO] TestAgent: Waiting for endpoints to shut down -42:20.037 [INFO] TestAgent: Endpoints down ---- PASS: TestHealthIngressServiceNodes (0.96s) - --- PASS: TestHealthIngressServiceNodes/associated_service (0.00s) - --- PASS: TestHealthIngressServiceNodes/non-associated_service (0.00s) -=== CONT TestHealthServiceNodes_NodeMetaFilter -2022-09-15T21:42:18.591Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:18.621Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:18.624 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b6152a40-a4f9-22b4-c97c-cfc517aefdb8 Address:127.0.0.1:13204}]" -42:18.624 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13204 [Follower]" leader= -42:18.625 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b6152a40-a4f9-22b4-c97c-cfc517aefdb8.dc1 127.0.0.1 -42:18.626 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b6152a40-a4f9-22b4-c97c-cfc517aefdb8 127.0.0.1 -2022-09-15T21:42:18.626Z [INFO] agent.router: Initializing LAN area manager -42:18.626 [INFO] TestAgent.server: Adding LAN server: server="Node-b6152a40-a4f9-22b4-c97c-cfc517aefdb8 (Addr: tcp/127.0.0.1:13204) (DC: dc1)" -42:18.626 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b6152a40-a4f9-22b4-c97c-cfc517aefdb8.dc1 area=wan -42:18.626 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13185 network=udp -42:18.626 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13185 network=tcp -42:18.627 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13186 network=tcp -42:18.627 [INFO] TestAgent: started state syncer -42:18.627 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13205 network=tcp -42:18.678 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:18.678 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13204 [Candidate]" term=2 -42:18.678 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:18.678 [DEBUG] TestAgent.server.raft: vote granted: from=b6152a40-a4f9-22b4-c97c-cfc517aefdb8 term=2 tally=1 -42:18.678 [INFO] TestAgent.server.raft: election won: tally=1 -42:18.678 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13204 [Leader]" -42:18.678 [INFO] TestAgent.server: cluster leadership acquired -42:18.679 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13204 -42:18.679 [INFO] TestAgent.server: New leader elected: payload=Node-b6152a40-a4f9-22b4-c97c-cfc517aefdb8 -42:18.680 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:18.683 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:18.683 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:18.683 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:18.683 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:18.683 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:18.683 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b6152a40-a4f9-22b4-c97c-cfc517aefdb8 -42:18.683 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b6152a40-a4f9-22b4-c97c-cfc517aefdb8 -42:18.686 [INFO] TestAgent.server: federation state anti-entropy synced -42:18.773 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:18.774 [INFO] TestAgent: Synced node info -42:18.774 [DEBUG] TestAgent: Node info in sync -42:19.066 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from= latency=3.62µs -42:19.066 [INFO] TestAgent: Requesting shutdown -42:19.066 [INFO] TestAgent.server: shutting down server -42:19.066 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:19.066 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:19.066 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:19.066 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:19.066 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:19.066 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:19.066 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:19.066 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:19.066 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:19.066 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:19.066 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:19.067Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:19.067Z [INFO] agent.router.manager: shutting down -42:19.067 [INFO] TestAgent: consul server down -42:19.067 [INFO] TestAgent: shutdown complete -42:19.067 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13185 network=tcp -42:19.067 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13185 network=udp -42:19.067 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13186 network=tcp -42:19.568 [INFO] TestAgent: Waiting for endpoints to shut down -42:19.568 [INFO] TestAgent: Endpoints down -2022-09-15T21:42:18.402Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:18.437Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:18.440 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6e86d35d-1604-d9db-d952-9403dd2a1a81 Address:127.0.0.1:13183}]" -42:18.440 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13183 [Follower]" leader= -42:18.441 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6e86d35d-1604-d9db-d952-9403dd2a1a81.dc1 127.0.0.1 -42:18.442 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6e86d35d-1604-d9db-d952-9403dd2a1a81 127.0.0.1 -2022-09-15T21:42:18.442Z [INFO] agent.router: Initializing LAN area manager -42:18.442 [INFO] TestAgent.server: Adding LAN server: server="Node-6e86d35d-1604-d9db-d952-9403dd2a1a81 (Addr: tcp/127.0.0.1:13183) (DC: dc1)" -42:18.442 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6e86d35d-1604-d9db-d952-9403dd2a1a81.dc1 area=wan -42:18.442 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13192 network=udp -42:18.443 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13192 network=tcp -42:18.443 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13193 network=tcp -42:18.443 [INFO] TestAgent: started state syncer -42:18.444 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13184 network=tcp -42:18.505 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:18.505 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13183 [Candidate]" term=2 -42:18.505 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:18.505 [DEBUG] TestAgent.server.raft: vote granted: from=6e86d35d-1604-d9db-d952-9403dd2a1a81 term=2 tally=1 -42:18.505 [INFO] TestAgent.server.raft: election won: tally=1 -42:18.505 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13183 [Leader]" -42:18.510 [INFO] TestAgent.server: cluster leadership acquired -42:18.510 [INFO] TestAgent.server: New leader elected: payload=Node-6e86d35d-1604-d9db-d952-9403dd2a1a81 -42:18.511 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13183 -42:18.512 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:18.515 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:18.515 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:18.515 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:18.515 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:18.515 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:18.515 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6e86d35d-1604-d9db-d952-9403dd2a1a81 -42:18.515 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6e86d35d-1604-d9db-d952-9403dd2a1a81 -42:18.517 [INFO] TestAgent.server: federation state anti-entropy synced -42:18.571 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from= latency=2.3µs -42:18.757 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:18.758 [INFO] TestAgent: Synced node info -42:19.568 [INFO] TestAgent: Requesting shutdown -42:19.568 [INFO] TestAgent.server: shutting down server -42:19.568 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:19.568 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:19.568 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:19.568 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:19.568 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:19.568 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:19.568 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:19.568 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:19.568 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:19.568 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:19.568 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:19.568Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:19.568Z [INFO] agent.router.manager: shutting down -42:19.569 [INFO] TestAgent: consul server down -42:19.569 [INFO] TestAgent: shutdown complete -42:19.569 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13192 network=tcp -42:19.569 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13192 network=udp -42:19.569 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13193 network=tcp -42:20.069 [INFO] TestAgent: Waiting for endpoints to shut down -42:20.069 [INFO] TestAgent: Endpoints down ---- PASS: TestHTTPAPI_TranslateAddrHeader (1.70s) -=== CONT TestHealthServiceNodes -=== RUN TestHealthServiceNodes_PassingFilter/bc_no_query_value -=== RUN TestHealthServiceNodes_PassingFilter/passing_true -=== RUN TestHealthServiceNodes_PassingFilter/passing_false -=== RUN TestHealthServiceNodes_PassingFilter/passing_bad -[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:19.818Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:19.840Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:19.843 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6db3abd3-edd3-28b1-1977-c3ade262dd0a Address:127.0.0.1:13253}]" -42:19.844 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6db3abd3-edd3-28b1-1977-c3ade262dd0a.dc1 127.0.0.1 -42:19.845 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6db3abd3-edd3-28b1-1977-c3ade262dd0a 127.0.0.1 -2022-09-15T21:42:19.845Z [INFO] agent.router: Initializing LAN area manager -42:19.845 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13248 network=udp -42:19.845 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13253 [Follower]" leader= -42:19.846 [INFO] TestAgent.server: Adding LAN server: server="Node-6db3abd3-edd3-28b1-1977-c3ade262dd0a (Addr: tcp/127.0.0.1:13253) (DC: dc1)" -42:19.846 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6db3abd3-edd3-28b1-1977-c3ade262dd0a.dc1 area=wan -42:19.846 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13248 network=tcp -42:19.847 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13249 network=tcp -42:19.847 [INFO] TestAgent: started state syncer -42:19.847 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13254 network=tcp -42:19.890 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:19.890 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13253 [Candidate]" term=2 -42:19.890 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:19.890 [DEBUG] TestAgent.server.raft: vote granted: from=6db3abd3-edd3-28b1-1977-c3ade262dd0a term=2 tally=1 -42:19.890 [INFO] TestAgent.server.raft: election won: tally=1 -42:19.890 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13253 [Leader]" -42:19.890 [INFO] TestAgent.server: cluster leadership acquired -42:19.891 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13253 -42:19.892 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:19.894 [INFO] TestAgent.server: New leader elected: payload=Node-6db3abd3-edd3-28b1-1977-c3ade262dd0a -42:19.895 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:19.895 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:19.895 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:19.895 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:19.895 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:19.895 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6db3abd3-edd3-28b1-1977-c3ade262dd0a -42:19.895 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6db3abd3-edd3-28b1-1977-c3ade262dd0a -42:19.896 [INFO] TestAgent.server: federation state anti-entropy synced -42:19.923 [INFO] TestAgent: Requesting shutdown -42:19.923 [INFO] TestAgent.server: shutting down server -42:19.923 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:19.923 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:19.923 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:19.923 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:19.923 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:19.924 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:19.924 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:19.924 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:19.924 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:19.924 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:19.924 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:19.924 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:19.924Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:19.924Z [INFO] agent.router.manager: shutting down -42:19.924 [INFO] TestAgent: consul server down -42:19.924 [INFO] TestAgent: shutdown complete -42:19.924 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13248 network=tcp -42:19.924 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13248 network=udp -42:19.924 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13249 network=tcp -42:20.425 [INFO] TestAgent: Waiting for endpoints to shut down -42:20.425 [INFO] TestAgent: Endpoints down ---- PASS: TestHealthConnectServiceNodes (0.63s) -=== CONT TestHealthServiceChecks_DistanceSort -[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:19.931Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:19.957Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:19.960 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:65b1ece6-b4a8-3ed8-589c-82113ec1a6cf Address:127.0.0.1:13274}]" -42:19.961 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-65b1ece6-b4a8-3ed8-589c-82113ec1a6cf.dc1 127.0.0.1 -42:19.962 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13274 [Follower]" leader= -42:19.962 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-65b1ece6-b4a8-3ed8-589c-82113ec1a6cf 127.0.0.1 -2022-09-15T21:42:19.962Z [INFO] agent.router: Initializing LAN area manager -42:19.963 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13262 network=udp -42:19.963 [INFO] TestAgent.server: Adding LAN server: server="Node-65b1ece6-b4a8-3ed8-589c-82113ec1a6cf (Addr: tcp/127.0.0.1:13274) (DC: dc1)" -42:19.963 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-65b1ece6-b4a8-3ed8-589c-82113ec1a6cf.dc1 area=wan -42:19.963 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13262 network=tcp -42:19.964 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13263 network=tcp -42:19.964 [INFO] TestAgent: started state syncer -42:19.964 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13275 network=tcp -42:20.006 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:20.007 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13274 [Candidate]" term=2 -42:20.007 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:20.007 [DEBUG] TestAgent.server.raft: vote granted: from=65b1ece6-b4a8-3ed8-589c-82113ec1a6cf term=2 tally=1 -42:20.008 [INFO] TestAgent.server.raft: election won: tally=1 -42:20.008 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13274 [Leader]" -42:20.008 [INFO] TestAgent.server: cluster leadership acquired -42:20.009 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13274 -42:20.010 [INFO] TestAgent.server: New leader elected: payload=Node-65b1ece6-b4a8-3ed8-589c-82113ec1a6cf -42:20.031 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:20.038 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:20.038 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:20.038 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:20.038 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:20.038 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:20.038 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-65b1ece6-b4a8-3ed8-589c-82113ec1a6cf -42:20.039 [INFO] TestAgent.server: member joined, marking health alive: member=Node-65b1ece6-b4a8-3ed8-589c-82113ec1a6cf -42:20.040 [INFO] TestAgent.server: federation state anti-entropy synced -42:20.067 [INFO] TestAgent: Requesting shutdown -42:20.067 [INFO] TestAgent.server: shutting down server -42:20.067 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:20.067 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:20.067 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:20.067 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:20.067 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:20.068 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:20.068 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:20.068 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:20.068 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:20.068 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:20.068 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:20.068 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:20.068Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:20.068Z [INFO] agent.router.manager: shutting down -42:20.068 [INFO] TestAgent: consul server down -42:20.068 [INFO] TestAgent: shutdown complete -42:20.068 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13262 network=tcp -42:20.068 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13262 network=udp -42:20.068 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13263 network=tcp -42:20.569 [INFO] TestAgent: Waiting for endpoints to shut down -42:20.569 [INFO] TestAgent: Endpoints down ---- PASS: TestHealthServiceNodes_CheckType (0.66s) -=== CONT TestHealthServiceChecks_Filtering -2022-09-15T21:42:19.969Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:19.986Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:19.988 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:032e689f-57b8-a130-7ea3-947340f742d5 Address:127.0.0.1:13267}]" -42:19.989 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13267 [Follower]" leader= -42:20.008 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-032e689f-57b8-a130-7ea3-947340f742d5.dc1 127.0.0.1 -42:20.021 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-032e689f-57b8-a130-7ea3-947340f742d5 127.0.0.1 -2022-09-15T21:42:20.021Z [INFO] agent.router: Initializing LAN area manager -42:20.026 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-032e689f-57b8-a130-7ea3-947340f742d5.dc1 area=wan -42:20.027 [INFO] TestAgent.server: Adding LAN server: server="Node-032e689f-57b8-a130-7ea3-947340f742d5 (Addr: tcp/127.0.0.1:13267) (DC: dc1)" -42:20.036 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13276 network=udp -42:20.041 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13276 network=tcp -42:20.042 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:20.042 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13267 [Candidate]" term=2 -42:20.045 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:20.045 [DEBUG] TestAgent.server.raft: vote granted: from=032e689f-57b8-a130-7ea3-947340f742d5 term=2 tally=1 -42:20.045 [INFO] TestAgent.server.raft: election won: tally=1 -42:20.045 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13267 [Leader]" -42:20.046 [INFO] TestAgent.server: cluster leadership acquired -42:20.046 [INFO] TestAgent.server: New leader elected: payload=Node-032e689f-57b8-a130-7ea3-947340f742d5 -42:20.048 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13277 network=tcp -42:20.048 [INFO] TestAgent: started state syncer -42:20.054 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13268 network=tcp -42:20.056 [WARN] TestAgent.server.fsm: EnsureRegistration failed: error="failed inserting check: Missing service registration on node "Node-032e689f-57b8-a130-7ea3-947340f742d5"" -42:20.056 [INFO] TestAgent: Synced node info -42:20.056 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13267 -42:20.057 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:20.064 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:20.064 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:20.064 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:20.064 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:20.064 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:20.064 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-032e689f-57b8-a130-7ea3-947340f742d5 -42:20.064 [INFO] TestAgent.server: member joined, marking health alive: member=Node-032e689f-57b8-a130-7ea3-947340f742d5 -42:20.065 [INFO] TestAgent.server: federation state anti-entropy synced -42:20.091 [INFO] TestAgent: Requesting shutdown -42:20.091 [INFO] TestAgent.server: shutting down server -42:20.091 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:20.091 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:20.091 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:20.091 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:20.091 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:20.091 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:20.091 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:20.091 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:20.091 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:20.091 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:20.091 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:20.091Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:20.091Z [INFO] agent.router.manager: shutting down -42:20.092 [INFO] TestAgent: consul server down -42:20.092 [INFO] TestAgent: shutdown complete -42:20.092 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13276 network=tcp -42:20.092 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13276 network=udp -42:20.092 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13277 network=tcp -42:20.594 [INFO] TestAgent: Waiting for endpoints to shut down -42:20.594 [INFO] TestAgent: Endpoints down ---- PASS: TestHealthServiceNodes_PassingFilter (0.66s) - --- PASS: TestHealthServiceNodes_PassingFilter/bc_no_query_value (0.00s) - --- PASS: TestHealthServiceNodes_PassingFilter/passing_true (0.00s) - --- PASS: TestHealthServiceNodes_PassingFilter/passing_false (0.00s) - --- PASS: TestHealthServiceNodes_PassingFilter/passing_bad (0.00s) -=== CONT TestHealthServiceChecks_NodeMetaFilter -[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:20.093Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:20.125Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:20.128 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c1977980-3fea-2d3c-40d0-df893d7ca0f5 Address:127.0.0.1:13295}]" -42:20.129 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c1977980-3fea-2d3c-40d0-df893d7ca0f5.dc1 127.0.0.1 -42:20.129 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13295 [Follower]" leader= -42:20.129 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c1977980-3fea-2d3c-40d0-df893d7ca0f5 127.0.0.1 -2022-09-15T21:42:20.129Z [INFO] agent.router: Initializing LAN area manager -42:20.130 [INFO] TestAgent.server: Adding LAN server: server="Node-c1977980-3fea-2d3c-40d0-df893d7ca0f5 (Addr: tcp/127.0.0.1:13295) (DC: dc1)" -42:20.130 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c1977980-3fea-2d3c-40d0-df893d7ca0f5.dc1 area=wan -42:20.131 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13290 network=udp -42:20.131 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13290 network=tcp -42:20.132 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13291 network=tcp -42:20.132 [INFO] TestAgent: started state syncer -42:20.132 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13296 network=tcp -42:20.193 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:20.193 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13295 [Candidate]" term=2 -42:20.193 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:20.193 [DEBUG] TestAgent.server.raft: vote granted: from=c1977980-3fea-2d3c-40d0-df893d7ca0f5 term=2 tally=1 -42:20.193 [INFO] TestAgent.server.raft: election won: tally=1 -42:20.193 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13295 [Leader]" -42:20.193 [INFO] TestAgent.server: cluster leadership acquired -42:20.194 [INFO] TestAgent.server: New leader elected: payload=Node-c1977980-3fea-2d3c-40d0-df893d7ca0f5 -42:20.194 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13295 -42:20.195 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:20.198 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:20.198 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:20.198 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:20.198 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:20.198 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:20.198 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c1977980-3fea-2d3c-40d0-df893d7ca0f5 -42:20.198 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c1977980-3fea-2d3c-40d0-df893d7ca0f5 -42:20.199 [INFO] TestAgent.server: federation state anti-entropy synced -42:20.279 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:20.279 [INFO] TestAgent: Synced node info -42:20.279 [DEBUG] TestAgent: Node info in sync -42:20.353 [INFO] TestAgent: Requesting shutdown -42:20.354 [INFO] TestAgent.server: shutting down server -42:20.355 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:20.355 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:20.355 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:20.355 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:20.355 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:20.355 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:20.355 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:20.355 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:20.355 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:20.355 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:20.355 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:20.355Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:20.355Z [INFO] agent.router.manager: shutting down -42:20.356 [INFO] TestAgent: consul server down -42:20.356 [INFO] TestAgent: shutdown complete -42:20.356 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13290 network=tcp -42:20.356 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13290 network=udp -42:20.356 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13291 network=tcp -42:20.858 [INFO] TestAgent: Waiting for endpoints to shut down -42:20.858 [INFO] TestAgent: Endpoints down ---- PASS: TestHealthServiceNodes_NodeMetaFilter (0.80s) -=== CONT TestHealthServiceChecks -2022-09-15T21:42:19.985Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:20.058Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:20.061 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3e126441-a5e9-f14a-46df-d7901dbbaaab Address:127.0.0.1:13281}]" -42:20.063 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13281 [Follower]" leader= -42:20.065 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3e126441-a5e9-f14a-46df-d7901dbbaaab.dc1 127.0.0.1 -42:20.066 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3e126441-a5e9-f14a-46df-d7901dbbaaab 127.0.0.1 -2022-09-15T21:42:20.066Z [INFO] agent.router: Initializing LAN area manager -42:20.067 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13269 network=udp -42:20.067 [INFO] TestAgent.server: Adding LAN server: server="Node-3e126441-a5e9-f14a-46df-d7901dbbaaab (Addr: tcp/127.0.0.1:13281) (DC: dc1)" -42:20.067 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3e126441-a5e9-f14a-46df-d7901dbbaaab.dc1 area=wan -42:20.067 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13269 network=tcp -42:20.068 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13270 network=tcp -42:20.068 [INFO] TestAgent: started state syncer -42:20.068 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13282 network=tcp -42:20.114 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:20.114 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13281 [Candidate]" term=2 -42:20.114 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:20.114 [DEBUG] TestAgent.server.raft: vote granted: from=3e126441-a5e9-f14a-46df-d7901dbbaaab term=2 tally=1 -42:20.114 [INFO] TestAgent.server.raft: election won: tally=1 -42:20.114 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13281 [Leader]" -42:20.114 [INFO] TestAgent.server: cluster leadership acquired -42:20.114 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13281 -42:20.116 [INFO] TestAgent.server: New leader elected: payload=Node-3e126441-a5e9-f14a-46df-d7901dbbaaab -42:20.116 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:20.119 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:20.119 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:20.119 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:20.119 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:20.119 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:20.119 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3e126441-a5e9-f14a-46df-d7901dbbaaab -42:20.119 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3e126441-a5e9-f14a-46df-d7901dbbaaab -42:20.120 [INFO] TestAgent.server: federation state anti-entropy synced -42:20.282 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:20.283 [INFO] TestAgent: Synced node info -42:20.283 [DEBUG] TestAgent: Node info in sync -42:20.380 [INFO] TestAgent: Requesting shutdown -42:20.380 [INFO] TestAgent.server: shutting down server -42:20.380 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:20.380 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:20.380 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:20.380 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:20.380 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:20.380 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:20.380 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:20.380 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:20.380 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:20.380 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:20.380 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:20.380Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:20.380Z [INFO] agent.router.manager: shutting down -42:20.381 [INFO] TestAgent: consul server down -42:20.381 [INFO] TestAgent: shutdown complete -42:20.381 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13269 network=tcp -42:20.381 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13269 network=udp -42:20.381 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13270 network=tcp -42:20.881 [INFO] TestAgent: Waiting for endpoints to shut down -42:20.881 [INFO] TestAgent: Endpoints down ---- PASS: TestHealthServiceNodes_DistanceSort (0.91s) -=== CONT TestHealthNodeChecks_Filtering -2022-09-15T21:42:20.102Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:20.121Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:20.128 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:685ff210-3594-e436-b3a8-672bad1d40d2 Address:127.0.0.1:13302}]" -42:20.129 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-685ff210-3594-e436-b3a8-672bad1d40d2.dc1 127.0.0.1 -42:20.129 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13302 [Follower]" leader= -42:20.131 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-685ff210-3594-e436-b3a8-672bad1d40d2 127.0.0.1 -2022-09-15T21:42:20.131Z [INFO] agent.router: Initializing LAN area manager -42:20.131 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13297 network=udp -42:20.131 [INFO] TestAgent.server: Adding LAN server: server="Node-685ff210-3594-e436-b3a8-672bad1d40d2 (Addr: tcp/127.0.0.1:13302) (DC: dc1)" -42:20.131 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-685ff210-3594-e436-b3a8-672bad1d40d2.dc1 area=wan -42:20.131 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13297 network=tcp -42:20.132 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13298 network=tcp -42:20.132 [INFO] TestAgent: started state syncer -42:20.132 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13303 network=tcp -42:20.171 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:20.171 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13302 [Candidate]" term=2 -42:20.171 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:20.171 [DEBUG] TestAgent.server.raft: vote granted: from=685ff210-3594-e436-b3a8-672bad1d40d2 term=2 tally=1 -42:20.171 [INFO] TestAgent.server.raft: election won: tally=1 -42:20.171 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13302 [Leader]" -42:20.171 [INFO] TestAgent.server: cluster leadership acquired -42:20.171 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13302 -42:20.172 [INFO] TestAgent.server: New leader elected: payload=Node-685ff210-3594-e436-b3a8-672bad1d40d2 -42:20.173 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:20.175 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:20.175 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:20.175 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:20.175 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:20.175 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:20.175 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-685ff210-3594-e436-b3a8-672bad1d40d2 -42:20.175 [INFO] TestAgent.server: member joined, marking health alive: member=Node-685ff210-3594-e436-b3a8-672bad1d40d2 -42:20.176 [INFO] TestAgent.server: federation state anti-entropy synced -42:20.185 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:20.185 [INFO] TestAgent: Synced node info -42:20.185 [DEBUG] TestAgent: Node info in sync -42:20.478 [INFO] TestAgent: Requesting shutdown -42:20.478 [INFO] TestAgent.server: shutting down server -42:20.478 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:20.478 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:20.478 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:20.478 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:20.478 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:20.478 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:20.478 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:20.479 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:20.479 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:20.479 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:20.479 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:20.479Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:20.479Z [INFO] agent.router.manager: shutting down -42:20.479 [INFO] TestAgent: consul server down -42:20.479 [INFO] TestAgent: shutdown complete -42:20.479 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13297 network=tcp -42:20.479 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13297 network=udp -42:20.479 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13298 network=tcp -42:20.979 [INFO] TestAgent: Waiting for endpoints to shut down -42:20.979 [INFO] TestAgent: Endpoints down ---- PASS: TestHealthServiceNodes (0.91s) -=== CONT TestHealthNodeChecks -2022-09-15T21:42:20.085Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:20.123Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:20.128 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f1ba5910-3537-90fa-3f58-7df5a985a69c Address:127.0.0.1:13288}]" -42:20.129 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f1ba5910-3537-90fa-3f58-7df5a985a69c.dc1 127.0.0.1 -42:20.130 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13288 [Follower]" leader= -42:20.131 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f1ba5910-3537-90fa-3f58-7df5a985a69c 127.0.0.1 -2022-09-15T21:42:20.131Z [INFO] agent.router: Initializing LAN area manager -42:20.131 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13283 network=udp -42:20.131 [INFO] TestAgent.server: Adding LAN server: server="Node-f1ba5910-3537-90fa-3f58-7df5a985a69c (Addr: tcp/127.0.0.1:13288) (DC: dc1)" -42:20.131 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f1ba5910-3537-90fa-3f58-7df5a985a69c.dc1 area=wan -42:20.131 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13283 network=tcp -42:20.132 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13284 network=tcp -42:20.132 [INFO] TestAgent: started state syncer -42:20.132 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13289 network=tcp -42:20.196 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:20.196 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13288 [Candidate]" term=2 -42:20.196 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:20.196 [DEBUG] TestAgent.server.raft: vote granted: from=f1ba5910-3537-90fa-3f58-7df5a985a69c term=2 tally=1 -42:20.196 [INFO] TestAgent.server.raft: election won: tally=1 -42:20.196 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13288 [Leader]" -42:20.197 [INFO] TestAgent.server: cluster leadership acquired -42:20.197 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13288 -42:20.198 [INFO] TestAgent.server: New leader elected: payload=Node-f1ba5910-3537-90fa-3f58-7df5a985a69c -42:20.198 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:20.201 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:20.201 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:20.201 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:20.201 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:20.201 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:20.201 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f1ba5910-3537-90fa-3f58-7df5a985a69c -42:20.201 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f1ba5910-3537-90fa-3f58-7df5a985a69c -42:20.202 [INFO] TestAgent.server: federation state anti-entropy synced -42:20.206 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:20.206 [INFO] TestAgent: Synced node info -42:20.483 [INFO] TestAgent: Requesting shutdown -42:20.483 [INFO] TestAgent.server: shutting down server -42:20.483 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:20.483 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:20.483 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:20.484 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:20.484 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:20.484 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:20.484 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:20.484 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:20.484 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:20.484 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:20.484 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:20.484Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:20.484Z [INFO] agent.router.manager: shutting down -42:20.484 [INFO] TestAgent: consul server down -42:20.484 [INFO] TestAgent: shutdown complete -42:20.484 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13283 network=tcp -42:20.484 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13283 network=udp -42:20.484 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13284 network=tcp -42:20.985 [INFO] TestAgent: Waiting for endpoints to shut down -42:20.985 [INFO] TestAgent: Endpoints down ---- PASS: TestHealthServiceNodes_Filter (0.97s) -=== CONT TestHealthChecksInState_DistanceSort -[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:20.442Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:20.460Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:20.463 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:df82f51b-2b13-5ea5-e343-dd5c344f907f Address:127.0.0.1:13316}]" -42:20.464 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-df82f51b-2b13-5ea5-e343-dd5c344f907f.dc1 127.0.0.1 -42:20.465 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-df82f51b-2b13-5ea5-e343-dd5c344f907f 127.0.0.1 -2022-09-15T21:42:20.465Z [INFO] agent.router: Initializing LAN area manager -42:20.465 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13311 network=udp -42:20.465 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13316 [Follower]" leader= -42:20.466 [INFO] TestAgent.server: Adding LAN server: server="Node-df82f51b-2b13-5ea5-e343-dd5c344f907f (Addr: tcp/127.0.0.1:13316) (DC: dc1)" -42:20.466 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-df82f51b-2b13-5ea5-e343-dd5c344f907f.dc1 area=wan -42:20.466 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13311 network=tcp -42:20.467 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13312 network=tcp -42:20.467 [INFO] TestAgent: started state syncer -42:20.467 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13317 network=tcp -42:20.521 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:20.521 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13316 [Candidate]" term=2 -42:20.521 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:20.521 [DEBUG] TestAgent.server.raft: vote granted: from=df82f51b-2b13-5ea5-e343-dd5c344f907f term=2 tally=1 -42:20.521 [INFO] TestAgent.server.raft: election won: tally=1 -42:20.521 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13316 [Leader]" -42:20.521 [INFO] TestAgent.server: cluster leadership acquired -42:20.522 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13316 -42:20.523 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:20.526 [INFO] TestAgent.server: New leader elected: payload=Node-df82f51b-2b13-5ea5-e343-dd5c344f907f -42:20.526 [INFO] TestAgent: Synced node info -42:20.526 [DEBUG] TestAgent: Node info in sync -42:20.528 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:20.528 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:20.528 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:20.528 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:20.528 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:20.528 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-df82f51b-2b13-5ea5-e343-dd5c344f907f -42:20.528 [INFO] TestAgent.server: member joined, marking health alive: member=Node-df82f51b-2b13-5ea5-e343-dd5c344f907f -42:20.529 [INFO] TestAgent.server: federation state anti-entropy synced -42:20.674 [INFO] TestAgent: Requesting shutdown -42:20.674 [INFO] TestAgent.server: shutting down server -42:20.674 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:20.674 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:20.674 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:20.674 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:20.674 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:20.674 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:20.675 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:20.675 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:20.675 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:20.675 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:20.675 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:20.675Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:20.675Z [INFO] agent.router.manager: shutting down -42:20.675 [INFO] TestAgent: consul server down -42:20.675 [INFO] TestAgent: shutdown complete -42:20.675 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13311 network=tcp -42:20.675 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13311 network=udp -42:20.675 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13312 network=tcp -42:21.175 [INFO] TestAgent: Waiting for endpoints to shut down -42:21.176 [INFO] TestAgent: Endpoints down ---- PASS: TestHealthServiceChecks_DistanceSort (0.75s) -=== CONT TestHealthChecksInState_Filter -[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:20.589Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:20.624Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:20.631 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d744b513-e60f-6f5f-a3b2-01ec414c8eec Address:127.0.0.1:13323}]" -42:20.632 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d744b513-e60f-6f5f-a3b2-01ec414c8eec.dc1 127.0.0.1 -42:20.633 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d744b513-e60f-6f5f-a3b2-01ec414c8eec 127.0.0.1 -2022-09-15T21:42:20.633Z [INFO] agent.router: Initializing LAN area manager -42:20.633 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13318 network=udp -42:20.633 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13323 [Follower]" leader= -42:20.634 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d744b513-e60f-6f5f-a3b2-01ec414c8eec.dc1 area=wan -42:20.634 [INFO] TestAgent.server: Adding LAN server: server="Node-d744b513-e60f-6f5f-a3b2-01ec414c8eec (Addr: tcp/127.0.0.1:13323) (DC: dc1)" -42:20.634 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13318 network=tcp -42:20.635 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13319 network=tcp -42:20.635 [INFO] TestAgent: started state syncer -42:20.635 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13324 network=tcp -42:20.678 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:20.678 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13323 [Candidate]" term=2 -42:20.678 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:20.678 [DEBUG] TestAgent.server.raft: vote granted: from=d744b513-e60f-6f5f-a3b2-01ec414c8eec term=2 tally=1 -42:20.678 [INFO] TestAgent.server.raft: election won: tally=1 -42:20.678 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13323 [Leader]" -42:20.678 [INFO] TestAgent.server: cluster leadership acquired -42:20.679 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13323 -42:20.680 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:20.682 [INFO] TestAgent.server: New leader elected: payload=Node-d744b513-e60f-6f5f-a3b2-01ec414c8eec -42:20.687 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:20.687 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:20.687 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:20.687 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:20.687 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:20.687 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d744b513-e60f-6f5f-a3b2-01ec414c8eec -42:20.687 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d744b513-e60f-6f5f-a3b2-01ec414c8eec -42:20.689 [INFO] TestAgent.server: federation state anti-entropy synced -42:20.788 [INFO] TestAgent: Requesting shutdown -42:20.788 [INFO] TestAgent.server: shutting down server -42:20.788 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:20.788 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:20.788 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:20.788 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:20.788 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:20.788 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:20.788 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:20.788 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:20.788 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:20.788 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:20.788 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:20.789 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:20.789Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:20.789Z [INFO] agent.router.manager: shutting down -42:20.789 [INFO] TestAgent: consul server down -42:20.789 [INFO] TestAgent: shutdown complete -42:20.789 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13318 network=tcp -42:20.789 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13318 network=udp -42:20.789 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13319 network=tcp -42:21.289 [INFO] TestAgent: Waiting for endpoints to shut down -42:21.289 [INFO] TestAgent: Endpoints down ---- PASS: TestHealthServiceChecks_Filtering (0.72s) -=== CONT TestHealthChecksInState_NodeMetaFilter -2022-09-15T21:42:20.316Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:20.316Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:20.363Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:20.363Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:20.371 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5beb38b0-806d-6cb6-e660-6f13430f3c15 Address:127.0.0.1:13309}]" -42:20.371 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13309 [Follower]" leader= -42:20.372 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5beb38b0-806d-6cb6-e660-6f13430f3c15.dc2 127.0.0.1 -42:20.373 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5beb38b0-806d-6cb6-e660-6f13430f3c15 127.0.0.1 -2022-09-15T21:42:20.373Z [INFO] agent.router: Initializing LAN area manager -42:20.373 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13304 network=udp -42:20.374 [INFO] TestAgent.server: Adding LAN server: server="Node-5beb38b0-806d-6cb6-e660-6f13430f3c15 (Addr: tcp/127.0.0.1:13309) (DC: dc2)" -42:20.374 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5beb38b0-806d-6cb6-e660-6f13430f3c15.dc2 area=wan -42:20.374 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13304 network=tcp -42:20.378 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13305 network=tcp -42:20.378 [INFO] TestAgent: started state syncer -42:20.378 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13310 network=tcp -42:20.424 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -42:20.437 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:20.437 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13309 [Candidate]" term=2 -42:20.438 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:20.438 [DEBUG] TestAgent.server.raft: vote granted: from=5beb38b0-806d-6cb6-e660-6f13430f3c15 term=2 tally=1 -42:20.438 [INFO] TestAgent.server.raft: election won: tally=1 -42:20.438 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13309 [Leader]" -42:20.438 [INFO] TestAgent.server: cluster leadership acquired -42:20.438 [INFO] TestAgent.server: initializing acls -42:20.439 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:20.439 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:20.439 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:20.439 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:20.440 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-5beb38b0-806d-6cb6-e660-6f13430f3c15 -42:20.440 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-5beb38b0-806d-6cb6-e660-6f13430f3c15.dc2 -42:20.441 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:20.442 [INFO] TestAgent.server: New leader elected: payload=Node-5beb38b0-806d-6cb6-e660-6f13430f3c15 -42:20.442 [INFO] TestAgent.server: Updating LAN server: server="Node-5beb38b0-806d-6cb6-e660-6f13430f3c15 (Addr: tcp/127.0.0.1:13309) (DC: dc2)" -42:20.442 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-5beb38b0-806d-6cb6-e660-6f13430f3c15.dc2 area=wan -42:20.444 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:20.444 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:20.444 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:20.444 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:20.444 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:20.444 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5beb38b0-806d-6cb6-e660-6f13430f3c15 -42:20.444 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5beb38b0-806d-6cb6-e660-6f13430f3c15 -42:20.445 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5beb38b0-806d-6cb6-e660-6f13430f3c15 -42:20.445 [INFO] TestAgent.server: federation state anti-entropy synced -42:20.450 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:20.451 [INFO] TestAgent: Synced node info -42:20.458 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:13259] -42:20.458 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:13259 -42:20.460 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f15f450c-5ef6-75df-9b5e-e7fe6befd9fc.dc1 127.0.0.1 -42:20.460 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 -42:20.462 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f15f450c-5ef6-75df-9b5e-e7fe6befd9fc.dc1 area=wan -42:20.470 [INFO] TestAgent: Requesting shutdown -42:20.470 [INFO] TestAgent.server: shutting down server -42:20.470 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:20.470 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:20.470 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:20.470 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:20.470 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:20.470 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:20.470 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:20.471 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:20.471 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:20.471 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:20.471 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:20.471 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:20.471 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:20.471 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:20.471 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:20.471Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:20.471Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:20.471Z [INFO] agent.router.manager: shutting down -42:20.471 [INFO] TestAgent: consul server down -42:20.471 [INFO] TestAgent: shutdown complete -42:20.471 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13304 network=tcp -42:20.471 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13304 network=udp -42:20.471 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13305 network=tcp -42:20.972 [INFO] TestAgent: Waiting for endpoints to shut down -42:20.972 [INFO] TestAgent: Endpoints down -2022-09-15T21:42:19.899Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:19.899Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:19.931Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:19.931Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:19.937 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f15f450c-5ef6-75df-9b5e-e7fe6befd9fc Address:127.0.0.1:13260}]" -42:19.938 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f15f450c-5ef6-75df-9b5e-e7fe6befd9fc.dc1 127.0.0.1 -42:19.939 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f15f450c-5ef6-75df-9b5e-e7fe6befd9fc 127.0.0.1 -2022-09-15T21:42:19.939Z [INFO] agent.router: Initializing LAN area manager -42:19.939 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13255 network=udp -42:19.939 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13260 [Follower]" leader= -42:19.940 [INFO] TestAgent.server: Adding LAN server: server="Node-f15f450c-5ef6-75df-9b5e-e7fe6befd9fc (Addr: tcp/127.0.0.1:13260) (DC: dc1)" -42:19.940 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f15f450c-5ef6-75df-9b5e-e7fe6befd9fc.dc1 area=wan -42:19.940 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13255 network=tcp -42:19.941 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13256 network=tcp -42:19.941 [INFO] TestAgent: started state syncer -42:19.941 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13261 network=tcp -42:19.998 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -42:19.998 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:19.998 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13260 [Candidate]" term=2 -42:19.999 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:19.999 [DEBUG] TestAgent.server.raft: vote granted: from=f15f450c-5ef6-75df-9b5e-e7fe6befd9fc term=2 tally=1 -42:19.999 [INFO] TestAgent.server.raft: election won: tally=1 -42:19.999 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13260 [Leader]" -42:20.000 [INFO] TestAgent.server: cluster leadership acquired -42:20.001 [INFO] TestAgent.server: initializing acls -42:20.002 [INFO] TestAgent.server: New leader elected: payload=Node-f15f450c-5ef6-75df-9b5e-e7fe6befd9fc -42:20.003 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:20.004 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:20.004 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:20.004 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:20.005 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-f15f450c-5ef6-75df-9b5e-e7fe6befd9fc -42:20.019 [INFO] TestAgent.server: Updating LAN server: server="Node-f15f450c-5ef6-75df-9b5e-e7fe6befd9fc (Addr: tcp/127.0.0.1:13260) (DC: dc1)" -42:20.022 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-f15f450c-5ef6-75df-9b5e-e7fe6befd9fc.dc1 -42:20.026 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-f15f450c-5ef6-75df-9b5e-e7fe6befd9fc.dc1 area=wan -42:20.029 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:20.049 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:20.049 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:20.049 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:20.049 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:20.049 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:20.049 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f15f450c-5ef6-75df-9b5e-e7fe6befd9fc -42:20.049 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f15f450c-5ef6-75df-9b5e-e7fe6befd9fc -42:20.056 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f15f450c-5ef6-75df-9b5e-e7fe6befd9fc -42:20.056 [INFO] TestAgent.server: federation state anti-entropy synced -42:20.251 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:20.252 [INFO] TestAgent: Synced node info -42:20.459 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:45304 -42:20.460 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5beb38b0-806d-6cb6-e660-6f13430f3c15.dc2 127.0.0.1 -42:20.460 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5beb38b0-806d-6cb6-e660-6f13430f3c15.dc2 area=wan -42:20.972 [INFO] TestAgent: Requesting shutdown -42:20.972 [INFO] TestAgent.server: shutting down server -42:20.972 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:20.972 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:20.972 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:20.972 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:20.972 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:20.972 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:20.972 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:20.973 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:20.973 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:20.973 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:20.973 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:20.973 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:20.973 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:20.973 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:20.973 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:20.973Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:20.973Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:20.973Z [INFO] agent.router.manager: shutting down -42:20.973 [INFO] TestAgent: consul server down -42:20.973 [INFO] TestAgent: shutdown complete -42:20.973 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13255 network=tcp -42:20.973 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13255 network=udp -42:20.973 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13256 network=tcp -42:21.474 [INFO] TestAgent: Waiting for endpoints to shut down -42:21.474 [INFO] TestAgent: Endpoints down ---- PASS: TestHealthServiceNodes_WanTranslation (1.60s) -=== CONT TestUUIDToUint64 ---- PASS: TestUUIDToUint64 (0.00s) -=== CONT TestEventList_EventBufOrder -[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:20.629Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:20.657Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:20.665 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:21ee32c7-a5c9-30f7-4336-2b0c619562ae Address:127.0.0.1:13330}]" -42:20.665 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13330 [Follower]" leader= -42:20.670 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-21ee32c7-a5c9-30f7-4336-2b0c619562ae.dc1 127.0.0.1 -42:20.671 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-21ee32c7-a5c9-30f7-4336-2b0c619562ae 127.0.0.1 -2022-09-15T21:42:20.671Z [INFO] agent.router: Initializing LAN area manager -42:20.672 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13325 network=udp -42:20.672 [INFO] TestAgent.server: Adding LAN server: server="Node-21ee32c7-a5c9-30f7-4336-2b0c619562ae (Addr: tcp/127.0.0.1:13330) (DC: dc1)" -42:20.672 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-21ee32c7-a5c9-30f7-4336-2b0c619562ae.dc1 area=wan -42:20.673 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13325 network=tcp -42:20.673 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13326 network=tcp -42:20.678 [INFO] TestAgent: started state syncer -42:20.678 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13331 network=tcp -42:20.718 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:20.718 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13330 [Candidate]" term=2 -42:20.718 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:20.718 [DEBUG] TestAgent.server.raft: vote granted: from=21ee32c7-a5c9-30f7-4336-2b0c619562ae term=2 tally=1 -42:20.718 [INFO] TestAgent.server.raft: election won: tally=1 -42:20.718 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13330 [Leader]" -42:20.718 [INFO] TestAgent.server: cluster leadership acquired -42:20.719 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13330 -42:20.720 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:20.722 [INFO] TestAgent.server: New leader elected: payload=Node-21ee32c7-a5c9-30f7-4336-2b0c619562ae -42:20.727 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:20.727 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:20.727 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:20.727 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:20.727 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:20.727 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-21ee32c7-a5c9-30f7-4336-2b0c619562ae -42:20.727 [INFO] TestAgent.server: member joined, marking health alive: member=Node-21ee32c7-a5c9-30f7-4336-2b0c619562ae -42:20.729 [INFO] TestAgent.server: federation state anti-entropy synced -42:20.951 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:20.951 [INFO] TestAgent: Synced node info -42:20.951 [DEBUG] TestAgent: Node info in sync -42:21.046 [INFO] TestAgent: Requesting shutdown -42:21.046 [INFO] TestAgent.server: shutting down server -42:21.046 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:21.046 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:21.046 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:21.046 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:21.046 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:21.046 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:21.046 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:21.046 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:21.046 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:21.046 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:21.046 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:21.047Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:21.047Z [INFO] agent.router.manager: shutting down -42:21.047 [INFO] TestAgent: consul server down -42:21.047 [INFO] TestAgent: shutdown complete -42:21.047 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13325 network=tcp -42:21.047 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13325 network=udp -42:21.047 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13326 network=tcp -42:21.548 [INFO] TestAgent: Waiting for endpoints to shut down -42:21.548 [INFO] TestAgent: Endpoints down ---- PASS: TestHealthServiceChecks_NodeMetaFilter (0.95s) -=== CONT TestEventList_Blocking -2022-09-15T21:42:20.905Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:20.944Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:20.947 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5a27dd72-5adc-62f6-6b4a-81b7130a67b5 Address:127.0.0.1:13337}]" -42:20.947 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13337 [Follower]" leader= -42:20.948 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5a27dd72-5adc-62f6-6b4a-81b7130a67b5.dc1 127.0.0.1 -42:20.948 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5a27dd72-5adc-62f6-6b4a-81b7130a67b5 127.0.0.1 -2022-09-15T21:42:20.948Z [INFO] agent.router: Initializing LAN area manager -42:20.949 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13332 network=udp -42:20.949 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5a27dd72-5adc-62f6-6b4a-81b7130a67b5.dc1 area=wan -42:20.949 [INFO] TestAgent.server: Adding LAN server: server="Node-5a27dd72-5adc-62f6-6b4a-81b7130a67b5 (Addr: tcp/127.0.0.1:13337) (DC: dc1)" -42:20.949 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13332 network=tcp -42:20.950 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13333 network=tcp -42:20.950 [INFO] TestAgent: started state syncer -42:20.950 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13338 network=tcp -42:20.990 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:20.990 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13337 [Candidate]" term=2 -42:20.990 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:20.990 [DEBUG] TestAgent.server.raft: vote granted: from=5a27dd72-5adc-62f6-6b4a-81b7130a67b5 term=2 tally=1 -42:20.990 [INFO] TestAgent.server.raft: election won: tally=1 -42:20.990 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13337 [Leader]" -42:20.990 [INFO] TestAgent.server: cluster leadership acquired -42:20.990 [INFO] TestAgent.server: New leader elected: payload=Node-5a27dd72-5adc-62f6-6b4a-81b7130a67b5 -42:20.990 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13337 -42:20.992 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:20.994 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:20.994 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:20.994 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:20.994 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:20.994 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:20.995 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5a27dd72-5adc-62f6-6b4a-81b7130a67b5 -42:20.995 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5a27dd72-5adc-62f6-6b4a-81b7130a67b5 -42:20.996 [INFO] TestAgent.server: federation state anti-entropy synced -42:21.067 [INFO] TestAgent: Requesting shutdown -42:21.067 [INFO] TestAgent.server: shutting down server -42:21.067 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:21.067 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:21.067 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:21.067 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:21.067 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:21.067 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:21.067 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:21.067 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:21.067 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:21.067 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:21.067 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:21.067 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:21.068Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:21.068Z [INFO] agent.router.manager: shutting down -42:21.068 [INFO] TestAgent: consul server down -42:21.068 [INFO] TestAgent: shutdown complete -42:21.068 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13332 network=tcp -42:21.068 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13332 network=udp -42:21.068 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13333 network=tcp -42:21.569 [INFO] TestAgent: Waiting for endpoints to shut down -42:21.569 [INFO] TestAgent: Endpoints down ---- PASS: TestHealthServiceChecks (0.71s) -=== CONT TestEventList_ACLFilter -=== RUN TestEventList_ACLFilter/no_token -=== RUN TestEventList_ACLFilter/root_token -[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:21.000Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:21.025Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:21.028 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:20d290bb-5b6c-45f9-1d06-7154fd84a734 Address:127.0.0.1:13351}]" -42:21.028 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13351 [Follower]" leader= -42:21.029 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-20d290bb-5b6c-45f9-1d06-7154fd84a734.dc1 127.0.0.1 -42:21.031 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-20d290bb-5b6c-45f9-1d06-7154fd84a734 127.0.0.1 -2022-09-15T21:42:21.031Z [INFO] agent.router: Initializing LAN area manager -42:21.031 [INFO] TestAgent.server: Adding LAN server: server="Node-20d290bb-5b6c-45f9-1d06-7154fd84a734 (Addr: tcp/127.0.0.1:13351) (DC: dc1)" -42:21.031 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13346 network=udp -42:21.031 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-20d290bb-5b6c-45f9-1d06-7154fd84a734.dc1 area=wan -42:21.031 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13346 network=tcp -42:21.032 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13347 network=tcp -42:21.032 [INFO] TestAgent: started state syncer -42:21.032 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13352 network=tcp -42:21.071 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:21.071 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13351 [Candidate]" term=2 -42:21.071 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:21.071 [DEBUG] TestAgent.server.raft: vote granted: from=20d290bb-5b6c-45f9-1d06-7154fd84a734 term=2 tally=1 -42:21.071 [INFO] TestAgent.server.raft: election won: tally=1 -42:21.071 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13351 [Leader]" -42:21.071 [INFO] TestAgent.server: cluster leadership acquired -42:21.072 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13351 -42:21.072 [INFO] TestAgent.server: New leader elected: payload=Node-20d290bb-5b6c-45f9-1d06-7154fd84a734 -42:21.073 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:21.076 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:21.076 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:21.076 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:21.076 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:21.076 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:21.076 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-20d290bb-5b6c-45f9-1d06-7154fd84a734 -42:21.077 [INFO] TestAgent.server: member joined, marking health alive: member=Node-20d290bb-5b6c-45f9-1d06-7154fd84a734 -42:21.078 [INFO] TestAgent.server: federation state anti-entropy synced -42:21.334 [INFO] TestAgent: Requesting shutdown -42:21.334 [INFO] TestAgent.server: shutting down server -42:21.334 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:21.334 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:21.334 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:21.334 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:21.334 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:21.334 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:21.334 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:21.334 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:21.334 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:21.334 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:21.334 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:21.334 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:21.335Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:21.335Z [INFO] agent.router.manager: shutting down -42:21.336 [INFO] TestAgent: consul server down -42:21.336 [INFO] TestAgent: shutdown complete -42:21.336 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13346 network=tcp -42:21.336 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13346 network=udp -42:21.336 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13347 network=tcp -42:21.837 [INFO] TestAgent: Waiting for endpoints to shut down -42:21.837 [INFO] TestAgent: Endpoints down ---- PASS: TestHealthNodeChecks (0.86s) -=== CONT TestEventList_Filter -2022-09-15T21:42:21.004Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:21.024Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:21.028 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:161f22cf-4720-a99b-f112-a378bb6d1de7 Address:127.0.0.1:13358}]" -42:21.028 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13358 [Follower]" leader= -42:21.029 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-161f22cf-4720-a99b-f112-a378bb6d1de7.dc1 127.0.0.1 -42:21.030 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-161f22cf-4720-a99b-f112-a378bb6d1de7 127.0.0.1 -2022-09-15T21:42:21.030Z [INFO] agent.router: Initializing LAN area manager -42:21.031 [INFO] TestAgent.server: Adding LAN server: server="Node-161f22cf-4720-a99b-f112-a378bb6d1de7 (Addr: tcp/127.0.0.1:13358) (DC: dc1)" -42:21.031 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-161f22cf-4720-a99b-f112-a378bb6d1de7.dc1 area=wan -42:21.031 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13353 network=udp -42:21.031 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13353 network=tcp -42:21.032 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13354 network=tcp -42:21.032 [INFO] TestAgent: started state syncer -42:21.032 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13359 network=tcp -42:21.091 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:21.091 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13358 [Candidate]" term=2 -42:21.091 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:21.091 [DEBUG] TestAgent.server.raft: vote granted: from=161f22cf-4720-a99b-f112-a378bb6d1de7 term=2 tally=1 -42:21.091 [INFO] TestAgent.server.raft: election won: tally=1 -42:21.091 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13358 [Leader]" -42:21.092 [INFO] TestAgent.server: cluster leadership acquired -42:21.092 [INFO] TestAgent.server: New leader elected: payload=Node-161f22cf-4720-a99b-f112-a378bb6d1de7 -42:21.092 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13358 -42:21.093 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:21.096 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:21.096 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:21.096 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:21.096 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:21.096 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:21.096 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-161f22cf-4720-a99b-f112-a378bb6d1de7 -42:21.096 [INFO] TestAgent.server: member joined, marking health alive: member=Node-161f22cf-4720-a99b-f112-a378bb6d1de7 -42:21.097 [INFO] TestAgent.server: federation state anti-entropy synced -42:21.103 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:21.103 [INFO] TestAgent: Synced node info -42:21.103 [DEBUG] TestAgent: Node info in sync -42:21.344 [INFO] TestAgent: Requesting shutdown -42:21.344 [INFO] TestAgent.server: shutting down server -42:21.344 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:21.344 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:21.344 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:21.344 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:21.344 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:21.345 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:21.345 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:21.345 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:21.345 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:21.345 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:21.345 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:21.345Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:21.345Z [INFO] agent.router.manager: shutting down -42:21.345 [INFO] TestAgent: consul server down -42:21.345 [INFO] TestAgent: shutdown complete -42:21.345 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13353 network=tcp -42:21.345 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13353 network=udp -42:21.345 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13354 network=tcp -42:21.846 [INFO] TestAgent: Waiting for endpoints to shut down -42:21.846 [INFO] TestAgent: Endpoints down ---- PASS: TestHealthChecksInState_DistanceSort (0.86s) -=== CONT TestEventList -2022-09-15T21:42:20.927Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:20.945Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:20.947 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:623a45eb-4b19-ee37-39ad-aa46e2a603b1 Address:127.0.0.1:13344}]" -42:20.948 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13344 [Follower]" leader= -42:20.948 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-623a45eb-4b19-ee37-39ad-aa46e2a603b1.dc1 127.0.0.1 -42:20.949 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-623a45eb-4b19-ee37-39ad-aa46e2a603b1 127.0.0.1 -2022-09-15T21:42:20.949Z [INFO] agent.router: Initializing LAN area manager -42:20.949 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-623a45eb-4b19-ee37-39ad-aa46e2a603b1.dc1 area=wan -42:20.949 [INFO] TestAgent.server: Adding LAN server: server="Node-623a45eb-4b19-ee37-39ad-aa46e2a603b1 (Addr: tcp/127.0.0.1:13344) (DC: dc1)" -42:20.949 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13339 network=udp -42:20.950 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13339 network=tcp -42:20.950 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13340 network=tcp -42:20.951 [INFO] TestAgent: started state syncer -42:20.951 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13345 network=tcp -42:21.011 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:21.011 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13344 [Candidate]" term=2 -42:21.012 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:21.012 [DEBUG] TestAgent.server.raft: vote granted: from=623a45eb-4b19-ee37-39ad-aa46e2a603b1 term=2 tally=1 -42:21.012 [INFO] TestAgent.server.raft: election won: tally=1 -42:21.012 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13344 [Leader]" -42:21.012 [INFO] TestAgent.server: cluster leadership acquired -42:21.012 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13344 -42:21.013 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:21.014 [INFO] TestAgent.server: New leader elected: payload=Node-623a45eb-4b19-ee37-39ad-aa46e2a603b1 -42:21.016 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:21.016 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:21.016 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:21.016 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:21.016 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:21.016 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-623a45eb-4b19-ee37-39ad-aa46e2a603b1 -42:21.016 [INFO] TestAgent.server: member joined, marking health alive: member=Node-623a45eb-4b19-ee37-39ad-aa46e2a603b1 -42:21.017 [INFO] TestAgent.server: federation state anti-entropy synced -42:21.040 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:21.040 [INFO] TestAgent: Synced node info -42:21.353 [INFO] TestAgent: Requesting shutdown -42:21.353 [INFO] TestAgent.server: shutting down server -42:21.353 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:21.353 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:21.353 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:21.353 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:21.353 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:21.353 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:21.353 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:21.353 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:21.353 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:21.353 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:21.354 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:21.354Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:21.354Z [INFO] agent.router.manager: shutting down -42:21.354 [INFO] TestAgent: consul server down -42:21.354 [INFO] TestAgent: shutdown complete -42:21.354 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13339 network=tcp -42:21.354 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13339 network=udp -42:21.354 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13340 network=tcp -42:21.858 [INFO] TestAgent: Waiting for endpoints to shut down -42:21.858 [INFO] TestAgent: Endpoints down ---- PASS: TestHealthNodeChecks_Filtering (0.98s) -=== CONT TestDNS_Compression_trimUDPResponse ---- PASS: TestDNS_Compression_trimUDPResponse (0.08s) -=== CONT TestEventFire_token -2022-09-15T21:42:21.194Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:21.211Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:21.214 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a229755c-9fa0-f07c-8528-07aaeda18a33 Address:127.0.0.1:13365}]" -42:21.215 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a229755c-9fa0-f07c-8528-07aaeda18a33.dc1 127.0.0.1 -42:21.216 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a229755c-9fa0-f07c-8528-07aaeda18a33 127.0.0.1 -2022-09-15T21:42:21.216Z [INFO] agent.router: Initializing LAN area manager -42:21.217 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13360 network=udp -42:21.217 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13365 [Follower]" leader= -42:21.217 [INFO] TestAgent.server: Adding LAN server: server="Node-a229755c-9fa0-f07c-8528-07aaeda18a33 (Addr: tcp/127.0.0.1:13365) (DC: dc1)" -42:21.217 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a229755c-9fa0-f07c-8528-07aaeda18a33.dc1 area=wan -42:21.217 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13360 network=tcp -42:21.218 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13361 network=tcp -42:21.219 [INFO] TestAgent: started state syncer -42:21.219 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13366 network=tcp -42:21.259 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:21.259 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13365 [Candidate]" term=2 -42:21.260 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:21.260 [DEBUG] TestAgent.server.raft: vote granted: from=a229755c-9fa0-f07c-8528-07aaeda18a33 term=2 tally=1 -42:21.260 [INFO] TestAgent.server.raft: election won: tally=1 -42:21.260 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13365 [Leader]" -42:21.260 [INFO] TestAgent.server: cluster leadership acquired -42:21.260 [INFO] TestAgent.server: New leader elected: payload=Node-a229755c-9fa0-f07c-8528-07aaeda18a33 -42:21.260 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13365 -42:21.261 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:21.272 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:21.272 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:21.272 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:21.272 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:21.272 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:21.272 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a229755c-9fa0-f07c-8528-07aaeda18a33 -42:21.272 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a229755c-9fa0-f07c-8528-07aaeda18a33 -42:21.276 [INFO] TestAgent.server: federation state anti-entropy synced -42:21.364 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:21.365 [INFO] TestAgent: Synced node info -42:21.365 [DEBUG] TestAgent: Node info in sync -42:21.464 [INFO] TestAgent: Requesting shutdown -42:21.465 [INFO] TestAgent.server: shutting down server -42:21.465 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:21.465 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:21.465 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:21.465 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:21.465 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:21.465 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:21.465 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:21.465 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:21.465 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:21.465 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:21.465 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:21.465Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:21.465Z [INFO] agent.router.manager: shutting down -42:21.465 [INFO] TestAgent: consul server down -42:21.465 [INFO] TestAgent: shutdown complete -42:21.465 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13360 network=tcp -42:21.465 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13360 network=udp -42:21.465 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13361 network=tcp -42:21.966 [INFO] TestAgent: Waiting for endpoints to shut down -42:21.966 [INFO] TestAgent: Endpoints down ---- PASS: TestHealthChecksInState_Filter (0.79s) -=== CONT TestEventFire -[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:21.329Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:21.353Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:21.357 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a8ffb00f-f5a8-00fe-d55c-1238b5639186 Address:127.0.0.1:13372}]" -42:21.357 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13372 [Follower]" leader= -42:21.358 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a8ffb00f-f5a8-00fe-d55c-1238b5639186.dc1 127.0.0.1 -42:21.359 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a8ffb00f-f5a8-00fe-d55c-1238b5639186 127.0.0.1 -2022-09-15T21:42:21.359Z [INFO] agent.router: Initializing LAN area manager -42:21.359 [INFO] TestAgent.server: Adding LAN server: server="Node-a8ffb00f-f5a8-00fe-d55c-1238b5639186 (Addr: tcp/127.0.0.1:13372) (DC: dc1)" -42:21.360 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a8ffb00f-f5a8-00fe-d55c-1238b5639186.dc1 area=wan -42:21.360 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13367 network=udp -42:21.360 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13367 network=tcp -42:21.361 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13368 network=tcp -42:21.361 [INFO] TestAgent: started state syncer -42:21.361 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13373 network=tcp -42:21.427 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:21.427 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13372 [Candidate]" term=2 -42:21.427 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:21.427 [DEBUG] TestAgent.server.raft: vote granted: from=a8ffb00f-f5a8-00fe-d55c-1238b5639186 term=2 tally=1 -42:21.427 [INFO] TestAgent.server.raft: election won: tally=1 -42:21.427 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13372 [Leader]" -42:21.428 [INFO] TestAgent.server: cluster leadership acquired -42:21.428 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13372 -42:21.428 [INFO] TestAgent.server: New leader elected: payload=Node-a8ffb00f-f5a8-00fe-d55c-1238b5639186 -42:21.429 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:21.432 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:21.432 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:21.432 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:21.432 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:21.432 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:21.432 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a8ffb00f-f5a8-00fe-d55c-1238b5639186 -42:21.432 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a8ffb00f-f5a8-00fe-d55c-1238b5639186 -42:21.432 [INFO] TestAgent.server: federation state anti-entropy synced -42:21.605 [INFO] TestAgent: Requesting shutdown -42:21.605 [INFO] TestAgent.server: shutting down server -42:21.605 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:21.605 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:21.605 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:21.605 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:21.606 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:21.606 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:21.606 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:21.606 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:21.606 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:21.606 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:21.606 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:21.607 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:21.607Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:21.607Z [INFO] agent.router.manager: shutting down -42:21.607 [INFO] TestAgent: consul server down -42:21.607 [INFO] TestAgent: shutdown complete -42:21.607 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13367 network=tcp -42:21.607 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13367 network=udp -42:21.607 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13368 network=tcp -42:22.107 [INFO] TestAgent: Waiting for endpoints to shut down -42:22.107 [INFO] TestAgent: Endpoints down ---- PASS: TestHealthChecksInState_NodeMetaFilter (0.82s) -=== CONT TestDNS_ReloadConfig_DuringQuery -2022-09-15T21:42:21.591Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:21.591Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:21.608Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:21.608Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:21.611 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:94e62a63-87ad-dfda-1697-d0735439f9ce Address:127.0.0.1:13400}]" -42:21.611 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13400 [Follower]" leader= -42:21.612 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-94e62a63-87ad-dfda-1697-d0735439f9ce.dc1 127.0.0.1 -42:21.613 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-94e62a63-87ad-dfda-1697-d0735439f9ce 127.0.0.1 -2022-09-15T21:42:21.613Z [INFO] agent.router: Initializing LAN area manager -42:21.613 [INFO] TestAgent.server: Adding LAN server: server="Node-94e62a63-87ad-dfda-1697-d0735439f9ce (Addr: tcp/127.0.0.1:13400) (DC: dc1)" -42:21.613 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-94e62a63-87ad-dfda-1697-d0735439f9ce.dc1 area=wan -42:21.613 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13388 network=udp -42:21.613 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13388 network=tcp -42:21.614 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13389 network=tcp -42:21.615 [INFO] TestAgent: started state syncer -42:21.615 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13401 network=tcp -42:21.663 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -42:21.677 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:21.677 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13400 [Candidate]" term=2 -42:21.677 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:21.677 [DEBUG] TestAgent.server.raft: vote granted: from=94e62a63-87ad-dfda-1697-d0735439f9ce term=2 tally=1 -42:21.677 [INFO] TestAgent.server.raft: election won: tally=1 -42:21.677 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13400 [Leader]" -42:21.678 [INFO] TestAgent.server: cluster leadership acquired -42:21.678 [INFO] TestAgent.server: initializing acls -42:21.678 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:21.678 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:21.679 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:21.680 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:21.680 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:21.680 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:21.680 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-94e62a63-87ad-dfda-1697-d0735439f9ce -42:21.680 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-94e62a63-87ad-dfda-1697-d0735439f9ce.dc1 -42:21.681 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:21.682 [INFO] TestAgent.server: New leader elected: payload=Node-94e62a63-87ad-dfda-1697-d0735439f9ce -42:21.682 [INFO] TestAgent.server: Updating LAN server: server="Node-94e62a63-87ad-dfda-1697-d0735439f9ce (Addr: tcp/127.0.0.1:13400) (DC: dc1)" -42:21.682 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-94e62a63-87ad-dfda-1697-d0735439f9ce.dc1 area=wan -42:21.684 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:21.684 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:21.684 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:21.684 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:21.684 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:21.684 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-94e62a63-87ad-dfda-1697-d0735439f9ce -42:21.684 [INFO] TestAgent.server: member joined, marking health alive: member=Node-94e62a63-87ad-dfda-1697-d0735439f9ce -42:21.685 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-94e62a63-87ad-dfda-1697-d0735439f9ce -42:21.686 [INFO] TestAgent.server: federation state anti-entropy synced -42:21.690 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-94e62a63-87ad-dfda-1697-d0735439f9ce -42:21.690 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-94e62a63-87ad-dfda-1697-d0735439f9ce -42:21.691 [DEBUG] TestAgent.server: User event: event=foo -42:21.691 [DEBUG] TestAgent: new event: event_name=foo event_id=17632d22-07ce-f359-7188-1e8585961db5 -42:21.691 [INFO] TestAgent: Requesting shutdown -42:21.691 [INFO] TestAgent.server: shutting down server -42:21.691 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:21.691 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:21.691 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:21.691 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:21.691 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:21.691 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:21.691 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:21.692 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:21.692 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:21.692 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:21.692 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:21.692 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:21.692 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:21.692 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:21.692 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:21.692 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:21.692Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:21.692Z [INFO] agent.router.manager: shutting down -42:21.692 [INFO] TestAgent: consul server down -42:21.692 [INFO] TestAgent: shutdown complete -42:21.692 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13388 network=tcp -42:21.692 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13388 network=udp -42:21.692 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13389 network=tcp -42:22.193 [INFO] TestAgent: Waiting for endpoints to shut down -42:22.193 [INFO] TestAgent: Endpoints down ---- PASS: TestEventList_ACLFilter (0.62s) - --- PASS: TestEventList_ACLFilter/no_token (0.00s) - --- PASS: TestEventList_ACLFilter/root_token (0.00s) -=== CONT TestDNS_ConfigReload -[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:21.494Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:21.532Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:21.538 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2ad6dc8f-a99f-a303-ad3d-368ebce5b0c5 Address:127.0.0.1:13379}]" -42:21.539 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2ad6dc8f-a99f-a303-ad3d-368ebce5b0c5.dc1 127.0.0.1 -42:21.540 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2ad6dc8f-a99f-a303-ad3d-368ebce5b0c5 127.0.0.1 -2022-09-15T21:42:21.540Z [INFO] agent.router: Initializing LAN area manager -42:21.541 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13374 network=udp -42:21.541 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13379 [Follower]" leader= -42:21.541 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2ad6dc8f-a99f-a303-ad3d-368ebce5b0c5.dc1 area=wan -42:21.541 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13374 network=tcp -42:21.541 [INFO] TestAgent.server: Adding LAN server: server="Node-2ad6dc8f-a99f-a303-ad3d-368ebce5b0c5 (Addr: tcp/127.0.0.1:13379) (DC: dc1)" -42:21.542 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13375 network=tcp -42:21.543 [INFO] TestAgent: started state syncer -42:21.543 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13380 network=tcp -42:21.606 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:21.606 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13379 [Candidate]" term=2 -42:21.606 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:21.606 [DEBUG] TestAgent.server.raft: vote granted: from=2ad6dc8f-a99f-a303-ad3d-368ebce5b0c5 term=2 tally=1 -42:21.606 [INFO] TestAgent.server.raft: election won: tally=1 -42:21.606 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13379 [Leader]" -42:21.606 [INFO] TestAgent.server: cluster leadership acquired -42:21.607 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13379 -42:21.607 [INFO] TestAgent.server: New leader elected: payload=Node-2ad6dc8f-a99f-a303-ad3d-368ebce5b0c5 -42:21.608 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:21.611 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:21.611 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:21.611 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:21.611 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:21.611 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:21.611 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2ad6dc8f-a99f-a303-ad3d-368ebce5b0c5 -42:21.611 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2ad6dc8f-a99f-a303-ad3d-368ebce5b0c5 -42:21.612 [INFO] TestAgent.server: federation state anti-entropy synced -42:21.622 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:21.622 [INFO] TestAgent: Synced node info -42:21.622 [DEBUG] TestAgent: Node info in sync -42:21.942 [DEBUG] TestAgent.server: User event: event=foo -42:21.943 [DEBUG] TestAgent.server: User event: event=bar -42:21.943 [DEBUG] TestAgent.server: User event: event=foo -42:21.943 [DEBUG] TestAgent.server: User event: event=foo -42:21.943 [DEBUG] TestAgent.server: User event: event=bar -42:21.943 [DEBUG] TestAgent: new event: event_name=foo event_id=ebfff603-d7fd-bbac-5071-0198f664e9c8 -42:21.943 [DEBUG] TestAgent: new event: event_name=bar event_id=fec9d7cf-5b0a-ab96-63ec-82e68d90029d -42:21.943 [DEBUG] TestAgent: new event: event_name=foo event_id=f699c294-26b3-c8c1-3d7a-71a73fc1bd72 -42:21.943 [DEBUG] TestAgent: new event: event_name=foo event_id=f813b8b3-4121-dc73-da5d-aad35fc4eb38 -42:21.943 [DEBUG] TestAgent: new event: event_name=bar event_id=aca32e34-4724-e656-4345-dc9fe2704eb7 -42:21.970 [INFO] TestAgent: Requesting shutdown -42:21.970 [INFO] TestAgent.server: shutting down server -42:21.970 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:21.970 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:21.970 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:21.970 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:21.970 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:21.970 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:21.970 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:21.970 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:21.970 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:21.970 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:21.970 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:21.971Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:21.971Z [INFO] agent.router.manager: shutting down -42:21.971 [INFO] TestAgent: consul server down -42:21.971 [INFO] TestAgent: shutdown complete -42:21.971 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13374 network=tcp -42:21.971 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13374 network=udp -42:21.971 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13375 network=tcp -42:22.471 [INFO] TestAgent: Waiting for endpoints to shut down -42:22.471 [INFO] TestAgent: Endpoints down ---- PASS: TestEventList_EventBufOrder (1.00s) -=== CONT TestDNS_Compression_Recurse -[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:21.566Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:21.583Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:21.586 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2ca9aa46-cfa6-bde9-118f-d8fee5cd9a31 Address:127.0.0.1:13386}]" -42:21.587 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13386 [Follower]" leader= -42:21.588 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2ca9aa46-cfa6-bde9-118f-d8fee5cd9a31.dc1 127.0.0.1 -42:21.589 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2ca9aa46-cfa6-bde9-118f-d8fee5cd9a31 127.0.0.1 -2022-09-15T21:42:21.589Z [INFO] agent.router: Initializing LAN area manager -42:21.589 [INFO] TestAgent.server: Adding LAN server: server="Node-2ca9aa46-cfa6-bde9-118f-d8fee5cd9a31 (Addr: tcp/127.0.0.1:13386) (DC: dc1)" -42:21.589 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2ca9aa46-cfa6-bde9-118f-d8fee5cd9a31.dc1 area=wan -42:21.589 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13381 network=udp -42:21.589 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13381 network=tcp -42:21.590 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13382 network=tcp -42:21.590 [INFO] TestAgent: started state syncer -42:21.590 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13387 network=tcp -42:21.649 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:21.649 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13386 [Candidate]" term=2 -42:21.649 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:21.649 [DEBUG] TestAgent.server.raft: vote granted: from=2ca9aa46-cfa6-bde9-118f-d8fee5cd9a31 term=2 tally=1 -42:21.649 [INFO] TestAgent.server.raft: election won: tally=1 -42:21.649 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13386 [Leader]" -42:21.649 [INFO] TestAgent.server: cluster leadership acquired -42:21.650 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13386 -42:21.651 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:21.654 [INFO] TestAgent.server: New leader elected: payload=Node-2ca9aa46-cfa6-bde9-118f-d8fee5cd9a31 -42:21.654 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:21.654 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:21.654 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:21.654 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:21.654 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:21.654 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2ca9aa46-cfa6-bde9-118f-d8fee5cd9a31 -42:21.654 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2ca9aa46-cfa6-bde9-118f-d8fee5cd9a31 -42:21.655 [INFO] TestAgent.server: federation state anti-entropy synced -42:21.818 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:21.818 [INFO] TestAgent: Synced node info -42:21.818 [DEBUG] TestAgent: Node info in sync -42:21.972 [DEBUG] TestAgent.server: User event: event=test -42:21.972 [DEBUG] TestAgent: new event: event_name=test event_id=d3d8d361-9b1a-5382-c90b-f705fa09c9cd -42:22.022 [DEBUG] TestAgent.server: User event: event=second -42:22.022 [DEBUG] TestAgent: new event: event_name=second event_id=4bc4ab4d-49fa-dee7-9872-603ac443499b -42:22.023 [INFO] TestAgent: Requesting shutdown -42:22.024 [INFO] TestAgent.server: shutting down server -42:22.024 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:22.024 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:22.024 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:22.024 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:22.024 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:22.024 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:22.024 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -42:22.024 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:22.024 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:22.024 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:22.024 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2022-09-15T21:42:22.024Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:22.024Z [INFO] agent.router.manager: shutting down -42:22.024 [INFO] TestAgent: consul server down -42:22.024 [INFO] TestAgent: shutdown complete -42:22.024 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13381 network=tcp -42:22.024 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13381 network=udp -42:22.024 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13382 network=tcp -42:22.525 [INFO] TestAgent: Waiting for endpoints to shut down -42:22.525 [INFO] TestAgent: Endpoints down ---- PASS: TestEventList_Blocking (0.98s) -=== CONT TestDNS_Compression_ReverseLookup -2022-09-15T21:42:21.936Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:21.954Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:21.957 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2ca00c98-c005-8a0e-90bc-7a941277cdab Address:127.0.0.1:13414}]" -42:21.958 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2ca00c98-c005-8a0e-90bc-7a941277cdab.dc1 127.0.0.1 -42:21.958 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13414 [Follower]" leader= -42:21.958 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2ca00c98-c005-8a0e-90bc-7a941277cdab 127.0.0.1 -2022-09-15T21:42:21.958Z [INFO] agent.router: Initializing LAN area manager -42:21.959 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13395 network=udp -42:21.959 [INFO] TestAgent.server: Adding LAN server: server="Node-2ca00c98-c005-8a0e-90bc-7a941277cdab (Addr: tcp/127.0.0.1:13414) (DC: dc1)" -42:21.959 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2ca00c98-c005-8a0e-90bc-7a941277cdab.dc1 area=wan -42:21.959 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13395 network=tcp -42:21.960 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13396 network=tcp -42:21.960 [INFO] TestAgent: started state syncer -42:21.960 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13415 network=tcp -42:22.007 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:22.007 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13414 [Candidate]" term=2 -42:22.007 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:22.007 [DEBUG] TestAgent.server.raft: vote granted: from=2ca00c98-c005-8a0e-90bc-7a941277cdab term=2 tally=1 -42:22.007 [INFO] TestAgent.server.raft: election won: tally=1 -42:22.007 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13414 [Leader]" -42:22.008 [INFO] TestAgent.server: cluster leadership acquired -42:22.008 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13414 -42:22.009 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:22.010 [INFO] TestAgent.server: New leader elected: payload=Node-2ca00c98-c005-8a0e-90bc-7a941277cdab -42:22.012 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:22.012 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:22.012 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:22.012 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:22.012 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:22.012 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2ca00c98-c005-8a0e-90bc-7a941277cdab -42:22.012 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2ca00c98-c005-8a0e-90bc-7a941277cdab -42:22.013 [INFO] TestAgent.server: federation state anti-entropy synced -42:22.102 [DEBUG] TestAgent.server: User event: event=test -42:22.102 [DEBUG] TestAgent: new event: event_name=test event_id=6a743dd9-acbe-c62c-fedc-6350bc61bffa -42:22.128 [INFO] TestAgent: Requesting shutdown -42:22.128 [INFO] TestAgent.server: shutting down server -42:22.128 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:22.128 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:22.128 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:22.128 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:22.128 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:22.128 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:22.128 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:22.129 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:22.129 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:22.129 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:22.129 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:22.129 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:22.129Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:22.129Z [INFO] agent.router.manager: shutting down -42:22.129 [INFO] TestAgent: consul server down -42:22.129 [INFO] TestAgent: shutdown complete -42:22.129 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13395 network=tcp -42:22.129 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13395 network=udp -42:22.129 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13396 network=tcp -42:22.630 [INFO] TestAgent: Waiting for endpoints to shut down -42:22.630 [INFO] TestAgent: Endpoints down ---- PASS: TestEventList (0.78s) -=== CONT TestDNS_Compression_Query -2022-09-15T21:42:21.893Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:21.948Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:21.955 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ce5b5b98-4ab9-1caa-bce7-28aec5f636d5 Address:127.0.0.1:13393}]" -42:21.956 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ce5b5b98-4ab9-1caa-bce7-28aec5f636d5.dc1 127.0.0.1 -42:21.957 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ce5b5b98-4ab9-1caa-bce7-28aec5f636d5 127.0.0.1 -2022-09-15T21:42:21.957Z [INFO] agent.router: Initializing LAN area manager -42:21.957 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13402 network=udp -42:21.958 [INFO] TestAgent.server: Adding LAN server: server="Node-ce5b5b98-4ab9-1caa-bce7-28aec5f636d5 (Addr: tcp/127.0.0.1:13393) (DC: dc1)" -42:21.958 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ce5b5b98-4ab9-1caa-bce7-28aec5f636d5.dc1 area=wan -42:21.958 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13402 network=tcp -42:21.959 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13403 network=tcp -42:21.959 [INFO] TestAgent: started state syncer -42:21.959 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13394 network=tcp -42:21.966 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13393 [Follower]" leader= -42:22.009 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:22.009 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13393 [Candidate]" term=2 -42:22.009 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:22.009 [DEBUG] TestAgent.server.raft: vote granted: from=ce5b5b98-4ab9-1caa-bce7-28aec5f636d5 term=2 tally=1 -42:22.009 [INFO] TestAgent.server.raft: election won: tally=1 -42:22.009 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13393 [Leader]" -42:22.009 [INFO] TestAgent.server: cluster leadership acquired -42:22.010 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13393 -42:22.010 [INFO] TestAgent.server: New leader elected: payload=Node-ce5b5b98-4ab9-1caa-bce7-28aec5f636d5 -42:22.011 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:22.013 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:22.013 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:22.013 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:22.013 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:22.013 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:22.013 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ce5b5b98-4ab9-1caa-bce7-28aec5f636d5 -42:22.013 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ce5b5b98-4ab9-1caa-bce7-28aec5f636d5 -42:22.014 [INFO] TestAgent.server: federation state anti-entropy synced -42:22.182 [DEBUG] TestAgent.server: User event: event=test -42:22.182 [DEBUG] TestAgent.server: User event: event=foo -42:22.182 [DEBUG] TestAgent: new event: event_name=test event_id=5eefbdff-a0b7-d9be-93ae-22bdc0e47c37 -42:22.182 [DEBUG] TestAgent: new event: event_name=foo event_id=666ed26d-a178-a3ee-3461-41dee55812d0 -42:22.210 [INFO] TestAgent: Requesting shutdown -42:22.210 [INFO] TestAgent.server: shutting down server -42:22.210 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:22.210 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:22.210 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:22.210 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:22.210 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:22.210 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:22.210 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:22.210 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:22.210 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:22.210 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:22.210 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:22.210 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:22.211Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:22.211Z [INFO] agent.router.manager: shutting down -42:22.211 [INFO] TestAgent: consul server down -42:22.211 [INFO] TestAgent: shutdown complete -42:22.211 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13402 network=tcp -42:22.211 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13402 network=udp -42:22.211 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13403 network=tcp -42:22.711 [INFO] TestAgent: Waiting for endpoints to shut down -42:22.712 [INFO] TestAgent: Endpoints down ---- PASS: TestEventList_Filter (0.87s) -=== CONT TestDNS_syncExtra ---- PASS: TestDNS_syncExtra (0.00s) -=== CONT TestDNS_trimUDPResponse_TrimSizeEDNS ---- PASS: TestDNS_trimUDPResponse_TrimSizeEDNS (0.03s) -=== CONT TestDNS_trimUDPResponse_TrimSize -[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:21.953Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:21.953Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:21.984Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:21.984Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:21.992 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:df914b45-0dd2-9a80-044f-f980859c3196 Address:127.0.0.1:13407}]" -42:21.993 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-df914b45-0dd2-9a80-044f-f980859c3196.dc1 127.0.0.1 -42:21.993 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13407 [Follower]" leader= -42:22.003 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-df914b45-0dd2-9a80-044f-f980859c3196 127.0.0.1 -2022-09-15T21:42:22.003Z [INFO] agent.router: Initializing LAN area manager -42:22.003 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13416 network=udp -42:22.003 [INFO] TestAgent.server: Adding LAN server: server="Node-df914b45-0dd2-9a80-044f-f980859c3196 (Addr: tcp/127.0.0.1:13407) (DC: dc1)" -42:22.003 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-df914b45-0dd2-9a80-044f-f980859c3196.dc1 area=wan -42:22.004 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13416 network=tcp -42:22.004 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13417 network=tcp -42:22.005 [INFO] TestAgent: started state syncer -42:22.005 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13408 network=tcp -42:22.043 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:22.043 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13407 [Candidate]" term=2 -42:22.043 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:22.043 [DEBUG] TestAgent.server.raft: vote granted: from=df914b45-0dd2-9a80-044f-f980859c3196 term=2 tally=1 -42:22.043 [INFO] TestAgent.server.raft: election won: tally=1 -42:22.043 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13407 [Leader]" -42:22.044 [INFO] TestAgent.server: cluster leadership acquired -42:22.044 [INFO] TestAgent.server: initializing acls -42:22.044 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:22.045 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:22.045 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:22.046 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:22.046 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:22.046 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:22.046 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-df914b45-0dd2-9a80-044f-f980859c3196 -42:22.046 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-df914b45-0dd2-9a80-044f-f980859c3196.dc1 -42:22.047 [INFO] TestAgent.server: New leader elected: payload=Node-df914b45-0dd2-9a80-044f-f980859c3196 -42:22.047 [INFO] TestAgent.server: Updating LAN server: server="Node-df914b45-0dd2-9a80-044f-f980859c3196 (Addr: tcp/127.0.0.1:13407) (DC: dc1)" -42:22.047 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-df914b45-0dd2-9a80-044f-f980859c3196.dc1 area=wan -42:22.047 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:22.050 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:22.050 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:22.050 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:22.050 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:22.050 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:22.050 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-df914b45-0dd2-9a80-044f-f980859c3196 -42:22.050 [INFO] TestAgent.server: member joined, marking health alive: member=Node-df914b45-0dd2-9a80-044f-f980859c3196 -42:22.051 [INFO] TestAgent.server: federation state anti-entropy synced -42:22.052 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-df914b45-0dd2-9a80-044f-f980859c3196 -42:22.257 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-df914b45-0dd2-9a80-044f-f980859c3196 -42:22.268 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-df914b45-0dd2-9a80-044f-f980859c3196 -42:22.270 [WARN] TestAgent.server.internal: user event blocked by ACLs: event=foo accessorID= -42:22.270 [WARN] TestAgent.server.internal: user event blocked by ACLs: event=bar accessorID= -42:22.270 [INFO] TestAgent: Requesting shutdown -42:22.270 [INFO] TestAgent.server: shutting down server -42:22.270 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:22.270 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:22.270 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:22.270 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:22.270 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:22.270 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:22.270 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:22.270 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:22.270 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:22.270 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:22.270 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:22.270 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:22.270 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:22.270 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:22.270 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:22.270 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:22.271Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:22.271Z [INFO] agent.router.manager: shutting down -42:22.271 [INFO] TestAgent: consul server down -42:22.271 [INFO] TestAgent: shutdown complete -42:22.271 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13416 network=tcp -42:22.271 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13416 network=udp -42:22.271 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13417 network=tcp -42:22.778 [INFO] TestAgent: Waiting for endpoints to shut down -42:22.778 [INFO] TestAgent: Endpoints down ---- PASS: TestEventFire_token (0.84s) -=== CONT TestDNS_trimUDPResponse_TrimLimit ---- PASS: TestDNS_trimUDPResponse_TrimSize (0.08s) -=== CONT TestDNS_trimUDPResponse_NoTrim ---- PASS: TestDNS_trimUDPResponse_TrimLimit (0.06s) -=== CONT TestDNS_PreparedQuery_AgentSource -2022-09-15T21:42:22.239Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:22.258Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:22.261 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bfd3f316-671e-3168-0795-b88e138c65e2 Address:127.0.0.1:13456}]" -42:22.262 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13456 [Follower]" leader= -42:22.265 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bfd3f316-671e-3168-0795-b88e138c65e2.dc1 127.0.0.1 -42:22.266 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bfd3f316-671e-3168-0795-b88e138c65e2 127.0.0.1 -2022-09-15T21:42:22.266Z [INFO] agent.router: Initializing LAN area manager -42:22.266 [DEBUG] TestAgent.dns: recursor enabled -42:22.266 [INFO] TestAgent.server: Adding LAN server: server="Node-bfd3f316-671e-3168-0795-b88e138c65e2 (Addr: tcp/127.0.0.1:13456) (DC: dc1)" -42:22.266 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bfd3f316-671e-3168-0795-b88e138c65e2.dc1 area=wan -42:22.266 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13430 network=udp -42:22.266 [DEBUG] TestAgent.dns: recursor enabled -42:22.266 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13430 network=tcp -42:22.267 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13431 network=tcp -42:22.267 [INFO] TestAgent: started state syncer -42:22.267 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13457 network=tcp -42:22.326 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:22.326 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13456 [Candidate]" term=2 -42:22.326 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:22.326 [DEBUG] TestAgent.server.raft: vote granted: from=bfd3f316-671e-3168-0795-b88e138c65e2 term=2 tally=1 -42:22.326 [INFO] TestAgent.server.raft: election won: tally=1 -42:22.326 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13456 [Leader]" -42:22.327 [INFO] TestAgent.server: cluster leadership acquired -42:22.327 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13456 -42:22.327 [INFO] TestAgent.server: New leader elected: payload=Node-bfd3f316-671e-3168-0795-b88e138c65e2 -42:22.328 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:22.331 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:22.331 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:22.331 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:22.331 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:22.331 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:22.332 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bfd3f316-671e-3168-0795-b88e138c65e2 -42:22.332 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bfd3f316-671e-3168-0795-b88e138c65e2 -42:22.333 [INFO] TestAgent.server: federation state anti-entropy synced -42:22.355 [INFO] TestAgent: Requesting shutdown -42:22.355 [INFO] TestAgent.server: shutting down server -42:22.355 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:22.355 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:22.355 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:22.356 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:22.356Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:22.356Z [INFO] agent.router.manager: shutting down -42:22.356 [INFO] TestAgent: consul server down -42:22.356 [INFO] TestAgent: shutdown complete -42:22.356 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13430 network=tcp -42:22.356 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13430 network=udp -42:22.356 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13431 network=tcp -42:22.858 [INFO] TestAgent: Waiting for endpoints to shut down -42:22.858 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ConfigReload (0.66s) -=== CONT TestDNS_InvalidQueries ---- PASS: TestDNS_trimUDPResponse_NoTrim (0.04s) -=== CONT TestDNS_PreparedQuery_AllowStale -2022-09-15T21:42:21.985Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:22.003Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:22.007 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c758a51e-3578-430c-fb35-2c4dfc94e65e Address:127.0.0.1:13421}]" -42:22.007 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13421 [Follower]" leader= -42:22.008 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c758a51e-3578-430c-fb35-2c4dfc94e65e.dc1 127.0.0.1 -42:22.008 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c758a51e-3578-430c-fb35-2c4dfc94e65e 127.0.0.1 -2022-09-15T21:42:22.008Z [INFO] agent.router: Initializing LAN area manager -42:22.009 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13409 network=udp -42:22.009 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c758a51e-3578-430c-fb35-2c4dfc94e65e.dc1 area=wan -42:22.009 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13409 network=tcp -42:22.009 [INFO] TestAgent.server: Adding LAN server: server="Node-c758a51e-3578-430c-fb35-2c4dfc94e65e (Addr: tcp/127.0.0.1:13421) (DC: dc1)" -42:22.010 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13410 network=tcp -42:22.010 [INFO] TestAgent: started state syncer -42:22.010 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13422 network=tcp -42:22.069 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:22.069 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13421 [Candidate]" term=2 -42:22.069 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:22.069 [DEBUG] TestAgent.server.raft: vote granted: from=c758a51e-3578-430c-fb35-2c4dfc94e65e term=2 tally=1 -42:22.069 [INFO] TestAgent.server.raft: election won: tally=1 -42:22.069 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13421 [Leader]" -42:22.069 [INFO] TestAgent.server: cluster leadership acquired -42:22.069 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13421 -42:22.071 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:22.073 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:22.073 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:22.073 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:22.073 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:22.073 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:22.074 [INFO] TestAgent.server: New leader elected: payload=Node-c758a51e-3578-430c-fb35-2c4dfc94e65e -42:22.074 [INFO] TestAgent.server: federation state anti-entropy synced -42:22.074 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c758a51e-3578-430c-fb35-2c4dfc94e65e -42:22.074 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c758a51e-3578-430c-fb35-2c4dfc94e65e -42:22.192 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:22.192 [INFO] TestAgent: Synced node info -42:22.192 [DEBUG] TestAgent: Node info in sync -42:22.478 [INFO] TestAgent: Requesting shutdown -42:22.478 [INFO] TestAgent.server: shutting down server -42:22.478 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:22.478 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:22.478 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:22.478 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:22.478 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:22.479 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:22.479 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:22.479 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:22.479 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:22.479 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:22.479 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:22.479Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:22.479Z [INFO] agent.router.manager: shutting down -42:22.479 [INFO] TestAgent: consul server down -42:22.479 [INFO] TestAgent: shutdown complete -42:22.479 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13409 network=tcp -42:22.479 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13409 network=udp -42:22.479 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13410 network=tcp -42:22.982 [INFO] TestAgent: Waiting for endpoints to shut down -42:22.982 [INFO] TestAgent: Endpoints down ---- PASS: TestEventFire (1.02s) -=== CONT TestDNS_AltDomains_Overlap -2022-09-15T21:42:22.126Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:22.143Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:22.146 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d1361ca3-5002-ae9f-23e3-acdcea3cd670 Address:127.0.0.1:13428}]" -42:22.146 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13428 [Follower]" leader= -42:22.147 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d1361ca3-5002-ae9f-23e3-acdcea3cd670.dc1 127.0.0.1 -42:22.148 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d1361ca3-5002-ae9f-23e3-acdcea3cd670 127.0.0.1 -2022-09-15T21:42:22.148Z [INFO] agent.router: Initializing LAN area manager -42:22.148 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13423 network=udp -42:22.149 [INFO] TestAgent.server: Adding LAN server: server="Node-d1361ca3-5002-ae9f-23e3-acdcea3cd670 (Addr: tcp/127.0.0.1:13428) (DC: dc1)" -42:22.149 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d1361ca3-5002-ae9f-23e3-acdcea3cd670.dc1 area=wan -42:22.149 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13423 network=tcp -42:22.150 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13424 network=tcp -42:22.150 [INFO] TestAgent: started state syncer -42:22.150 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13429 network=tcp -42:22.190 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:22.190 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13428 [Candidate]" term=2 -42:22.190 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:22.190 [DEBUG] TestAgent.server.raft: vote granted: from=d1361ca3-5002-ae9f-23e3-acdcea3cd670 term=2 tally=1 -42:22.190 [INFO] TestAgent.server.raft: election won: tally=1 -42:22.190 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13428 [Leader]" -42:22.190 [INFO] TestAgent.server: cluster leadership acquired -42:22.191 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13428 -42:22.192 [INFO] TestAgent.server: New leader elected: payload=Node-d1361ca3-5002-ae9f-23e3-acdcea3cd670 -42:22.193 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:22.211 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:22.211 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:22.211 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:22.211 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:22.211 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:22.211 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d1361ca3-5002-ae9f-23e3-acdcea3cd670 -42:22.211 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d1361ca3-5002-ae9f-23e3-acdcea3cd670 -42:22.212 [INFO] TestAgent.server: federation state anti-entropy synced -42:22.292 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:22.292 [INFO] TestAgent: Synced node info -42:22.293 [DEBUG] TestAgent: Node info in sync -42:22.381 [WARN] TestAgent.server: endpoint injected; this should only be used for testing -42:22.486 [INFO] TestAgent: Requesting shutdown -42:22.486 [INFO] TestAgent.server: shutting down server -42:22.486 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:22.486 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:22.486 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:22.487Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:22.487Z [INFO] agent.router.manager: shutting down -42:22.487 [INFO] TestAgent: consul server down -42:22.487 [INFO] TestAgent: shutdown complete -42:22.487 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13423 network=tcp -42:22.487 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13423 network=udp -42:22.487 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13424 network=tcp -42:22.990 [INFO] TestAgent: Waiting for endpoints to shut down -42:22.990 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ReloadConfig_DuringQuery (0.88s) -=== CONT TestDNS_AltDomains_SOA -[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:22.649Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:22.667Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:22.670 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e2073877-1f42-8d74-723c-8d51e8d28d8d Address:127.0.0.1:13442}]" -42:22.671 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13442 [Follower]" leader= -42:22.672 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e2073877-1f42-8d74-723c-8d51e8d28d8d.dc1 127.0.0.1 -42:22.672 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e2073877-1f42-8d74-723c-8d51e8d28d8d 127.0.0.1 -2022-09-15T21:42:22.673Z [INFO] agent.router: Initializing LAN area manager -42:22.673 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13486 network=udp -42:22.673 [INFO] TestAgent.server: Adding LAN server: server="Node-e2073877-1f42-8d74-723c-8d51e8d28d8d (Addr: tcp/127.0.0.1:13442) (DC: dc1)" -42:22.673 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e2073877-1f42-8d74-723c-8d51e8d28d8d.dc1 area=wan -42:22.674 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13486 network=tcp -42:22.675 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13487 network=tcp -42:22.675 [INFO] TestAgent: started state syncer -42:22.675 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13443 network=tcp -42:22.722 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:22.722 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13442 [Candidate]" term=2 -42:22.722 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:22.722 [DEBUG] TestAgent.server.raft: vote granted: from=e2073877-1f42-8d74-723c-8d51e8d28d8d term=2 tally=1 -42:22.722 [INFO] TestAgent.server.raft: election won: tally=1 -42:22.722 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13442 [Leader]" -42:22.730 [INFO] TestAgent.server: cluster leadership acquired -42:22.731 [INFO] TestAgent.server: New leader elected: payload=Node-e2073877-1f42-8d74-723c-8d51e8d28d8d -42:22.732 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13442 -42:22.737 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:22.768 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:22.768 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:22.768 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:22.768 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:22.768 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:22.768 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e2073877-1f42-8d74-723c-8d51e8d28d8d -42:22.768 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e2073877-1f42-8d74-723c-8d51e8d28d8d -42:22.774 [INFO] TestAgent.server: federation state anti-entropy synced -42:22.842 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=401.38µs client=127.0.0.1:57543 client_network=udp -42:22.843 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=247.86µs client=127.0.0.1:57543 client_network=udp -42:22.844 [DEBUG] TestAgent.dns: request served from client: name=27442946-5263-00f0-3f40-212a84d9d52f.query.consul. type=SRV class=IN latency=203.3µs client=127.0.0.1:43592 client_network=udp -42:22.844 [DEBUG] TestAgent.dns: request served from client: name=27442946-5263-00f0-3f40-212a84d9d52f.query.consul. type=SRV class=IN latency=164.08µs client=127.0.0.1:43592 client_network=udp -42:22.844 [INFO] TestAgent: Requesting shutdown -42:22.844 [INFO] TestAgent.server: shutting down server -42:22.844 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:22.844 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:22.844 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:22.844 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:22.844 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:22.845 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:22.845 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:22.845 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:22.845 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:22.845 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:22.845 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:22.845 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:22.845Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:22.845Z [INFO] agent.router.manager: shutting down -42:22.845 [INFO] TestAgent: consul server down -42:22.845 [INFO] TestAgent: shutdown complete -42:22.845 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13486 network=tcp -42:22.845 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13486 network=udp -42:22.845 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13487 network=tcp -42:23.346 [INFO] TestAgent: Waiting for endpoints to shut down -42:23.346 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_Compression_Query (0.72s) -=== CONT TestDNS_AltDomains_Service -2022-09-15T21:42:22.494Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:22.513Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:22.516 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:017edf55-8131-99ac-0790-c78c55e641a8 Address:127.0.0.1:13435}]" -42:22.517 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-017edf55-8131-99ac-0790-c78c55e641a8.dc1 127.0.0.1 -42:22.518 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13435 [Follower]" leader= -42:22.518 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-017edf55-8131-99ac-0790-c78c55e641a8 127.0.0.1 -2022-09-15T21:42:22.518Z [INFO] agent.router: Initializing LAN area manager -42:22.518 [DEBUG] TestAgent.dns: recursor enabled -42:22.519 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13458 network=udp -42:22.519 [INFO] TestAgent.server: Adding LAN server: server="Node-017edf55-8131-99ac-0790-c78c55e641a8 (Addr: tcp/127.0.0.1:13435) (DC: dc1)" -42:22.519 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-017edf55-8131-99ac-0790-c78c55e641a8.dc1 area=wan -42:22.519 [DEBUG] TestAgent.dns: recursor enabled -42:22.519 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13458 network=tcp -42:22.520 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13459 network=tcp -42:22.520 [INFO] TestAgent: started state syncer -42:22.520 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13436 network=tcp -42:22.554 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:22.554 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13435 [Candidate]" term=2 -42:22.554 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:22.554 [DEBUG] TestAgent.server.raft: vote granted: from=017edf55-8131-99ac-0790-c78c55e641a8 term=2 tally=1 -42:22.554 [INFO] TestAgent.server.raft: election won: tally=1 -42:22.554 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13435 [Leader]" -42:22.554 [INFO] TestAgent.server: cluster leadership acquired -42:22.554 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13435 -42:22.556 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:22.558 [INFO] TestAgent.server: New leader elected: payload=Node-017edf55-8131-99ac-0790-c78c55e641a8 -42:22.559 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:22.559 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:22.559 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:22.559 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:22.559 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:22.559 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-017edf55-8131-99ac-0790-c78c55e641a8 -42:22.559 [INFO] TestAgent.server: member joined, marking health alive: member=Node-017edf55-8131-99ac-0790-c78c55e641a8 -42:22.560 [INFO] TestAgent.server: federation state anti-entropy synced -42:22.879 [DEBUG] TestAgent.dns: recurse succeeded for question: question="{apple.com. 255 1}" rtt=231.2µs recursor=127.0.0.1:45684 -42:22.879 [DEBUG] TestAgent.dns: request served from client: question="{apple.com. 255 1}" network=udp latency=497.96µs client=127.0.0.1:47326 client_network=udp -42:22.880 [DEBUG] TestAgent.dns: recurse succeeded for question: question="{apple.com. 255 1}" rtt=148.12µs recursor=127.0.0.1:45684 -42:22.880 [DEBUG] TestAgent.dns: request served from client: question="{apple.com. 255 1}" network=udp latency=325.8µs client=127.0.0.1:47326 client_network=udp -42:22.880 [INFO] TestAgent: Requesting shutdown -42:22.880 [INFO] TestAgent.server: shutting down server -42:22.880 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:22.880 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:22.880 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:22.880 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:22.880 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:22.880 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:22.880 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:22.880 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:22.880 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:22.880 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:22.880 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:22.880 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:22.880Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:22.881Z [INFO] agent.router.manager: shutting down -42:22.881 [INFO] TestAgent: consul server down -42:22.881 [INFO] TestAgent: shutdown complete -42:22.881 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13458 network=tcp -42:22.881 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13458 network=udp -42:22.881 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13459 network=tcp -42:23.382 [INFO] TestAgent: Waiting for endpoints to shut down -42:23.382 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_Compression_Recurse (0.91s) -=== CONT TestDNS_NonExistingLookupEmptyAorAAAA -2022-09-15T21:42:22.545Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:22.571Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:22.578 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7b51b914-0e46-b9c8-234d-e152cd75a02c Address:127.0.0.1:13484}]" -42:22.579 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7b51b914-0e46-b9c8-234d-e152cd75a02c.dc1 127.0.0.1 -42:22.580 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7b51b914-0e46-b9c8-234d-e152cd75a02c 127.0.0.1 -2022-09-15T21:42:22.580Z [INFO] agent.router: Initializing LAN area manager -42:22.581 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13437 network=udp -42:22.581 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13484 [Follower]" leader= -42:22.581 [INFO] TestAgent.server: Adding LAN server: server="Node-7b51b914-0e46-b9c8-234d-e152cd75a02c (Addr: tcp/127.0.0.1:13484) (DC: dc1)" -42:22.581 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7b51b914-0e46-b9c8-234d-e152cd75a02c.dc1 area=wan -42:22.582 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13437 network=tcp -42:22.583 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13438 network=tcp -42:22.583 [INFO] TestAgent: started state syncer -42:22.583 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13485 network=tcp -42:22.648 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:22.648 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13484 [Candidate]" term=2 -42:22.648 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:22.648 [DEBUG] TestAgent.server.raft: vote granted: from=7b51b914-0e46-b9c8-234d-e152cd75a02c term=2 tally=1 -42:22.648 [INFO] TestAgent.server.raft: election won: tally=1 -42:22.648 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13484 [Leader]" -42:22.648 [INFO] TestAgent.server: cluster leadership acquired -42:22.648 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13484 -42:22.650 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:22.652 [INFO] TestAgent.server: New leader elected: payload=Node-7b51b914-0e46-b9c8-234d-e152cd75a02c -42:22.652 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:22.652 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:22.652 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:22.652 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:22.652 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:22.653 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7b51b914-0e46-b9c8-234d-e152cd75a02c -42:22.653 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7b51b914-0e46-b9c8-234d-e152cd75a02c -42:22.655 [INFO] TestAgent.server: federation state anti-entropy synced -42:22.842 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:22.843 [INFO] TestAgent: Synced node info -42:23.009 [DEBUG] TestAgent.dns: request served from client: question="{2.0.0.127.in-addr.arpa. 255 1}" latency=147.26µs client=127.0.0.1:40526 client_network=udp -42:23.010 [DEBUG] TestAgent.dns: request served from client: question="{2.0.0.127.in-addr.arpa. 255 1}" latency=128.66µs client=127.0.0.1:40526 client_network=udp -42:23.010 [INFO] TestAgent: Requesting shutdown -42:23.010 [INFO] TestAgent.server: shutting down server -42:23.010 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:23.010 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:23.010 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:23.010 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:23.010 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:23.010 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:23.010 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:23.010 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:23.010 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:23.010 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:23.010 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:23.010Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:23.010Z [INFO] agent.router.manager: shutting down -42:23.011 [INFO] TestAgent: consul server down -42:23.011 [INFO] TestAgent: shutdown complete -42:23.011 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13437 network=tcp -42:23.011 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13437 network=udp -42:23.011 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13438 network=tcp -42:23.518 [INFO] TestAgent: Waiting for endpoints to shut down -42:23.518 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_Compression_ReverseLookup (0.99s) -=== CONT TestDNS_NonExistingLookup -[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:22.908Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:22.927Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:22.932 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:313eb43f-4307-7857-8a43-c0e62f4dd77f Address:127.0.0.1:13463}]" -42:22.933 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13463 [Follower]" leader= -42:22.933 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-313eb43f-4307-7857-8a43-c0e62f4dd77f.dc1 127.0.0.1 -42:22.934 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-313eb43f-4307-7857-8a43-c0e62f4dd77f 127.0.0.1 -2022-09-15T21:42:22.934Z [INFO] agent.router: Initializing LAN area manager -42:22.934 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13444 network=udp -42:22.934 [INFO] TestAgent.server: Adding LAN server: server="Node-313eb43f-4307-7857-8a43-c0e62f4dd77f (Addr: tcp/127.0.0.1:13463) (DC: dc1)" -42:22.934 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-313eb43f-4307-7857-8a43-c0e62f4dd77f.dc1 area=wan -42:22.935 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13444 network=tcp -42:22.936 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13445 network=tcp -42:22.936 [INFO] TestAgent: started state syncer -42:22.936 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13464 network=tcp -42:22.986 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:22.986 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13463 [Candidate]" term=2 -42:22.986 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:22.986 [DEBUG] TestAgent.server.raft: vote granted: from=313eb43f-4307-7857-8a43-c0e62f4dd77f term=2 tally=1 -42:22.986 [INFO] TestAgent.server.raft: election won: tally=1 -42:22.986 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13463 [Leader]" -42:22.986 [INFO] TestAgent.server: cluster leadership acquired -42:22.987 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13463 -42:22.988 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:22.990 [INFO] TestAgent.server: New leader elected: payload=Node-313eb43f-4307-7857-8a43-c0e62f4dd77f -42:23.007 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:23.007 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:23.007 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:23.007 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:23.007 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:23.007 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-313eb43f-4307-7857-8a43-c0e62f4dd77f -42:23.007 [INFO] TestAgent.server: member joined, marking health alive: member=Node-313eb43f-4307-7857-8a43-c0e62f4dd77f -42:23.008 [INFO] TestAgent.server: federation state anti-entropy synced -42:23.154 [WARN] TestAgent.server: endpoint injected; this should only be used for testing -42:23.158 [DEBUG] TestAgent.dns: request served from client: name=foo.query.consul. type=SRV class=IN latency=211.18µs client=127.0.0.1:50370 client_network=udp -42:23.158 [INFO] TestAgent: Requesting shutdown -42:23.158 [INFO] TestAgent.server: shutting down server -42:23.158 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:23.158 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:23.158 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:23.158 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:23.158 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:23.159 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:23.159 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:23.159 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:23.159 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:23.159 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:23.159 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:23.159 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:23.159Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:23.159Z [INFO] agent.router.manager: shutting down -42:23.159 [INFO] TestAgent: consul server down -42:23.160 [INFO] TestAgent: shutdown complete -42:23.160 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13444 network=tcp -42:23.160 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13444 network=udp -42:23.160 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13445 network=tcp -42:23.660 [INFO] TestAgent: Waiting for endpoints to shut down -42:23.660 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_PreparedQuery_AgentSource (0.82s) -=== CONT TestDNS_NonExistentDC_RPC -2022-09-15T21:42:22.901Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:22.937Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:22.951 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:66df8bd0-fce7-eb46-dbf4-a6640ae0a531 Address:127.0.0.1:13449}]" -42:22.952 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-66df8bd0-fce7-eb46-dbf4-a6640ae0a531.dc1 127.0.0.1 -42:22.953 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-66df8bd0-fce7-eb46-dbf4-a6640ae0a531 127.0.0.1 -2022-09-15T21:42:22.953Z [INFO] agent.router: Initializing LAN area manager -42:22.953 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13465 network=udp -42:22.953 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13449 [Follower]" leader= -42:22.954 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-66df8bd0-fce7-eb46-dbf4-a6640ae0a531.dc1 area=wan -42:22.954 [INFO] TestAgent.server: Adding LAN server: server="Node-66df8bd0-fce7-eb46-dbf4-a6640ae0a531 (Addr: tcp/127.0.0.1:13449) (DC: dc1)" -42:22.954 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13465 network=tcp -42:22.955 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13466 network=tcp -42:22.955 [INFO] TestAgent: started state syncer -42:22.955 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13450 network=tcp -42:23.016 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:23.016 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13449 [Candidate]" term=2 -42:23.016 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:23.016 [DEBUG] TestAgent.server.raft: vote granted: from=66df8bd0-fce7-eb46-dbf4-a6640ae0a531 term=2 tally=1 -42:23.016 [INFO] TestAgent.server.raft: election won: tally=1 -42:23.016 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13449 [Leader]" -42:23.017 [INFO] TestAgent.server: cluster leadership acquired -42:23.017 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13449 -42:23.018 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:23.021 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:23.021 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:23.021 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:23.021 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:23.021 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:23.021 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-66df8bd0-fce7-eb46-dbf4-a6640ae0a531 -42:23.021 [INFO] TestAgent.server: member joined, marking health alive: member=Node-66df8bd0-fce7-eb46-dbf4-a6640ae0a531 -42:23.022 [INFO] TestAgent.server: New leader elected: payload=Node-66df8bd0-fce7-eb46-dbf4-a6640ae0a531 -42:23.047 [INFO] TestAgent: Synced node info -42:23.047 [INFO] TestAgent.server: federation state anti-entropy synced -42:23.258 [WARN] TestAgent.dns: QName invalid: qname= -42:23.259 [DEBUG] TestAgent.dns: request served from client: name=consul. type=SRV class=IN latency=93.32µs client=127.0.0.1:52712 client_network=udp -42:23.259 [WARN] TestAgent.dns: QName invalid: qname=node. -42:23.259 [DEBUG] TestAgent.dns: request served from client: name=node.consul. type=SRV class=IN latency=66.66µs client=127.0.0.1:53744 client_network=udp -42:23.259 [WARN] TestAgent.dns: QName invalid: qname=service. -42:23.259 [DEBUG] TestAgent.dns: request served from client: name=service.consul. type=SRV class=IN latency=61.28µs client=127.0.0.1:45078 client_network=udp -42:23.259 [WARN] TestAgent.dns: QName invalid: qname=query. -42:23.259 [DEBUG] TestAgent.dns: request served from client: name=query.consul. type=SRV class=IN latency=59.1µs client=127.0.0.1:52393 client_network=udp -42:23.260 [WARN] TestAgent.dns: QName invalid: qname=foo.node.dc1.extra.more. -42:23.260 [DEBUG] TestAgent.dns: request served from client: name=foo.node.dc1.extra.more.consul. type=SRV class=IN latency=67.86µs client=127.0.0.1:53967 client_network=udp -42:23.260 [WARN] TestAgent.dns: QName invalid: qname=foo.service.dc1.extra.more. -42:23.260 [DEBUG] TestAgent.dns: request served from client: name=foo.service.dc1.extra.more.consul. type=SRV class=IN latency=75.78µs client=127.0.0.1:50201 client_network=udp -42:23.260 [WARN] TestAgent.dns: QName invalid: qname=foo.query.dc1.extra.more. -42:23.260 [DEBUG] TestAgent.dns: request served from client: name=foo.query.dc1.extra.more.consul. type=SRV class=IN latency=70.8µs client=127.0.0.1:55851 client_network=udp -42:23.260 [INFO] TestAgent: Requesting shutdown -42:23.261 [INFO] TestAgent.server: shutting down server -42:23.261 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:23.261 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:23.261 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:23.261 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:23.261 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:23.261 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:23.261 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:23.261 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:23.261 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:23.261 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:23.261 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:23.261Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:23.261Z [INFO] agent.router.manager: shutting down -42:23.262 [INFO] TestAgent: consul server down -42:23.262 [INFO] TestAgent: shutdown complete -42:23.262 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13465 network=tcp -42:23.262 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13465 network=udp -42:23.262 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13466 network=tcp -42:23.764 [INFO] TestAgent: Waiting for endpoints to shut down -42:23.764 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_InvalidQueries (0.91s) -=== CONT TestDNS_NonExistentDC_Server -[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:22.932Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:22.981Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:23.011 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b9856f04-1b16-10ed-42e9-9d0bd4a4f8f1 Address:127.0.0.1:13477}]" -42:23.012 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b9856f04-1b16-10ed-42e9-9d0bd4a4f8f1.dc1 127.0.0.1 -42:23.013 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b9856f04-1b16-10ed-42e9-9d0bd4a4f8f1 127.0.0.1 -2022-09-15T21:42:23.013Z [INFO] agent.router: Initializing LAN area manager -42:23.014 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13477 [Follower]" leader= -42:23.014 [INFO] TestAgent.server: Adding LAN server: server="Node-b9856f04-1b16-10ed-42e9-9d0bd4a4f8f1 (Addr: tcp/127.0.0.1:13477) (DC: dc1)" -42:23.014 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b9856f04-1b16-10ed-42e9-9d0bd4a4f8f1.dc1 area=wan -42:23.015 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13451 network=udp -42:23.015 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13451 network=tcp -42:23.016 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13452 network=tcp -42:23.016 [INFO] TestAgent: started state syncer -42:23.016 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13478 network=tcp -42:23.062 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:23.062 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13477 [Candidate]" term=2 -42:23.062 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:23.062 [DEBUG] TestAgent.server.raft: vote granted: from=b9856f04-1b16-10ed-42e9-9d0bd4a4f8f1 term=2 tally=1 -42:23.062 [INFO] TestAgent.server.raft: election won: tally=1 -42:23.062 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13477 [Leader]" -42:23.062 [INFO] TestAgent.server: cluster leadership acquired -42:23.063 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13477 -42:23.064 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:23.066 [INFO] TestAgent.server: New leader elected: payload=Node-b9856f04-1b16-10ed-42e9-9d0bd4a4f8f1 -42:23.069 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:23.069 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:23.069 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:23.069 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:23.069 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:23.069 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b9856f04-1b16-10ed-42e9-9d0bd4a4f8f1 -42:23.069 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b9856f04-1b16-10ed-42e9-9d0bd4a4f8f1 -42:23.098 [INFO] TestAgent.server: federation state anti-entropy synced -42:23.285 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:23.286 [INFO] TestAgent: Synced node info -42:23.286 [DEBUG] TestAgent: Node info in sync -42:23.294 [WARN] TestAgent.server: endpoint injected; this should only be used for testing -42:23.295 [WARN] TestAgent.dns: Query results too stale, re-requesting -42:23.295 [DEBUG] TestAgent.dns: request served from client: name=nope.query.consul. type=SRV class=IN latency=197.02µs client=127.0.0.1:40789 client_network=udp -42:23.295 [INFO] TestAgent: Requesting shutdown -42:23.295 [INFO] TestAgent.server: shutting down server -42:23.295 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:23.295 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:23.295 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:23.295 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:23.295 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:23.295 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:23.295 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:23.295 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:23.295 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:23.295 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:23.295 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:23.296Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:23.296Z [INFO] agent.router.manager: shutting down -42:23.296 [INFO] TestAgent: consul server down -42:23.296 [INFO] TestAgent: shutdown complete -42:23.296 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13451 network=tcp -42:23.296 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13451 network=udp -42:23.296 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13452 network=tcp -42:23.798 [INFO] TestAgent: Waiting for endpoints to shut down -42:23.798 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_PreparedQuery_AllowStale (0.94s) -=== CONT TestDNS_AddressLookupIPV6 -2022-09-15T21:42:23.066Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:23.185Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:23.188 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c4a6a6b0-3d9b-97fc-1408-d8207dcb709c Address:127.0.0.1:13498}]" -42:23.189 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -42:23.190 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13498 [Follower]" leader= -42:23.194 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2022-09-15T21:42:23.194Z [INFO] agent.router: Initializing LAN area manager -42:23.194 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13472 network=udp -42:23.195 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:13498) (DC: dc1)" -42:23.195 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -42:23.195 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13472 network=tcp -42:23.196 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13473 network=tcp -42:23.196 [INFO] TestAgent: started state syncer -42:23.196 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13499 network=tcp -42:23.258 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:23.258 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13498 [Candidate]" term=2 -42:23.258 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:23.258 [DEBUG] TestAgent.server.raft: vote granted: from=c4a6a6b0-3d9b-97fc-1408-d8207dcb709c term=2 tally=1 -42:23.258 [INFO] TestAgent.server.raft: election won: tally=1 -42:23.258 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13498 [Leader]" -42:23.258 [INFO] TestAgent.server: cluster leadership acquired -42:23.259 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13498 -42:23.260 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:23.262 [INFO] TestAgent.server: New leader elected: payload=test-node -42:23.267 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:23.267 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:23.267 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:23.267 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:23.267 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:23.267 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -42:23.267 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -42:23.269 [INFO] TestAgent.server: federation state anti-entropy synced -42:23.414 [DEBUG] TestAgent.dns: request served from client: name=test-node.node.consul. type=SOA class=IN latency=241.32µs client=127.0.0.1:36245 client_network=udp -42:23.415 [DEBUG] TestAgent.dns: request served from client: name=test-node.node.test-domain. type=SOA class=IN latency=148.86µs client=127.0.0.1:32829 client_network=udp -42:23.415 [INFO] TestAgent: Requesting shutdown -42:23.415 [INFO] TestAgent.server: shutting down server -42:23.415 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:23.415 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:23.415 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:23.415 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:23.415 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:23.415 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:23.415 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:23.415 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:23.415 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:23.415 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:23.415 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:23.415 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:23.416Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:23.416Z [INFO] agent.router.manager: shutting down -42:23.416 [INFO] TestAgent: consul server down -42:23.416 [INFO] TestAgent: shutdown complete -42:23.416 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13472 network=tcp -42:23.416 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13472 network=udp -42:23.416 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13473 network=tcp -42:23.921 [INFO] TestAgent: Waiting for endpoints to shut down -42:23.921 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_AltDomains_SOA (0.93s) -=== CONT TestDNS_AddressLookup -2022-09-15T21:42:23.053Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:23.181Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:23.185 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:32554a62-f262-b843-d518-8c06654f3e80 Address:127.0.0.1:13470}]" -42:23.186 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13470 [Follower]" leader= -42:23.196 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -42:23.197 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2022-09-15T21:42:23.197Z [INFO] agent.router: Initializing LAN area manager -42:23.197 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13479 network=udp -42:23.197 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -42:23.197 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:13470) (DC: dc1)" -42:23.198 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13479 network=tcp -42:23.199 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13480 network=tcp -42:23.199 [INFO] TestAgent: started state syncer -42:23.199 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13471 network=tcp -42:23.254 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:23.254 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13470 [Candidate]" term=2 -42:23.254 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:23.254 [DEBUG] TestAgent.server.raft: vote granted: from=32554a62-f262-b843-d518-8c06654f3e80 term=2 tally=1 -42:23.254 [INFO] TestAgent.server.raft: election won: tally=1 -42:23.254 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13470 [Leader]" -42:23.254 [INFO] TestAgent.server: cluster leadership acquired -42:23.255 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13470 -42:23.256 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:23.258 [INFO] TestAgent.server: New leader elected: payload=test-node -42:23.267 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:23.267 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:23.267 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:23.267 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:23.267 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:23.267 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -42:23.267 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -42:23.269 [INFO] TestAgent.server: federation state anti-entropy synced -42:23.326 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:23.327 [INFO] TestAgent: Synced node info -42:23.327 [DEBUG] TestAgent: Node info in sync -42:23.487 [DEBUG] TestAgent.dns: request served from client: name=test-node.node.consul. type=A class=IN latency=270.86µs client=127.0.0.1:34476 client_network=udp -42:23.487 [DEBUG] TestAgent.dns: request served from client: name=test-node.node.test.consul. type=A class=IN latency=175.78µs client=127.0.0.1:60070 client_network=udp -42:23.488 [DEBUG] TestAgent.dns: request served from client: name=test-node.node.dc1.consul. type=A class=IN latency=209µs client=127.0.0.1:37733 client_network=udp -42:23.488 [DEBUG] TestAgent.dns: request served from client: name=test-node.node.dc1.test.consul. type=A class=IN latency=181.88µs client=127.0.0.1:35352 client_network=udp -42:23.488 [INFO] TestAgent: Requesting shutdown -42:23.488 [INFO] TestAgent.server: shutting down server -42:23.489 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:23.489 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:23.489 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:23.489 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:23.489 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:23.489 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:23.489 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:23.489 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:23.489 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:23.489 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:23.489 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:23.489Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:23.489Z [INFO] agent.router.manager: shutting down -42:23.489 [INFO] TestAgent: consul server down -42:23.489 [INFO] TestAgent: shutdown complete -42:23.489 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13479 network=tcp -42:23.489 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13479 network=udp -42:23.489 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13480 network=tcp -42:23.994 [INFO] TestAgent: Waiting for endpoints to shut down -42:23.994 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_AltDomains_Overlap (1.01s) -=== CONT TestDNS_ServiceLookup_FilterACL -=== RUN TestDNS_ServiceLookup_FilterACL/ACLToken_==_root -[WARN] freeport: 4 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:23.412Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:23.430Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:23.433 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:abc668c0-36f7-4565-7497-d6909fa85c5d Address:127.0.0.1:13505}]" -42:23.434 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13505 [Follower]" leader= -42:23.434 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-abc668c0-36f7-4565-7497-d6909fa85c5d.dc1 127.0.0.1 -42:23.435 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-abc668c0-36f7-4565-7497-d6909fa85c5d 127.0.0.1 -2022-09-15T21:42:23.435Z [INFO] agent.router: Initializing LAN area manager -42:23.436 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13493 network=udp -42:23.436 [INFO] TestAgent.server: Adding LAN server: server="Node-abc668c0-36f7-4565-7497-d6909fa85c5d (Addr: tcp/127.0.0.1:13505) (DC: dc1)" -42:23.436 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-abc668c0-36f7-4565-7497-d6909fa85c5d.dc1 area=wan -42:23.436 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13493 network=tcp -42:23.437 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13494 network=tcp -42:23.437 [INFO] TestAgent: started state syncer -42:23.438 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13506 network=tcp -42:23.483 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:23.483 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13505 [Candidate]" term=2 -42:23.483 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:23.483 [DEBUG] TestAgent.server.raft: vote granted: from=abc668c0-36f7-4565-7497-d6909fa85c5d term=2 tally=1 -42:23.483 [INFO] TestAgent.server.raft: election won: tally=1 -42:23.483 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13505 [Leader]" -42:23.483 [INFO] TestAgent.server: cluster leadership acquired -42:23.483 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13505 -42:23.485 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:23.486 [INFO] TestAgent.server: New leader elected: payload=Node-abc668c0-36f7-4565-7497-d6909fa85c5d -42:23.491 [INFO] TestAgent: Synced node info -42:23.491 [DEBUG] TestAgent: Node info in sync -42:23.494 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:23.494 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:23.494 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:23.494 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:23.494 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:23.494 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-abc668c0-36f7-4565-7497-d6909fa85c5d -42:23.494 [INFO] TestAgent.server: member joined, marking health alive: member=Node-abc668c0-36f7-4565-7497-d6909fa85c5d -42:23.495 [INFO] TestAgent.server: federation state anti-entropy synced -42:23.585 [DEBUG] TestAgent.dns: request served from client: name=webv4.service.consul. type=AAAA class=IN latency=330.44µs client=127.0.0.1:35988 client_network=udp -42:23.586 [DEBUG] TestAgent.dns: request served from client: name=webv4.query.consul. type=AAAA class=IN latency=206.4µs client=127.0.0.1:33026 client_network=udp -42:23.587 [DEBUG] TestAgent.dns: request served from client: name=webv6.service.consul. type=A class=IN latency=245.12µs client=127.0.0.1:60869 client_network=udp -42:23.587 [DEBUG] TestAgent.dns: request served from client: name=webv6.query.consul. type=A class=IN latency=162.68µs client=127.0.0.1:45740 client_network=udp -42:23.587 [INFO] TestAgent: Requesting shutdown -42:23.588 [INFO] TestAgent.server: shutting down server -42:23.588 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:23.588 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:23.588 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:23.588 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:23.588 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:23.588 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:23.588 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:23.588 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:23.588 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:23.588 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:23.588 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:23.588Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:23.588Z [INFO] agent.router.manager: shutting down -42:23.589 [INFO] TestAgent: consul server down -42:23.589 [INFO] TestAgent: shutdown complete -42:23.589 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13493 network=tcp -42:23.589 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13493 network=udp -42:23.589 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13494 network=tcp -42:24.102 [INFO] TestAgent: Waiting for endpoints to shut down -42:24.103 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_NonExistingLookupEmptyAorAAAA (0.73s) -=== CONT TestDNS_ServiceLookup_SRV_RFC_TCP_Default -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:23.365Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:23.384Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:23.387 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:af554fc7-6030-65df-5a8d-f25d7c1a75bc Address:127.0.0.1:13491}]" -42:23.388 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-af554fc7-6030-65df-5a8d-f25d7c1a75bc.dc1 127.0.0.1 -42:23.389 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-af554fc7-6030-65df-5a8d-f25d7c1a75bc 127.0.0.1 -2022-09-15T21:42:23.389Z [INFO] agent.router: Initializing LAN area manager -42:23.390 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13491 [Follower]" leader= -42:23.390 [INFO] TestAgent.server: Adding LAN server: server="Node-af554fc7-6030-65df-5a8d-f25d7c1a75bc (Addr: tcp/127.0.0.1:13491) (DC: dc1)" -42:23.390 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-af554fc7-6030-65df-5a8d-f25d7c1a75bc.dc1 area=wan -42:23.391 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13500 network=tcp -42:23.410 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13500 network=udp -42:23.411 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13501 network=tcp -42:23.411 [INFO] TestAgent: started state syncer -42:23.411 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13492 network=tcp -42:23.430 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:23.430 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13491 [Candidate]" term=2 -42:23.430 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:23.430 [DEBUG] TestAgent.server.raft: vote granted: from=af554fc7-6030-65df-5a8d-f25d7c1a75bc term=2 tally=1 -42:23.430 [INFO] TestAgent.server.raft: election won: tally=1 -42:23.430 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13491 [Leader]" -42:23.430 [INFO] TestAgent.server: cluster leadership acquired -42:23.431 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13491 -42:23.432 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:23.434 [INFO] TestAgent.server: New leader elected: payload=Node-af554fc7-6030-65df-5a8d-f25d7c1a75bc -42:23.434 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:23.434 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:23.434 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:23.434 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:23.434 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:23.435 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-af554fc7-6030-65df-5a8d-f25d7c1a75bc -42:23.435 [INFO] TestAgent.server: member joined, marking health alive: member=Node-af554fc7-6030-65df-5a8d-f25d7c1a75bc -42:23.436 [INFO] TestAgent.server: federation state anti-entropy synced -42:23.482 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:23.483 [INFO] TestAgent: Synced node info -42:23.483 [DEBUG] TestAgent: Node info in sync -42:23.820 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=442.62µs client=127.0.0.1:34812 client_network=udp -42:23.821 [DEBUG] TestAgent.dns: request served from client: name=db.service.test-domain. type=SRV class=IN latency=216.12µs client=127.0.0.1:39335 client_network=udp -42:23.821 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc1.consul. type=SRV class=IN latency=201.58µs client=127.0.0.1:59599 client_network=udp -42:23.821 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc1.test-domain. type=SRV class=IN latency=190.2µs client=127.0.0.1:47519 client_network=udp -42:23.822 [INFO] TestAgent: Requesting shutdown -42:23.822 [INFO] TestAgent.server: shutting down server -42:23.822 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:23.822 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:23.822 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:23.822 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:23.822 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:23.822 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:23.822 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:23.822 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:23.822 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:23.822 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:23.822 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:23.822Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:23.822Z [INFO] agent.router.manager: shutting down -42:23.823 [INFO] TestAgent: consul server down -42:23.823 [INFO] TestAgent: shutdown complete -42:23.823 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13500 network=tcp -42:23.823 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13500 network=udp -42:23.823 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13501 network=tcp -42:24.326 [INFO] TestAgent: Waiting for endpoints to shut down -42:24.326 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_AltDomains_Service (0.98s) -=== CONT TestDNS_ServiceLookup_SRV_RFC -2022-09-15T21:42:23.568Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:23.595Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:23.598 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a82e1eb8-b030-277b-d84e-b5c539ff664b Address:127.0.0.1:13512}]" -42:23.600 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a82e1eb8-b030-277b-d84e-b5c539ff664b.dc1 127.0.0.1 -42:23.600 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a82e1eb8-b030-277b-d84e-b5c539ff664b 127.0.0.1 -2022-09-15T21:42:23.600Z [INFO] agent.router: Initializing LAN area manager -42:23.601 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13507 network=udp -42:23.601 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13512 [Follower]" leader= -42:23.602 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a82e1eb8-b030-277b-d84e-b5c539ff664b.dc1 area=wan -42:23.602 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13507 network=tcp -42:23.603 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13508 network=tcp -42:23.603 [INFO] TestAgent: started state syncer -42:23.603 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13513 network=tcp -42:23.602 [INFO] TestAgent.server: Adding LAN server: server="Node-a82e1eb8-b030-277b-d84e-b5c539ff664b (Addr: tcp/127.0.0.1:13512) (DC: dc1)" -42:23.641 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:23.641 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13512 [Candidate]" term=2 -42:23.642 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:23.642 [DEBUG] TestAgent.server.raft: vote granted: from=a82e1eb8-b030-277b-d84e-b5c539ff664b term=2 tally=1 -42:23.642 [INFO] TestAgent.server.raft: election won: tally=1 -42:23.642 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13512 [Leader]" -42:23.642 [INFO] TestAgent.server: cluster leadership acquired -42:23.642 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13512 -42:23.644 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:23.646 [INFO] TestAgent.server: New leader elected: payload=Node-a82e1eb8-b030-277b-d84e-b5c539ff664b -42:23.650 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:23.650 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:23.650 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:23.650 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:23.650 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:23.651 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a82e1eb8-b030-277b-d84e-b5c539ff664b -42:23.651 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a82e1eb8-b030-277b-d84e-b5c539ff664b -42:23.652 [INFO] TestAgent.server: federation state anti-entropy synced -42:23.786 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:23.795 [INFO] TestAgent: Synced node info -42:23.962 [WARN] TestAgent.dns: QName invalid: qname=nonexisting. -42:23.962 [DEBUG] TestAgent.dns: request served from client: name=nonexisting.consul. type=ANY class=IN latency=114.76µs client=127.0.0.1:39867 client_network=udp -42:23.963 [INFO] TestAgent: Requesting shutdown -42:23.963 [INFO] TestAgent.server: shutting down server -42:23.963 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:23.963 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:23.963 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:23.963 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:23.963 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:23.963 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:23.963 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:23.963 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:23.963 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:23.963 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:23.963 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:23.963Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:23.963Z [INFO] agent.router.manager: shutting down -42:23.963 [INFO] TestAgent: consul server down -42:23.963 [INFO] TestAgent: shutdown complete -42:23.963 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13507 network=tcp -42:23.963 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13507 network=udp -42:23.964 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13508 network=tcp -42:24.464 [INFO] TestAgent: Waiting for endpoints to shut down -42:24.464 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_NonExistingLookup (0.95s) -=== CONT TestDNS_PreparedQuery_Failover -2022-09-15T21:42:23.851Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:23.889Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:23.893 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9497be20-fb09-45b1-033b-eeba4f175e2a Address:127.0.0.1:13526}]" -42:23.898 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13526 [Follower]" leader= -42:23.899 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9497be20-fb09-45b1-033b-eeba4f175e2a.dc1 127.0.0.1 -42:23.900 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9497be20-fb09-45b1-033b-eeba4f175e2a 127.0.0.1 -2022-09-15T21:42:23.900Z [INFO] agent.router: Initializing LAN area manager -42:23.901 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13535 network=udp -42:23.901 [INFO] TestAgent.server: Adding LAN server: server="Node-9497be20-fb09-45b1-033b-eeba4f175e2a (Addr: tcp/127.0.0.1:13526) (DC: dc1)" -42:23.901 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9497be20-fb09-45b1-033b-eeba4f175e2a.dc1 area=wan -42:23.902 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13535 network=tcp -42:23.903 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13536 network=tcp -42:23.903 [INFO] TestAgent: started state syncer -42:23.903 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13527 network=tcp -42:23.970 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:23.970 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13526 [Candidate]" term=2 -42:23.970 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:23.970 [DEBUG] TestAgent.server.raft: vote granted: from=9497be20-fb09-45b1-033b-eeba4f175e2a term=2 tally=1 -42:23.970 [INFO] TestAgent.server.raft: election won: tally=1 -42:23.970 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13526 [Leader]" -42:23.970 [INFO] TestAgent.server: cluster leadership acquired -42:23.970 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13526 -42:23.972 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:23.975 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:23.975 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:23.975 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:23.975 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:23.975 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:23.975 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9497be20-fb09-45b1-033b-eeba4f175e2a -42:23.975 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9497be20-fb09-45b1-033b-eeba4f175e2a -42:23.976 [INFO] TestAgent.server: federation state anti-entropy synced -42:23.977 [INFO] TestAgent.server: New leader elected: payload=Node-9497be20-fb09-45b1-033b-eeba4f175e2a -42:23.978 [DEBUG] TestAgent.dns: request served from client: name=2607002040050808000000000000200e.addr.consul. type=SRV class=IN latency=88.48µs client=127.0.0.1:35591 client_network=udp -42:23.979 [DEBUG] TestAgent.dns: request served from client: name=2607112040051808ffffffffffff200e.addr.consul. type=SRV class=IN latency=59.74µs client=127.0.0.1:48831 client_network=udp -42:23.979 [INFO] TestAgent: Requesting shutdown -42:23.979 [INFO] TestAgent.server: shutting down server -42:23.979 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:23.979 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:23.979 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:23.979 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:23.979 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:23.979 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:23.979 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:23.979 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:23.979 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:23.979 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:23.979 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:23.979 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:23.980Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:23.980Z [INFO] agent.router.manager: shutting down -42:23.980 [INFO] TestAgent: consul server down -42:23.980 [INFO] TestAgent: shutdown complete -42:23.980 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13535 network=tcp -42:23.980 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13535 network=udp -42:23.980 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13536 network=tcp -42:24.482 [INFO] TestAgent: Waiting for endpoints to shut down -42:24.482 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_AddressLookupIPV6 (0.67s) -=== CONT TestDNS_PreparedQuery_TTL -[WARN] freeport: 4 out of 25 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:23.824Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:23.868Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:23.876 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a1fbe707-6f3a-9154-036d-1153a94a16a9 Address:127.0.0.1:13533}]" -42:23.877 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a1fbe707-6f3a-9154-036d-1153a94a16a9.dc1 127.0.0.1 -42:23.878 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13533 [Follower]" leader= -42:23.882 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a1fbe707-6f3a-9154-036d-1153a94a16a9 127.0.0.1 -2022-09-15T21:42:23.882Z [INFO] agent.router: Initializing LAN area manager -42:23.882 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13521 network=udp -42:23.883 [INFO] TestAgent.server: Adding LAN server: server="Node-a1fbe707-6f3a-9154-036d-1153a94a16a9 (Addr: tcp/127.0.0.1:13533) (DC: dc1)" -42:23.883 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a1fbe707-6f3a-9154-036d-1153a94a16a9.dc1 area=wan -42:23.883 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13521 network=tcp -42:23.884 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13522 network=tcp -42:23.884 [INFO] TestAgent: started state syncer -42:23.884 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13534 network=tcp -42:23.938 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:23.938 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13533 [Candidate]" term=2 -42:23.938 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:23.938 [DEBUG] TestAgent.server.raft: vote granted: from=a1fbe707-6f3a-9154-036d-1153a94a16a9 term=2 tally=1 -42:23.938 [INFO] TestAgent.server.raft: election won: tally=1 -42:23.938 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13533 [Leader]" -42:23.938 [INFO] TestAgent.server: cluster leadership acquired -42:23.939 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13533 -42:23.939 [INFO] TestAgent.server: New leader elected: payload=Node-a1fbe707-6f3a-9154-036d-1153a94a16a9 -42:23.940 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:23.943 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:23.943 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:23.943 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:23.943 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:23.943 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:23.943 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a1fbe707-6f3a-9154-036d-1153a94a16a9 -42:23.943 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a1fbe707-6f3a-9154-036d-1153a94a16a9 -42:23.945 [INFO] TestAgent.server: federation state anti-entropy synced -42:24.082 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:24.083 [INFO] TestAgent: Synced node info -42:24.131 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc2 method=Health.ServiceNodes -42:24.131 [ERROR] TestAgent.dns: rpc error: error="No path to datacenter" -42:24.131 [DEBUG] TestAgent.dns: request served from client: name=consul.service.dc2.consul. type=ANY class=IN latency=189.14µs client=127.0.0.1:53071 client_network=udp -42:24.132 [INFO] TestAgent: Requesting shutdown -42:24.132 [INFO] TestAgent.server: shutting down server -42:24.132 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:24.132 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:24.132 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:24.132 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:24.132 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:24.132 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:24.132 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:24.132 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:24.132 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:24.132 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:24.132 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:24.133Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:24.133Z [INFO] agent.router.manager: shutting down -42:24.133 [INFO] TestAgent: consul server down -42:24.133 [INFO] TestAgent: shutdown complete -42:24.133 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13521 network=tcp -42:24.133 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13521 network=udp -42:24.133 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13522 network=tcp -42:24.634 [INFO] TestAgent: Waiting for endpoints to shut down -42:24.634 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_NonExistentDC_Server (0.87s) -=== CONT TestDNS_ServiceLookup_TTL -2022-09-15T21:42:23.941Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:23.966Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:23.969 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6fed4cdf-9d8b-911f-d8e7-fdcb8fd81460 Address:127.0.0.1:13540}]" -42:23.972 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6fed4cdf-9d8b-911f-d8e7-fdcb8fd81460.dc1 127.0.0.1 -42:23.973 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6fed4cdf-9d8b-911f-d8e7-fdcb8fd81460 127.0.0.1 -2022-09-15T21:42:23.973Z [INFO] agent.router: Initializing LAN area manager -42:23.977 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13540 [Follower]" leader= -42:23.977 [INFO] TestAgent.server: Adding LAN server: server="Node-6fed4cdf-9d8b-911f-d8e7-fdcb8fd81460 (Addr: tcp/127.0.0.1:13540) (DC: dc1)" -42:23.977 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6fed4cdf-9d8b-911f-d8e7-fdcb8fd81460.dc1 area=wan -42:23.982 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13528 network=udp -42:24.002 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13528 network=tcp -42:24.003 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13529 network=tcp -42:24.003 [INFO] TestAgent: started state syncer -42:24.003 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13541 network=tcp -42:24.018 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:24.018 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13540 [Candidate]" term=2 -42:24.018 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:24.018 [DEBUG] TestAgent.server.raft: vote granted: from=6fed4cdf-9d8b-911f-d8e7-fdcb8fd81460 term=2 tally=1 -42:24.018 [INFO] TestAgent.server.raft: election won: tally=1 -42:24.018 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13540 [Leader]" -42:24.018 [INFO] TestAgent.server: cluster leadership acquired -42:24.019 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13540 -42:24.020 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:24.022 [INFO] TestAgent.server: New leader elected: payload=Node-6fed4cdf-9d8b-911f-d8e7-fdcb8fd81460 -42:24.022 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:24.023 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:24.023 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:24.023 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:24.023 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:24.023 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6fed4cdf-9d8b-911f-d8e7-fdcb8fd81460 -42:24.023 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6fed4cdf-9d8b-911f-d8e7-fdcb8fd81460 -42:24.024 [INFO] TestAgent.server: federation state anti-entropy synced -42:24.158 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:24.159 [INFO] TestAgent: Synced node info -42:24.170 [DEBUG] TestAgent.dns: request served from client: name=7f000001.addr.dc1.consul. type=SRV class=IN latency=131.98µs client=127.0.0.1:54905 client_network=udp -42:24.170 [INFO] TestAgent: Requesting shutdown -42:24.170 [INFO] TestAgent.server: shutting down server -42:24.170 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:24.170 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:24.170 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:24.170 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:24.170 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:24.170 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:24.171 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:24.171 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:24.171 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:24.171 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:24.171 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:24.171Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:24.171Z [INFO] agent.router.manager: shutting down -42:24.171 [INFO] TestAgent: consul server down -42:24.171 [INFO] TestAgent: shutdown complete -42:24.171 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13528 network=tcp -42:24.171 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13528 network=udp -42:24.171 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13529 network=tcp -42:24.674 [INFO] TestAgent: Waiting for endpoints to shut down -42:24.674 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_AddressLookup (0.75s) -=== CONT TestDNS_NodeLookup_TTL -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_PreparedQuery_TTL/db.query.consul. -=== RUN TestDNS_PreparedQuery_TTL/db-ttl.query.consul. -=== RUN TestDNS_PreparedQuery_TTL/dblb.query.consul. -=== RUN TestDNS_PreparedQuery_TTL/dblb-ttl.query.consul. -=== RUN TestDNS_PreparedQuery_TTL/dk.query.consul. -=== RUN TestDNS_PreparedQuery_TTL/dk-ttl.query.consul. -=== RUN TestDNS_PreparedQuery_TTL/api.query.consul. -=== RUN TestDNS_PreparedQuery_TTL/api-ttl.query.consul. -=== RUN TestDNS_ServiceLookup_TTL/db.service.consul. -=== RUN TestDNS_ServiceLookup_TTL/dblb.service.consul. -=== RUN TestDNS_ServiceLookup_TTL/dk.service.consul. -=== RUN TestDNS_ServiceLookup_TTL/api.service.consul. -2022-09-15T21:42:24.039Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:24.039Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:24.073Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:24.073Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:24.081 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2f3e93b1-e570-6176-226f-fb25d2b533da Address:127.0.0.1:13556}]" -42:24.082 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13556 [Follower]" leader= -42:24.086 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2f3e93b1-e570-6176-226f-fb25d2b533da.dc1 127.0.0.1 -42:24.116 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2f3e93b1-e570-6176-226f-fb25d2b533da 127.0.0.1 -2022-09-15T21:42:24.116Z [INFO] agent.router: Initializing LAN area manager -42:24.117 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13549 network=udp -42:24.117 [INFO] TestAgent.server: Adding LAN server: server="Node-2f3e93b1-e570-6176-226f-fb25d2b533da (Addr: tcp/127.0.0.1:13556) (DC: dc1)" -42:24.117 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2f3e93b1-e570-6176-226f-fb25d2b533da.dc1 area=wan -42:24.117 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13549 network=tcp -42:24.119 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13550 network=tcp -42:24.119 [INFO] TestAgent: started state syncer -42:24.119 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13568 network=tcp -42:24.150 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:24.150 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13556 [Candidate]" term=2 -42:24.154 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:24.154 [DEBUG] TestAgent.server.raft: vote granted: from=2f3e93b1-e570-6176-226f-fb25d2b533da term=2 tally=1 -42:24.154 [INFO] TestAgent.server.raft: election won: tally=1 -42:24.154 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13556 [Leader]" -42:24.155 [INFO] TestAgent.server: cluster leadership acquired -42:24.155 [INFO] TestAgent.server: initializing acls -42:24.156 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:24.156 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:24.157 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:24.157 [INFO] TestAgent.server: New leader elected: payload=Node-2f3e93b1-e570-6176-226f-fb25d2b533da -42:24.157 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:24.157 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:24.157 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:24.158 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-2f3e93b1-e570-6176-226f-fb25d2b533da -42:24.158 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-2f3e93b1-e570-6176-226f-fb25d2b533da.dc1 -42:24.159 [INFO] TestAgent.server: Updating LAN server: server="Node-2f3e93b1-e570-6176-226f-fb25d2b533da (Addr: tcp/127.0.0.1:13556) (DC: dc1)" -42:24.159 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-2f3e93b1-e570-6176-226f-fb25d2b533da.dc1 area=wan -42:24.160 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:24.168 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:24.168 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:24.168 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:24.168 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:24.168 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:24.168 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2f3e93b1-e570-6176-226f-fb25d2b533da -42:24.168 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2f3e93b1-e570-6176-226f-fb25d2b533da -42:24.195 [INFO] TestAgent.server: federation state anti-entropy synced -42:24.196 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2f3e93b1-e570-6176-226f-fb25d2b533da -42:24.227 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:24.227 [INFO] TestAgent: Synced node info -42:24.228 [DEBUG] TestAgent: Node info in sync -42:24.471 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-2f3e93b1-e570-6176-226f-fb25d2b533da -42:24.471 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-2f3e93b1-e570-6176-226f-fb25d2b533da -42:24.473 [DEBUG] TestAgent.dns: request served from client: name=foo.service.consul. type=A class=IN latency=550.24µs client=127.0.0.1:59021 client_network=udp -42:24.474 [INFO] TestAgent: Requesting shutdown -42:24.474 [INFO] TestAgent.server: shutting down server -42:24.474 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:24.474 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:24.474 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:24.474 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:24.474 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:24.474 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:24.474 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:24.474 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:24.474 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:24.474 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:24.474 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:24.474 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:24.474 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:24.474 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:24.474 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:24.474Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:24.474Z [INFO] agent.router.manager: shutting down -42:24.475 [INFO] TestAgent: consul server down -42:24.475 [INFO] TestAgent: shutdown complete -42:24.475 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13549 network=tcp -42:24.475 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13549 network=udp -42:24.475 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13550 network=tcp -42:24.982 [INFO] TestAgent: Waiting for endpoints to shut down -42:24.982 [INFO] TestAgent: Endpoints down -=== RUN TestDNS_ServiceLookup_FilterACL/ACLToken_==_anonymous -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:24.214Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:24.243Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:24.246 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:88ab7a45-814e-f52c-a2ba-e9c70ea2d40b Address:127.0.0.1:13574}]" -42:24.247 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-88ab7a45-814e-f52c-a2ba-e9c70ea2d40b.dc1 127.0.0.1 -42:24.248 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-88ab7a45-814e-f52c-a2ba-e9c70ea2d40b 127.0.0.1 -2022-09-15T21:42:24.248Z [INFO] agent.router: Initializing LAN area manager -42:24.249 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13569 network=udp -42:24.249 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13574 [Follower]" leader= -42:24.249 [INFO] TestAgent.server: Adding LAN server: server="Node-88ab7a45-814e-f52c-a2ba-e9c70ea2d40b (Addr: tcp/127.0.0.1:13574) (DC: dc1)" -42:24.249 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-88ab7a45-814e-f52c-a2ba-e9c70ea2d40b.dc1 area=wan -42:24.250 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13569 network=tcp -42:24.251 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13570 network=tcp -42:24.251 [INFO] TestAgent: started state syncer -42:24.251 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13561 network=tcp -42:24.290 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:24.290 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13574 [Candidate]" term=2 -42:24.290 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:24.290 [DEBUG] TestAgent.server.raft: vote granted: from=88ab7a45-814e-f52c-a2ba-e9c70ea2d40b term=2 tally=1 -42:24.290 [INFO] TestAgent.server.raft: election won: tally=1 -42:24.290 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13574 [Leader]" -42:24.290 [INFO] TestAgent.server: cluster leadership acquired -42:24.291 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13574 -42:24.292 [INFO] TestAgent.server: New leader elected: payload=Node-88ab7a45-814e-f52c-a2ba-e9c70ea2d40b -42:24.292 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:24.295 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:24.295 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:24.295 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:24.295 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:24.295 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:24.295 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-88ab7a45-814e-f52c-a2ba-e9c70ea2d40b -42:24.295 [INFO] TestAgent.server: member joined, marking health alive: member=Node-88ab7a45-814e-f52c-a2ba-e9c70ea2d40b -42:24.296 [INFO] TestAgent.server: federation state anti-entropy synced -42:24.367 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:24.368 [INFO] TestAgent: Synced node info -42:24.535 [DEBUG] TestAgent.dns: request served from client: name=_db._tcp.service.dc1.consul. type=SRV class=IN latency=304.74µs client=127.0.0.1:49641 client_network=udp -42:24.536 [DEBUG] TestAgent.dns: request served from client: name=_db._tcp.service.consul. type=SRV class=IN latency=210.62µs client=127.0.0.1:51846 client_network=udp -42:24.536 [DEBUG] TestAgent.dns: request served from client: name=_db._tcp.dc1.consul. type=SRV class=IN latency=203.5µs client=127.0.0.1:37160 client_network=udp -42:24.537 [DEBUG] TestAgent.dns: request served from client: name=_db._tcp.consul. type=SRV class=IN latency=207.06µs client=127.0.0.1:38218 client_network=udp -42:24.537 [INFO] TestAgent: Requesting shutdown -42:24.537 [INFO] TestAgent.server: shutting down server -42:24.537 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:24.537 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:24.537 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:24.537 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:24.537 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:24.537 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:24.537 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:24.537 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:24.537 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:24.537 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:24.537 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:24.538Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:24.538Z [INFO] agent.router.manager: shutting down -42:24.538 [INFO] TestAgent: consul server down -42:24.538 [INFO] TestAgent: shutdown complete -42:24.538 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13569 network=tcp -42:24.538 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13569 network=udp -42:24.538 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13570 network=tcp -42:25.042 [INFO] TestAgent: Waiting for endpoints to shut down -42:25.042 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_SRV_RFC_TCP_Default (0.93s) -=== CONT TestDNS_ServiceLookup_ServiceAddress_CNAME -42:24.053 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: test-client 127.0.0.1 -2022-09-15T21:42:24.053Z [INFO] agent.router: Initializing LAN area manager -42:24.058 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13542 network=udp -42:24.058 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13542 network=tcp -42:24.059 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13543 network=tcp -42:24.059 [INFO] TestAgent: started state syncer -42:24.059 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:13517] -42:24.060 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13548 network=tcp -2022-09-15T21:42:24.060Z [WARN] agent.router.manager: No servers available -42:24.060 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" -42:24.062 [DEBUG] TestAgent.client.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:13517 -42:24.065 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: test-server 127.0.0.1 -42:24.065 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 -42:24.065 [DEBUG] TestAgent: systemd notify failed: error="No socket" -2022-09-15T21:42:24.065Z [WARN] agent.router.manager: No servers available -42:24.065 [INFO] TestAgent.client: adding server: server="test-server (Addr: tcp/127.0.0.1:13519) (DC: dc1)" -42:24.121 [ERROR] TestAgent.client: RPC failed to server: method=Health.ServiceNodes server=127.0.0.1:13519 error="rpc error making call: No path to datacenter" -42:24.121 [ERROR] TestAgent.dns: rpc error: error="rpc error making call: No path to datacenter" -42:24.121 [DEBUG] TestAgent.dns: request served from client: name=consul.service.dc2.consul. type=ANY class=IN latency=764.02µs client=127.0.0.1:49714 client_network=udp -42:24.121 [INFO] TestAgent: Requesting shutdown -42:24.121 [INFO] TestAgent.client: shutting down client -42:24.121 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave -42:24.121 [INFO] TestAgent: consul client down -42:24.121 [INFO] TestAgent: shutdown complete -42:24.121 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13542 network=tcp -42:24.130 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13542 network=udp -42:24.130 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13543 network=tcp -42:24.638 [INFO] TestAgent: Waiting for endpoints to shut down -42:24.638 [INFO] TestAgent: Endpoints down -2022-09-15T21:42:23.699Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:23.772Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:23.776 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3d9b6ba6-7d8a-0733-480f-455e0cdbc2f8 Address:127.0.0.1:13519}]" -42:23.778 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13519 [Follower]" leader= -42:23.782 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-server.dc1 127.0.0.1 -42:23.783 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-server 127.0.0.1 -2022-09-15T21:42:23.783Z [INFO] agent.router: Initializing LAN area manager -42:23.784 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13514 network=udp -42:23.784 [INFO] TestAgent.server: Adding LAN server: server="test-server (Addr: tcp/127.0.0.1:13519) (DC: dc1)" -42:23.784 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-server.dc1 area=wan -42:23.785 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13514 network=tcp -42:23.795 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13515 network=tcp -42:23.795 [INFO] TestAgent: started state syncer -42:23.795 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13520 network=tcp -42:23.850 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:23.850 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13519 [Candidate]" term=2 -42:23.850 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:23.850 [DEBUG] TestAgent.server.raft: vote granted: from=3d9b6ba6-7d8a-0733-480f-455e0cdbc2f8 term=2 tally=1 -42:23.850 [INFO] TestAgent.server.raft: election won: tally=1 -42:23.850 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13519 [Leader]" -42:23.850 [INFO] TestAgent.server: cluster leadership acquired -42:23.851 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13519 -42:23.852 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:23.854 [INFO] TestAgent.server: New leader elected: payload=test-server -42:23.855 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:23.855 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:23.855 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:23.855 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:23.855 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:23.855 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-server -42:23.855 [INFO] TestAgent.server: member joined, marking health alive: member=test-server -42:23.857 [INFO] TestAgent.server: federation state anti-entropy synced -42:24.062 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:56232 -42:24.063 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-client 127.0.0.1 -42:24.063 [INFO] TestAgent.server: member joined, marking health alive: member=test-client -42:24.120 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc2 method=Health.ServiceNodes -42:24.166 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:24.167 [INFO] TestAgent: Synced node info -42:24.638 [INFO] TestAgent: Requesting shutdown -42:24.638 [INFO] TestAgent.server: shutting down server -42:24.638 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:24.638 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:24.638 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:24.638 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:24.638 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:24.638 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:24.638 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:24.638 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:24.638 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:24.638 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:24.638 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:24.638Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:24.639Z [INFO] agent.router.manager: shutting down -42:24.639 [INFO] TestAgent: consul server down -42:24.639 [INFO] TestAgent: shutdown complete -42:24.639 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13514 network=tcp -42:24.639 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13514 network=udp -42:24.639 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13515 network=tcp -42:25.139 [INFO] TestAgent: Waiting for endpoints to shut down -42:25.139 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_NonExistentDC_RPC (1.48s) -=== CONT TestDNS_ServiceLookup_AnswerLimits -=== RUN TestDNS_ServiceLookup_AnswerLimits/A_lookup_{0_0_0_0_0_0_0_0_0_0_0} -=== PAUSE TestDNS_ServiceLookup_AnswerLimits/A_lookup_{0_0_0_0_0_0_0_0_0_0_0} -=== RUN TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{0_0_0_0_0_0_0_0_0_0_0} -=== PAUSE TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{0_0_0_0_0_0_0_0_0_0_0} -=== RUN TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{0_0_0_0_0_0_0_0_0_0_0} -=== PAUSE TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{0_0_0_0_0_0_0_0_0_0_0} -=== RUN TestDNS_ServiceLookup_AnswerLimits/A_lookup_{1_1_1_1_1_1_1_1_1_1_1} -=== PAUSE TestDNS_ServiceLookup_AnswerLimits/A_lookup_{1_1_1_1_1_1_1_1_1_1_1} -=== RUN TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{1_1_1_1_1_1_1_1_1_1_1} -=== PAUSE TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{1_1_1_1_1_1_1_1_1_1_1} -=== RUN TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{1_1_1_1_1_1_1_1_1_1_1} -=== PAUSE TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{1_1_1_1_1_1_1_1_1_1_1} -=== RUN TestDNS_ServiceLookup_AnswerLimits/A_lookup_{2_2_2_2_2_2_2_2_2_2_2} -=== PAUSE TestDNS_ServiceLookup_AnswerLimits/A_lookup_{2_2_2_2_2_2_2_2_2_2_2} -=== RUN TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{2_2_2_2_2_2_2_2_2_2_2} -=== PAUSE TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{2_2_2_2_2_2_2_2_2_2_2} -=== RUN TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{2_2_2_2_2_2_2_2_2_2_2} -=== PAUSE TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{2_2_2_2_2_2_2_2_2_2_2} -=== RUN TestDNS_ServiceLookup_AnswerLimits/A_lookup_{3_3_3_3_3_3_3_3_3_3_3} -=== PAUSE TestDNS_ServiceLookup_AnswerLimits/A_lookup_{3_3_3_3_3_3_3_3_3_3_3} -=== RUN TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{3_3_3_3_3_3_3_3_3_3_3} -=== PAUSE TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{3_3_3_3_3_3_3_3_3_3_3} -=== RUN TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{3_3_3_3_3_3_3_3_3_3_3} -=== PAUSE TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{3_3_3_3_3_3_3_3_3_3_3} -=== RUN TestDNS_ServiceLookup_AnswerLimits/A_lookup_{4_4_4_4_4_4_4_4_4_4_4} -=== PAUSE TestDNS_ServiceLookup_AnswerLimits/A_lookup_{4_4_4_4_4_4_4_4_4_4_4} -=== RUN TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{4_4_4_4_4_4_4_4_4_4_4} -=== PAUSE TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{4_4_4_4_4_4_4_4_4_4_4} -=== RUN TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{4_4_4_4_4_4_4_4_4_4_4} -=== PAUSE TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{4_4_4_4_4_4_4_4_4_4_4} -=== RUN TestDNS_ServiceLookup_AnswerLimits/A_lookup_{5_5_5_5_5_5_5_5_5_5_5} -=== PAUSE TestDNS_ServiceLookup_AnswerLimits/A_lookup_{5_5_5_5_5_5_5_5_5_5_5} -=== RUN TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{5_5_5_5_5_5_5_5_5_5_5} -=== PAUSE TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{5_5_5_5_5_5_5_5_5_5_5} -=== RUN TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{5_5_5_5_5_5_5_5_5_5_5} -=== PAUSE TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{5_5_5_5_5_5_5_5_5_5_5} -=== RUN TestDNS_ServiceLookup_AnswerLimits/A_lookup_{6_6_6_6_6_6_6_5_6_6_-5} -=== PAUSE TestDNS_ServiceLookup_AnswerLimits/A_lookup_{6_6_6_6_6_6_6_5_6_6_-5} -=== RUN TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{6_6_6_6_6_6_6_5_6_6_-5} -=== PAUSE TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{6_6_6_6_6_6_6_5_6_6_-5} -=== RUN TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{6_6_6_6_6_6_6_5_6_6_-5} -=== PAUSE TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{6_6_6_6_6_6_6_5_6_6_-5} -=== RUN TestDNS_ServiceLookup_AnswerLimits/A_lookup_{7_7_7_7_6_7_7_5_7_7_-5} -=== PAUSE TestDNS_ServiceLookup_AnswerLimits/A_lookup_{7_7_7_7_6_7_7_5_7_7_-5} -=== RUN TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{7_7_7_7_6_7_7_5_7_7_-5} -=== PAUSE TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{7_7_7_7_6_7_7_5_7_7_-5} -=== RUN TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{7_7_7_7_6_7_7_5_7_7_-5} -=== PAUSE TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{7_7_7_7_6_7_7_5_7_7_-5} -=== RUN TestDNS_ServiceLookup_AnswerLimits/A_lookup_{8_8_8_8_6_8_8_5_8_8_-5} -=== PAUSE TestDNS_ServiceLookup_AnswerLimits/A_lookup_{8_8_8_8_6_8_8_5_8_8_-5} -=== RUN TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{8_8_8_8_6_8_8_5_8_8_-5} -=== PAUSE TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{8_8_8_8_6_8_8_5_8_8_-5} -=== RUN TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{8_8_8_8_6_8_8_5_8_8_-5} -=== PAUSE TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{8_8_8_8_6_8_8_5_8_8_-5} -=== RUN TestDNS_ServiceLookup_AnswerLimits/A_lookup_{9_9_8_8_6_8_8_5_8_8_-5} -=== PAUSE TestDNS_ServiceLookup_AnswerLimits/A_lookup_{9_9_8_8_6_8_8_5_8_8_-5} -=== RUN TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{9_9_8_8_6_8_8_5_8_8_-5} -=== PAUSE TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{9_9_8_8_6_8_8_5_8_8_-5} -=== RUN TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{9_9_8_8_6_8_8_5_8_8_-5} -=== PAUSE TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{9_9_8_8_6_8_8_5_8_8_-5} -=== RUN TestDNS_ServiceLookup_AnswerLimits/A_lookup_{20_20_8_8_6_8_8_5_8_-5_-5} -=== PAUSE TestDNS_ServiceLookup_AnswerLimits/A_lookup_{20_20_8_8_6_8_8_5_8_-5_-5} -=== RUN TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{20_20_8_8_6_8_8_5_8_-5_-5} -=== PAUSE TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{20_20_8_8_6_8_8_5_8_-5_-5} -=== RUN TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{20_20_8_8_6_8_8_5_8_-5_-5} -=== PAUSE TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{20_20_8_8_6_8_8_5_8_-5_-5} -=== RUN TestDNS_ServiceLookup_AnswerLimits/A_lookup_{30_30_8_8_6_8_8_5_8_-5_-5} -=== PAUSE TestDNS_ServiceLookup_AnswerLimits/A_lookup_{30_30_8_8_6_8_8_5_8_-5_-5} -=== RUN TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{30_30_8_8_6_8_8_5_8_-5_-5} -=== PAUSE TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{30_30_8_8_6_8_8_5_8_-5_-5} -=== RUN TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{30_30_8_8_6_8_8_5_8_-5_-5} -=== PAUSE TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{30_30_8_8_6_8_8_5_8_-5_-5} -=== CONT TestDNS_ServiceLookup_LargeResponses -[WARN] freeport: 4 out of 25 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:24.504Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:24.525Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:24.528 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:89ae07dd-7e1f-a149-73b6-3dce49da5d32 Address:127.0.0.1:13581}]" -42:24.529 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-89ae07dd-7e1f-a149-73b6-3dce49da5d32.dc1 127.0.0.1 -42:24.530 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13581 [Follower]" leader= -42:24.539 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-89ae07dd-7e1f-a149-73b6-3dce49da5d32 127.0.0.1 -2022-09-15T21:42:24.539Z [INFO] agent.router: Initializing LAN area manager -42:24.539 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13576 network=udp -42:24.540 [INFO] TestAgent.server: Adding LAN server: server="Node-89ae07dd-7e1f-a149-73b6-3dce49da5d32 (Addr: tcp/127.0.0.1:13581) (DC: dc1)" -42:24.540 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-89ae07dd-7e1f-a149-73b6-3dce49da5d32.dc1 area=wan -42:24.540 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13576 network=tcp -42:24.541 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13577 network=tcp -42:24.541 [INFO] TestAgent: started state syncer -42:24.541 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13589 network=tcp -42:24.590 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:24.590 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13581 [Candidate]" term=2 -42:24.590 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:24.590 [DEBUG] TestAgent.server.raft: vote granted: from=89ae07dd-7e1f-a149-73b6-3dce49da5d32 term=2 tally=1 -42:24.590 [INFO] TestAgent.server.raft: election won: tally=1 -42:24.590 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13581 [Leader]" -42:24.590 [INFO] TestAgent.server: cluster leadership acquired -42:24.591 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13581 -42:24.592 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:24.594 [INFO] TestAgent.server: New leader elected: payload=Node-89ae07dd-7e1f-a149-73b6-3dce49da5d32 -42:24.595 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:24.595 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:24.595 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:24.595 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:24.595 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:24.595 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-89ae07dd-7e1f-a149-73b6-3dce49da5d32 -42:24.595 [INFO] TestAgent.server: member joined, marking health alive: member=Node-89ae07dd-7e1f-a149-73b6-3dce49da5d32 -42:24.597 [INFO] TestAgent.server: federation state anti-entropy synced -42:24.799 [DEBUG] TestAgent.dns: request served from client: name=db.query.consul. type=SRV class=IN latency=256.28µs client=127.0.0.1:49879 client_network=udp -42:24.800 [DEBUG] TestAgent.dns: request served from client: name=db-ttl.query.consul. type=SRV class=IN latency=170.56µs client=127.0.0.1:57154 client_network=udp -42:24.800 [DEBUG] TestAgent.dns: request served from client: name=dblb.query.consul. type=SRV class=IN latency=165.96µs client=127.0.0.1:52174 client_network=udp -42:24.801 [DEBUG] TestAgent.dns: request served from client: name=dblb-ttl.query.consul. type=SRV class=IN latency=163.34µs client=127.0.0.1:49968 client_network=udp -42:24.801 [DEBUG] TestAgent.dns: request served from client: name=dk.query.consul. type=SRV class=IN latency=162.2µs client=127.0.0.1:40866 client_network=udp -42:24.803 [DEBUG] TestAgent.dns: request served from client: name=dk-ttl.query.consul. type=SRV class=IN latency=210.6µs client=127.0.0.1:37285 client_network=udp -42:24.804 [DEBUG] TestAgent.dns: request served from client: name=api.query.consul. type=SRV class=IN latency=164.66µs client=127.0.0.1:55849 client_network=udp -42:24.804 [DEBUG] TestAgent.dns: request served from client: name=api-ttl.query.consul. type=SRV class=IN latency=152.36µs client=127.0.0.1:45330 client_network=udp -42:24.804 [INFO] TestAgent: Requesting shutdown -42:24.804 [INFO] TestAgent.server: shutting down server -42:24.804 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:24.804 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:24.804 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:24.804 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:24.804 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:24.805 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:24.805 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:24.805 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:24.805 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:24.805 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:24.805 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:24.805 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:24.805Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:24.805Z [INFO] agent.router.manager: shutting down -42:24.805 [INFO] TestAgent: consul server down -42:24.805 [INFO] TestAgent: shutdown complete -42:24.805 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13576 network=tcp -42:24.805 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13576 network=udp -42:24.805 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13577 network=tcp -42:25.305 [INFO] TestAgent: Waiting for endpoints to shut down -42:25.305 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_PreparedQuery_TTL (0.82s) - --- PASS: TestDNS_PreparedQuery_TTL/db.query.consul. (0.00s) - --- PASS: TestDNS_PreparedQuery_TTL/db-ttl.query.consul. (0.00s) - --- PASS: TestDNS_PreparedQuery_TTL/dblb.query.consul. (0.00s) - --- PASS: TestDNS_PreparedQuery_TTL/dblb-ttl.query.consul. (0.00s) - --- PASS: TestDNS_PreparedQuery_TTL/dk.query.consul. (0.00s) - --- PASS: TestDNS_PreparedQuery_TTL/dk-ttl.query.consul. (0.00s) - --- PASS: TestDNS_PreparedQuery_TTL/api.query.consul. (0.00s) - --- PASS: TestDNS_PreparedQuery_TTL/api-ttl.query.consul. (0.00s) -=== CONT TestDNS_ServiceLookup_Truncate -2022-09-15T21:42:24.358Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:24.397Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:24.405 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3c55832e-f9a2-8107-b410-c5755648082e Address:127.0.0.1:13567}]" -42:24.406 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13567 [Follower]" leader= -42:24.414 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3c55832e-f9a2-8107-b410-c5755648082e.dc1 127.0.0.1 -42:24.415 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3c55832e-f9a2-8107-b410-c5755648082e 127.0.0.1 -2022-09-15T21:42:24.415Z [INFO] agent.router: Initializing LAN area manager -42:24.416 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13562 network=udp -42:24.416 [INFO] TestAgent.server: Adding LAN server: server="Node-3c55832e-f9a2-8107-b410-c5755648082e (Addr: tcp/127.0.0.1:13567) (DC: dc1)" -42:24.416 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3c55832e-f9a2-8107-b410-c5755648082e.dc1 area=wan -42:24.417 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13562 network=tcp -42:24.417 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13563 network=tcp -42:24.422 [INFO] TestAgent: started state syncer -42:24.422 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13582 network=tcp -42:24.475 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:24.475 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13567 [Candidate]" term=2 -42:24.475 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:24.475 [DEBUG] TestAgent.server.raft: vote granted: from=3c55832e-f9a2-8107-b410-c5755648082e term=2 tally=1 -42:24.475 [INFO] TestAgent.server.raft: election won: tally=1 -42:24.475 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13567 [Leader]" -42:24.475 [INFO] TestAgent.server: cluster leadership acquired -42:24.476 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13567 -42:24.477 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:24.482 [INFO] TestAgent.server: New leader elected: payload=Node-3c55832e-f9a2-8107-b410-c5755648082e -42:24.498 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:24.498 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:24.498 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:24.498 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:24.498 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:24.498 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3c55832e-f9a2-8107-b410-c5755648082e -42:24.498 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3c55832e-f9a2-8107-b410-c5755648082e -42:24.500 [INFO] TestAgent.server: federation state anti-entropy synced -42:24.506 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:24.507 [INFO] TestAgent: Synced node info -42:24.845 [DEBUG] TestAgent.dns: request served from client: name=_db._primary.service.dc1.consul. type=SRV class=IN latency=255.18µs client=127.0.0.1:44239 client_network=udp -42:24.846 [DEBUG] TestAgent.dns: request served from client: name=_db._primary.service.consul. type=SRV class=IN latency=202.68µs client=127.0.0.1:32973 client_network=udp -42:24.850 [DEBUG] TestAgent.dns: request served from client: name=_db._primary.dc1.consul. type=SRV class=IN latency=334.84µs client=127.0.0.1:45472 client_network=udp -42:24.854 [DEBUG] TestAgent.dns: request served from client: name=_db._primary.consul. type=SRV class=IN latency=264.52µs client=127.0.0.1:44092 client_network=udp -42:24.858 [INFO] TestAgent: Requesting shutdown -42:24.858 [INFO] TestAgent.server: shutting down server -42:24.858 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:24.858 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:24.858 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:24.858 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:24.858 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:24.858 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:24.858 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:24.858 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:24.858 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:24.858 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:24.858 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:24.858Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:24.859Z [INFO] agent.router.manager: shutting down -42:24.859 [INFO] TestAgent: consul server down -42:24.859 [INFO] TestAgent: shutdown complete -42:24.859 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13562 network=tcp -42:24.859 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13562 network=udp -42:24.859 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13563 network=tcp -42:25.359 [INFO] TestAgent: Waiting for endpoints to shut down -42:25.359 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_SRV_RFC (1.03s) -=== CONT TestDNS_TCP_and_UDP_Truncate -2022-09-15T21:42:24.673Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:24.774Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:24.785 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:312c89ba-d7d5-f827-29b4-f10fabe73a2f Address:127.0.0.1:13595}]" -42:24.786 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13595 [Follower]" leader= -42:24.786 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-312c89ba-d7d5-f827-29b4-f10fabe73a2f.dc1 127.0.0.1 -42:24.787 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-312c89ba-d7d5-f827-29b4-f10fabe73a2f 127.0.0.1 -2022-09-15T21:42:24.787Z [INFO] agent.router: Initializing LAN area manager -42:24.787 [INFO] TestAgent.server: Adding LAN server: server="Node-312c89ba-d7d5-f827-29b4-f10fabe73a2f (Addr: tcp/127.0.0.1:13595) (DC: dc1)" -42:24.787 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-312c89ba-d7d5-f827-29b4-f10fabe73a2f.dc1 area=wan -42:24.788 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13590 network=tcp -42:24.788 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13590 network=udp -42:24.789 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13591 network=tcp -42:24.789 [INFO] TestAgent: started state syncer -42:24.789 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13596 network=tcp -42:24.821 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:24.821 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13595 [Candidate]" term=2 -42:24.822 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:24.822 [DEBUG] TestAgent.server.raft: vote granted: from=312c89ba-d7d5-f827-29b4-f10fabe73a2f term=2 tally=1 -42:24.822 [INFO] TestAgent.server.raft: election won: tally=1 -42:24.822 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13595 [Leader]" -42:24.822 [INFO] TestAgent.server: cluster leadership acquired -42:24.822 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13595 -42:24.823 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:24.826 [INFO] TestAgent.server: New leader elected: payload=Node-312c89ba-d7d5-f827-29b4-f10fabe73a2f -42:24.826 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:24.826 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:24.826 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:24.826 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:24.826 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:24.827 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-312c89ba-d7d5-f827-29b4-f10fabe73a2f -42:24.827 [INFO] TestAgent.server: member joined, marking health alive: member=Node-312c89ba-d7d5-f827-29b4-f10fabe73a2f -42:24.827 [INFO] TestAgent.server: federation state anti-entropy synced -42:24.851 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:24.852 [INFO] TestAgent: Synced node info -42:24.852 [DEBUG] TestAgent: Node info in sync -42:24.887 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=342.08µs client=127.0.0.1:36466 client_network=udp -42:24.888 [DEBUG] TestAgent.dns: request served from client: name=dblb.service.consul. type=SRV class=IN latency=214.5µs client=127.0.0.1:33596 client_network=udp -42:24.888 [DEBUG] TestAgent.dns: request served from client: name=dk.service.consul. type=SRV class=IN latency=211.92µs client=127.0.0.1:58850 client_network=udp -42:24.889 [DEBUG] TestAgent.dns: request served from client: name=api.service.consul. type=SRV class=IN latency=204.8µs client=127.0.0.1:57103 client_network=udp -42:24.889 [INFO] TestAgent: Requesting shutdown -42:24.889 [INFO] TestAgent.server: shutting down server -42:24.889 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:24.889 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:24.889 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:24.889 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:24.889 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:24.889 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:24.889 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:24.889 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:24.889 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:24.889 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:24.889 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:24.889Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:24.889Z [INFO] agent.router.manager: shutting down -42:24.890 [INFO] TestAgent: consul server down -42:24.890 [INFO] TestAgent: shutdown complete -42:24.890 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13590 network=tcp -42:24.890 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13590 network=udp -42:24.890 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13591 network=tcp -42:25.390 [INFO] TestAgent: Waiting for endpoints to shut down -42:25.390 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_TTL (0.76s) - --- PASS: TestDNS_ServiceLookup_TTL/db.service.consul. (0.00s) - --- PASS: TestDNS_ServiceLookup_TTL/dblb.service.consul. (0.00s) - --- PASS: TestDNS_ServiceLookup_TTL/dk.service.consul. (0.00s) - --- PASS: TestDNS_ServiceLookup_TTL/api.service.consul. (0.00s) -=== CONT TestBinarySearch -=== RUN TestBinarySearch/binarySearch_12 -=== RUN TestBinarySearch/binarySearch_256 -=== RUN TestBinarySearch/binarySearch_512 -=== RUN TestBinarySearch/binarySearch_8192 -=== RUN TestBinarySearch/binarySearch_65535 -=== RUN TestBinarySearch/binarySearch_12#01 -=== RUN TestBinarySearch/binarySearch_256#01 -[WARN] freeport: 4 out of 25 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestBinarySearch/binarySearch_512#01 -=== RUN TestBinarySearch/binarySearch_8192#01 -=== RUN TestBinarySearch/binarySearch_65535#01 ---- PASS: TestBinarySearch (0.20s) - --- PASS: TestBinarySearch/binarySearch_12 (0.01s) - --- PASS: TestBinarySearch/binarySearch_256 (0.02s) - --- PASS: TestBinarySearch/binarySearch_512 (0.01s) - --- PASS: TestBinarySearch/binarySearch_8192 (0.01s) - --- PASS: TestBinarySearch/binarySearch_65535 (0.03s) - --- PASS: TestBinarySearch/binarySearch_12#01 (0.03s) - --- PASS: TestBinarySearch/binarySearch_256#01 (0.02s) - --- PASS: TestBinarySearch/binarySearch_512#01 (0.01s) - --- PASS: TestBinarySearch/binarySearch_8192#01 (0.01s) - --- PASS: TestBinarySearch/binarySearch_65535#01 (0.03s) -=== CONT TestDNS_ServiceLookup_Randomize -2022-09-15T21:42:24.755Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:24.792Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:24.799 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a6eb2908-d28e-7528-97ca-e7105579c6f6 Address:127.0.0.1:13602}]" -42:24.800 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a6eb2908-d28e-7528-97ca-e7105579c6f6.dc1 127.0.0.1 -42:24.801 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a6eb2908-d28e-7528-97ca-e7105579c6f6 127.0.0.1 -2022-09-15T21:42:24.801Z [INFO] agent.router: Initializing LAN area manager -42:24.802 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13602 [Follower]" leader= -42:24.802 [INFO] TestAgent.server: Adding LAN server: server="Node-a6eb2908-d28e-7528-97ca-e7105579c6f6 (Addr: tcp/127.0.0.1:13602) (DC: dc1)" -42:24.802 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a6eb2908-d28e-7528-97ca-e7105579c6f6.dc1 area=wan -42:24.803 [DEBUG] TestAgent.dns: recursor enabled -42:24.803 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13597 network=udp -42:24.803 [DEBUG] TestAgent.dns: recursor enabled -42:24.803 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13597 network=tcp -42:24.804 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13598 network=tcp -42:24.804 [INFO] TestAgent: started state syncer -42:24.804 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13603 network=tcp -42:24.859 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:24.859 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13602 [Candidate]" term=2 -42:24.859 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:24.860 [DEBUG] TestAgent.server.raft: vote granted: from=a6eb2908-d28e-7528-97ca-e7105579c6f6 term=2 tally=1 -42:24.860 [INFO] TestAgent.server.raft: election won: tally=1 -42:24.860 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13602 [Leader]" -42:24.860 [INFO] TestAgent.server: cluster leadership acquired -42:24.860 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13602 -42:24.861 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:24.862 [INFO] TestAgent.server: New leader elected: payload=Node-a6eb2908-d28e-7528-97ca-e7105579c6f6 -42:24.865 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:24.865 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:24.865 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:24.865 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:24.865 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:24.865 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a6eb2908-d28e-7528-97ca-e7105579c6f6 -42:24.865 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a6eb2908-d28e-7528-97ca-e7105579c6f6 -42:24.866 [INFO] TestAgent.server: federation state anti-entropy synced -42:24.893 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:24.893 [INFO] TestAgent: Synced node info -42:24.893 [DEBUG] TestAgent: Node info in sync -42:25.147 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=ANY class=IN latency=169.88µs client=127.0.0.1:43540 client_network=udp -42:25.148 [DEBUG] TestAgent.dns: request served from client: name=bar.node.consul. type=ANY class=IN latency=111.14µs client=127.0.0.1:59327 client_network=udp -42:25.148 [DEBUG] TestAgent.dns: cname recurse RTT for name: name=www.google.com. rtt=173.64µs -42:25.148 [DEBUG] TestAgent.dns: request served from client: name=google.node.consul. type=ANY class=IN latency=416.6µs client=127.0.0.1:36375 client_network=udp -42:25.149 [INFO] TestAgent: Requesting shutdown -42:25.149 [INFO] TestAgent.server: shutting down server -42:25.149 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:25.149 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:25.149 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:25.149 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:25.149 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:25.149 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:25.149 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:25.149 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:25.149 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:25.149 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:25.149 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:25.149Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:25.149Z [INFO] agent.router.manager: shutting down -42:25.149 [INFO] TestAgent: consul server down -42:25.149 [INFO] TestAgent: shutdown complete -42:25.149 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13597 network=tcp -42:25.149 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13597 network=udp -42:25.150 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13598 network=tcp -42:25.650 [INFO] TestAgent: Waiting for endpoints to shut down -42:25.650 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_NodeLookup_TTL (0.98s) -=== CONT TestDNS_ServiceLookup_OnlyPassing -2022-09-15T21:42:25.020Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:25.020Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:25.051Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:25.051Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:25.055 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a361cef0-b4d2-9b23-c4b9-70ad59cb9e7f Address:127.0.0.1:13616}]" -42:25.056 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a361cef0-b4d2-9b23-c4b9-70ad59cb9e7f.dc1 127.0.0.1 -42:25.056 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a361cef0-b4d2-9b23-c4b9-70ad59cb9e7f 127.0.0.1 -2022-09-15T21:42:25.056Z [INFO] agent.router: Initializing LAN area manager -42:25.057 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13611 network=udp -42:25.057 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13616 [Follower]" leader= -42:25.057 [INFO] TestAgent.server: Adding LAN server: server="Node-a361cef0-b4d2-9b23-c4b9-70ad59cb9e7f (Addr: tcp/127.0.0.1:13616) (DC: dc1)" -42:25.057 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a361cef0-b4d2-9b23-c4b9-70ad59cb9e7f.dc1 area=wan -42:25.066 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13611 network=tcp -42:25.067 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13612 network=tcp -42:25.067 [INFO] TestAgent: started state syncer -42:25.067 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13624 network=tcp -42:25.098 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:25.098 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13616 [Candidate]" term=2 -42:25.098 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:25.098 [DEBUG] TestAgent.server.raft: vote granted: from=a361cef0-b4d2-9b23-c4b9-70ad59cb9e7f term=2 tally=1 -42:25.098 [INFO] TestAgent.server.raft: election won: tally=1 -42:25.098 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13616 [Leader]" -42:25.098 [INFO] TestAgent.server: cluster leadership acquired -42:25.099 [INFO] TestAgent.server: initializing acls -42:25.099 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:25.099 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:25.100 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:25.100 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:25.100 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:25.100 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:25.101 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-a361cef0-b4d2-9b23-c4b9-70ad59cb9e7f -42:25.101 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-a361cef0-b4d2-9b23-c4b9-70ad59cb9e7f.dc1 -42:25.102 [INFO] TestAgent.server: New leader elected: payload=Node-a361cef0-b4d2-9b23-c4b9-70ad59cb9e7f -42:25.102 [INFO] TestAgent.server: Updating LAN server: server="Node-a361cef0-b4d2-9b23-c4b9-70ad59cb9e7f (Addr: tcp/127.0.0.1:13616) (DC: dc1)" -42:25.102 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-a361cef0-b4d2-9b23-c4b9-70ad59cb9e7f.dc1 area=wan -42:25.103 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:25.105 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:25.105 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:25.105 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:25.105 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:25.105 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:25.105 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a361cef0-b4d2-9b23-c4b9-70ad59cb9e7f -42:25.105 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a361cef0-b4d2-9b23-c4b9-70ad59cb9e7f -42:25.112 [INFO] TestAgent.server: federation state anti-entropy synced -42:25.112 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a361cef0-b4d2-9b23-c4b9-70ad59cb9e7f -42:25.160 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-a361cef0-b4d2-9b23-c4b9-70ad59cb9e7f -42:25.160 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-a361cef0-b4d2-9b23-c4b9-70ad59cb9e7f -42:25.165 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=foo -42:25.165 [DEBUG] TestAgent.dns: request served from client: name=foo.service.consul. type=A class=IN latency=450.08µs client=127.0.0.1:45717 client_network=udp -42:25.165 [INFO] TestAgent: Requesting shutdown -42:25.170 [INFO] TestAgent.server: shutting down server -42:25.170 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:25.170 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:25.170 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:25.170 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:25.170 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:25.170 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:25.170 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:25.171 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:25.171 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:25.171 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:25.171 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:25.171 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:25.171 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:25.171 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:25.171 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:25.171 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:25.171Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:25.171Z [INFO] agent.router.manager: shutting down -42:25.172 [INFO] TestAgent: consul server down -42:25.172 [INFO] TestAgent: shutdown complete -42:25.172 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13611 network=tcp -42:25.172 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13611 network=udp -42:25.172 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13612 network=tcp -42:25.673 [INFO] TestAgent: Waiting for endpoints to shut down -42:25.673 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_FilterACL (1.68s) - --- PASS: TestDNS_ServiceLookup_FilterACL/ACLToken_==_root (0.99s) - --- PASS: TestDNS_ServiceLookup_FilterACL/ACLToken_==_anonymous (0.69s) -=== CONT TestDNS_ServiceLookup_OnlyFailing -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=255)_compressed=true -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=255)_compressed=false -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=255)_compressed=true -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=255)_compressed=false -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._tcp_(qType:=255)_compressed=true -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._tcp_(qType:=255)_compressed=false -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._udp_(qType:=255)_compressed=true -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._udp_(qType:=255)_compressed=false -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=1)_compressed=true -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=1)_compressed=false -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=1)_compressed=true -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=1)_compressed=false -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._tcp_(qType:=1)_compressed=true -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._tcp_(qType:=1)_compressed=false -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._udp_(qType:=1)_compressed=true -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._udp_(qType:=1)_compressed=false -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=33)_compressed=true -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=33)_compressed=false -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=33)_compressed=true -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=33)_compressed=false -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._tcp_(qType:=33)_compressed=true -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._tcp_(qType:=33)_compressed=false -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._udp_(qType:=33)_compressed=true -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._udp_(qType:=33)_compressed=false -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=255)_compressed=true#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=255)_compressed=false#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=255)_compressed=true#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=255)_compressed=false#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._tcp_(qType:=255)_compressed=true#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._tcp_(qType:=255)_compressed=false#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._udp_(qType:=255)_compressed=true#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._udp_(qType:=255)_compressed=false#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=1)_compressed=true#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=1)_compressed=false#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=1)_compressed=true#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=1)_compressed=false#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._tcp_(qType:=1)_compressed=true#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._tcp_(qType:=1)_compressed=false#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._udp_(qType:=1)_compressed=true#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._udp_(qType:=1)_compressed=false#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=33)_compressed=true#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=33)_compressed=false#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=33)_compressed=true#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=33)_compressed=false#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._tcp_(qType:=33)_compressed=true#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._tcp_(qType:=33)_compressed=false#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._udp_(qType:=33)_compressed=true#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._udp_(qType:=33)_compressed=false#01 -2022-09-15T21:42:25.189Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:25.209Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:25.212 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c2905bae-87ee-d0e0-0461-8abb3316207e Address:127.0.0.1:13623}]" -42:25.213 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c2905bae-87ee-d0e0-0461-8abb3316207e.dc1 127.0.0.1 -42:25.214 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13623 [Follower]" leader= -42:25.215 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c2905bae-87ee-d0e0-0461-8abb3316207e 127.0.0.1 -2022-09-15T21:42:25.215Z [INFO] agent.router: Initializing LAN area manager -42:25.215 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13618 network=udp -42:25.215 [INFO] TestAgent.server: Adding LAN server: server="Node-c2905bae-87ee-d0e0-0461-8abb3316207e (Addr: tcp/127.0.0.1:13623) (DC: dc1)" -42:25.215 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c2905bae-87ee-d0e0-0461-8abb3316207e.dc1 area=wan -42:25.215 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13618 network=tcp -42:25.216 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13619 network=tcp -42:25.216 [INFO] TestAgent: started state syncer -42:25.217 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13638 network=tcp -42:25.255 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:25.255 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13623 [Candidate]" term=2 -42:25.255 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:25.255 [DEBUG] TestAgent.server.raft: vote granted: from=c2905bae-87ee-d0e0-0461-8abb3316207e term=2 tally=1 -42:25.255 [INFO] TestAgent.server.raft: election won: tally=1 -42:25.255 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13623 [Leader]" -42:25.255 [INFO] TestAgent.server: cluster leadership acquired -42:25.256 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13623 -42:25.256 [INFO] TestAgent.server: New leader elected: payload=Node-c2905bae-87ee-d0e0-0461-8abb3316207e -42:25.257 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:25.259 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:25.259 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:25.259 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:25.259 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:25.259 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:25.260 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c2905bae-87ee-d0e0-0461-8abb3316207e -42:25.260 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c2905bae-87ee-d0e0-0461-8abb3316207e -42:25.261 [INFO] TestAgent.server: federation state anti-entropy synced -42:25.362 [DEBUG] TestAgent.dns: request served from client: name=_this-is-a-very-very-very-very-very-long-name-for-a-service._primary.service.consul. type=SRV class=IN latency=343.56µs client=127.0.0.1:57056 client_network=udp -42:25.363 [DEBUG] TestAgent.dns: request served from client: name=this-is-a-very-very-very-very-very-long-name-for-a-service.query.consul. type=SRV class=IN latency=277.34µs client=127.0.0.1:59217 client_network=udp -42:25.363 [INFO] TestAgent: Requesting shutdown -42:25.363 [INFO] TestAgent.server: shutting down server -42:25.363 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:25.363 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:25.363 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:25.363 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:25.363 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:25.364 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:25.364 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:25.364 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:25.364 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:25.364 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:25.364 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:25.364 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:25.364Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:25.364Z [INFO] agent.router.manager: shutting down -42:25.364 [INFO] TestAgent: consul server down -42:25.364 [INFO] TestAgent: shutdown complete -42:25.364 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13618 network=tcp -42:25.364 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13618 network=udp -42:25.364 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13619 network=tcp -42:25.866 [INFO] TestAgent: Waiting for endpoints to shut down -42:25.866 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_LargeResponses (0.72s) -=== CONT TestDNS_ServiceLookup_FilterCritical -2022-09-15T21:42:25.064Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:25.084Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:25.087 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:70f0f3f4-3da1-0cdf-d6b5-cefc41bfc682 Address:127.0.0.1:13630}]" -42:25.088 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-70f0f3f4-3da1-0cdf-d6b5-cefc41bfc682.dc1 127.0.0.1 -42:25.089 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-70f0f3f4-3da1-0cdf-d6b5-cefc41bfc682 127.0.0.1 -2022-09-15T21:42:25.089Z [INFO] agent.router: Initializing LAN area manager -42:25.090 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13630 [Follower]" leader= -42:25.090 [INFO] TestAgent.server: Adding LAN server: server="Node-70f0f3f4-3da1-0cdf-d6b5-cefc41bfc682 (Addr: tcp/127.0.0.1:13630) (DC: dc1)" -42:25.090 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-70f0f3f4-3da1-0cdf-d6b5-cefc41bfc682.dc1 area=wan -42:25.090 [DEBUG] TestAgent.dns: recursor enabled -42:25.090 [DEBUG] TestAgent.dns: recursor enabled -42:25.090 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13625 network=tcp -42:25.090 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13625 network=udp -42:25.091 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13626 network=tcp -42:25.091 [INFO] TestAgent: started state syncer -42:25.091 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13617 network=tcp -42:25.158 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:25.158 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13630 [Candidate]" term=2 -42:25.159 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:25.159 [DEBUG] TestAgent.server.raft: vote granted: from=70f0f3f4-3da1-0cdf-d6b5-cefc41bfc682 term=2 tally=1 -42:25.159 [INFO] TestAgent.server.raft: election won: tally=1 -42:25.159 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13630 [Leader]" -42:25.162 [INFO] TestAgent.server: cluster leadership acquired -42:25.163 [INFO] TestAgent.server: New leader elected: payload=Node-70f0f3f4-3da1-0cdf-d6b5-cefc41bfc682 -42:25.163 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13630 -42:25.165 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:25.176 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:25.176 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:25.176 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:25.176 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:25.176 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:25.177 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-70f0f3f4-3da1-0cdf-d6b5-cefc41bfc682 -42:25.177 [INFO] TestAgent.server: member joined, marking health alive: member=Node-70f0f3f4-3da1-0cdf-d6b5-cefc41bfc682 -42:25.179 [INFO] TestAgent.server: federation state anti-entropy synced -42:25.428 [DEBUG] TestAgent.dns: cname recurse RTT for name: name=www.google.com. rtt=250.42µs -42:25.428 [DEBUG] TestAgent.dns: request served from client: name=search.service.consul. type=ANY class=IN latency=781.44µs client=127.0.0.1:48337 client_network=udp -42:25.429 [DEBUG] TestAgent.dns: cname recurse RTT for name: name=www.google.com. rtt=166.42µs -42:25.429 [DEBUG] TestAgent.dns: request served from client: name=e7b65d63-fdec-82ea-43bd-7225d5307402.query.consul. type=ANY class=IN latency=506.4µs client=127.0.0.1:53629 client_network=udp -42:25.429 [INFO] TestAgent: Requesting shutdown -42:25.429 [INFO] TestAgent.server: shutting down server -42:25.429 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:25.429 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:25.429 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:25.429 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:25.429 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:25.429 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:25.430 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:25.430 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:25.430 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:25.430 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:25.430 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -42:25.430 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2022-09-15T21:42:25.430Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:25.430Z [INFO] agent.router.manager: shutting down -42:25.430 [INFO] TestAgent: consul server down -42:25.430 [INFO] TestAgent: shutdown complete -42:25.430 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13625 network=tcp -42:25.430 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13625 network=udp -42:25.430 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13626 network=tcp -42:25.931 [INFO] TestAgent: Waiting for endpoints to shut down -42:25.931 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_ServiceAddress_CNAME (0.89s) -=== CONT TestDNS_RecursorTimeout -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:25.325Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:25.352Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:25.355 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3f7c1561-a4fe-e5b9-7489-b13d167d21b7 Address:127.0.0.1:13644}]" -42:25.355 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13644 [Follower]" leader= -42:25.356 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3f7c1561-a4fe-e5b9-7489-b13d167d21b7.dc1 127.0.0.1 -42:25.357 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3f7c1561-a4fe-e5b9-7489-b13d167d21b7 127.0.0.1 -2022-09-15T21:42:25.357Z [INFO] agent.router: Initializing LAN area manager -42:25.357 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13639 network=udp -42:25.357 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3f7c1561-a4fe-e5b9-7489-b13d167d21b7.dc1 area=wan -42:25.357 [INFO] TestAgent.server: Adding LAN server: server="Node-3f7c1561-a4fe-e5b9-7489-b13d167d21b7 (Addr: tcp/127.0.0.1:13644) (DC: dc1)" -42:25.357 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13639 network=tcp -42:25.358 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13640 network=tcp -42:25.359 [INFO] TestAgent: started state syncer -42:25.359 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13631 network=tcp -42:25.418 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:25.418 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13644 [Candidate]" term=2 -42:25.418 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:25.418 [DEBUG] TestAgent.server.raft: vote granted: from=3f7c1561-a4fe-e5b9-7489-b13d167d21b7 term=2 tally=1 -42:25.418 [INFO] TestAgent.server.raft: election won: tally=1 -42:25.418 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13644 [Leader]" -42:25.418 [INFO] TestAgent.server: cluster leadership acquired -42:25.418 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13644 -42:25.420 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:25.422 [INFO] TestAgent.server: New leader elected: payload=Node-3f7c1561-a4fe-e5b9-7489-b13d167d21b7 -42:25.423 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:25.423 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:25.423 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:25.423 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:25.423 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:25.423 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3f7c1561-a4fe-e5b9-7489-b13d167d21b7 -42:25.423 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3f7c1561-a4fe-e5b9-7489-b13d167d21b7 -42:25.425 [INFO] TestAgent.server: federation state anti-entropy synced -42:25.482 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:25.483 [INFO] TestAgent: Synced node info -42:25.483 [DEBUG] TestAgent: Node info in sync -42:25.557 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=2.00378ms client=127.0.0.1:41993 client_network=udp -42:25.560 [DEBUG] TestAgent.dns: request served from client: name=cb89fb46-c552-4bb4-d76e-ff2d0d394f34.query.consul. type=ANY class=IN latency=1.96344ms client=127.0.0.1:36620 client_network=udp -42:25.560 [INFO] TestAgent: Requesting shutdown -42:25.560 [INFO] TestAgent.server: shutting down server -42:25.560 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:25.560 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:25.560 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:25.560 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:25.560 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:25.560 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:25.560 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:25.560 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:25.560 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:25.560 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:25.560 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:25.561Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:25.561Z [INFO] agent.router.manager: shutting down -42:25.561 [INFO] TestAgent: consul server down -42:25.561 [INFO] TestAgent: shutdown complete -42:25.561 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13639 network=tcp -42:25.561 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13639 network=udp -42:25.561 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13640 network=tcp -42:26.062 [INFO] TestAgent: Waiting for endpoints to shut down -42:26.062 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_Truncate (0.76s) -=== CONT TestDNS_Recurse_Truncation -2022-09-15T21:42:24.952Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:24.952Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:24.996Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:24.996Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:24.999 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b896266e-e833-cadf-a0e9-bea0d187e9a3 Address:127.0.0.1:13609}]" -42:25.000 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b896266e-e833-cadf-a0e9-bea0d187e9a3.dc2 127.0.0.1 -42:25.000 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13609 [Follower]" leader= -42:25.001 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b896266e-e833-cadf-a0e9-bea0d187e9a3 127.0.0.1 -2022-09-15T21:42:25.001Z [INFO] agent.router: Initializing LAN area manager -42:25.001 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13604 network=udp -42:25.002 [INFO] TestAgent.server: Adding LAN server: server="Node-b896266e-e833-cadf-a0e9-bea0d187e9a3 (Addr: tcp/127.0.0.1:13609) (DC: dc2)" -42:25.002 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b896266e-e833-cadf-a0e9-bea0d187e9a3.dc2 area=wan -42:25.002 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13604 network=tcp -42:25.003 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13605 network=tcp -42:25.003 [INFO] TestAgent: started state syncer -42:25.003 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13610 network=tcp -42:25.053 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -42:25.053 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:25.054 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13609 [Candidate]" term=2 -42:25.058 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:25.058 [DEBUG] TestAgent.server.raft: vote granted: from=b896266e-e833-cadf-a0e9-bea0d187e9a3 term=2 tally=1 -42:25.058 [INFO] TestAgent.server.raft: election won: tally=1 -42:25.058 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13609 [Leader]" -42:25.058 [INFO] TestAgent.server: cluster leadership acquired -42:25.058 [INFO] TestAgent.server: initializing acls -42:25.059 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:25.059 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:25.059 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:25.059 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:25.059 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-b896266e-e833-cadf-a0e9-bea0d187e9a3 -42:25.060 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-b896266e-e833-cadf-a0e9-bea0d187e9a3.dc2 -42:25.061 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:25.062 [INFO] TestAgent.server: New leader elected: payload=Node-b896266e-e833-cadf-a0e9-bea0d187e9a3 -42:25.062 [INFO] TestAgent.server: Updating LAN server: server="Node-b896266e-e833-cadf-a0e9-bea0d187e9a3 (Addr: tcp/127.0.0.1:13609) (DC: dc2)" -42:25.062 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-b896266e-e833-cadf-a0e9-bea0d187e9a3.dc2 area=wan -42:25.064 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:25.064 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:25.064 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:25.064 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:25.064 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:25.064 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b896266e-e833-cadf-a0e9-bea0d187e9a3 -42:25.064 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b896266e-e833-cadf-a0e9-bea0d187e9a3 -42:25.065 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b896266e-e833-cadf-a0e9-bea0d187e9a3 -42:25.070 [INFO] TestAgent.server: federation state anti-entropy synced -42:25.116 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:25.116 [INFO] TestAgent: Synced node info -42:25.258 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:13587] -42:25.258 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:13587 -42:25.260 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-cc7f7f3e-8fca-f24c-ead6-cec9e6594a00.dc1 127.0.0.1 -42:25.260 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 -42:25.261 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-cc7f7f3e-8fca-f24c-ead6-cec9e6594a00.dc1 area=wan -42:25.264 [INFO] TestAgent: Requesting shutdown -42:25.264 [INFO] TestAgent.server: shutting down server -42:25.264 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:25.264 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:25.264 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:25.264 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:25.264 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:25.264 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:25.264 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:25.264 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:25.265 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:25.265 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:25.265 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:25.265 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:25.265 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:25.265 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:25.265 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:25.265Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:25.265Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:25.265Z [INFO] agent.router.manager: shutting down -42:25.265 [INFO] TestAgent: consul server down -42:25.265 [INFO] TestAgent: shutdown complete -42:25.265 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13604 network=tcp -42:25.265 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13604 network=udp -42:25.265 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13605 network=tcp -42:25.765 [INFO] TestAgent: Waiting for endpoints to shut down -42:25.765 [INFO] TestAgent: Endpoints down -2022-09-15T21:42:24.525Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:24.525Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:24.567Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:24.567Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:24.570 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cc7f7f3e-8fca-f24c-ead6-cec9e6594a00 Address:127.0.0.1:13588}]" -42:24.571 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13588 [Follower]" leader= -42:24.572 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-cc7f7f3e-8fca-f24c-ead6-cec9e6594a00.dc1 127.0.0.1 -42:24.573 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-cc7f7f3e-8fca-f24c-ead6-cec9e6594a00 127.0.0.1 -2022-09-15T21:42:24.573Z [INFO] agent.router: Initializing LAN area manager -42:24.573 [INFO] TestAgent.server: Adding LAN server: server="Node-cc7f7f3e-8fca-f24c-ead6-cec9e6594a00 (Addr: tcp/127.0.0.1:13588) (DC: dc1)" -42:24.573 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-cc7f7f3e-8fca-f24c-ead6-cec9e6594a00.dc1 area=wan -42:24.573 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13583 network=udp -42:24.573 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13583 network=tcp -42:24.582 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13584 network=tcp -42:24.582 [INFO] TestAgent: started state syncer -42:24.583 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13575 network=tcp -42:24.624 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -42:24.624 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:24.624 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13588 [Candidate]" term=2 -42:24.624 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:24.624 [DEBUG] TestAgent.server.raft: vote granted: from=cc7f7f3e-8fca-f24c-ead6-cec9e6594a00 term=2 tally=1 -42:24.624 [INFO] TestAgent.server.raft: election won: tally=1 -42:24.624 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13588 [Leader]" -42:24.625 [INFO] TestAgent.server: cluster leadership acquired -42:24.625 [INFO] TestAgent.server: initializing acls -42:24.625 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:24.626 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:24.626 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:24.626 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:24.626 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-cc7f7f3e-8fca-f24c-ead6-cec9e6594a00 -42:24.627 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-cc7f7f3e-8fca-f24c-ead6-cec9e6594a00.dc1 -42:24.628 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:24.630 [INFO] TestAgent.server: New leader elected: payload=Node-cc7f7f3e-8fca-f24c-ead6-cec9e6594a00 -42:24.630 [INFO] TestAgent.server: Updating LAN server: server="Node-cc7f7f3e-8fca-f24c-ead6-cec9e6594a00 (Addr: tcp/127.0.0.1:13588) (DC: dc1)" -42:24.630 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-cc7f7f3e-8fca-f24c-ead6-cec9e6594a00.dc1 area=wan -42:24.631 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:24.631 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:24.631 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:24.631 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:24.631 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:24.631 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cc7f7f3e-8fca-f24c-ead6-cec9e6594a00 -42:24.631 [INFO] TestAgent.server: member joined, marking health alive: member=Node-cc7f7f3e-8fca-f24c-ead6-cec9e6594a00 -42:24.632 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cc7f7f3e-8fca-f24c-ead6-cec9e6594a00 -42:24.633 [INFO] TestAgent.server: federation state anti-entropy synced -42:24.712 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:24.713 [INFO] TestAgent: Synced node info -42:24.713 [DEBUG] TestAgent: Node info in sync -42:25.138 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:25.138 [DEBUG] TestAgent: Node info in sync -42:25.259 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:47244 -42:25.259 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b896266e-e833-cadf-a0e9-bea0d187e9a3.dc2 127.0.0.1 -42:25.260 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b896266e-e833-cadf-a0e9-bea0d187e9a3.dc2 area=wan -42:25.264 [DEBUG] TestAgent.dns: request served from client: name=my-query.query.consul. type=SRV class=IN latency=2.11686ms client=127.0.0.1:46873 client_network=udp -42:25.765 [INFO] TestAgent: Requesting shutdown -42:25.766 [INFO] TestAgent.server: shutting down server -42:25.766 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:25.766 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:25.766 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:25.766 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:25.766 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:25.766 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:25.766 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:25.766 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:25.766 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:25.766 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:25.766 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:25.766 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:25.766 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:25.766 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:25.766 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:25.766Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:25.766Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:25.767Z [INFO] agent.router.manager: shutting down -42:25.767 [INFO] TestAgent: consul server down -42:25.767 [INFO] TestAgent: shutdown complete -42:25.767 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13583 network=tcp -42:25.767 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13583 network=udp -42:25.767 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13584 network=tcp -42:26.267 [INFO] TestAgent: Waiting for endpoints to shut down -42:26.267 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_PreparedQuery_Failover (1.80s) -=== CONT TestDNS_Recurse -[WARN] freeport: 4 out of 25 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:25.707Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:25.730Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:25.733 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:36b7314c-879e-934a-3a3f-e9e8c7b977a8 Address:127.0.0.1:13665}]" -42:25.733 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13665 [Follower]" leader= -42:25.734 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-36b7314c-879e-934a-3a3f-e9e8c7b977a8.dc1 127.0.0.1 -42:25.735 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-36b7314c-879e-934a-3a3f-e9e8c7b977a8 127.0.0.1 -2022-09-15T21:42:25.735Z [INFO] agent.router: Initializing LAN area manager -42:25.735 [INFO] TestAgent.server: Adding LAN server: server="Node-36b7314c-879e-934a-3a3f-e9e8c7b977a8 (Addr: tcp/127.0.0.1:13665) (DC: dc1)" -42:25.735 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-36b7314c-879e-934a-3a3f-e9e8c7b977a8.dc1 area=wan -42:25.735 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13660 network=udp -42:25.735 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13660 network=tcp -42:25.736 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13661 network=tcp -42:25.736 [INFO] TestAgent: started state syncer -42:25.736 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13645 network=tcp -42:25.769 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:25.769 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13665 [Candidate]" term=2 -42:25.769 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:25.769 [DEBUG] TestAgent.server.raft: vote granted: from=36b7314c-879e-934a-3a3f-e9e8c7b977a8 term=2 tally=1 -42:25.769 [INFO] TestAgent.server.raft: election won: tally=1 -42:25.769 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13665 [Leader]" -42:25.769 [INFO] TestAgent.server: cluster leadership acquired -42:25.770 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13665 -42:25.771 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:25.771 [INFO] TestAgent.server: New leader elected: payload=Node-36b7314c-879e-934a-3a3f-e9e8c7b977a8 -42:25.775 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:25.775 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:25.775 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:25.775 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:25.775 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:25.775 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-36b7314c-879e-934a-3a3f-e9e8c7b977a8 -42:25.775 [INFO] TestAgent.server: member joined, marking health alive: member=Node-36b7314c-879e-934a-3a3f-e9e8c7b977a8 -42:25.776 [INFO] TestAgent.server: federation state anti-entropy synced -42:25.817 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:25.818 [INFO] TestAgent: Synced node info -42:25.818 [DEBUG] TestAgent: Node info in sync -42:26.014 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=ANY class=IN latency=364.62µs client=127.0.0.1:39029 client_network=udp -42:26.015 [DEBUG] TestAgent.dns: request served from client: name=44a539c9-fc95-e17c-3906-c469685ccd47.query.consul. type=ANY class=IN latency=372.28µs client=127.0.0.1:47059 client_network=udp -42:26.015 [INFO] TestAgent: Requesting shutdown -42:26.015 [INFO] TestAgent.server: shutting down server -42:26.015 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:26.015 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:26.015 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:26.015 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:26.015 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:26.015 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:26.015 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:26.015 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:26.015 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:26.015 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:26.016 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:26.016Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:26.016Z [INFO] agent.router.manager: shutting down -42:26.016 [INFO] TestAgent: consul server down -42:26.016 [INFO] TestAgent: shutdown complete -42:26.016 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13660 network=tcp -42:26.016 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13660 network=udp -42:26.016 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13661 network=tcp -42:26.516 [INFO] TestAgent: Waiting for endpoints to shut down -42:26.516 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_OnlyFailing (0.84s) -=== CONT TestDNS_ServiceLookup_Dedup_SRV -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:25.904Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:25.922Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:25.925 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:84de5867-97bf-5805-46a3-548c36d22495 Address:127.0.0.1:13651}]" -42:25.926 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13651 [Follower]" leader= -42:25.926 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-84de5867-97bf-5805-46a3-548c36d22495.dc1 127.0.0.1 -42:25.927 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-84de5867-97bf-5805-46a3-548c36d22495 127.0.0.1 -2022-09-15T21:42:25.927Z [INFO] agent.router: Initializing LAN area manager -42:25.928 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13646 network=udp -42:25.928 [INFO] TestAgent.server: Adding LAN server: server="Node-84de5867-97bf-5805-46a3-548c36d22495 (Addr: tcp/127.0.0.1:13651) (DC: dc1)" -42:25.928 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-84de5867-97bf-5805-46a3-548c36d22495.dc1 area=wan -42:25.928 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13646 network=tcp -42:25.929 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13647 network=tcp -42:25.929 [INFO] TestAgent: started state syncer -42:25.929 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13673 network=tcp -42:25.993 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:25.993 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13651 [Candidate]" term=2 -42:25.993 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:25.993 [DEBUG] TestAgent.server.raft: vote granted: from=84de5867-97bf-5805-46a3-548c36d22495 term=2 tally=1 -42:25.993 [INFO] TestAgent.server.raft: election won: tally=1 -42:25.993 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13651 [Leader]" -42:25.993 [INFO] TestAgent.server: cluster leadership acquired -42:25.994 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13651 -42:25.995 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:25.997 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:25.997 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:25.997 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:25.997 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:25.997 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:25.994 [INFO] TestAgent.server: New leader elected: payload=Node-84de5867-97bf-5805-46a3-548c36d22495 -42:25.998 [INFO] TestAgent.server: federation state anti-entropy synced -42:26.006 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-84de5867-97bf-5805-46a3-548c36d22495 -42:26.006 [INFO] TestAgent.server: member joined, marking health alive: member=Node-84de5867-97bf-5805-46a3-548c36d22495 -42:26.118 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=ANY class=IN latency=1.03442ms client=127.0.0.1:52672 client_network=udp -42:26.135 [DEBUG] TestAgent.dns: request served from client: name=ae659b23-bfc7-533b-077b-0052e0085f75.query.consul. type=ANY class=IN latency=414.22µs client=127.0.0.1:42141 client_network=udp -42:26.135 [INFO] TestAgent: Requesting shutdown -42:26.135 [INFO] TestAgent.server: shutting down server -42:26.135 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:26.135 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:26.135 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:26.135 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:26.135 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:26.136 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:26.136 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:26.136 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:26.136 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:26.136 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:26.136 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:26.136 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:26.136Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:26.136Z [INFO] agent.router.manager: shutting down -42:26.136 [INFO] TestAgent: consul server down -42:26.136 [INFO] TestAgent: shutdown complete -42:26.136 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13646 network=tcp -42:26.136 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13646 network=udp -42:26.136 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13647 network=tcp -42:26.636 [INFO] TestAgent: Waiting for endpoints to shut down -42:26.636 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_FilterCritical (0.77s) -=== CONT TestDNS_ServiceLookup_Dedup -2022-09-15T21:42:25.673Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:25.710Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:25.713 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:725883f4-9023-33d6-b0d4-534e5847a66c Address:127.0.0.1:13658}]" -42:25.713 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13658 [Follower]" leader= -42:25.718 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-725883f4-9023-33d6-b0d4-534e5847a66c.dc1 127.0.0.1 -42:25.719 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-725883f4-9023-33d6-b0d4-534e5847a66c 127.0.0.1 -2022-09-15T21:42:25.719Z [INFO] agent.router: Initializing LAN area manager -42:25.720 [INFO] TestAgent.server: Adding LAN server: server="Node-725883f4-9023-33d6-b0d4-534e5847a66c (Addr: tcp/127.0.0.1:13658) (DC: dc1)" -42:25.720 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-725883f4-9023-33d6-b0d4-534e5847a66c.dc1 area=wan -42:25.720 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13653 network=udp -42:25.720 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13653 network=tcp -42:25.721 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13654 network=tcp -42:25.721 [INFO] TestAgent: started state syncer -42:25.721 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13659 network=tcp -42:25.782 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:25.782 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13658 [Candidate]" term=2 -42:25.782 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:25.782 [DEBUG] TestAgent.server.raft: vote granted: from=725883f4-9023-33d6-b0d4-534e5847a66c term=2 tally=1 -42:25.782 [INFO] TestAgent.server.raft: election won: tally=1 -42:25.782 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13658 [Leader]" -42:25.782 [INFO] TestAgent.server: cluster leadership acquired -42:25.783 [INFO] TestAgent.server: New leader elected: payload=Node-725883f4-9023-33d6-b0d4-534e5847a66c -42:25.783 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13658 -42:25.784 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:25.787 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:25.787 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:25.787 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:25.787 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:25.787 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:25.787 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-725883f4-9023-33d6-b0d4-534e5847a66c -42:25.787 [INFO] TestAgent.server: member joined, marking health alive: member=Node-725883f4-9023-33d6-b0d4-534e5847a66c -42:25.788 [INFO] TestAgent.server: federation state anti-entropy synced -42:25.817 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:25.817 [INFO] TestAgent: Synced node info -42:26.151 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=ANY class=IN latency=359.5µs client=127.0.0.1:38209 client_network=udp -42:26.151 [DEBUG] TestAgent.dns: request served from client: name=83a0e5ef-7344-9669-93a8-43bf7649a8dd.query.consul. type=ANY class=IN latency=205.52µs client=127.0.0.1:34837 client_network=udp -42:26.152 [INFO] TestAgent: Requesting shutdown -42:26.152 [INFO] TestAgent.server: shutting down server -42:26.152 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:26.152 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:26.152 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:26.153Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:26.153Z [INFO] agent.router.manager: shutting down -42:26.153 [INFO] TestAgent: consul server down -42:26.153 [INFO] TestAgent: shutdown complete -42:26.153 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13653 network=tcp -42:26.153 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13653 network=udp -42:26.153 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13654 network=tcp -42:26.653 [INFO] TestAgent: Waiting for endpoints to shut down -42:26.653 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_OnlyPassing (1.00s) -=== CONT TestDNS_ServiceLookup_PreparedQueryNamePeriod -2022-09-15T21:42:25.609Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:25.626Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:25.629 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a7f3a99b-2044-1058-8e3c-f573f7289f73 Address:127.0.0.1:13672}]" -42:25.630 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13672 [Follower]" leader= -42:25.630 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a7f3a99b-2044-1058-8e3c-f573f7289f73.dc1 127.0.0.1 -42:25.631 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a7f3a99b-2044-1058-8e3c-f573f7289f73 127.0.0.1 -2022-09-15T21:42:25.631Z [INFO] agent.router: Initializing LAN area manager -42:25.632 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13667 network=udp -42:25.632 [INFO] TestAgent.server: Adding LAN server: server="Node-a7f3a99b-2044-1058-8e3c-f573f7289f73 (Addr: tcp/127.0.0.1:13672) (DC: dc1)" -42:25.632 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a7f3a99b-2044-1058-8e3c-f573f7289f73.dc1 area=wan -42:25.633 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13667 network=tcp -42:25.634 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13668 network=tcp -42:25.634 [INFO] TestAgent: started state syncer -42:25.634 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13652 network=tcp -42:25.673 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:25.673 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13672 [Candidate]" term=2 -42:25.673 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:25.673 [DEBUG] TestAgent.server.raft: vote granted: from=a7f3a99b-2044-1058-8e3c-f573f7289f73 term=2 tally=1 -42:25.673 [INFO] TestAgent.server.raft: election won: tally=1 -42:25.673 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13672 [Leader]" -42:25.674 [INFO] TestAgent.server: cluster leadership acquired -42:25.674 [INFO] TestAgent.server: New leader elected: payload=Node-a7f3a99b-2044-1058-8e3c-f573f7289f73 -42:25.674 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13672 -42:25.675 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:25.678 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:25.678 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:25.678 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:25.678 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:25.678 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:25.678 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a7f3a99b-2044-1058-8e3c-f573f7289f73 -42:25.678 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a7f3a99b-2044-1058-8e3c-f573f7289f73 -42:25.679 [INFO] TestAgent.server: federation state anti-entropy synced -42:26.075 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:26.087 [INFO] TestAgent: Synced node info -42:26.087 [DEBUG] TestAgent: Node info in sync -42:26.105 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:26.105 [DEBUG] TestAgent: Node info in sync -42:26.164 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=2.24028ms client=127.0.0.1:42708 client_network=udp -42:26.166 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=1.87668ms client=127.0.0.1:47874 client_network=udp -42:26.177 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=2.06356ms client=127.0.0.1:32962 client_network=udp -42:26.179 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=1.88522ms client=127.0.0.1:59731 client_network=udp -42:26.181 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=1.8757ms client=127.0.0.1:43715 client_network=udp -42:26.184 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=1.87632ms client=127.0.0.1:36902 client_network=udp -42:26.186 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=1.89618ms client=127.0.0.1:53544 client_network=udp -42:26.189 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=1.79528ms client=127.0.0.1:54300 client_network=udp -42:26.191 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=1.92364ms client=127.0.0.1:51893 client_network=udp -42:26.196 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=2.1612ms client=127.0.0.1:34230 client_network=udp -42:26.198 [DEBUG] TestAgent.dns: request served from client: name=5d1802cd-187c-5ed9-7ff1-4833c240d32d.query.consul. type=ANY class=IN latency=2.01154ms client=127.0.0.1:55370 client_network=udp -42:26.201 [DEBUG] TestAgent.dns: request served from client: name=5d1802cd-187c-5ed9-7ff1-4833c240d32d.query.consul. type=ANY class=IN latency=1.90584ms client=127.0.0.1:33705 client_network=udp -42:26.203 [DEBUG] TestAgent.dns: request served from client: name=5d1802cd-187c-5ed9-7ff1-4833c240d32d.query.consul. type=ANY class=IN latency=1.94852ms client=127.0.0.1:33904 client_network=udp -42:26.205 [DEBUG] TestAgent.dns: request served from client: name=5d1802cd-187c-5ed9-7ff1-4833c240d32d.query.consul. type=ANY class=IN latency=1.97054ms client=127.0.0.1:49707 client_network=udp -42:26.215 [DEBUG] TestAgent.dns: request served from client: name=5d1802cd-187c-5ed9-7ff1-4833c240d32d.query.consul. type=ANY class=IN latency=4.595ms client=127.0.0.1:42670 client_network=udp -42:26.217 [DEBUG] TestAgent.dns: request served from client: name=5d1802cd-187c-5ed9-7ff1-4833c240d32d.query.consul. type=ANY class=IN latency=1.9789ms client=127.0.0.1:59019 client_network=udp -42:26.219 [DEBUG] TestAgent.dns: request served from client: name=5d1802cd-187c-5ed9-7ff1-4833c240d32d.query.consul. type=ANY class=IN latency=2.01778ms client=127.0.0.1:35875 client_network=udp -42:26.222 [DEBUG] TestAgent.dns: request served from client: name=5d1802cd-187c-5ed9-7ff1-4833c240d32d.query.consul. type=ANY class=IN latency=2.14296ms client=127.0.0.1:46080 client_network=udp -42:26.224 [DEBUG] TestAgent.dns: request served from client: name=5d1802cd-187c-5ed9-7ff1-4833c240d32d.query.consul. type=ANY class=IN latency=1.9552ms client=127.0.0.1:46191 client_network=udp -42:26.227 [DEBUG] TestAgent.dns: request served from client: name=5d1802cd-187c-5ed9-7ff1-4833c240d32d.query.consul. type=ANY class=IN latency=2.04048ms client=127.0.0.1:41329 client_network=udp -42:26.227 [INFO] TestAgent: Requesting shutdown -42:26.227 [INFO] TestAgent.server: shutting down server -42:26.227 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:26.227 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:26.227 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:26.227 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:26.227 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:26.227 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:26.227 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:26.227 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:26.227 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:26.227 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:26.227 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:26.228Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:26.228Z [INFO] agent.router.manager: shutting down -42:26.228 [INFO] TestAgent: consul server down -42:26.228 [INFO] TestAgent: shutdown complete -42:26.228 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13667 network=tcp -42:26.228 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13667 network=udp -42:26.228 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13668 network=tcp -42:26.729 [INFO] TestAgent: Waiting for endpoints to shut down -42:26.729 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_Randomize (1.15s) -=== CONT TestDNS_PreparedQueryNearIP -[WARN] freeport: 4 out of 25 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:26.153Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:26.177Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:26.180 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7430b787-68da-efdf-9baf-a5379b3f21a3 Address:127.0.0.1:13686}]" -42:26.181 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13686 [Follower]" leader= -42:26.181 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7430b787-68da-efdf-9baf-a5379b3f21a3.dc1 127.0.0.1 -42:26.184 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7430b787-68da-efdf-9baf-a5379b3f21a3 127.0.0.1 -2022-09-15T21:42:26.184Z [INFO] agent.router: Initializing LAN area manager -42:26.184 [DEBUG] TestAgent.dns: recursor enabled -42:26.184 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13681 network=udp -42:26.184 [INFO] TestAgent.server: Adding LAN server: server="Node-7430b787-68da-efdf-9baf-a5379b3f21a3 (Addr: tcp/127.0.0.1:13686) (DC: dc1)" -42:26.185 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7430b787-68da-efdf-9baf-a5379b3f21a3.dc1 area=wan -42:26.185 [DEBUG] TestAgent.dns: recursor enabled -42:26.185 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13681 network=tcp -42:26.186 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13682 network=tcp -42:26.186 [INFO] TestAgent: started state syncer -42:26.186 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13694 network=tcp -42:26.233 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:26.233 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13686 [Candidate]" term=2 -42:26.233 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:26.233 [DEBUG] TestAgent.server.raft: vote granted: from=7430b787-68da-efdf-9baf-a5379b3f21a3 term=2 tally=1 -42:26.233 [INFO] TestAgent.server.raft: election won: tally=1 -42:26.233 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13686 [Leader]" -42:26.234 [INFO] TestAgent.server: cluster leadership acquired -42:26.234 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13686 -42:26.234 [INFO] TestAgent.server: New leader elected: payload=Node-7430b787-68da-efdf-9baf-a5379b3f21a3 -42:26.235 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:26.238 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:26.238 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:26.238 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:26.238 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:26.238 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:26.238 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7430b787-68da-efdf-9baf-a5379b3f21a3 -42:26.238 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7430b787-68da-efdf-9baf-a5379b3f21a3 -42:26.238 [INFO] TestAgent.server: federation state anti-entropy synced -42:26.271 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:26.272 [INFO] TestAgent: Synced node info -42:26.272 [DEBUG] TestAgent: Node info in sync -42:26.294 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:26.294 [DEBUG] TestAgent: Node info in sync -42:26.372 [DEBUG] TestAgent.dns: recurse succeeded for question: question="{apple.com. 255 1}" rtt=205.74µs recursor=127.0.0.1:56092 -42:26.372 [DEBUG] TestAgent.dns: request served from client: question="{apple.com. 255 1}" network=udp latency=450.32µs client=127.0.0.1:37565 client_network=udp -42:26.372 [INFO] TestAgent: Requesting shutdown -42:26.372 [INFO] TestAgent.server: shutting down server -42:26.372 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:26.372 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:26.372 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:26.372 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:26.372 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:26.373 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:26.373 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:26.373 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:26.373 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:26.373 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:26.373 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:26.373Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:26.373Z [INFO] agent.router.manager: shutting down -42:26.373 [INFO] TestAgent: consul server down -42:26.373 [INFO] TestAgent: shutdown complete -42:26.373 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13681 network=tcp -42:26.373 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13681 network=udp -42:26.373 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13682 network=tcp -42:26.874 [INFO] TestAgent: Waiting for endpoints to shut down -42:26.874 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_Recurse_Truncation (0.80s) -=== CONT TestDNS_PreparedQueryNearIPEDNS -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:26.315Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:26.341Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:26.344 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1662c868-5839-f386-bd41-c444d6f3f873 Address:127.0.0.1:13700}]" -42:26.345 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1662c868-5839-f386-bd41-c444d6f3f873.dc1 127.0.0.1 -42:26.346 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13700 [Follower]" leader= -42:26.346 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1662c868-5839-f386-bd41-c444d6f3f873 127.0.0.1 -2022-09-15T21:42:26.346Z [INFO] agent.router: Initializing LAN area manager -42:26.347 [DEBUG] TestAgent.dns: recursor enabled -42:26.347 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13695 network=udp -42:26.347 [INFO] TestAgent.server: Adding LAN server: server="Node-1662c868-5839-f386-bd41-c444d6f3f873 (Addr: tcp/127.0.0.1:13700) (DC: dc1)" -42:26.347 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1662c868-5839-f386-bd41-c444d6f3f873.dc1 area=wan -42:26.347 [DEBUG] TestAgent.dns: recursor enabled -42:26.347 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13695 network=tcp -42:26.348 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13696 network=tcp -42:26.348 [INFO] TestAgent: started state syncer -42:26.348 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13687 network=tcp -42:26.411 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:26.411 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13700 [Candidate]" term=2 -42:26.412 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:26.412 [DEBUG] TestAgent.server.raft: vote granted: from=1662c868-5839-f386-bd41-c444d6f3f873 term=2 tally=1 -42:26.412 [INFO] TestAgent.server.raft: election won: tally=1 -42:26.412 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13700 [Leader]" -42:26.412 [INFO] TestAgent.server: cluster leadership acquired -42:26.412 [INFO] TestAgent.server: New leader elected: payload=Node-1662c868-5839-f386-bd41-c444d6f3f873 -42:26.412 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13700 -42:26.413 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:26.416 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:26.416 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:26.416 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:26.416 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:26.416 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:26.416 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1662c868-5839-f386-bd41-c444d6f3f873 -42:26.416 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1662c868-5839-f386-bd41-c444d6f3f873 -42:26.417 [INFO] TestAgent.server: federation state anti-entropy synced -42:26.659 [DEBUG] TestAgent.dns: recurse succeeded for question: question="{apple.com. 255 1}" rtt=195.02µs recursor=127.0.0.1:56359 -42:26.659 [DEBUG] TestAgent.dns: request served from client: question="{apple.com. 255 1}" network=udp latency=424.04µs client=127.0.0.1:41493 client_network=udp -42:26.659 [INFO] TestAgent: Requesting shutdown -42:26.659 [INFO] TestAgent.server: shutting down server -42:26.659 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:26.659 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:26.659 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:26.659 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:26.659 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:26.659 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:26.659 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:26.659 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:26.659 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:26.659 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:26.659 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:26.659 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:26.660Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:26.660Z [INFO] agent.router.manager: shutting down -42:26.660 [INFO] TestAgent: consul server down -42:26.660 [INFO] TestAgent: shutdown complete -42:26.660 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13695 network=tcp -42:26.660 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13695 network=udp -42:26.660 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13696 network=tcp -42:27.162 [INFO] TestAgent: Waiting for endpoints to shut down -42:27.162 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_Recurse (0.89s) -=== CONT TestDNS_ServiceLookup_TagPeriod -Added 3 service nodes -2022-09-15T21:42:26.560Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:26.596Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:26.599 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:eabe3876-0fd1-0172-69c0-bc37d052d8b6 Address:127.0.0.1:13693}]" -42:26.599 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13693 [Follower]" leader= -42:26.600 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-eabe3876-0fd1-0172-69c0-bc37d052d8b6.dc1 127.0.0.1 -42:26.600 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-eabe3876-0fd1-0172-69c0-bc37d052d8b6 127.0.0.1 -2022-09-15T21:42:26.601Z [INFO] agent.router: Initializing LAN area manager -42:26.601 [INFO] TestAgent.server: Adding LAN server: server="Node-eabe3876-0fd1-0172-69c0-bc37d052d8b6 (Addr: tcp/127.0.0.1:13693) (DC: dc1)" -42:26.601 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-eabe3876-0fd1-0172-69c0-bc37d052d8b6.dc1 area=wan -42:26.601 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13688 network=udp -42:26.601 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13688 network=tcp -42:26.602 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13689 network=tcp -42:26.602 [INFO] TestAgent: started state syncer -42:26.602 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13715 network=tcp -42:26.666 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:26.666 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13693 [Candidate]" term=2 -42:26.667 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:26.667 [DEBUG] TestAgent.server.raft: vote granted: from=eabe3876-0fd1-0172-69c0-bc37d052d8b6 term=2 tally=1 -42:26.667 [INFO] TestAgent.server.raft: election won: tally=1 -42:26.667 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13693 [Leader]" -42:26.667 [INFO] TestAgent.server: cluster leadership acquired -42:26.667 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13693 -42:26.668 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:26.670 [INFO] TestAgent.server: New leader elected: payload=Node-eabe3876-0fd1-0172-69c0-bc37d052d8b6 -42:26.671 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:26.671 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:26.671 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:26.671 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:26.671 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:26.672 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-eabe3876-0fd1-0172-69c0-bc37d052d8b6 -42:26.672 [INFO] TestAgent.server: member joined, marking health alive: member=Node-eabe3876-0fd1-0172-69c0-bc37d052d8b6 -42:26.673 [INFO] TestAgent.server: federation state anti-entropy synced -42:26.710 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=362.2µs client=127.0.0.1:45606 client_network=udp -42:26.711 [DEBUG] TestAgent.dns: request served from client: name=360578c9-84e3-bb70-96a2-578c557e62e4.query.consul. type=SRV class=IN latency=234.7µs client=127.0.0.1:42242 client_network=udp -42:26.711 [INFO] TestAgent: Requesting shutdown -42:26.711 [INFO] TestAgent.server: shutting down server -42:26.711 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:26.711 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:26.711 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:26.711 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:26.711 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:26.712 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:26.712 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:26.712 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:26.712 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:26.712 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:26.712 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:26.712 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:26.712Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:26.712Z [INFO] agent.router.manager: shutting down -42:26.712 [INFO] TestAgent: consul server down -42:26.712 [INFO] TestAgent: shutdown complete -42:26.712 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13688 network=tcp -42:26.712 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13688 network=udp -42:26.712 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13689 network=tcp -42:27.213 [INFO] TestAgent: Waiting for endpoints to shut down -42:27.213 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_Dedup_SRV (0.70s) -=== CONT TestDNS_CaseInsensitiveServiceLookup -Added 3 service nodes -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:26.671Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:26.697Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:26.700 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:810027c8-f44d-efff-140f-70e1dbee29f9 Address:127.0.0.1:13707}]" -42:26.700 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13707 [Follower]" leader= -42:26.701 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-810027c8-f44d-efff-140f-70e1dbee29f9.dc1 127.0.0.1 -42:26.702 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-810027c8-f44d-efff-140f-70e1dbee29f9 127.0.0.1 -2022-09-15T21:42:26.702Z [INFO] agent.router: Initializing LAN area manager -42:26.702 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13702 network=udp -42:26.702 [INFO] TestAgent.server: Adding LAN server: server="Node-810027c8-f44d-efff-140f-70e1dbee29f9 (Addr: tcp/127.0.0.1:13707) (DC: dc1)" -42:26.702 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-810027c8-f44d-efff-140f-70e1dbee29f9.dc1 area=wan -42:26.703 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13702 network=tcp -42:26.704 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13703 network=tcp -42:26.704 [INFO] TestAgent: started state syncer -42:26.704 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13722 network=tcp -42:26.754 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:26.754 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13707 [Candidate]" term=2 -42:26.754 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:26.754 [DEBUG] TestAgent.server.raft: vote granted: from=810027c8-f44d-efff-140f-70e1dbee29f9 term=2 tally=1 -42:26.754 [INFO] TestAgent.server.raft: election won: tally=1 -42:26.754 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13707 [Leader]" -42:26.755 [INFO] TestAgent.server: cluster leadership acquired -42:26.755 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13707 -42:26.756 [INFO] TestAgent.server: New leader elected: payload=Node-810027c8-f44d-efff-140f-70e1dbee29f9 -42:26.760 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:26.765 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:26.765 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:26.765 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:26.765 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:26.765 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:26.766 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-810027c8-f44d-efff-140f-70e1dbee29f9 -42:26.766 [INFO] TestAgent.server: member joined, marking health alive: member=Node-810027c8-f44d-efff-140f-70e1dbee29f9 -42:26.766 [INFO] TestAgent.server: federation state anti-entropy synced -42:26.783 [DEBUG] TestAgent.dns: request served from client: name=some.query.we.like.query.consul. type=SRV class=IN latency=259.04µs client=127.0.0.1:46477 client_network=udp -42:26.783 [INFO] TestAgent: Requesting shutdown -42:26.783 [INFO] TestAgent.server: shutting down server -42:26.783 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:26.783 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:26.783 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:26.783 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:26.783 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:26.783 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:26.783 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:26.783 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:26.783 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:26.783 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:26.783 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:26.784 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:26.784Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:26.784Z [INFO] agent.router.manager: shutting down -42:26.784 [INFO] TestAgent: consul server down -42:26.784 [INFO] TestAgent: shutdown complete -42:26.784 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13702 network=tcp -42:26.784 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13702 network=udp -42:26.784 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13703 network=tcp -42:27.284 [INFO] TestAgent: Waiting for endpoints to shut down -42:27.284 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_PreparedQueryNamePeriod (0.63s) -=== CONT TestDNS_Lookup_TaggedIPAddresses -2022-09-15T21:42:26.655Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:26.671Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:26.674 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1162fdc7-8f9e-4903-dafc-11df0b0e91f1 Address:127.0.0.1:13721}]" -42:26.674 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13721 [Follower]" leader= -42:26.675 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1162fdc7-8f9e-4903-dafc-11df0b0e91f1.dc1 127.0.0.1 -42:26.676 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1162fdc7-8f9e-4903-dafc-11df0b0e91f1 127.0.0.1 -2022-09-15T21:42:26.676Z [INFO] agent.router: Initializing LAN area manager -42:26.677 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13716 network=udp -42:26.677 [INFO] TestAgent.server: Adding LAN server: server="Node-1162fdc7-8f9e-4903-dafc-11df0b0e91f1 (Addr: tcp/127.0.0.1:13721) (DC: dc1)" -42:26.677 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1162fdc7-8f9e-4903-dafc-11df0b0e91f1.dc1 area=wan -42:26.677 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13716 network=tcp -42:26.678 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13717 network=tcp -42:26.678 [INFO] TestAgent: started state syncer -42:26.678 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13701 network=tcp -42:26.712 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:26.712 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13721 [Candidate]" term=2 -42:26.713 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:26.713 [DEBUG] TestAgent.server.raft: vote granted: from=1162fdc7-8f9e-4903-dafc-11df0b0e91f1 term=2 tally=1 -42:26.713 [INFO] TestAgent.server.raft: election won: tally=1 -42:26.713 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13721 [Leader]" -42:26.713 [INFO] TestAgent.server: cluster leadership acquired -42:26.713 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13721 -42:26.714 [INFO] TestAgent.server: New leader elected: payload=Node-1162fdc7-8f9e-4903-dafc-11df0b0e91f1 -42:26.718 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:26.721 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:26.721 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:26.721 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:26.721 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:26.721 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:26.721 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1162fdc7-8f9e-4903-dafc-11df0b0e91f1 -42:26.721 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1162fdc7-8f9e-4903-dafc-11df0b0e91f1 -42:26.721 [INFO] TestAgent.server: federation state anti-entropy synced -42:26.740 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:26.740 [INFO] TestAgent: Synced node info -42:26.740 [DEBUG] TestAgent: Node info in sync -42:26.982 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=ANY class=IN latency=439.92µs client=127.0.0.1:38238 client_network=udp -42:26.986 [DEBUG] TestAgent.dns: request served from client: name=3966d8dd-7e2f-bd61-9646-955bf0c74247.query.consul. type=ANY class=IN latency=334.88µs client=127.0.0.1:48055 client_network=udp -42:26.987 [INFO] TestAgent: Requesting shutdown -42:26.987 [INFO] TestAgent.server: shutting down server -42:26.987 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:26.987 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:26.987 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:26.987 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:26.987 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:26.987 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:26.987 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:26.987 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:26.987 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:26.987 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:26.987 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:26.988Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:26.988Z [INFO] agent.router.manager: shutting down -42:26.991 [INFO] TestAgent: consul server down -42:26.991 [INFO] TestAgent: shutdown complete -42:26.991 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13716 network=tcp -42:26.991 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13716 network=udp -42:27.002 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13717 network=tcp -42:27.506 [INFO] TestAgent: Waiting for endpoints to shut down -42:27.506 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_Dedup (0.87s) -=== CONT TestDNS_ServiceLookup_WanTranslation -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_Lookup_TaggedIPAddresses/simple-ipv4 -=== RUN TestDNS_Lookup_TaggedIPAddresses/simple-ipv6 -=== RUN TestDNS_Lookup_TaggedIPAddresses/ipv4-with-tagged-ipv6 -=== RUN TestDNS_Lookup_TaggedIPAddresses/ipv6-with-tagged-ipv4 -2022-09-15T21:42:26.892Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:26.912Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:26.915 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:928bd683-ca6f-7cef-baee-f44cd31e2d25 Address:127.0.0.1:13714}]" -42:26.915 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13714 [Follower]" leader= -42:26.916 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-928bd683-ca6f-7cef-baee-f44cd31e2d25.dc1 127.0.0.1 -42:26.917 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-928bd683-ca6f-7cef-baee-f44cd31e2d25 127.0.0.1 -2022-09-15T21:42:26.917Z [INFO] agent.router: Initializing LAN area manager -42:26.917 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13709 network=udp -42:26.917 [INFO] TestAgent.server: Adding LAN server: server="Node-928bd683-ca6f-7cef-baee-f44cd31e2d25 (Addr: tcp/127.0.0.1:13714) (DC: dc1)" -42:26.917 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-928bd683-ca6f-7cef-baee-f44cd31e2d25.dc1 area=wan -42:26.917 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13709 network=tcp -42:26.918 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13710 network=tcp -42:26.918 [INFO] TestAgent: started state syncer -42:26.919 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13736 network=tcp -42:26.978 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:26.978 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13714 [Candidate]" term=2 -42:26.978 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:26.978 [DEBUG] TestAgent.server.raft: vote granted: from=928bd683-ca6f-7cef-baee-f44cd31e2d25 term=2 tally=1 -42:26.978 [INFO] TestAgent.server.raft: election won: tally=1 -42:26.978 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13714 [Leader]" -42:26.978 [INFO] TestAgent.server: cluster leadership acquired -42:26.979 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13714 -42:26.980 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:26.982 [INFO] TestAgent.server: New leader elected: payload=Node-928bd683-ca6f-7cef-baee-f44cd31e2d25 -42:26.983 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:26.983 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:26.983 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:26.983 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:26.983 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:26.983 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-928bd683-ca6f-7cef-baee-f44cd31e2d25 -42:26.983 [INFO] TestAgent.server: member joined, marking health alive: member=Node-928bd683-ca6f-7cef-baee-f44cd31e2d25 -42:26.985 [INFO] TestAgent.server: federation state anti-entropy synced -42:27.023 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:27.024 [INFO] TestAgent: Synced node info -42:27.024 [DEBUG] TestAgent: Node info in sync -42:27.033 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:27.033 [DEBUG] TestAgent: Node info in sync -42:27.171 [DEBUG] TestAgent.dns: request served from client: name=some.query.we.like.query.consul. type=A class=IN latency=291.1µs client=127.0.0.1:45125 client_network=udp -42:27.171 [INFO] TestAgent: Requesting shutdown -42:27.171 [INFO] TestAgent.server: shutting down server -42:27.171 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:27.171 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:27.171 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:27.171 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:27.171 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:27.171 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:27.171 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:27.171 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:27.171 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:27.171 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:27.171 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:27.171Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:27.171Z [INFO] agent.router.manager: shutting down -42:27.171 [INFO] TestAgent: consul server down -42:27.171 [INFO] TestAgent: shutdown complete -42:27.171 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13709 network=tcp -42:27.171 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13709 network=udp -42:27.171 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13710 network=tcp -42:27.674 [INFO] TestAgent: Waiting for endpoints to shut down -42:27.674 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_PreparedQueryNearIPEDNS (0.80s) -=== CONT TestDNS_ServiceLookup_ServiceAddressIPV6 -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:26.781Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:26.802Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:26.805 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5ed0fcf0-3efd-ab5a-2f49-f1fd5562fd5f Address:127.0.0.1:13728}]" -42:26.805 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13728 [Follower]" leader= -42:26.807 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5ed0fcf0-3efd-ab5a-2f49-f1fd5562fd5f.dc1 127.0.0.1 -42:26.808 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5ed0fcf0-3efd-ab5a-2f49-f1fd5562fd5f 127.0.0.1 -2022-09-15T21:42:26.808Z [INFO] agent.router: Initializing LAN area manager -42:26.808 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13723 network=udp -42:26.808 [INFO] TestAgent.server: Adding LAN server: server="Node-5ed0fcf0-3efd-ab5a-2f49-f1fd5562fd5f (Addr: tcp/127.0.0.1:13728) (DC: dc1)" -42:26.808 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5ed0fcf0-3efd-ab5a-2f49-f1fd5562fd5f.dc1 area=wan -42:26.808 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13723 network=tcp -42:26.809 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13724 network=tcp -42:26.809 [INFO] TestAgent: started state syncer -42:26.809 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13708 network=tcp -42:26.866 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:26.866 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13728 [Candidate]" term=2 -42:26.867 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:26.867 [DEBUG] TestAgent.server.raft: vote granted: from=5ed0fcf0-3efd-ab5a-2f49-f1fd5562fd5f term=2 tally=1 -42:26.867 [INFO] TestAgent.server.raft: election won: tally=1 -42:26.867 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13728 [Leader]" -42:26.870 [INFO] TestAgent.server: cluster leadership acquired -42:26.870 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13728 -42:26.871 [INFO] TestAgent.server: New leader elected: payload=Node-5ed0fcf0-3efd-ab5a-2f49-f1fd5562fd5f -42:26.871 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:26.874 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:26.874 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:26.874 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:26.874 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:26.874 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:26.874 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5ed0fcf0-3efd-ab5a-2f49-f1fd5562fd5f -42:26.874 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5ed0fcf0-3efd-ab5a-2f49-f1fd5562fd5f -42:26.875 [INFO] TestAgent.server: federation state anti-entropy synced -42:27.215 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:27.216 [INFO] TestAgent: Synced node info -42:27.216 [DEBUG] TestAgent: Node info in sync -42:27.260 [DEBUG] TestAgent.dns: request served from client: name=some.query.we.like.query.consul. type=A class=IN latency=850.26µs client=127.0.0.1:39656 client_network=udp -42:27.287 [DEBUG] TestAgent.dns: request served from client: name=some.query.we.like.query.consul. type=A class=IN latency=328.1µs client=127.0.0.1:43016 client_network=udp -42:27.315 [DEBUG] TestAgent.dns: request served from client: name=some.query.we.like.query.consul. type=A class=IN latency=279.82µs client=127.0.0.1:49439 client_network=udp -42:27.316 [INFO] TestAgent: Requesting shutdown -42:27.316 [INFO] TestAgent.server: shutting down server -42:27.316 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:27.316 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:27.316 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:27.316 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:27.316 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:27.316 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:27.316 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:27.316 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:27.316 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:27.316 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:27.316 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:27.316Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:27.316Z [INFO] agent.router.manager: shutting down -42:27.316 [INFO] TestAgent: consul server down -42:27.316 [INFO] TestAgent: shutdown complete -42:27.316 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13723 network=tcp -42:27.317 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13723 network=udp -42:27.317 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13724 network=tcp -42:27.817 [INFO] TestAgent: Waiting for endpoints to shut down -42:27.817 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_PreparedQueryNearIP (1.08s) -=== CONT TestDNS_ServiceLookup_ServiceAddress_SRV -=== RUN TestDNS_ServiceLookup_WanTranslation/service-wan-from-dc1 -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:27.179Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:27.196Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:27.199 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:92754f4b-d642-8d8e-48f3-15fc87231671 Address:127.0.0.1:13742}]" -42:27.199 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13742 [Follower]" leader= -42:27.200 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-92754f4b-d642-8d8e-48f3-15fc87231671.dc1 127.0.0.1 -42:27.201 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-92754f4b-d642-8d8e-48f3-15fc87231671 127.0.0.1 -2022-09-15T21:42:27.201Z [INFO] agent.router: Initializing LAN area manager -42:27.201 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13737 network=udp -42:27.201 [INFO] TestAgent.server: Adding LAN server: server="Node-92754f4b-d642-8d8e-48f3-15fc87231671 (Addr: tcp/127.0.0.1:13742) (DC: dc1)" -42:27.201 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-92754f4b-d642-8d8e-48f3-15fc87231671.dc1 area=wan -42:27.201 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13737 network=tcp -42:27.202 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13738 network=tcp -42:27.202 [INFO] TestAgent: started state syncer -42:27.202 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13729 network=tcp -42:27.250 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:27.250 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13742 [Candidate]" term=2 -42:27.252 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:27.252 [DEBUG] TestAgent.server.raft: vote granted: from=92754f4b-d642-8d8e-48f3-15fc87231671 term=2 tally=1 -42:27.252 [INFO] TestAgent.server.raft: election won: tally=1 -42:27.252 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13742 [Leader]" -42:27.252 [INFO] TestAgent.server: cluster leadership acquired -42:27.254 [INFO] TestAgent.server: New leader elected: payload=Node-92754f4b-d642-8d8e-48f3-15fc87231671 -42:27.256 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13742 -42:27.263 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:27.283 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:27.283 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:27.283 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:27.283 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:27.283 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:27.284 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-92754f4b-d642-8d8e-48f3-15fc87231671 -42:27.284 [INFO] TestAgent.server: member joined, marking health alive: member=Node-92754f4b-d642-8d8e-48f3-15fc87231671 -42:27.285 [INFO] TestAgent.server: federation state anti-entropy synced -42:27.340 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:27.341 [INFO] TestAgent: Synced node info -42:27.525 [DEBUG] TestAgent.dns: request served from client: name=v1.primary2.db.service.consul. type=SRV class=IN latency=198.42µs client=127.0.0.1:40103 client_network=udp -42:27.525 [DEBUG] TestAgent.dns: request served from client: name=v1.primary.db.service.consul. type=SRV class=IN latency=208.8µs client=127.0.0.1:41517 client_network=udp -42:27.525 [INFO] TestAgent: Requesting shutdown -42:27.525 [INFO] TestAgent.server: shutting down server -42:27.525 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:27.525 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:27.525 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:27.525 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:27.525 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:27.526 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:27.526 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:27.526 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:27.526 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:27.526 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:27.526 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:27.526Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:27.526Z [INFO] agent.router.manager: shutting down -42:27.526 [INFO] TestAgent: consul server down -42:27.526 [INFO] TestAgent: shutdown complete -42:27.526 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13737 network=tcp -42:27.526 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13737 network=udp -42:27.526 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13738 network=tcp -42:28.026 [INFO] TestAgent: Waiting for endpoints to shut down -42:28.026 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_TagPeriod (0.86s) -=== CONT TestDNS_ServiceLookup_ServiceAddress_A -=== RUN TestDNS_ServiceLookup_WanTranslation/node-addr-from-dc2 -=== RUN TestDNS_ServiceLookup_WanTranslation/node-wan-from-dc2 -=== RUN TestDNS_ServiceLookup_WanTranslation/service-addr-from-dc2 -=== RUN TestDNS_ServiceLookup_WanTranslation/service-wan-from-dc2 -=== RUN TestDNS_ServiceLookup_WanTranslation/node-addr-from-dc1 -=== RUN TestDNS_ServiceLookup_WanTranslation/node-wan-from-dc1 -2022-09-15T21:42:27.237Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:27.290Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:27.292 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:337d78a6-e6c5-c158-6841-152014771736 Address:127.0.0.1:13735}]" -42:27.293 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-337d78a6-e6c5-c158-6841-152014771736.dc1 127.0.0.1 -42:27.294 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13735 [Follower]" leader= -42:27.294 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-337d78a6-e6c5-c158-6841-152014771736 127.0.0.1 -2022-09-15T21:42:27.294Z [INFO] agent.router: Initializing LAN area manager -42:27.294 [INFO] TestAgent.server: Adding LAN server: server="Node-337d78a6-e6c5-c158-6841-152014771736 (Addr: tcp/127.0.0.1:13735) (DC: dc1)" -42:27.294 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-337d78a6-e6c5-c158-6841-152014771736.dc1 area=wan -42:27.294 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13730 network=udp -42:27.295 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13730 network=tcp -42:27.295 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13731 network=tcp -42:27.295 [INFO] TestAgent: started state syncer -42:27.296 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13750 network=tcp -42:27.358 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:27.358 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13735 [Candidate]" term=2 -42:27.358 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:27.358 [DEBUG] TestAgent.server.raft: vote granted: from=337d78a6-e6c5-c158-6841-152014771736 term=2 tally=1 -42:27.358 [INFO] TestAgent.server.raft: election won: tally=1 -42:27.358 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13735 [Leader]" -42:27.358 [INFO] TestAgent.server: cluster leadership acquired -42:27.358 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13735 -42:27.360 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:27.360 [INFO] TestAgent.server: New leader elected: payload=Node-337d78a6-e6c5-c158-6841-152014771736 -42:27.363 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:27.363 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:27.363 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:27.363 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:27.363 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:27.363 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-337d78a6-e6c5-c158-6841-152014771736 -42:27.363 [INFO] TestAgent.server: member joined, marking health alive: member=Node-337d78a6-e6c5-c158-6841-152014771736 -42:27.364 [INFO] TestAgent.server: federation state anti-entropy synced -42:27.436 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:27.436 [INFO] TestAgent: Synced node info -42:27.593 [DEBUG] TestAgent.dns: request served from client: name=primary.db.service.consul. type=SRV class=IN latency=183.1µs client=127.0.0.1:34348 client_network=udp -42:27.593 [DEBUG] TestAgent.dns: request served from client: name=pRIMARY.dB.service.consul. type=SRV class=IN latency=148.58µs client=127.0.0.1:36662 client_network=udp -42:27.594 [DEBUG] TestAgent.dns: request served from client: name=PRIMARY.dB.service.consul. type=SRV class=IN latency=138.8µs client=127.0.0.1:33200 client_network=udp -42:27.594 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=202.66µs client=127.0.0.1:41136 client_network=udp -42:27.595 [DEBUG] TestAgent.dns: request served from client: name=DB.service.consul. type=SRV class=IN latency=246.8µs client=127.0.0.1:47372 client_network=udp -42:27.595 [DEBUG] TestAgent.dns: request served from client: name=Db.service.consul. type=SRV class=IN latency=269.18µs client=127.0.0.1:40286 client_network=udp -42:27.596 [DEBUG] TestAgent.dns: request served from client: name=somequery.query.consul. type=SRV class=IN latency=157.62µs client=127.0.0.1:50222 client_network=udp -42:27.596 [DEBUG] TestAgent.dns: request served from client: name=SomeQuery.query.consul. type=SRV class=IN latency=148.3µs client=127.0.0.1:38308 client_network=udp -42:27.597 [DEBUG] TestAgent.dns: request served from client: name=SOMEQUERY.query.consul. type=SRV class=IN latency=128.94µs client=127.0.0.1:57008 client_network=udp -42:27.597 [INFO] TestAgent: Requesting shutdown -42:27.597 [INFO] TestAgent.server: shutting down server -42:27.597 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:27.597 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:27.597 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:27.597 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:27.597 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:27.597 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:27.597 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:27.597 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:27.597 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:27.597 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:27.597 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:27.597Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:27.597Z [INFO] agent.router.manager: shutting down -42:27.598 [INFO] TestAgent: consul server down -42:27.598 [INFO] TestAgent: shutdown complete -42:27.598 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13730 network=tcp -42:27.598 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13730 network=udp -42:27.598 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13731 network=tcp -42:28.098 [INFO] TestAgent: Waiting for endpoints to shut down -42:28.098 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_CaseInsensitiveServiceLookup (0.89s) -=== CONT TestDNS_ExternalServiceToConsulCNAMENestedLookup -=== RUN TestDNS_ServiceLookup_WanTranslation/service-addr-from-dc1 -2022-09-15T21:42:27.308Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:27.324Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:27.327 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:55a3415e-9a0f-b062-ac2e-7b959091087c Address:127.0.0.1:13756}]" -42:27.327 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13756 [Follower]" leader= -42:27.328 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-55a3415e-9a0f-b062-ac2e-7b959091087c.dc1 127.0.0.1 -42:27.329 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-55a3415e-9a0f-b062-ac2e-7b959091087c 127.0.0.1 -2022-09-15T21:42:27.329Z [INFO] agent.router: Initializing LAN area manager -42:27.329 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13751 network=udp -42:27.329 [INFO] TestAgent.server: Adding LAN server: server="Node-55a3415e-9a0f-b062-ac2e-7b959091087c (Addr: tcp/127.0.0.1:13756) (DC: dc1)" -42:27.329 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-55a3415e-9a0f-b062-ac2e-7b959091087c.dc1 area=wan -42:27.330 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13751 network=tcp -42:27.330 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13752 network=tcp -42:27.331 [INFO] TestAgent: started state syncer -42:27.331 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13743 network=tcp -42:27.398 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:27.398 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13756 [Candidate]" term=2 -42:27.398 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:27.398 [DEBUG] TestAgent.server.raft: vote granted: from=55a3415e-9a0f-b062-ac2e-7b959091087c term=2 tally=1 -42:27.398 [INFO] TestAgent.server.raft: election won: tally=1 -42:27.398 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13756 [Leader]" -42:27.398 [INFO] TestAgent.server: cluster leadership acquired -42:27.398 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13756 -42:27.400 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:27.402 [INFO] TestAgent.server: New leader elected: payload=Node-55a3415e-9a0f-b062-ac2e-7b959091087c -42:27.402 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:27.402 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:27.402 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:27.402 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:27.402 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:27.402 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-55a3415e-9a0f-b062-ac2e-7b959091087c -42:27.402 [INFO] TestAgent.server: member joined, marking health alive: member=Node-55a3415e-9a0f-b062-ac2e-7b959091087c -42:27.404 [INFO] TestAgent.server: federation state anti-entropy synced -42:27.487 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:27.487 [INFO] TestAgent: Synced node info -42:27.576 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=A class=IN latency=276.74µs client=127.0.0.1:46027 client_network=udp -42:27.576 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=AAAA class=IN latency=191.96µs client=127.0.0.1:41596 client_network=udp -42:27.577 [DEBUG] TestAgent.dns: request served from client: name=d40de0ba-d881-bcd2-544d-3f5e0d822bf2.query.consul. type=A class=IN latency=157.9µs client=127.0.0.1:38561 client_network=udp -42:27.577 [DEBUG] TestAgent.dns: request served from client: name=d40de0ba-d881-bcd2-544d-3f5e0d822bf2.query.consul. type=AAAA class=IN latency=133.26µs client=127.0.0.1:50100 client_network=udp -42:27.577 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=A class=IN latency=114.86µs client=127.0.0.1:55640 client_network=udp -42:27.578 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=AAAA class=IN latency=107.48µs client=127.0.0.1:36361 client_network=udp -42:27.586 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=A class=IN latency=330.94µs client=127.0.0.1:58169 client_network=udp -42:27.587 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=AAAA class=IN latency=211.8µs client=127.0.0.1:57322 client_network=udp -42:27.587 [DEBUG] TestAgent.dns: request served from client: name=d40de0ba-d881-bcd2-544d-3f5e0d822bf2.query.consul. type=A class=IN latency=183.88µs client=127.0.0.1:53526 client_network=udp -42:27.588 [DEBUG] TestAgent.dns: request served from client: name=d40de0ba-d881-bcd2-544d-3f5e0d822bf2.query.consul. type=AAAA class=IN latency=164.78µs client=127.0.0.1:60323 client_network=udp -42:27.588 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=A class=IN latency=112.74µs client=127.0.0.1:47232 client_network=udp -42:27.588 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=AAAA class=IN latency=104.96µs client=127.0.0.1:53288 client_network=udp -42:27.590 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=A class=IN latency=302.72µs client=127.0.0.1:53921 client_network=udp -42:27.597 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=AAAA class=IN latency=239.88µs client=127.0.0.1:49122 client_network=udp -42:27.597 [DEBUG] TestAgent.dns: request served from client: name=d40de0ba-d881-bcd2-544d-3f5e0d822bf2.query.consul. type=A class=IN latency=167.26µs client=127.0.0.1:57824 client_network=udp -42:27.598 [DEBUG] TestAgent.dns: request served from client: name=d40de0ba-d881-bcd2-544d-3f5e0d822bf2.query.consul. type=AAAA class=IN latency=200.44µs client=127.0.0.1:55075 client_network=udp -42:27.598 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=A class=IN latency=126.48µs client=127.0.0.1:56564 client_network=udp -42:27.599 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=AAAA class=IN latency=113.46µs client=127.0.0.1:33314 client_network=udp -42:27.600 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=A class=IN latency=241.28µs client=127.0.0.1:46147 client_network=udp -42:27.601 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=AAAA class=IN latency=201.9µs client=127.0.0.1:37257 client_network=udp -42:27.601 [DEBUG] TestAgent.dns: request served from client: name=d40de0ba-d881-bcd2-544d-3f5e0d822bf2.query.consul. type=A class=IN latency=179.6µs client=127.0.0.1:55169 client_network=udp -42:27.603 [DEBUG] TestAgent.dns: request served from client: name=d40de0ba-d881-bcd2-544d-3f5e0d822bf2.query.consul. type=AAAA class=IN latency=190.42µs client=127.0.0.1:55963 client_network=udp -42:27.604 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=A class=IN latency=105.94µs client=127.0.0.1:44147 client_network=udp -42:27.604 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=AAAA class=IN latency=91.24µs client=127.0.0.1:45189 client_network=udp -42:27.604 [INFO] TestAgent: Requesting shutdown -42:27.604 [INFO] TestAgent.server: shutting down server -42:27.604 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:27.604 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:27.604 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:27.604 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:27.604 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:27.604 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:27.604 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:27.605 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:27.605 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:27.605 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:27.605 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:27.605Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:27.605Z [INFO] agent.router.manager: shutting down -42:27.605 [INFO] TestAgent: consul server down -42:27.605 [INFO] TestAgent: shutdown complete -42:27.605 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13751 network=tcp -42:27.605 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13751 network=udp -42:27.605 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13752 network=tcp -42:28.105 [INFO] TestAgent: Waiting for endpoints to shut down -42:28.105 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_Lookup_TaggedIPAddresses (0.83s) - --- PASS: TestDNS_Lookup_TaggedIPAddresses/simple-ipv4 (0.00s) - --- PASS: TestDNS_Lookup_TaggedIPAddresses/simple-ipv6 (0.01s) - --- PASS: TestDNS_Lookup_TaggedIPAddresses/ipv4-with-tagged-ipv6 (0.01s) - --- PASS: TestDNS_Lookup_TaggedIPAddresses/ipv6-with-tagged-ipv4 (0.01s) -=== CONT TestDNS_NSRecords_IPV6 -[WARN] freeport: 4 out of 25 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:27.690Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:27.712Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:27.715 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a56e99a8-8a3c-081a-4a6c-2975fb1e0608 Address:127.0.0.1:13770}]" -42:27.715 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13770 [Follower]" leader= -42:27.716 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a56e99a8-8a3c-081a-4a6c-2975fb1e0608.dc1 127.0.0.1 -42:27.717 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a56e99a8-8a3c-081a-4a6c-2975fb1e0608 127.0.0.1 -2022-09-15T21:42:27.717Z [INFO] agent.router: Initializing LAN area manager -42:27.717 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13765 network=udp -42:27.717 [INFO] TestAgent.server: Adding LAN server: server="Node-a56e99a8-8a3c-081a-4a6c-2975fb1e0608 (Addr: tcp/127.0.0.1:13770) (DC: dc1)" -42:27.717 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a56e99a8-8a3c-081a-4a6c-2975fb1e0608.dc1 area=wan -42:27.718 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13765 network=tcp -42:27.719 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13766 network=tcp -42:27.719 [INFO] TestAgent: started state syncer -42:27.719 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13757 network=tcp -42:27.782 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:27.782 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13770 [Candidate]" term=2 -42:27.782 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:27.783 [DEBUG] TestAgent.server.raft: vote granted: from=a56e99a8-8a3c-081a-4a6c-2975fb1e0608 term=2 tally=1 -42:27.783 [INFO] TestAgent.server.raft: election won: tally=1 -42:27.783 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13770 [Leader]" -42:27.783 [INFO] TestAgent.server: cluster leadership acquired -42:27.784 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13770 -42:27.784 [INFO] TestAgent.server: New leader elected: payload=Node-a56e99a8-8a3c-081a-4a6c-2975fb1e0608 -42:27.843 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:27.849 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:27.850 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:27.850 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:27.850 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:27.850 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:27.850 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a56e99a8-8a3c-081a-4a6c-2975fb1e0608 -42:27.850 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a56e99a8-8a3c-081a-4a6c-2975fb1e0608 -42:27.852 [INFO] TestAgent.server: federation state anti-entropy synced -42:27.978 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:27.978 [INFO] TestAgent: Synced node info -42:27.989 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=304.22µs client=127.0.0.1:49593 client_network=udp -42:27.989 [DEBUG] TestAgent.dns: request served from client: name=5c090741-7196-287e-ace7-8f833fb36268.query.consul. type=SRV class=IN latency=180.62µs client=127.0.0.1:47434 client_network=udp -42:27.989 [INFO] TestAgent: Requesting shutdown -42:27.989 [INFO] TestAgent.server: shutting down server -42:27.989 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:27.989 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:27.989 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:27.989 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:27.989 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:27.990 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:27.990 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:27.990 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -42:27.990 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:27.990 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:27.990 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2022-09-15T21:42:27.990Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:27.990Z [INFO] agent.router.manager: shutting down -42:27.990 [INFO] TestAgent: consul server down -42:27.990 [INFO] TestAgent: shutdown complete -42:27.990 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13765 network=tcp -42:27.990 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13765 network=udp -42:27.990 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13766 network=tcp -42:28.493 [INFO] TestAgent: Waiting for endpoints to shut down -42:28.493 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_ServiceAddressIPV6 (0.82s) -=== CONT TestDNS_NSRecords -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:27.865Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:27.886Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:27.889 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:01375e04-ec7f-dc12-ec7f-4e1aea682076 Address:127.0.0.1:13763}]" -42:27.889 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13763 [Follower]" leader= -42:27.890 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-01375e04-ec7f-dc12-ec7f-4e1aea682076.dc1 127.0.0.1 -42:27.891 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-01375e04-ec7f-dc12-ec7f-4e1aea682076 127.0.0.1 -2022-09-15T21:42:27.891Z [INFO] agent.router: Initializing LAN area manager -42:27.891 [DEBUG] TestAgent.dns: recursor enabled -42:27.892 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13758 network=udp -42:27.892 [INFO] TestAgent.server: Adding LAN server: server="Node-01375e04-ec7f-dc12-ec7f-4e1aea682076 (Addr: tcp/127.0.0.1:13763) (DC: dc1)" -42:27.892 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-01375e04-ec7f-dc12-ec7f-4e1aea682076.dc1 area=wan -42:27.892 [DEBUG] TestAgent.dns: recursor enabled -42:27.892 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13758 network=tcp -42:27.893 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13759 network=tcp -42:27.893 [INFO] TestAgent: started state syncer -42:27.893 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13778 network=tcp -42:27.942 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:27.942 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13763 [Candidate]" term=2 -42:27.942 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:27.942 [DEBUG] TestAgent.server.raft: vote granted: from=01375e04-ec7f-dc12-ec7f-4e1aea682076 term=2 tally=1 -42:27.942 [INFO] TestAgent.server.raft: election won: tally=1 -42:27.942 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13763 [Leader]" -42:27.942 [INFO] TestAgent.server: cluster leadership acquired -42:27.942 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13763 -42:27.943 [INFO] TestAgent.server: New leader elected: payload=Node-01375e04-ec7f-dc12-ec7f-4e1aea682076 -42:27.944 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:27.946 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:27.946 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:27.946 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:27.946 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:27.946 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:27.946 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-01375e04-ec7f-dc12-ec7f-4e1aea682076 -42:27.947 [INFO] TestAgent.server: member joined, marking health alive: member=Node-01375e04-ec7f-dc12-ec7f-4e1aea682076 -42:27.948 [INFO] TestAgent.server: federation state anti-entropy synced -42:28.166 [DEBUG] TestAgent.dns: cname recurse RTT for name: name=www.google.com. rtt=270.28µs -42:28.166 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=733.2µs client=127.0.0.1:51854 client_network=udp -42:28.167 [DEBUG] TestAgent.dns: cname recurse RTT for name: name=www.google.com. rtt=138.32µs -42:28.167 [DEBUG] TestAgent.dns: request served from client: name=76b2d03d-898c-e220-c298-b2ef86520e6a.query.consul. type=SRV class=IN latency=424.94µs client=127.0.0.1:43988 client_network=udp -42:28.167 [INFO] TestAgent: Requesting shutdown -42:28.167 [INFO] TestAgent.server: shutting down server -42:28.167 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:28.167 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:28.167 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:28.167 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:28.167 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:28.167 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:28.168 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:28.168 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:28.168 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:28.168 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:28.168Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:28.168Z [INFO] agent.router.manager: shutting down -42:28.168 [INFO] TestAgent: consul server down -42:28.168 [INFO] TestAgent: shutdown complete -42:28.168 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13758 network=tcp -42:28.168 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13758 network=udp -42:28.168 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13759 network=tcp -42:28.168 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:28.168 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:28.674 [INFO] TestAgent: Waiting for endpoints to shut down -42:28.674 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_ServiceAddress_SRV (0.86s) -=== CONT TestDNS_ExternalServiceToConsulCNAMELookup -2022-09-15T21:42:28.127Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:28.145Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:28.147 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7709640c-7e5d-dde3-71e5-4977f35886c4 Address:[::1]:13798}]" -42:28.148 [INFO] TestAgent.server.raft: entering follower state: follower="Node at [::1]:13798 [Follower]" leader= -42:28.148 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: server1.dc1 ::1 -42:28.149 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: server1 ::1 -2022-09-15T21:42:28.149Z [INFO] agent.router: Initializing LAN area manager -42:28.150 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13793 network=udp -42:28.150 [INFO] TestAgent.server: Adding LAN server: server="server1 (Addr: tcp/[::1]:13798) (DC: dc1)" -42:28.150 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13793 network=tcp -42:28.150 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=server1.dc1 area=wan -42:28.151 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13794 network=tcp -42:28.151 [INFO] TestAgent: started state syncer -42:28.151 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13799 network=tcp -42:28.186 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:28.186 [INFO] TestAgent.server.raft: entering candidate state: node="Node at [::1]:13798 [Candidate]" term=2 -42:28.186 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:28.186 [DEBUG] TestAgent.server.raft: vote granted: from=7709640c-7e5d-dde3-71e5-4977f35886c4 term=2 tally=1 -42:28.186 [INFO] TestAgent.server.raft: election won: tally=1 -42:28.186 [INFO] TestAgent.server.raft: entering leader state: leader="Node at [::1]:13798 [Leader]" -42:28.186 [INFO] TestAgent.server: cluster leadership acquired -42:28.186 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=[::1]:13798 -42:28.187 [INFO] TestAgent.server: New leader elected: payload=server1 -42:28.188 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:28.194 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:28.194 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:28.194 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:28.194 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:28.194 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:28.194 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=server1 -42:28.194 [INFO] TestAgent.server: member joined, marking health alive: member=server1 -42:28.195 [INFO] TestAgent.server: federation state anti-entropy synced -42:28.198 [DEBUG] TestAgent.dns: request served from client: name=server1.node.dc1.consul. type=NS class=IN latency=203.54µs client=127.0.0.1:54094 client_network=udp -42:28.198 [INFO] TestAgent: Requesting shutdown -42:28.198 [INFO] TestAgent.server: shutting down server -42:28.198 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:28.198 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:28.198 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:28.198 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:28.198 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:28.198 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:28.198 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:28.199 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:28.199 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:28.199 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:28.199 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:28.199 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:28.199Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:28.199Z [INFO] agent.router.manager: shutting down -42:28.199 [INFO] TestAgent: consul server down -42:28.199 [INFO] TestAgent: shutdown complete -42:28.199 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13793 network=tcp -42:28.199 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13793 network=udp -42:28.199 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13794 network=tcp -42:28.701 [INFO] TestAgent: Waiting for endpoints to shut down -42:28.701 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_NSRecords_IPV6 (0.59s) -=== CONT TestDNS_ConnectServiceLookup -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:28.085Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:28.127Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:28.130 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:372726f7-4bb8-34dd-d022-345483cd26b0 Address:127.0.0.1:13777}]" -42:28.130 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13777 [Follower]" leader= -42:28.131 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-372726f7-4bb8-34dd-d022-345483cd26b0.dc1 127.0.0.1 -42:28.132 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-372726f7-4bb8-34dd-d022-345483cd26b0 127.0.0.1 -2022-09-15T21:42:28.132Z [INFO] agent.router: Initializing LAN area manager -42:28.132 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13772 network=udp -42:28.132 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-372726f7-4bb8-34dd-d022-345483cd26b0.dc1 area=wan -42:28.132 [INFO] TestAgent.server: Adding LAN server: server="Node-372726f7-4bb8-34dd-d022-345483cd26b0 (Addr: tcp/127.0.0.1:13777) (DC: dc1)" -42:28.133 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13772 network=tcp -42:28.134 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13773 network=tcp -42:28.134 [INFO] TestAgent: started state syncer -42:28.134 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13785 network=tcp -42:28.175 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:28.175 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13777 [Candidate]" term=2 -42:28.175 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:28.175 [DEBUG] TestAgent.server.raft: vote granted: from=372726f7-4bb8-34dd-d022-345483cd26b0 term=2 tally=1 -42:28.175 [INFO] TestAgent.server.raft: election won: tally=1 -42:28.175 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13777 [Leader]" -42:28.175 [INFO] TestAgent.server: cluster leadership acquired -42:28.176 [INFO] TestAgent.server: New leader elected: payload=Node-372726f7-4bb8-34dd-d022-345483cd26b0 -42:28.176 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13777 -42:28.177 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:28.179 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:28.179 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:28.179 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:28.179 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:28.179 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:28.180 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-372726f7-4bb8-34dd-d022-345483cd26b0 -42:28.180 [INFO] TestAgent.server: member joined, marking health alive: member=Node-372726f7-4bb8-34dd-d022-345483cd26b0 -42:28.181 [INFO] TestAgent.server: federation state anti-entropy synced -42:28.278 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=341.22µs client=127.0.0.1:38309 client_network=udp -42:28.280 [DEBUG] TestAgent.dns: request served from client: name=18cd404e-7300-e603-d3ac-eb248dd28ff5.query.consul. type=SRV class=IN latency=245.4µs client=127.0.0.1:34948 client_network=udp -42:28.280 [INFO] TestAgent: Requesting shutdown -42:28.280 [INFO] TestAgent.server: shutting down server -42:28.280 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:28.280 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:28.280 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:28.280 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:28.280 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:28.280 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:28.280 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:28.280 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:28.280 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:28.280 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:28.280 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:28.280 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:28.281Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:28.281Z [INFO] agent.router.manager: shutting down -42:28.281 [INFO] TestAgent: consul server down -42:28.281 [INFO] TestAgent: shutdown complete -42:28.281 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13772 network=tcp -42:28.281 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13772 network=udp -42:28.281 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13773 network=tcp -42:28.781 [INFO] TestAgent: Waiting for endpoints to shut down -42:28.781 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_ServiceAddress_A (0.75s) -=== CONT TestDNS_InifiniteRecursion -2022-09-15T21:42:28.127Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:28.145Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:28.148 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8955291b-6092-a018-467d-37fe7247ee1c Address:127.0.0.1:13791}]" -42:28.148 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13791 [Follower]" leader= -42:28.148 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -42:28.150 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2022-09-15T21:42:28.150Z [INFO] agent.router: Initializing LAN area manager -42:28.150 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13786 network=udp -42:28.150 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:13791) (DC: dc1)" -42:28.150 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -42:28.150 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13786 network=tcp -42:28.151 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13787 network=tcp -42:28.151 [INFO] TestAgent: started state syncer -42:28.151 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13792 network=tcp -42:28.218 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:28.218 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13791 [Candidate]" term=2 -42:28.218 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:28.218 [DEBUG] TestAgent.server.raft: vote granted: from=8955291b-6092-a018-467d-37fe7247ee1c term=2 tally=1 -42:28.218 [INFO] TestAgent.server.raft: election won: tally=1 -42:28.218 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13791 [Leader]" -42:28.218 [INFO] TestAgent.server: cluster leadership acquired -42:28.218 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13791 -42:28.220 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:28.222 [INFO] TestAgent.server: New leader elected: payload=test-node -42:28.222 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:28.222 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:28.222 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:28.222 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:28.222 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:28.222 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -42:28.222 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -42:28.223 [INFO] TestAgent.server: federation state anti-entropy synced -42:28.326 [DEBUG] TestAgent.dns: request served from client: name=alias2.service.consul. type=SRV class=IN latency=603.96µs client=127.0.0.1:54995 client_network=udp -42:28.330 [INFO] TestAgent: Requesting shutdown -42:28.330 [INFO] TestAgent.server: shutting down server -42:28.330 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:28.330 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:28.330 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:28.330 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:28.330 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:28.330 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:28.330 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:28.330 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:28.330 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:28.330 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:28.330 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:28.330 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:28.331Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:28.331Z [INFO] agent.router.manager: shutting down -42:28.331 [INFO] TestAgent: consul server down -42:28.331 [INFO] TestAgent: shutdown complete -42:28.331 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13786 network=tcp -42:28.331 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13786 network=udp -42:28.331 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13787 network=tcp -42:28.831 [INFO] TestAgent: Waiting for endpoints to shut down -42:28.831 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ExternalServiceToConsulCNAMENestedLookup (0.73s) -=== CONT TestDNS_ExternalServiceLookup -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:27.887Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:27.887Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:27.905Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:27.905Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:27.908 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:990e4182-f6ad-031e-664a-c91cc14045ee Address:127.0.0.1:13784}]" -42:27.909 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-990e4182-f6ad-031e-664a-c91cc14045ee.dc2 127.0.0.1 -42:27.910 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13784 [Follower]" leader= -42:27.911 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-990e4182-f6ad-031e-664a-c91cc14045ee 127.0.0.1 -2022-09-15T21:42:27.911Z [INFO] agent.router: Initializing LAN area manager -42:27.911 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13779 network=udp -42:27.911 [INFO] TestAgent.server: Adding LAN server: server="Node-990e4182-f6ad-031e-664a-c91cc14045ee (Addr: tcp/127.0.0.1:13784) (DC: dc2)" -42:27.911 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-990e4182-f6ad-031e-664a-c91cc14045ee.dc2 area=wan -42:27.911 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13779 network=tcp -42:27.912 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13780 network=tcp -42:27.913 [INFO] TestAgent: started state syncer -42:27.913 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13771 network=tcp -42:27.962 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -42:27.962 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:27.962 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13784 [Candidate]" term=2 -42:27.962 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:27.962 [DEBUG] TestAgent.server.raft: vote granted: from=990e4182-f6ad-031e-664a-c91cc14045ee term=2 tally=1 -42:27.962 [INFO] TestAgent.server.raft: election won: tally=1 -42:27.962 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13784 [Leader]" -42:27.962 [INFO] TestAgent.server: cluster leadership acquired -42:27.962 [INFO] TestAgent.server: New leader elected: payload=Node-990e4182-f6ad-031e-664a-c91cc14045ee -42:27.962 [INFO] TestAgent.server: initializing acls -42:27.963 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:27.963 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:27.963 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:27.963 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:27.964 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-990e4182-f6ad-031e-664a-c91cc14045ee -42:27.964 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-990e4182-f6ad-031e-664a-c91cc14045ee.dc2 -42:27.964 [INFO] TestAgent.server: Updating LAN server: server="Node-990e4182-f6ad-031e-664a-c91cc14045ee (Addr: tcp/127.0.0.1:13784) (DC: dc2)" -42:27.964 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-990e4182-f6ad-031e-664a-c91cc14045ee.dc2 area=wan -42:27.965 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:27.968 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:27.968 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:27.968 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:27.968 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:27.968 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:27.968 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-990e4182-f6ad-031e-664a-c91cc14045ee -42:27.968 [INFO] TestAgent.server: member joined, marking health alive: member=Node-990e4182-f6ad-031e-664a-c91cc14045ee -42:27.969 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-990e4182-f6ad-031e-664a-c91cc14045ee -42:27.969 [INFO] TestAgent.server: federation state anti-entropy synced -42:28.020 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:13748] -42:28.020 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:13748 -42:28.022 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c83b09cb-c46b-b0ff-484d-1f588623e040.dc1 127.0.0.1 -42:28.022 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 -42:28.022 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c83b09cb-c46b-b0ff-484d-1f588623e040.dc1 area=wan -42:28.032 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=SRV class=IN latency=307.88µs client=127.0.0.1:54021 client_network=udp -42:28.033 [DEBUG] TestAgent.dns: request served from client: name=91a14020-2fca-9298-f2c6-19cb23e1d1e8.query.dc2.consul. type=SRV class=IN latency=198.22µs client=127.0.0.1:58937 client_network=udp -42:28.033 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=A class=IN latency=245.24µs client=127.0.0.1:36768 client_network=udp -42:28.034 [DEBUG] TestAgent.dns: request served from client: name=91a14020-2fca-9298-f2c6-19cb23e1d1e8.query.dc2.consul. type=A class=IN latency=303.56µs client=127.0.0.1:47210 client_network=udp -42:28.041 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=SRV class=IN latency=434.06µs client=127.0.0.1:47635 client_network=udp -42:28.050 [DEBUG] TestAgent.dns: request served from client: name=91a14020-2fca-9298-f2c6-19cb23e1d1e8.query.dc2.consul. type=SRV class=IN latency=503.48µs client=127.0.0.1:54760 client_network=udp -42:28.051 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=A class=IN latency=363.94µs client=127.0.0.1:51028 client_network=udp -42:28.052 [DEBUG] TestAgent.dns: request served from client: name=91a14020-2fca-9298-f2c6-19cb23e1d1e8.query.dc2.consul. type=A class=IN latency=251.56µs client=127.0.0.1:48710 client_network=udp -42:28.054 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=SRV class=IN latency=442.5µs client=127.0.0.1:52923 client_network=udp -42:28.055 [DEBUG] TestAgent.dns: request served from client: name=91a14020-2fca-9298-f2c6-19cb23e1d1e8.query.dc2.consul. type=SRV class=IN latency=269.32µs client=127.0.0.1:45732 client_network=udp -42:28.056 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=A class=IN latency=295µs client=127.0.0.1:38621 client_network=udp -42:28.056 [DEBUG] TestAgent.dns: request served from client: name=91a14020-2fca-9298-f2c6-19cb23e1d1e8.query.dc2.consul. type=A class=IN latency=236.26µs client=127.0.0.1:49915 client_network=udp -42:28.075 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=SRV class=IN latency=430.72µs client=127.0.0.1:45255 client_network=udp -42:28.076 [DEBUG] TestAgent.dns: request served from client: name=91a14020-2fca-9298-f2c6-19cb23e1d1e8.query.dc2.consul. type=SRV class=IN latency=251.82µs client=127.0.0.1:54926 client_network=udp -42:28.077 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=A class=IN latency=288.66µs client=127.0.0.1:57933 client_network=udp -42:28.077 [DEBUG] TestAgent.dns: request served from client: name=91a14020-2fca-9298-f2c6-19cb23e1d1e8.query.dc2.consul. type=A class=IN latency=246.24µs client=127.0.0.1:53283 client_network=udp -42:28.108 [INFO] TestAgent: Requesting shutdown -42:28.108 [INFO] TestAgent.server: shutting down server -42:28.108 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:28.108 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:28.108 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:28.108 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:28.108 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:28.108 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:28.108 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:28.109 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:28.109 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:28.109 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:28.109 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:28.109 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:28.109 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:28.109 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:28.109 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:28.109 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:28.109Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:28.109Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:28.109Z [INFO] agent.router.manager: shutting down -42:28.109 [INFO] TestAgent: consul server down -42:28.109 [INFO] TestAgent: shutdown complete -42:28.109 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13779 network=tcp -42:28.109 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13779 network=udp -42:28.109 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13780 network=tcp -42:28.610 [INFO] TestAgent: Waiting for endpoints to shut down -42:28.610 [INFO] TestAgent: Endpoints down -2022-09-15T21:42:27.562Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:27.568Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:27.588Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:27.588Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:27.591 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c83b09cb-c46b-b0ff-484d-1f588623e040 Address:127.0.0.1:13749}]" -42:27.592 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c83b09cb-c46b-b0ff-484d-1f588623e040.dc1 127.0.0.1 -42:27.593 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c83b09cb-c46b-b0ff-484d-1f588623e040 127.0.0.1 -2022-09-15T21:42:27.593Z [INFO] agent.router: Initializing LAN area manager -42:27.593 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13744 network=udp -42:27.593 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13749 [Follower]" leader= -42:27.594 [INFO] TestAgent.server: Adding LAN server: server="Node-c83b09cb-c46b-b0ff-484d-1f588623e040 (Addr: tcp/127.0.0.1:13749) (DC: dc1)" -42:27.594 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c83b09cb-c46b-b0ff-484d-1f588623e040.dc1 area=wan -42:27.594 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13744 network=tcp -42:27.596 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13745 network=tcp -42:27.596 [INFO] TestAgent: started state syncer -42:27.596 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13764 network=tcp -42:27.645 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -42:27.655 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:27.655 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13749 [Candidate]" term=2 -42:27.655 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:27.655 [DEBUG] TestAgent.server.raft: vote granted: from=c83b09cb-c46b-b0ff-484d-1f588623e040 term=2 tally=1 -42:27.655 [INFO] TestAgent.server.raft: election won: tally=1 -42:27.655 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13749 [Leader]" -42:27.655 [INFO] TestAgent.server: cluster leadership acquired -42:27.655 [INFO] TestAgent.server: initializing acls -42:27.656 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:27.656 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:27.656 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:27.656 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:27.657 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-c83b09cb-c46b-b0ff-484d-1f588623e040 -42:27.657 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-c83b09cb-c46b-b0ff-484d-1f588623e040.dc1 -42:27.658 [INFO] TestAgent.server: New leader elected: payload=Node-c83b09cb-c46b-b0ff-484d-1f588623e040 -42:27.658 [INFO] TestAgent.server: Updating LAN server: server="Node-c83b09cb-c46b-b0ff-484d-1f588623e040 (Addr: tcp/127.0.0.1:13749) (DC: dc1)" -42:27.658 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-c83b09cb-c46b-b0ff-484d-1f588623e040.dc1 area=wan -42:27.658 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:27.660 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:27.660 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:27.660 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:27.660 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:27.660 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:27.660 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c83b09cb-c46b-b0ff-484d-1f588623e040 -42:27.660 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c83b09cb-c46b-b0ff-484d-1f588623e040 -42:27.661 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c83b09cb-c46b-b0ff-484d-1f588623e040 -42:27.662 [INFO] TestAgent.server: federation state anti-entropy synced -42:28.021 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:51132 -42:28.021 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-990e4182-f6ad-031e-664a-c91cc14045ee.dc2 127.0.0.1 -42:28.022 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-990e4182-f6ad-031e-664a-c91cc14045ee.dc2 area=wan -42:28.023 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:28.024 [INFO] TestAgent: Synced node info -42:28.024 [DEBUG] TestAgent: Node info in sync -42:28.026 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=SRV class=IN latency=1.74348ms client=127.0.0.1:47038 client_network=udp -42:28.027 [DEBUG] TestAgent.dns: request served from client: name=91a14020-2fca-9298-f2c6-19cb23e1d1e8.query.dc2.consul. type=SRV class=IN latency=840.5µs client=127.0.0.1:42183 client_network=udp -42:28.028 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=A class=IN latency=721.88µs client=127.0.0.1:38416 client_network=udp -42:28.029 [DEBUG] TestAgent.dns: request served from client: name=91a14020-2fca-9298-f2c6-19cb23e1d1e8.query.dc2.consul. type=A class=IN latency=706.22µs client=127.0.0.1:40815 client_network=udp -42:28.080 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=SRV class=IN latency=1.31344ms client=127.0.0.1:48318 client_network=udp -42:28.086 [DEBUG] TestAgent.dns: request served from client: name=91a14020-2fca-9298-f2c6-19cb23e1d1e8.query.dc2.consul. type=SRV class=IN latency=5.0222ms client=127.0.0.1:47454 client_network=udp -42:28.087 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=A class=IN latency=939.32µs client=127.0.0.1:57085 client_network=udp -42:28.089 [DEBUG] TestAgent.dns: request served from client: name=91a14020-2fca-9298-f2c6-19cb23e1d1e8.query.dc2.consul. type=A class=IN latency=1.17082ms client=127.0.0.1:41628 client_network=udp -42:28.098 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=SRV class=IN latency=1.2854ms client=127.0.0.1:57786 client_network=udp -42:28.099 [DEBUG] TestAgent.dns: request served from client: name=91a14020-2fca-9298-f2c6-19cb23e1d1e8.query.dc2.consul. type=SRV class=IN latency=775.18µs client=127.0.0.1:60570 client_network=udp -42:28.100 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=A class=IN latency=695.22µs client=127.0.0.1:38686 client_network=udp -42:28.101 [DEBUG] TestAgent.dns: request served from client: name=91a14020-2fca-9298-f2c6-19cb23e1d1e8.query.dc2.consul. type=A class=IN latency=687µs client=127.0.0.1:50612 client_network=udp -42:28.104 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=SRV class=IN latency=835.18µs client=127.0.0.1:34447 client_network=udp -42:28.105 [DEBUG] TestAgent.dns: request served from client: name=91a14020-2fca-9298-f2c6-19cb23e1d1e8.query.dc2.consul. type=SRV class=IN latency=746.2µs client=127.0.0.1:34554 client_network=udp -42:28.107 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=A class=IN latency=1.0172ms client=127.0.0.1:42657 client_network=udp -42:28.108 [DEBUG] TestAgent.dns: request served from client: name=91a14020-2fca-9298-f2c6-19cb23e1d1e8.query.dc2.consul. type=A class=IN latency=717.6µs client=127.0.0.1:37917 client_network=udp -42:28.610 [INFO] TestAgent: Requesting shutdown -42:28.610 [INFO] TestAgent.server: shutting down server -42:28.610 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:28.610 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:28.610 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:28.610 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:28.610 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:28.610 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:28.610 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:28.610 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:28.610 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:28.610 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:28.610 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:28.610 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:28.610 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:28.611Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:28.611Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:28.611Z [INFO] agent.router.manager: shutting down -42:28.611 [INFO] TestAgent: consul server down -42:28.611 [INFO] TestAgent: shutdown complete -42:28.611 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13744 network=tcp -42:28.611 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13744 network=udp -42:28.611 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13745 network=tcp -42:28.610 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:28.630 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:29.112 [INFO] TestAgent: Waiting for endpoints to shut down -42:29.112 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_WanTranslation (1.61s) - --- PASS: TestDNS_ServiceLookup_WanTranslation/service-wan-from-dc1 (0.01s) - --- PASS: TestDNS_ServiceLookup_WanTranslation/node-addr-from-dc2 (0.01s) - --- PASS: TestDNS_ServiceLookup_WanTranslation/node-wan-from-dc2 (0.02s) - --- PASS: TestDNS_ServiceLookup_WanTranslation/service-addr-from-dc2 (0.00s) - --- PASS: TestDNS_ServiceLookup_WanTranslation/service-wan-from-dc2 (0.02s) - --- PASS: TestDNS_ServiceLookup_WanTranslation/node-addr-from-dc1 (0.01s) - --- PASS: TestDNS_ServiceLookup_WanTranslation/node-wan-from-dc1 (0.01s) - --- PASS: TestDNS_ServiceLookup_WanTranslation/service-addr-from-dc1 (0.01s) -=== CONT TestDNS_IngressServiceLookup -2022-09-15T21:42:28.511Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:28.536Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:28.540 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cf32e7c3-1459-6aa4-7e27-2ba666ee19f4 Address:127.0.0.1:13805}]" -42:28.541 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13805 [Follower]" leader= -42:28.542 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: server1.dc1 127.0.0.1 -42:28.543 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: server1 127.0.0.1 -2022-09-15T21:42:28.543Z [INFO] agent.router: Initializing LAN area manager -42:28.543 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13800 network=udp -42:28.544 [INFO] TestAgent.server: Adding LAN server: server="server1 (Addr: tcp/127.0.0.1:13805) (DC: dc1)" -42:28.544 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=server1.dc1 area=wan -42:28.544 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13800 network=tcp -42:28.545 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13801 network=tcp -42:28.545 [INFO] TestAgent: started state syncer -42:28.545 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13806 network=tcp -42:28.581 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:28.581 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13805 [Candidate]" term=2 -42:28.582 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:28.582 [DEBUG] TestAgent.server.raft: vote granted: from=cf32e7c3-1459-6aa4-7e27-2ba666ee19f4 term=2 tally=1 -42:28.582 [INFO] TestAgent.server.raft: election won: tally=1 -42:28.582 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13805 [Leader]" -42:28.582 [INFO] TestAgent.server: cluster leadership acquired -42:28.583 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13805 -42:28.585 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:28.586 [INFO] TestAgent.server: New leader elected: payload=server1 -42:28.588 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:28.588 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:28.588 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:28.588 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:28.588 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:28.588 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=server1 -42:28.588 [INFO] TestAgent.server: member joined, marking health alive: member=server1 -42:28.590 [INFO] TestAgent.server: federation state anti-entropy synced -42:28.714 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:28.714 [INFO] TestAgent: Synced node info -42:28.714 [DEBUG] TestAgent: Node info in sync -42:28.717 [DEBUG] TestAgent.dns: request served from client: name=something.node.consul. type=NS class=IN latency=296.28µs client=127.0.0.1:53479 client_network=udp -42:28.717 [INFO] TestAgent: Requesting shutdown -42:28.718 [INFO] TestAgent.server: shutting down server -42:28.718 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:28.718 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:28.718 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:28.718 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:28.718 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:28.718 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:28.718 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:28.718 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:28.718 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:28.718 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:28.718 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:28.718Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:28.718Z [INFO] agent.router.manager: shutting down -42:28.719 [INFO] TestAgent: consul server down -42:28.719 [INFO] TestAgent: shutdown complete -42:28.719 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13800 network=tcp -42:28.719 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13800 network=udp -42:28.719 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13801 network=tcp -42:29.222 [INFO] TestAgent: Waiting for endpoints to shut down -42:29.222 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_NSRecords (0.73s) -=== CONT TestDNS_ServiceLookupWithInternalServiceAddress -[WARN] freeport: 4 out of 25 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:28.695Z [WARN] agent: Node name "test node" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. -2022-09-15T21:42:28.695Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:28.725Z [WARN] agent.auto_config: Node name "test node" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. -2022-09-15T21:42:28.725Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:28.728 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c4308ed2-5a08-f3ea-c75d-a70927d02187 Address:127.0.0.1:13812}]" -42:28.729 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test node.dc1 127.0.0.1 -42:28.734 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13812 [Follower]" leader= -42:28.742 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test node 127.0.0.1 -2022-09-15T21:42:28.742Z [INFO] agent.router: Initializing LAN area manager -42:28.742 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13807 network=udp -42:28.743 [INFO] TestAgent.server: Adding LAN server: server="test node (Addr: tcp/127.0.0.1:13812) (DC: dc1)" -42:28.743 [INFO] TestAgent.server: Handled event for server in area: event=member-join server="test node.dc1" area=wan -42:28.743 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13807 network=tcp -42:28.744 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13808 network=tcp -42:28.744 [INFO] TestAgent: started state syncer -42:28.744 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13813 network=tcp -42:28.786 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:28.786 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13812 [Candidate]" term=2 -42:28.786 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:28.786 [DEBUG] TestAgent.server.raft: vote granted: from=c4308ed2-5a08-f3ea-c75d-a70927d02187 term=2 tally=1 -42:28.786 [INFO] TestAgent.server.raft: election won: tally=1 -42:28.786 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13812 [Leader]" -42:28.786 [INFO] TestAgent.server: cluster leadership acquired -42:28.786 [INFO] TestAgent.server: New leader elected: payload="test node" -42:28.786 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13812 -42:28.788 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:28.790 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:28.790 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:28.790 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:28.790 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:28.790 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:28.790 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node="test node" -42:28.790 [INFO] TestAgent.server: member joined, marking health alive: member="test node" -42:28.791 [INFO] TestAgent.server: federation state anti-entropy synced -42:28.842 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:28.843 [INFO] TestAgent: Synced node info -42:28.869 [DEBUG] TestAgent.dns: request served from client: name=alias.service.consul. type=SRV class=IN latency=509.08µs client=127.0.0.1:35795 client_network=udp -42:28.878 [DEBUG] TestAgent.dns: request served from client: name=alias.service.CoNsUl. type=SRV class=IN latency=9.15256ms client=127.0.0.1:40623 client_network=udp -42:28.879 [INFO] TestAgent: Requesting shutdown -42:28.879 [INFO] TestAgent.server: shutting down server -42:28.879 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:28.879 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:28.879 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:28.879 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:28.879 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:28.879 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:28.879 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:28.879 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:28.879 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:28.879 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:28.879 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:28.880Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:28.880Z [INFO] agent.router.manager: shutting down -42:28.880 [INFO] TestAgent: consul server down -42:28.880 [INFO] TestAgent: shutdown complete -42:28.880 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13807 network=tcp -42:28.880 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13807 network=udp -42:28.880 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13808 network=tcp -42:29.380 [INFO] TestAgent: Waiting for endpoints to shut down -42:29.380 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ExternalServiceToConsulCNAMELookup (0.71s) -=== CONT TestDNS_ServiceLookup -2022-09-15T21:42:28.799Z [WARN] agent: Node name "test node" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. -2022-09-15T21:42:28.799Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:28.816Z [WARN] agent.auto_config: Node name "test node" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. -2022-09-15T21:42:28.816Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:28.819 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3a4b6c1b-590d-7bae-8049-522ca776ad93 Address:127.0.0.1:13826}]" -42:28.819 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13826 [Follower]" leader= -42:28.820 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test node.dc1 127.0.0.1 -42:28.821 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test node 127.0.0.1 -2022-09-15T21:42:28.821Z [INFO] agent.router: Initializing LAN area manager -42:28.821 [INFO] TestAgent.server: Adding LAN server: server="test node (Addr: tcp/127.0.0.1:13826) (DC: dc1)" -42:28.821 [INFO] TestAgent.server: Handled event for server in area: event=member-join server="test node.dc1" area=wan -42:28.821 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13821 network=tcp -42:28.821 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13821 network=udp -42:28.822 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13822 network=tcp -42:28.822 [INFO] TestAgent: started state syncer -42:28.822 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13834 network=tcp -42:28.866 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:28.866 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13826 [Candidate]" term=2 -42:28.866 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:28.866 [DEBUG] TestAgent.server.raft: vote granted: from=3a4b6c1b-590d-7bae-8049-522ca776ad93 term=2 tally=1 -42:28.866 [INFO] TestAgent.server.raft: election won: tally=1 -42:28.866 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13826 [Leader]" -42:28.866 [INFO] TestAgent.server: cluster leadership acquired -42:28.867 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13826 -42:28.869 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:28.870 [INFO] TestAgent.server: New leader elected: payload="test node" -42:28.872 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:28.872 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:28.872 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:28.872 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:28.872 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:28.873 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node="test node" -42:28.873 [INFO] TestAgent.server: member joined, marking health alive: member="test node" -42:28.878 [INFO] TestAgent.server: federation state anti-entropy synced -42:28.954 [ERROR] TestAgent.dns: Infinite recursion detected for name, won't perform any CNAME resolution.: name=web.service.consul. -42:28.954 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=A class=IN latency=605.32µs client=127.0.0.1:50056 client_network=udp -42:28.954 [INFO] TestAgent: Requesting shutdown -42:28.954 [INFO] TestAgent.server: shutting down server -42:28.954 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:28.954 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:28.954 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:28.954 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:28.954 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:28.955 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:28.955 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:28.955 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:28.955 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:28.955 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:28.955 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:28.955 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:28.955Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:28.955Z [INFO] agent.router.manager: shutting down -42:28.955 [INFO] TestAgent: consul server down -42:28.955 [INFO] TestAgent: shutdown complete -42:28.955 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13821 network=tcp -42:28.955 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13821 network=udp -42:28.955 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13822 network=tcp -42:29.458 [INFO] TestAgent: Waiting for endpoints to shut down -42:29.458 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_InifiniteRecursion (0.68s) -=== CONT TestDNS_ServiceLookupMultiAddrNoCNAME -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_IngressServiceLookup/api.ingress.consul. -=== RUN TestDNS_IngressServiceLookup/api.ingress.dc1.consul. -=== RUN TestDNS_IngressServiceLookup/db.ingress.consul. -=== RUN TestDNS_IngressServiceLookup/db.ingress.dc1.consul. -2022-09-15T21:42:28.902Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:28.918Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:28.921 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1d7bd4b3-a72b-3f38-ff24-1e9d1d8bc54a Address:127.0.0.1:13840}]" -42:28.921 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13840 [Follower]" leader= -42:28.922 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1d7bd4b3-a72b-3f38-ff24-1e9d1d8bc54a.dc1 127.0.0.1 -42:28.923 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1d7bd4b3-a72b-3f38-ff24-1e9d1d8bc54a 127.0.0.1 -2022-09-15T21:42:28.923Z [INFO] agent.router: Initializing LAN area manager -42:28.923 [INFO] TestAgent.server: Adding LAN server: server="Node-1d7bd4b3-a72b-3f38-ff24-1e9d1d8bc54a (Addr: tcp/127.0.0.1:13840) (DC: dc1)" -42:28.923 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1d7bd4b3-a72b-3f38-ff24-1e9d1d8bc54a.dc1 area=wan -42:28.923 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13835 network=udp -42:28.924 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13835 network=tcp -42:28.924 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13836 network=tcp -42:28.925 [INFO] TestAgent: started state syncer -42:28.925 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13827 network=tcp -42:28.986 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:28.986 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13840 [Candidate]" term=2 -42:28.987 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:28.987 [DEBUG] TestAgent.server.raft: vote granted: from=1d7bd4b3-a72b-3f38-ff24-1e9d1d8bc54a term=2 tally=1 -42:28.987 [INFO] TestAgent.server.raft: election won: tally=1 -42:28.987 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13840 [Leader]" -42:28.987 [INFO] TestAgent.server: cluster leadership acquired -42:28.987 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13840 -42:28.987 [INFO] TestAgent.server: New leader elected: payload=Node-1d7bd4b3-a72b-3f38-ff24-1e9d1d8bc54a -42:28.988 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:28.991 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:28.991 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:28.991 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:28.991 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:28.991 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:28.991 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1d7bd4b3-a72b-3f38-ff24-1e9d1d8bc54a -42:28.991 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1d7bd4b3-a72b-3f38-ff24-1e9d1d8bc54a -42:28.992 [INFO] TestAgent.server: federation state anti-entropy synced -42:29.089 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=272.06µs client=127.0.0.1:55469 client_network=udp -42:29.089 [INFO] TestAgent: Requesting shutdown -42:29.089 [INFO] TestAgent.server: shutting down server -42:29.089 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:29.089 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:29.089 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:29.089 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:29.089 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:29.089 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:29.089 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:29.089 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:29.089 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:29.089 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:29.089 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:29.089 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:29.090Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:29.090Z [INFO] agent.router.manager: shutting down -42:29.090 [INFO] TestAgent: consul server down -42:29.090 [INFO] TestAgent: shutdown complete -42:29.090 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13835 network=tcp -42:29.090 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13835 network=udp -42:29.090 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13836 network=tcp -42:29.590 [INFO] TestAgent: Waiting for endpoints to shut down -42:29.590 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ExternalServiceLookup (0.75s) -=== CONT TestDNS_ServiceLookupPreferNoCNAME -2022-09-15T21:42:28.732Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:28.752Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:28.755 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9011c58e-cb7d-6834-2951-5b189fac93d5 Address:127.0.0.1:13819}]" -42:28.755 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13819 [Follower]" leader= -42:28.756 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9011c58e-cb7d-6834-2951-5b189fac93d5.dc1 127.0.0.1 -42:28.757 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9011c58e-cb7d-6834-2951-5b189fac93d5 127.0.0.1 -2022-09-15T21:42:28.757Z [INFO] agent.router: Initializing LAN area manager -42:28.758 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13814 network=udp -42:28.758 [INFO] TestAgent.server: Adding LAN server: server="Node-9011c58e-cb7d-6834-2951-5b189fac93d5 (Addr: tcp/127.0.0.1:13819) (DC: dc1)" -42:28.758 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9011c58e-cb7d-6834-2951-5b189fac93d5.dc1 area=wan -42:28.758 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13814 network=tcp -42:28.759 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13815 network=tcp -42:28.759 [INFO] TestAgent: started state syncer -42:28.759 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13820 network=tcp -42:28.793 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:28.793 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13819 [Candidate]" term=2 -42:28.793 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:28.793 [DEBUG] TestAgent.server.raft: vote granted: from=9011c58e-cb7d-6834-2951-5b189fac93d5 term=2 tally=1 -42:28.793 [INFO] TestAgent.server.raft: election won: tally=1 -42:28.793 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13819 [Leader]" -42:28.794 [INFO] TestAgent.server: cluster leadership acquired -42:28.794 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13819 -42:28.795 [INFO] TestAgent.server: New leader elected: payload=Node-9011c58e-cb7d-6834-2951-5b189fac93d5 -42:28.795 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:28.797 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:28.797 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:28.797 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:28.797 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:28.797 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:28.798 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9011c58e-cb7d-6834-2951-5b189fac93d5 -42:28.798 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9011c58e-cb7d-6834-2951-5b189fac93d5 -42:28.799 [INFO] TestAgent.server: federation state anti-entropy synced -42:28.902 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:28.903 [INFO] TestAgent: Synced node info -42:28.903 [DEBUG] TestAgent: Node info in sync -42:29.120 [DEBUG] TestAgent.dns: request served from client: name=db.connect.consul. type=SRV class=IN latency=313.88µs client=127.0.0.1:59905 client_network=udp -42:29.120 [INFO] TestAgent: Requesting shutdown -42:29.120 [INFO] TestAgent.server: shutting down server -42:29.120 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:29.120 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:29.120 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:29.120 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:29.120 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:29.120 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:29.120 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:29.121 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:29.121 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:29.121 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:29.121 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:29.121Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:29.121Z [INFO] agent.router.manager: shutting down -42:29.121 [INFO] TestAgent: consul server down -42:29.121 [INFO] TestAgent: shutdown complete -42:29.121 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13814 network=tcp -42:29.121 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13814 network=udp -42:29.121 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13815 network=tcp -42:29.621 [INFO] TestAgent: Waiting for endpoints to shut down -42:29.621 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ConnectServiceLookup (0.92s) -=== CONT TestDNS_ServiceLookupNoMultiCNAME -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:25.950Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:25.970Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:25.973 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3e65f97c-b14b-c08b-26b2-ba7d7637da3b Address:127.0.0.1:13679}]" -42:25.974 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13679 [Follower]" leader= -42:25.974 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3e65f97c-b14b-c08b-26b2-ba7d7637da3b.dc1 127.0.0.1 -42:25.975 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3e65f97c-b14b-c08b-26b2-ba7d7637da3b 127.0.0.1 -2022-09-15T21:42:25.975Z [INFO] agent.router: Initializing LAN area manager -42:25.976 [DEBUG] TestAgent.dns: recursor enabled -42:25.976 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13674 network=udp -42:25.976 [INFO] TestAgent.server: Adding LAN server: server="Node-3e65f97c-b14b-c08b-26b2-ba7d7637da3b (Addr: tcp/127.0.0.1:13679) (DC: dc1)" -42:25.976 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3e65f97c-b14b-c08b-26b2-ba7d7637da3b.dc1 area=wan -42:25.976 [DEBUG] TestAgent.dns: recursor enabled -42:25.976 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13674 network=tcp -42:25.977 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13675 network=tcp -42:25.977 [INFO] TestAgent: started state syncer -42:25.978 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13680 network=tcp -42:26.027 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:26.027 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13679 [Candidate]" term=2 -42:26.027 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:26.027 [DEBUG] TestAgent.server.raft: vote granted: from=3e65f97c-b14b-c08b-26b2-ba7d7637da3b term=2 tally=1 -42:26.027 [INFO] TestAgent.server.raft: election won: tally=1 -42:26.027 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13679 [Leader]" -42:26.027 [INFO] TestAgent.server: cluster leadership acquired -42:26.027 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13679 -42:26.029 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:26.030 [INFO] TestAgent.server: New leader elected: payload=Node-3e65f97c-b14b-c08b-26b2-ba7d7637da3b -42:26.032 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:26.032 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:26.032 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:26.032 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:26.032 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:26.032 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3e65f97c-b14b-c08b-26b2-ba7d7637da3b -42:26.032 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3e65f97c-b14b-c08b-26b2-ba7d7637da3b -42:26.033 [INFO] TestAgent.server: federation state anti-entropy synced -42:26.086 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:26.087 [INFO] TestAgent: Synced node info -42:28.678 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:28.678 [DEBUG] TestAgent: Node info in sync -42:28.678 [DEBUG] TestAgent: Node info in sync -42:29.366 [ERROR] TestAgent.dns: recurse failed: error="read udp 127.0.0.1:38896->127.0.0.1:60155: i/o timeout" -42:29.366 [ERROR] TestAgent.dns: all resolvers failed for question from client: question="{apple.com. 255 1}" client=127.0.0.1:39377 client_network=udp -42:29.366 [DEBUG] TestAgent.dns: request served from client: question="{apple.com. 255 1}" network=udp latency=3.00158772s client=127.0.0.1:39377 client_network=udp -42:29.370 [INFO] TestAgent: Requesting shutdown -42:29.370 [INFO] TestAgent.server: shutting down server -42:29.370 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:29.370 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:29.370 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:29.370 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:29.370 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:29.370 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:29.370 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:29.370 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:29.370 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:29.370 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:29.370 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:29.370Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:29.370Z [INFO] agent.router.manager: shutting down -42:29.371 [INFO] TestAgent: consul server down -42:29.371 [INFO] TestAgent: shutdown complete -42:29.371 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13674 network=tcp -42:29.371 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13674 network=udp -42:29.371 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13675 network=tcp -42:29.871 [INFO] TestAgent: Waiting for endpoints to shut down -42:29.871 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_RecursorTimeout (3.94s) -=== CONT TestDNS_ServiceReverseLookupNodeAddress -2022-09-15T21:42:29.243Z [WARN] agent: Node name "my.test-node" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. -2022-09-15T21:42:29.243Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:29.261Z [WARN] agent.auto_config: Node name "my.test-node" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. -2022-09-15T21:42:29.261Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:29.264 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0ababcea-e1bf-ed17-0654-147b5a4b9e0e Address:127.0.0.1:13854}]" -42:29.264 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13854 [Follower]" leader= -42:29.265 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: my.test-node.dc1 127.0.0.1 -42:29.274 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: my.test-node 127.0.0.1 -2022-09-15T21:42:29.274Z [INFO] agent.router: Initializing LAN area manager -42:29.274 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13849 network=udp -42:29.275 [INFO] TestAgent.server: Adding LAN server: server="my.test-node (Addr: tcp/127.0.0.1:13854) (DC: dc1)" -42:29.275 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=my.test-node.dc1 area=wan -42:29.275 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13849 network=tcp -42:29.276 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13850 network=tcp -42:29.276 [INFO] TestAgent: started state syncer -42:29.276 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13841 network=tcp -42:29.316 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:29.316 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13854 [Candidate]" term=2 -42:29.316 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:29.316 [DEBUG] TestAgent.server.raft: vote granted: from=0ababcea-e1bf-ed17-0654-147b5a4b9e0e term=2 tally=1 -42:29.316 [INFO] TestAgent.server.raft: election won: tally=1 -42:29.316 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13854 [Leader]" -42:29.316 [INFO] TestAgent.server: cluster leadership acquired -42:29.316 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13854 -42:29.317 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:29.320 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:29.320 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:29.320 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:29.320 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:29.320 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:29.320 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=my.test-node -42:29.320 [INFO] TestAgent.server: member joined, marking health alive: member=my.test-node -42:29.321 [INFO] TestAgent.server: New leader elected: payload=my.test-node -42:29.321 [INFO] TestAgent.server: federation state anti-entropy synced -42:29.371 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=280.78µs client=127.0.0.1:51733 client_network=udp -42:29.371 [INFO] TestAgent: Requesting shutdown -42:29.371 [INFO] TestAgent.server: shutting down server -42:29.371 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:29.371 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:29.371 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:29.371 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:29.371 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:29.371 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:29.371 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:29.371 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:29.371 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:29.371 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:29.371 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:29.371 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:29.372Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:29.372Z [INFO] agent.router.manager: shutting down -42:29.372 [INFO] TestAgent: consul server down -42:29.372 [INFO] TestAgent: shutdown complete -42:29.372 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13849 network=tcp -42:29.372 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13849 network=udp -42:29.372 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13850 network=tcp -42:29.872 [INFO] TestAgent: Waiting for endpoints to shut down -42:29.872 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookupWithInternalServiceAddress (0.66s) -=== CONT TestDNS_SOA_Settings -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:29.180Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:29.203Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:29.205 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6d50d173-6ebe-09c3-4c12-b0f6a97a52a4 Address:127.0.0.1:13833}]" -42:29.206 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13833 [Follower]" leader= -42:29.206 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6d50d173-6ebe-09c3-4c12-b0f6a97a52a4.dc1 127.0.0.1 -42:29.207 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6d50d173-6ebe-09c3-4c12-b0f6a97a52a4 127.0.0.1 -2022-09-15T21:42:29.207Z [INFO] agent.router: Initializing LAN area manager -42:29.207 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6d50d173-6ebe-09c3-4c12-b0f6a97a52a4.dc1 area=wan -42:29.207 [INFO] TestAgent.server: Adding LAN server: server="Node-6d50d173-6ebe-09c3-4c12-b0f6a97a52a4 (Addr: tcp/127.0.0.1:13833) (DC: dc1)" -42:29.208 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13828 network=udp -42:29.208 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13828 network=tcp -42:29.209 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13829 network=tcp -42:29.209 [INFO] TestAgent: started state syncer -42:29.209 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13848 network=tcp -42:29.249 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:29.249 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13833 [Candidate]" term=2 -42:29.249 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:29.249 [DEBUG] TestAgent.server.raft: vote granted: from=6d50d173-6ebe-09c3-4c12-b0f6a97a52a4 term=2 tally=1 -42:29.249 [INFO] TestAgent.server.raft: election won: tally=1 -42:29.249 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13833 [Leader]" -42:29.249 [INFO] TestAgent.server: cluster leadership acquired -42:29.249 [INFO] TestAgent.server: New leader elected: payload=Node-6d50d173-6ebe-09c3-4c12-b0f6a97a52a4 -42:29.249 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13833 -42:29.251 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:29.253 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:29.253 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:29.253 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:29.253 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:29.253 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:29.253 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6d50d173-6ebe-09c3-4c12-b0f6a97a52a4 -42:29.253 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6d50d173-6ebe-09c3-4c12-b0f6a97a52a4 -42:29.254 [INFO] TestAgent.server: federation state anti-entropy synced -42:29.418 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:29.419 [INFO] TestAgent: Synced node info -42:29.585 [DEBUG] TestAgent.dns: request served from client: name=api.ingress.consul. type=A class=IN latency=351.78µs client=127.0.0.1:47428 client_network=udp -42:29.586 [DEBUG] TestAgent.dns: request served from client: name=api.ingress.dc1.consul. type=A class=IN latency=250.14µs client=127.0.0.1:58101 client_network=udp -42:29.586 [DEBUG] TestAgent.dns: request served from client: name=db.ingress.consul. type=A class=IN latency=205.58µs client=127.0.0.1:59817 client_network=udp -42:29.587 [DEBUG] TestAgent.dns: request served from client: name=db.ingress.dc1.consul. type=A class=IN latency=219.28µs client=127.0.0.1:48091 client_network=udp -42:29.587 [INFO] TestAgent: Requesting shutdown -42:29.587 [INFO] TestAgent.server: shutting down server -42:29.587 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:29.587 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:29.587 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:29.587 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:29.587 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:29.587 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:29.587 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:29.587 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:29.587 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:29.587 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:29.588 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:29.588Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:29.588Z [INFO] agent.router.manager: shutting down -42:29.588 [INFO] TestAgent: consul server down -42:29.588 [INFO] TestAgent: shutdown complete -42:29.588 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13828 network=tcp -42:29.588 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13828 network=udp -42:29.588 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13829 network=tcp -42:30.088 [INFO] TestAgent: Waiting for endpoints to shut down -42:30.088 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_IngressServiceLookup (0.98s) - --- PASS: TestDNS_IngressServiceLookup/api.ingress.consul. (0.00s) - --- PASS: TestDNS_IngressServiceLookup/api.ingress.dc1.consul. (0.00s) - --- PASS: TestDNS_IngressServiceLookup/db.ingress.consul. (0.00s) - --- PASS: TestDNS_IngressServiceLookup/db.ingress.dc1.consul. (0.00s) -=== CONT TestDNS_ServiceReverseLookup_CustomDomain -2022-09-15T21:42:29.493Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:29.510Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:29.513 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4e2d0223-693b-5f19-74bf-525cf63efe80 Address:127.0.0.1:13868}]" -42:29.513 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13868 [Follower]" leader= -42:29.514 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4e2d0223-693b-5f19-74bf-525cf63efe80.dc1 127.0.0.1 -42:29.514 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4e2d0223-693b-5f19-74bf-525cf63efe80 127.0.0.1 -2022-09-15T21:42:29.515Z [INFO] agent.router: Initializing LAN area manager -42:29.515 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13863 network=udp -42:29.515 [INFO] TestAgent.server: Adding LAN server: server="Node-4e2d0223-693b-5f19-74bf-525cf63efe80 (Addr: tcp/127.0.0.1:13868) (DC: dc1)" -42:29.515 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4e2d0223-693b-5f19-74bf-525cf63efe80.dc1 area=wan -42:29.515 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13863 network=tcp -42:29.516 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13864 network=tcp -42:29.516 [INFO] TestAgent: started state syncer -42:29.516 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13855 network=tcp -42:29.558 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:29.558 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13868 [Candidate]" term=2 -42:29.558 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:29.558 [DEBUG] TestAgent.server.raft: vote granted: from=4e2d0223-693b-5f19-74bf-525cf63efe80 term=2 tally=1 -42:29.558 [INFO] TestAgent.server.raft: election won: tally=1 -42:29.558 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13868 [Leader]" -42:29.558 [INFO] TestAgent.server: cluster leadership acquired -42:29.558 [INFO] TestAgent.server: New leader elected: payload=Node-4e2d0223-693b-5f19-74bf-525cf63efe80 -42:29.558 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13868 -42:29.560 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:29.573 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:29.573 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:29.573 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:29.573 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:29.573 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:29.573 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4e2d0223-693b-5f19-74bf-525cf63efe80 -42:29.573 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4e2d0223-693b-5f19-74bf-525cf63efe80 -42:29.574 [INFO] TestAgent.server: federation state anti-entropy synced -42:29.576 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:29.576 [INFO] TestAgent: Synced node info -42:29.576 [DEBUG] TestAgent: Node info in sync -42:29.594 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=ANY class=IN latency=474.96µs client=127.0.0.1:39308 client_network=udp -42:29.594 [INFO] TestAgent: Requesting shutdown -42:29.595 [INFO] TestAgent.server: shutting down server -42:29.595 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:29.595 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:29.595 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:29.595 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:29.595 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:29.595 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:29.595 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:29.595 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:29.595 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:29.595 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:29.595 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:29.595Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:29.595Z [INFO] agent.router.manager: shutting down -42:29.595 [INFO] TestAgent: consul server down -42:29.595 [INFO] TestAgent: shutdown complete -42:29.595 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13863 network=tcp -42:29.595 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13863 network=udp -42:29.595 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13864 network=tcp -42:30.096 [INFO] TestAgent: Waiting for endpoints to shut down -42:30.096 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookupMultiAddrNoCNAME (0.63s) -=== CONT TestDNS_ServiceReverseLookup_IPV6 -2022-09-15T21:42:29.407Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:29.455Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:29.483 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:61503a65-b623-91a0-7f53-eeb2afc48445 Address:127.0.0.1:13847}]" -42:29.484 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-61503a65-b623-91a0-7f53-eeb2afc48445.dc1 127.0.0.1 -42:29.485 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-61503a65-b623-91a0-7f53-eeb2afc48445 127.0.0.1 -2022-09-15T21:42:29.485Z [INFO] agent.router: Initializing LAN area manager -42:29.485 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13842 network=udp -42:29.485 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13847 [Follower]" leader= -42:29.486 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-61503a65-b623-91a0-7f53-eeb2afc48445.dc1 area=wan -42:29.486 [INFO] TestAgent.server: Adding LAN server: server="Node-61503a65-b623-91a0-7f53-eeb2afc48445 (Addr: tcp/127.0.0.1:13847) (DC: dc1)" -42:29.486 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13842 network=tcp -42:29.487 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13843 network=tcp -42:29.487 [INFO] TestAgent: started state syncer -42:29.487 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13862 network=tcp -42:29.538 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:29.538 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13847 [Candidate]" term=2 -42:29.538 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:29.538 [DEBUG] TestAgent.server.raft: vote granted: from=61503a65-b623-91a0-7f53-eeb2afc48445 term=2 tally=1 -42:29.538 [INFO] TestAgent.server.raft: election won: tally=1 -42:29.538 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13847 [Leader]" -42:29.538 [INFO] TestAgent.server: cluster leadership acquired -42:29.538 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13847 -42:29.540 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:29.542 [INFO] TestAgent.server: New leader elected: payload=Node-61503a65-b623-91a0-7f53-eeb2afc48445 -42:29.542 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:29.542 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:29.542 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:29.542 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:29.542 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:29.542 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-61503a65-b623-91a0-7f53-eeb2afc48445 -42:29.542 [INFO] TestAgent.server: member joined, marking health alive: member=Node-61503a65-b623-91a0-7f53-eeb2afc48445 -42:29.543 [INFO] TestAgent.server: federation state anti-entropy synced -42:29.594 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=342.78µs client=127.0.0.1:44937 client_network=udp -42:29.595 [DEBUG] TestAgent.dns: request served from client: name=fe00a76c-0e99-8dbc-07fc-6640b764d837.query.consul. type=SRV class=IN latency=184.96µs client=127.0.0.1:48351 client_network=udp -42:29.595 [DEBUG] TestAgent.dns: request served from client: name=nodb.service.consul. type=SRV class=IN latency=128.06µs client=127.0.0.1:43598 client_network=udp -42:29.595 [DEBUG] TestAgent.dns: request served from client: name=nope.query.consul. type=SRV class=IN latency=103.4µs client=127.0.0.1:39671 client_network=udp -42:29.595 [INFO] TestAgent: Requesting shutdown -42:29.595 [INFO] TestAgent.server: shutting down server -42:29.595 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:29.595 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:29.595 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:29.595 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:29.596 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:29.596 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:29.596 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:29.596 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:29.596 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:29.596 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:29.596 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:29.596 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:29.596Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:29.596Z [INFO] agent.router.manager: shutting down -42:29.596 [INFO] TestAgent: consul server down -42:29.596 [INFO] TestAgent: shutdown complete -42:29.596 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13842 network=tcp -42:29.596 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13842 network=udp -42:29.596 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13843 network=tcp -42:30.098 [INFO] TestAgent: Waiting for endpoints to shut down -42:30.098 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup (0.72s) -=== CONT TestDNS_ServiceReverseLookup -[WARN] freeport: 4 out of 25 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:29.639Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:29.655Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:29.657 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2557e534-0cec-d2db-c7fc-4723ab3ade8d Address:127.0.0.1:13872}]" -42:29.658 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13872 [Follower]" leader= -42:29.658 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2557e534-0cec-d2db-c7fc-4723ab3ade8d.dc1 127.0.0.1 -42:29.659 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2557e534-0cec-d2db-c7fc-4723ab3ade8d 127.0.0.1 -2022-09-15T21:42:29.659Z [INFO] agent.router: Initializing LAN area manager -42:29.660 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13880 network=udp -42:29.660 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2557e534-0cec-d2db-c7fc-4723ab3ade8d.dc1 area=wan -42:29.660 [INFO] TestAgent.server: Adding LAN server: server="Node-2557e534-0cec-d2db-c7fc-4723ab3ade8d (Addr: tcp/127.0.0.1:13872) (DC: dc1)" -42:29.660 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13880 network=tcp -42:29.661 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13881 network=tcp -42:29.661 [INFO] TestAgent: started state syncer -42:29.661 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13873 network=tcp -42:29.718 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:29.718 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13872 [Candidate]" term=2 -42:29.718 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:29.718 [DEBUG] TestAgent.server.raft: vote granted: from=2557e534-0cec-d2db-c7fc-4723ab3ade8d term=2 tally=1 -42:29.718 [INFO] TestAgent.server.raft: election won: tally=1 -42:29.718 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13872 [Leader]" -42:29.719 [INFO] TestAgent.server: cluster leadership acquired -42:29.719 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13872 -42:29.721 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:29.722 [INFO] TestAgent.server: New leader elected: payload=Node-2557e534-0cec-d2db-c7fc-4723ab3ade8d -42:29.737 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:29.737 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:29.737 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:29.737 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:29.737 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:29.737 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2557e534-0cec-d2db-c7fc-4723ab3ade8d -42:29.737 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2557e534-0cec-d2db-c7fc-4723ab3ade8d -42:29.740 [INFO] TestAgent.server: federation state anti-entropy synced -42:29.774 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=ANY class=IN latency=472.96µs client=127.0.0.1:58041 client_network=udp -42:29.774 [INFO] TestAgent: Requesting shutdown -42:29.774 [INFO] TestAgent.server: shutting down server -42:29.775 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:29.775 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:29.775 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:29.775 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:29.775 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:29.775 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:29.775 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:29.775 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:29.775 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:29.775 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:29.775 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:29.775 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:29.775Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:29.775Z [INFO] agent.router.manager: shutting down -42:29.775 [INFO] TestAgent: consul server down -42:29.775 [INFO] TestAgent: shutdown complete -42:29.775 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13880 network=tcp -42:29.775 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13880 network=udp -42:29.775 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13881 network=tcp -42:30.276 [INFO] TestAgent: Waiting for endpoints to shut down -42:30.276 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookupNoMultiCNAME (0.65s) -=== CONT TestDNS_ReverseLookup_IPV6 -2022-09-15T21:42:29.608Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:29.626Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:29.629 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bd5c343a-0fad-6045-63cd-f8dcd8179b95 Address:127.0.0.1:13861}]" -42:29.630 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13861 [Follower]" leader= -42:29.630 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bd5c343a-0fad-6045-63cd-f8dcd8179b95.dc1 127.0.0.1 -42:29.631 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bd5c343a-0fad-6045-63cd-f8dcd8179b95 127.0.0.1 -2022-09-15T21:42:29.631Z [INFO] agent.router: Initializing LAN area manager -42:29.631 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13856 network=udp -42:29.631 [INFO] TestAgent.server: Adding LAN server: server="Node-bd5c343a-0fad-6045-63cd-f8dcd8179b95 (Addr: tcp/127.0.0.1:13861) (DC: dc1)" -42:29.631 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13856 network=tcp -42:29.632 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13857 network=tcp -42:29.632 [INFO] TestAgent: started state syncer -42:29.632 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13878 network=tcp -42:29.633 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bd5c343a-0fad-6045-63cd-f8dcd8179b95.dc1 area=wan -42:29.690 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:29.690 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13861 [Candidate]" term=2 -42:29.690 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:29.690 [DEBUG] TestAgent.server.raft: vote granted: from=bd5c343a-0fad-6045-63cd-f8dcd8179b95 term=2 tally=1 -42:29.690 [INFO] TestAgent.server.raft: election won: tally=1 -42:29.690 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13861 [Leader]" -42:29.690 [INFO] TestAgent.server: cluster leadership acquired -42:29.691 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13861 -42:29.693 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:29.694 [INFO] TestAgent.server: New leader elected: payload=Node-bd5c343a-0fad-6045-63cd-f8dcd8179b95 -42:29.696 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:29.702 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:29.702 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:29.702 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:29.702 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:29.702 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bd5c343a-0fad-6045-63cd-f8dcd8179b95 -42:29.702 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bd5c343a-0fad-6045-63cd-f8dcd8179b95 -42:29.703 [INFO] TestAgent.server: federation state anti-entropy synced -42:29.884 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=ANY class=IN latency=464.58µs client=127.0.0.1:36827 client_network=udp -42:29.884 [INFO] TestAgent: Requesting shutdown -42:29.885 [INFO] TestAgent.server: shutting down server -42:29.885 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:29.885 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:29.885 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:29.885 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:29.885 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:29.885 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:29.885 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:29.885 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:29.885 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:29.885 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:29.885 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:29.885 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:29.885Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:29.885Z [INFO] agent.router.manager: shutting down -42:29.892 [INFO] TestAgent: consul server down -42:29.892 [INFO] TestAgent: shutdown complete -42:29.892 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13856 network=tcp -42:29.892 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13856 network=udp -42:29.893 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13857 network=tcp -42:30.394 [INFO] TestAgent: Waiting for endpoints to shut down -42:30.394 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookupPreferNoCNAME (0.80s) -=== CONT TestDNS_ReverseLookup_CustomDomain -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=255)_compressed=true -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:29.909Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:29.946Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:29.949 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bae62e3d-80ef-50c8-21b8-b91075982584 Address:127.0.0.1:13893}]" -42:29.950 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13893 [Follower]" leader= -42:29.955 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bae62e3d-80ef-50c8-21b8-b91075982584.dc1 127.0.0.1 -42:29.956 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bae62e3d-80ef-50c8-21b8-b91075982584 127.0.0.1 -2022-09-15T21:42:29.962Z [INFO] agent.router: Initializing LAN area manager -42:29.962 [INFO] TestAgent.server: Adding LAN server: server="Node-bae62e3d-80ef-50c8-21b8-b91075982584 (Addr: tcp/127.0.0.1:13893) (DC: dc1)" -42:29.962 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bae62e3d-80ef-50c8-21b8-b91075982584.dc1 area=wan -42:29.962 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13874 network=udp -42:29.962 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13874 network=tcp -42:29.963 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13875 network=tcp -42:29.963 [INFO] TestAgent: started state syncer -42:29.963 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13894 network=tcp -42:30.006 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:30.006 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13893 [Candidate]" term=2 -42:30.006 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:30.006 [DEBUG] TestAgent.server.raft: vote granted: from=bae62e3d-80ef-50c8-21b8-b91075982584 term=2 tally=1 -42:30.006 [INFO] TestAgent.server.raft: election won: tally=1 -42:30.006 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13893 [Leader]" -42:30.006 [INFO] TestAgent.server: cluster leadership acquired -42:30.007 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13893 -42:30.008 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:30.010 [INFO] TestAgent.server: New leader elected: payload=Node-bae62e3d-80ef-50c8-21b8-b91075982584 -42:30.011 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:30.011 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:30.011 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:30.011 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:30.011 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:30.011 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bae62e3d-80ef-50c8-21b8-b91075982584 -42:30.011 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bae62e3d-80ef-50c8-21b8-b91075982584 -42:30.012 [INFO] TestAgent.server: federation state anti-entropy synced -42:30.149 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:30.149 [INFO] TestAgent: Synced node info -42:30.185 [DEBUG] TestAgent.dns: request served from client: question="{1.0.0.127.in-addr.arpa. 255 1}" latency=219.36µs client=127.0.0.1:41858 client_network=udp -42:30.185 [INFO] TestAgent: Requesting shutdown -42:30.185 [INFO] TestAgent.server: shutting down server -42:30.185 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:30.185 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:30.185 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:30.185 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:30.185 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:30.185 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:30.185 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:30.185 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:30.185 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:30.185 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:30.185 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:30.186Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:30.186Z [INFO] agent.router.manager: shutting down -42:30.186 [INFO] TestAgent: consul server down -42:30.186 [INFO] TestAgent: shutdown complete -42:30.186 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13874 network=tcp -42:30.186 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13874 network=udp -42:30.186 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13875 network=tcp -42:30.686 [INFO] TestAgent: Waiting for endpoints to shut down -42:30.686 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceReverseLookupNodeAddress (0.82s) -=== CONT TestDNS_ReverseLookup -2022-09-15T21:42:30.111Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:30.129Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:30.132 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:836b9a65-bf3e-f16a-6210-e183ed7d7586 Address:127.0.0.1:13900}]" -42:30.133 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-836b9a65-bf3e-f16a-6210-e183ed7d7586.dc1 127.0.0.1 -42:30.134 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13900 [Follower]" leader= -42:30.134 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-836b9a65-bf3e-f16a-6210-e183ed7d7586 127.0.0.1 -2022-09-15T21:42:30.134Z [INFO] agent.router: Initializing LAN area manager -42:30.135 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13888 network=udp -42:30.135 [INFO] TestAgent.server: Adding LAN server: server="Node-836b9a65-bf3e-f16a-6210-e183ed7d7586 (Addr: tcp/127.0.0.1:13900) (DC: dc1)" -42:30.135 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-836b9a65-bf3e-f16a-6210-e183ed7d7586.dc1 area=wan -42:30.135 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13888 network=tcp -42:30.136 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13889 network=tcp -42:30.136 [INFO] TestAgent: started state syncer -42:30.136 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13901 network=tcp -42:30.183 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:30.183 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13900 [Candidate]" term=2 -42:30.183 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:30.183 [DEBUG] TestAgent.server.raft: vote granted: from=836b9a65-bf3e-f16a-6210-e183ed7d7586 term=2 tally=1 -42:30.183 [INFO] TestAgent.server.raft: election won: tally=1 -42:30.183 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13900 [Leader]" -42:30.184 [INFO] TestAgent.server: cluster leadership acquired -42:30.184 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13900 -42:30.186 [INFO] TestAgent.server: New leader elected: payload=Node-836b9a65-bf3e-f16a-6210-e183ed7d7586 -42:30.187 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:30.190 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:30.190 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:30.190 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:30.190 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:30.190 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:30.190 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-836b9a65-bf3e-f16a-6210-e183ed7d7586 -42:30.190 [INFO] TestAgent.server: member joined, marking health alive: member=Node-836b9a65-bf3e-f16a-6210-e183ed7d7586 -42:30.193 [INFO] TestAgent.server: federation state anti-entropy synced -42:30.231 [DEBUG] TestAgent.dns: request served from client: question="{2.0.0.127.in-addr.arpa. 255 1}" latency=255.5µs client=127.0.0.1:44471 client_network=udp -42:30.238 [INFO] TestAgent: Requesting shutdown -42:30.238 [INFO] TestAgent.server: shutting down server -42:30.238 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:30.238 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:30.238 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:30.238 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:30.238 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:30.238 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:30.238 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:30.238 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:30.238 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:30.238 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:30.238 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -42:30.238 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -2022-09-15T21:42:30.238Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:30.238Z [INFO] agent.router.manager: shutting down -42:30.239 [INFO] TestAgent: consul server down -42:30.239 [INFO] TestAgent: shutdown complete -42:30.239 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13888 network=tcp -42:30.239 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13888 network=udp -42:30.239 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13889 network=tcp -42:30.749 [INFO] TestAgent: Waiting for endpoints to shut down -42:30.749 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceReverseLookup_CustomDomain (0.67s) -=== CONT TestDNS_EDNS0_ECS -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:30.115Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:30.131Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:30.134 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2d0db7b7-66a5-866f-362a-b04e98ad4127 Address:127.0.0.1:13907}]" -42:30.135 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2d0db7b7-66a5-866f-362a-b04e98ad4127.dc1 127.0.0.1 -42:30.136 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2d0db7b7-66a5-866f-362a-b04e98ad4127 127.0.0.1 -2022-09-15T21:42:30.136Z [INFO] agent.router: Initializing LAN area manager -42:30.136 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13902 network=udp -42:30.136 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13907 [Follower]" leader= -42:30.136 [INFO] TestAgent.server: Adding LAN server: server="Node-2d0db7b7-66a5-866f-362a-b04e98ad4127 (Addr: tcp/127.0.0.1:13907) (DC: dc1)" -42:30.137 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13902 network=tcp -42:30.138 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13903 network=tcp -42:30.138 [INFO] TestAgent: started state syncer -42:30.138 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13908 network=tcp -42:30.138 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2d0db7b7-66a5-866f-362a-b04e98ad4127.dc1 area=wan -42:30.193 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:30.193 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13907 [Candidate]" term=2 -42:30.193 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:30.193 [DEBUG] TestAgent.server.raft: vote granted: from=2d0db7b7-66a5-866f-362a-b04e98ad4127 term=2 tally=1 -42:30.193 [INFO] TestAgent.server.raft: election won: tally=1 -42:30.193 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13907 [Leader]" -42:30.193 [INFO] TestAgent.server: cluster leadership acquired -42:30.193 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13907 -42:30.194 [INFO] TestAgent.server: New leader elected: payload=Node-2d0db7b7-66a5-866f-362a-b04e98ad4127 -42:30.195 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:30.202 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:30.202 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:30.202 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:30.202 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:30.202 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:30.202 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2d0db7b7-66a5-866f-362a-b04e98ad4127 -42:30.202 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2d0db7b7-66a5-866f-362a-b04e98ad4127 -42:30.203 [INFO] TestAgent.server: federation state anti-entropy synced -42:30.227 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:30.227 [INFO] TestAgent: Synced node info -42:30.227 [DEBUG] TestAgent: Node info in sync -42:30.330 [DEBUG] TestAgent.dns: request served from client: question="{9.2.3.8.2.4.0.0.0.0.f.f.0.0.0.0.0.0.0.0.0.0.0.0.8.b.d.0.1.0.0.2.ip6.arpa. 255 1}" latency=404.86µs client=127.0.0.1:51413 client_network=udp -42:30.330 [INFO] TestAgent: Requesting shutdown -42:30.330 [INFO] TestAgent.server: shutting down server -42:30.330 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:30.330 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:30.330 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:30.330 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:30.331 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:30.331 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:30.331 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:30.331 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:30.331 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:30.331 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:30.331 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:30.331Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:30.331Z [INFO] agent.router.manager: shutting down -42:30.332 [INFO] TestAgent: consul server down -42:30.332 [INFO] TestAgent: shutdown complete -42:30.332 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13902 network=tcp -42:30.332 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13902 network=udp -42:30.341 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13903 network=tcp -42:30.845 [INFO] TestAgent: Waiting for endpoints to shut down -42:30.850 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceReverseLookup_IPV6 (0.75s) -=== CONT TestDNS_EDNS0 -2022-09-15T21:42:30.119Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:30.137Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:30.140 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cd7be3b4-c943-aa47-070e-721f64f39d0b Address:127.0.0.1:13914}]" -42:30.140 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13914 [Follower]" leader= -42:30.141 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-cd7be3b4-c943-aa47-070e-721f64f39d0b.dc1 127.0.0.1 -42:30.142 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-cd7be3b4-c943-aa47-070e-721f64f39d0b 127.0.0.1 -2022-09-15T21:42:30.142Z [INFO] agent.router: Initializing LAN area manager -42:30.142 [INFO] TestAgent.server: Adding LAN server: server="Node-cd7be3b4-c943-aa47-070e-721f64f39d0b (Addr: tcp/127.0.0.1:13914) (DC: dc1)" -42:30.142 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13909 network=udp -42:30.142 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-cd7be3b4-c943-aa47-070e-721f64f39d0b.dc1 area=wan -42:30.143 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13909 network=tcp -42:30.143 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13910 network=tcp -42:30.144 [INFO] TestAgent: started state syncer -42:30.144 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13915 network=tcp -42:30.177 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:30.177 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13914 [Candidate]" term=2 -42:30.178 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:30.178 [DEBUG] TestAgent.server.raft: vote granted: from=cd7be3b4-c943-aa47-070e-721f64f39d0b term=2 tally=1 -42:30.178 [INFO] TestAgent.server.raft: election won: tally=1 -42:30.178 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13914 [Leader]" -42:30.178 [INFO] TestAgent.server: cluster leadership acquired -42:30.178 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13914 -42:30.180 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:30.178 [INFO] TestAgent.server: New leader elected: payload=Node-cd7be3b4-c943-aa47-070e-721f64f39d0b -42:30.183 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:30.183 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:30.183 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:30.183 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:30.183 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:30.183 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cd7be3b4-c943-aa47-070e-721f64f39d0b -42:30.183 [INFO] TestAgent.server: member joined, marking health alive: member=Node-cd7be3b4-c943-aa47-070e-721f64f39d0b -42:30.185 [INFO] TestAgent.server: federation state anti-entropy synced -42:30.242 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:30.243 [INFO] TestAgent: Synced node info -42:30.495 [DEBUG] TestAgent.dns: request served from client: question="{2.0.0.127.in-addr.arpa. 255 1}" latency=188.6µs client=127.0.0.1:34689 client_network=udp -42:30.495 [INFO] TestAgent: Requesting shutdown -42:30.495 [INFO] TestAgent.server: shutting down server -42:30.495 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:30.495 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:30.495 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:30.495 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:30.495 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:30.495 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:30.495 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:30.495 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:30.495 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:30.495 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:30.496 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:30.496Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:30.496Z [INFO] agent.router.manager: shutting down -42:30.496 [INFO] TestAgent: consul server down -42:30.496 [INFO] TestAgent: shutdown complete -42:30.496 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13909 network=tcp -42:30.496 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13909 network=udp -42:30.496 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13910 network=tcp -42:31.010 [INFO] TestAgent: Waiting for endpoints to shut down -42:31.010 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceReverseLookup (0.91s) -=== CONT TestDNS_NodeLookup_CNAME -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:30.414Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:30.432Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:30.435 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7484cf44-891c-1708-beb9-2f902d5f9d88 Address:127.0.0.1:13928}]" -42:30.436 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7484cf44-891c-1708-beb9-2f902d5f9d88.dc1 127.0.0.1 -42:30.437 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7484cf44-891c-1708-beb9-2f902d5f9d88 127.0.0.1 -2022-09-15T21:42:30.437Z [INFO] agent.router: Initializing LAN area manager -42:30.437 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13923 network=udp -42:30.437 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13928 [Follower]" leader= -42:30.438 [INFO] TestAgent.server: Adding LAN server: server="Node-7484cf44-891c-1708-beb9-2f902d5f9d88 (Addr: tcp/127.0.0.1:13928) (DC: dc1)" -42:30.438 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7484cf44-891c-1708-beb9-2f902d5f9d88.dc1 area=wan -42:30.438 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13923 network=tcp -42:30.439 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13924 network=tcp -42:30.439 [INFO] TestAgent: started state syncer -42:30.439 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13929 network=tcp -42:30.493 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:30.493 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13928 [Candidate]" term=2 -42:30.494 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:30.494 [DEBUG] TestAgent.server.raft: vote granted: from=7484cf44-891c-1708-beb9-2f902d5f9d88 term=2 tally=1 -42:30.494 [INFO] TestAgent.server.raft: election won: tally=1 -42:30.494 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13928 [Leader]" -42:30.494 [INFO] TestAgent.server: cluster leadership acquired -42:30.494 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13928 -42:30.495 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:30.496 [INFO] TestAgent.server: New leader elected: payload=Node-7484cf44-891c-1708-beb9-2f902d5f9d88 -42:30.499 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:30.499 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:30.499 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:30.499 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:30.499 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:30.499 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7484cf44-891c-1708-beb9-2f902d5f9d88 -42:30.499 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7484cf44-891c-1708-beb9-2f902d5f9d88 -42:30.500 [INFO] TestAgent.server: federation state anti-entropy synced -42:30.546 [DEBUG] TestAgent.dns: request served from client: question="{2.0.0.127.in-addr.arpa. 255 1}" latency=160.46µs client=127.0.0.1:60719 client_network=udp -42:30.546 [INFO] TestAgent: Requesting shutdown -42:30.546 [INFO] TestAgent.server: shutting down server -42:30.546 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:30.546 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:30.546 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:30.546 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:30.546 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:30.546 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:30.546 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:30.546 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:30.547 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:30.547 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:30.547 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:30.547 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:30.547Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:30.547Z [INFO] agent.router.manager: shutting down -42:30.547 [INFO] TestAgent: consul server down -42:30.547 [INFO] TestAgent: shutdown complete -42:30.547 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13923 network=tcp -42:30.547 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13923 network=udp -42:30.547 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13924 network=tcp -42:31.048 [INFO] TestAgent: Waiting for endpoints to shut down -42:31.049 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ReverseLookup_CustomDomain (0.66s) -=== CONT TestDNSCycleRecursorCheck -2022-09-15T21:42:30.317Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:30.376Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:30.380 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cc179073-1053-a54b-b5ea-d6f81a3ad4b6 Address:127.0.0.1:13921}]" -42:30.380 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13921 [Follower]" leader= -42:30.381 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-cc179073-1053-a54b-b5ea-d6f81a3ad4b6.dc1 127.0.0.1 -42:30.381 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-cc179073-1053-a54b-b5ea-d6f81a3ad4b6 127.0.0.1 -2022-09-15T21:42:30.381Z [INFO] agent.router: Initializing LAN area manager -42:30.382 [INFO] TestAgent.server: Adding LAN server: server="Node-cc179073-1053-a54b-b5ea-d6f81a3ad4b6 (Addr: tcp/127.0.0.1:13921) (DC: dc1)" -42:30.382 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-cc179073-1053-a54b-b5ea-d6f81a3ad4b6.dc1 area=wan -42:30.382 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13916 network=udp -42:30.382 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13916 network=tcp -42:30.383 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13917 network=tcp -42:30.383 [INFO] TestAgent: started state syncer -42:30.383 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13922 network=tcp -42:30.434 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:30.434 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13921 [Candidate]" term=2 -42:30.434 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:30.434 [DEBUG] TestAgent.server.raft: vote granted: from=cc179073-1053-a54b-b5ea-d6f81a3ad4b6 term=2 tally=1 -42:30.434 [INFO] TestAgent.server.raft: election won: tally=1 -42:30.434 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13921 [Leader]" -42:30.434 [INFO] TestAgent.server: cluster leadership acquired -42:30.435 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13921 -42:30.436 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:30.439 [INFO] TestAgent.server: New leader elected: payload=Node-cc179073-1053-a54b-b5ea-d6f81a3ad4b6 -42:30.445 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:30.445 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:30.445 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:30.445 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:30.445 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:30.445 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cc179073-1053-a54b-b5ea-d6f81a3ad4b6 -42:30.445 [INFO] TestAgent.server: member joined, marking health alive: member=Node-cc179073-1053-a54b-b5ea-d6f81a3ad4b6 -42:30.446 [INFO] TestAgent.server: federation state anti-entropy synced -42:30.665 [DEBUG] TestAgent.dns: request served from client: question="{2.4.2.4.2.4.2.4.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa. 255 1}" latency=200.7µs client=127.0.0.1:35225 client_network=udp -42:30.665 [INFO] TestAgent: Requesting shutdown -42:30.665 [INFO] TestAgent.server: shutting down server -42:30.665 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:30.665 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:30.665 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:30.665 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:30.665 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:30.666 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:30.666 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:30.666 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:30.666 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:30.666 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:30.666 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:30.666 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:30.666Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:30.666Z [INFO] agent.router.manager: shutting down -42:30.666 [INFO] TestAgent: consul server down -42:30.666 [INFO] TestAgent: shutdown complete -42:30.666 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13916 network=tcp -42:30.666 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13916 network=udp -42:30.666 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13917 network=tcp -42:31.171 [INFO] TestAgent: Waiting for endpoints to shut down -42:31.171 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ReverseLookup_IPV6 (0.94s) -=== CONT TestDNS_NodeLookup_AAAA -=== RUN TestDNS_EDNS0_ECS/global -=== RUN TestDNS_EDNS0_ECS/query -=== RUN TestDNS_EDNS0_ECS/query-subnet -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=255)_compressed=false -[WARN] freeport: 4 out of 4 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:30.786Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:30.941Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:30.948 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8e1fd4f4-ebc0-e0f0-adde-31f561502bf5 Address:127.0.0.1:13935}]" -42:30.950 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13935 [Follower]" leader= -42:30.952 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8e1fd4f4-ebc0-e0f0-adde-31f561502bf5.dc1 127.0.0.1 -42:30.954 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8e1fd4f4-ebc0-e0f0-adde-31f561502bf5 127.0.0.1 -2022-09-15T21:42:30.954Z [INFO] agent.router: Initializing LAN area manager -42:30.955 [INFO] TestAgent.server: Adding LAN server: server="Node-8e1fd4f4-ebc0-e0f0-adde-31f561502bf5 (Addr: tcp/127.0.0.1:13935) (DC: dc1)" -42:30.956 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8e1fd4f4-ebc0-e0f0-adde-31f561502bf5.dc1 area=wan -42:30.957 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13930 network=udp -42:30.957 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13930 network=tcp -42:30.959 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13931 network=tcp -42:30.959 [INFO] TestAgent: started state syncer -42:30.959 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13936 network=tcp -42:30.995 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:30.995 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13935 [Candidate]" term=2 -42:30.995 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:30.995 [DEBUG] TestAgent.server.raft: vote granted: from=8e1fd4f4-ebc0-e0f0-adde-31f561502bf5 term=2 tally=1 -42:30.995 [INFO] TestAgent.server.raft: election won: tally=1 -42:30.995 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13935 [Leader]" -42:30.996 [INFO] TestAgent.server: cluster leadership acquired -42:30.996 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13935 -42:31.006 [INFO] TestAgent.server: New leader elected: payload=Node-8e1fd4f4-ebc0-e0f0-adde-31f561502bf5 -42:31.006 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:31.009 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:31.009 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:31.009 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:31.009 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:31.009 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:31.009 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8e1fd4f4-ebc0-e0f0-adde-31f561502bf5 -42:31.009 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8e1fd4f4-ebc0-e0f0-adde-31f561502bf5 -42:31.018 [INFO] TestAgent.server: federation state anti-entropy synced -42:31.213 [DEBUG] TestAgent.dns: request served from client: question="{2.0.0.127.in-addr.arpa. 255 1}" latency=139.66µs client=127.0.0.1:41606 client_network=udp -42:31.213 [INFO] TestAgent: Requesting shutdown -42:31.213 [INFO] TestAgent.server: shutting down server -42:31.213 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:31.213 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:31.213 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:31.213 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:31.213 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:31.213 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:31.213 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:31.213 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:31.213 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:31.213 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:31.213 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:31.213 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:31.213Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:31.213Z [INFO] agent.router.manager: shutting down -42:31.214 [INFO] TestAgent: consul server down -42:31.214 [INFO] TestAgent: shutdown complete -42:31.214 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13930 network=tcp -42:31.214 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13930 network=udp -42:31.214 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13931 network=tcp -42:31.722 [INFO] TestAgent: Waiting for endpoints to shut down -42:31.722 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ReverseLookup (1.03s) -=== CONT TestDNS_NodeLookup_PeriodName -2022-09-15T21:42:30.803Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:30.906Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:30.938 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a10224e9-9e49-4607-926a-34e252649218 Address:127.0.0.1:13942}]" -42:30.939 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a10224e9-9e49-4607-926a-34e252649218.dc1 127.0.0.1 -42:30.940 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13942 [Follower]" leader= -42:30.940 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a10224e9-9e49-4607-926a-34e252649218 127.0.0.1 -2022-09-15T21:42:30.940Z [INFO] agent.router: Initializing LAN area manager -42:30.941 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13937 network=udp -42:30.941 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a10224e9-9e49-4607-926a-34e252649218.dc1 area=wan -42:30.941 [INFO] TestAgent.server: Adding LAN server: server="Node-a10224e9-9e49-4607-926a-34e252649218 (Addr: tcp/127.0.0.1:13942) (DC: dc1)" -42:30.941 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13937 network=tcp -42:30.944 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13938 network=tcp -42:30.944 [INFO] TestAgent: started state syncer -42:30.944 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13943 network=tcp -42:30.995 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:30.995 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13942 [Candidate]" term=2 -42:30.996 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:30.996 [DEBUG] TestAgent.server.raft: vote granted: from=a10224e9-9e49-4607-926a-34e252649218 term=2 tally=1 -42:30.996 [INFO] TestAgent.server.raft: election won: tally=1 -42:30.996 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13942 [Leader]" -42:31.006 [INFO] TestAgent.server: cluster leadership acquired -42:31.006 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13942 -42:31.008 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:31.011 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:31.011 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:31.011 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:31.011 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:31.011 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:31.013 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a10224e9-9e49-4607-926a-34e252649218 -42:31.013 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a10224e9-9e49-4607-926a-34e252649218 -42:31.015 [INFO] TestAgent.server: New leader elected: payload=Node-a10224e9-9e49-4607-926a-34e252649218 -42:31.019 [INFO] TestAgent.server: federation state anti-entropy synced -42:31.022 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:31.025 [INFO] TestAgent: Synced node info -42:31.025 [DEBUG] TestAgent: Node info in sync -42:31.216 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=A class=IN latency=271.84µs client=127.0.0.1:50289 client_network=udp -42:31.217 [DEBUG] TestAgent.dns: request served from client: name=test.query.consul. type=A class=IN latency=192.56µs client=127.0.0.1:54270 client_network=udp -42:31.217 [DEBUG] TestAgent.dns: request served from client: name=test.query.consul. type=A class=IN latency=220.54µs client=127.0.0.1:54301 client_network=udp -42:31.218 [INFO] TestAgent: Requesting shutdown -42:31.218 [INFO] TestAgent.server: shutting down server -42:31.218 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:31.218 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:31.218 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:31.218 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:31.218 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:31.218 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:31.218 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:31.218 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:31.218 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:31.218 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:31.218 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:31.219Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:31.219Z [INFO] agent.router.manager: shutting down -42:31.219 [INFO] TestAgent: consul server down -42:31.219 [INFO] TestAgent: shutdown complete -42:31.219 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13937 network=tcp -42:31.219 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13937 network=udp -42:31.219 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13938 network=tcp -42:31.730 [INFO] TestAgent: Waiting for endpoints to shut down -42:31.730 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_EDNS0_ECS (0.97s) - --- PASS: TestDNS_EDNS0_ECS/global (0.00s) - --- PASS: TestDNS_EDNS0_ECS/query (0.00s) - --- PASS: TestDNS_EDNS0_ECS/query-subnet (0.00s) -=== CONT TestDNS_CaseInsensitiveNodeLookup -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:31.132Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:31.196Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:31.207 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a25628a7-c618-88bf-cb95-354f17954c5c Address:127.0.0.1:13963}]" -42:31.208 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a25628a7-c618-88bf-cb95-354f17954c5c.dc1 127.0.0.1 -42:31.208 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13963 [Follower]" leader= -42:31.209 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a25628a7-c618-88bf-cb95-354f17954c5c 127.0.0.1 -2022-09-15T21:42:31.209Z [INFO] agent.router: Initializing LAN area manager -42:31.209 [DEBUG] TestAgent.dns: recursor enabled -42:31.209 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13958 network=udp -42:31.210 [INFO] TestAgent.server: Adding LAN server: server="Node-a25628a7-c618-88bf-cb95-354f17954c5c (Addr: tcp/127.0.0.1:13963) (DC: dc1)" -42:31.210 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a25628a7-c618-88bf-cb95-354f17954c5c.dc1 area=wan -42:31.210 [DEBUG] TestAgent.dns: recursor enabled -42:31.210 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13958 network=tcp -42:31.211 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13959 network=tcp -42:31.211 [INFO] TestAgent: started state syncer -42:31.211 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13964 network=tcp -42:31.270 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:31.271 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13963 [Candidate]" term=2 -42:31.271 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:31.271 [DEBUG] TestAgent.server.raft: vote granted: from=a25628a7-c618-88bf-cb95-354f17954c5c term=2 tally=1 -42:31.271 [INFO] TestAgent.server.raft: election won: tally=1 -42:31.271 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13963 [Leader]" -42:31.271 [INFO] TestAgent.server: cluster leadership acquired -42:31.279 [INFO] TestAgent.server: New leader elected: payload=Node-a25628a7-c618-88bf-cb95-354f17954c5c -42:31.282 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13963 -42:31.284 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:31.291 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:31.291 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:31.291 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:31.291 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:31.291 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:31.291 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a25628a7-c618-88bf-cb95-354f17954c5c -42:31.291 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a25628a7-c618-88bf-cb95-354f17954c5c -42:31.292 [INFO] TestAgent.server: federation state anti-entropy synced -42:31.312 [DEBUG] TestAgent.dns: cname recurse RTT for name: name=www.google.com. rtt=479.78µs -42:31.312 [DEBUG] TestAgent.dns: request served from client: name=google.node.consul. type=ANY class=IN latency=860.94µs client=127.0.0.1:56616 client_network=udp -42:31.312 [INFO] TestAgent: Requesting shutdown -42:31.313 [INFO] TestAgent.server: shutting down server -42:31.313 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:31.313 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:31.313 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:31.313 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:31.313 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:31.313 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:31.313 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:31.313 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:31.313 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:31.318 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:31.318 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:31.318 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:31.318Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:31.318Z [INFO] agent.router.manager: shutting down -42:31.322 [INFO] TestAgent: consul server down -42:31.322 [INFO] TestAgent: shutdown complete -42:31.322 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13958 network=tcp -42:31.322 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13958 network=udp -42:31.322 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13959 network=tcp -42:31.822 [INFO] TestAgent: Waiting for endpoints to shut down -42:31.822 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_NodeLookup_CNAME (0.81s) -=== CONT TestDNS_NodeLookup -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=255)_compressed=true -2022-09-15T21:42:30.963Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:31.053Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:31.067 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8717c14a-0d49-cfe4-de56-c77c2abe895d Address:127.0.0.1:13949}]" -42:31.068 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8717c14a-0d49-cfe4-de56-c77c2abe895d.dc1 127.0.0.1 -42:31.068 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13949 [Follower]" leader= -42:31.069 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8717c14a-0d49-cfe4-de56-c77c2abe895d 127.0.0.1 -2022-09-15T21:42:31.069Z [INFO] agent.router: Initializing LAN area manager -42:31.070 [INFO] TestAgent.server: Adding LAN server: server="Node-8717c14a-0d49-cfe4-de56-c77c2abe895d (Addr: tcp/127.0.0.1:13949) (DC: dc1)" -42:31.072 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8717c14a-0d49-cfe4-de56-c77c2abe895d.dc1 area=wan -42:31.074 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13944 network=tcp -42:31.079 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13944 network=udp -42:31.080 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13945 network=tcp -42:31.080 [INFO] TestAgent: started state syncer -42:31.081 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13950 network=tcp -42:31.118 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:31.118 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13949 [Candidate]" term=2 -42:31.118 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:31.118 [DEBUG] TestAgent.server.raft: vote granted: from=8717c14a-0d49-cfe4-de56-c77c2abe895d term=2 tally=1 -42:31.118 [INFO] TestAgent.server.raft: election won: tally=1 -42:31.118 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13949 [Leader]" -42:31.118 [INFO] TestAgent.server: cluster leadership acquired -42:31.119 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13949 -42:31.120 [INFO] TestAgent.server: New leader elected: payload=Node-8717c14a-0d49-cfe4-de56-c77c2abe895d -42:31.130 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:31.133 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:31.133 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:31.133 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:31.133 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:31.133 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:31.134 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8717c14a-0d49-cfe4-de56-c77c2abe895d -42:31.134 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8717c14a-0d49-cfe4-de56-c77c2abe895d -42:31.134 [INFO] TestAgent.server: federation state anti-entropy synced -42:31.207 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:31.208 [INFO] TestAgent: Synced node info -42:31.208 [DEBUG] TestAgent: Node info in sync -42:31.451 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:31.451 [DEBUG] TestAgent: Node info in sync -42:31.463 [DEBUG] TestAgent.dns: request served from client: name=foo.node.dc1.consul. type=ANY class=IN latency=186.62µs client=127.0.0.1:58954 client_network=udp -42:31.464 [INFO] TestAgent: Requesting shutdown -42:31.464 [INFO] TestAgent.server: shutting down server -42:31.464 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:31.464 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:31.464 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:31.464 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:31.464 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:31.464 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:31.464 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:31.464 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:31.464 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:31.464 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:31.464 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:31.464Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:31.464Z [INFO] agent.router.manager: shutting down -42:31.464 [INFO] TestAgent: consul server down -42:31.464 [INFO] TestAgent: shutdown complete -42:31.464 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13944 network=tcp -42:31.464 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13944 network=udp -42:31.465 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13945 network=tcp -42:31.966 [INFO] TestAgent: Waiting for endpoints to shut down -42:31.966 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_EDNS0 (1.11s) -=== CONT TestDNS_EmptyAltDomain -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:31.112Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:31.200Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:31.204 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5ccdc8eb-f48e-becc-337c-8651c76e0027 Address:127.0.0.1:13970}]" -42:31.207 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13970 [Follower]" leader= -42:31.209 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5ccdc8eb-f48e-becc-337c-8651c76e0027.dc1 127.0.0.1 -42:31.210 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5ccdc8eb-f48e-becc-337c-8651c76e0027 127.0.0.1 -2022-09-15T21:42:31.210Z [INFO] agent.router: Initializing LAN area manager -42:31.210 [DEBUG] TestAgent.dns: recursor enabled -42:31.210 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13965 network=udp -42:31.211 [INFO] TestAgent.server: Adding LAN server: server="Node-5ccdc8eb-f48e-becc-337c-8651c76e0027 (Addr: tcp/127.0.0.1:13970) (DC: dc1)" -42:31.211 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5ccdc8eb-f48e-becc-337c-8651c76e0027.dc1 area=wan -42:31.211 [DEBUG] TestAgent.dns: recursor enabled -42:31.211 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13965 network=tcp -42:31.212 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13966 network=tcp -42:31.212 [INFO] TestAgent: started state syncer -42:31.212 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13971 network=tcp -42:31.279 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:31.280 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13970 [Candidate]" term=2 -42:31.280 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:31.280 [DEBUG] TestAgent.server.raft: vote granted: from=5ccdc8eb-f48e-becc-337c-8651c76e0027 term=2 tally=1 -42:31.280 [INFO] TestAgent.server.raft: election won: tally=1 -42:31.280 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13970 [Leader]" -42:31.280 [INFO] TestAgent.server: cluster leadership acquired -42:31.280 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13970 -42:31.282 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:31.286 [INFO] TestAgent.server: New leader elected: payload=Node-5ccdc8eb-f48e-becc-337c-8651c76e0027 -42:31.287 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:31.287 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:31.287 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:31.287 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:31.287 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:31.287 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5ccdc8eb-f48e-becc-337c-8651c76e0027 -42:31.287 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5ccdc8eb-f48e-becc-337c-8651c76e0027 -42:31.289 [INFO] TestAgent.server: federation state anti-entropy synced -42:31.557 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:31.558 [INFO] TestAgent: Synced node info -42:31.560 [DEBUG] TestAgent.dns: recurse failed for question: question="{google.com. 1 1}" rtt=1.83656ms recursor=127.0.0.1:44313 rcode=SERVFAIL -42:31.561 [DEBUG] TestAgent.dns: recurse succeeded for question: question="{google.com. 1 1}" rtt=387.5µs recursor=127.0.0.1:51621 -42:31.561 [DEBUG] TestAgent.dns: request served from client: question="{google.com. 1 1}" network=udp latency=2.65288ms client=127.0.0.1:41529 client_network=udp -42:31.561 [INFO] TestAgent: Requesting shutdown -42:31.561 [INFO] TestAgent.server: shutting down server -42:31.561 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:31.562 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:31.562 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:31.562 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:31.562 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:31.562 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:31.563 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:31.563 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:31.563 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:31.563 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -42:31.563 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -2022-09-15T21:42:31.563Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:31.563Z [INFO] agent.router.manager: shutting down -42:31.564 [INFO] TestAgent: consul server down -42:31.564 [INFO] TestAgent: shutdown complete -42:31.564 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13965 network=tcp -42:31.564 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13965 network=udp -42:31.564 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13966 network=tcp -42:32.064 [INFO] TestAgent: Waiting for endpoints to shut down -42:32.064 [INFO] TestAgent: Endpoints down ---- PASS: TestDNSCycleRecursorCheck (1.01s) -=== CONT TestDNS_Over_TCP -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=255)_compressed=false -2022-09-15T21:42:31.286Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:31.335Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:31.340 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ca42245e-7cf0-3173-63fe-3e76549786ec Address:127.0.0.1:13977}]" -42:31.341 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ca42245e-7cf0-3173-63fe-3e76549786ec.dc1 127.0.0.1 -42:31.342 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13977 [Follower]" leader= -42:31.347 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ca42245e-7cf0-3173-63fe-3e76549786ec 127.0.0.1 -2022-09-15T21:42:31.347Z [INFO] agent.router: Initializing LAN area manager -42:31.348 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13972 network=udp -42:31.348 [INFO] TestAgent.server: Adding LAN server: server="Node-ca42245e-7cf0-3173-63fe-3e76549786ec (Addr: tcp/127.0.0.1:13977) (DC: dc1)" -42:31.348 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ca42245e-7cf0-3173-63fe-3e76549786ec.dc1 area=wan -42:31.348 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13972 network=tcp -42:31.354 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13973 network=tcp -42:31.354 [INFO] TestAgent: started state syncer -42:31.354 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13978 network=tcp -42:31.407 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:31.407 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13977 [Candidate]" term=2 -42:31.408 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:31.408 [DEBUG] TestAgent.server.raft: vote granted: from=ca42245e-7cf0-3173-63fe-3e76549786ec term=2 tally=1 -42:31.408 [INFO] TestAgent.server.raft: election won: tally=1 -42:31.408 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13977 [Leader]" -42:31.408 [INFO] TestAgent.server: cluster leadership acquired -42:31.408 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13977 -42:31.409 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:31.410 [INFO] TestAgent.server: New leader elected: payload=Node-ca42245e-7cf0-3173-63fe-3e76549786ec -42:31.414 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:31.414 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:31.414 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:31.414 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:31.414 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:31.414 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ca42245e-7cf0-3173-63fe-3e76549786ec -42:31.414 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ca42245e-7cf0-3173-63fe-3e76549786ec -42:31.416 [INFO] TestAgent.server: federation state anti-entropy synced -42:31.594 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:31.595 [INFO] TestAgent: Synced node info -42:31.595 [DEBUG] TestAgent: Node info in sync -42:31.670 [DEBUG] TestAgent.dns: request served from client: name=bar.node.consul. type=AAAA class=IN latency=308.38µs client=127.0.0.1:57602 client_network=udp -42:31.670 [INFO] TestAgent: Requesting shutdown -42:31.670 [INFO] TestAgent.server: shutting down server -42:31.670 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:31.670 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:31.670 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:31.670 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:31.670 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:31.671 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:31.671 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:31.671 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:31.671 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:31.671 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:31.671 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:31.671Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:31.671Z [INFO] agent.router.manager: shutting down -42:31.671 [INFO] TestAgent: consul server down -42:31.671 [INFO] TestAgent: shutdown complete -42:31.671 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13972 network=tcp -42:31.671 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13972 network=udp -42:31.671 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13973 network=tcp -42:32.174 [INFO] TestAgent: Waiting for endpoints to shut down -42:32.174 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_NodeLookup_AAAA (0.96s) -=== CONT TestRecursorAddr ---- PASS: TestRecursorAddr (0.00s) -=== CONT TestDiscoveryChainRead -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._tcp_(qType:=255)_compressed=true -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:31.779Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:31.839Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:31.853 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b535e230-4bfe-4b80-dfa6-2471e3e370a2 Address:127.0.0.1:13984}]" -42:31.855 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b535e230-4bfe-4b80-dfa6-2471e3e370a2.dc1 127.0.0.1 -42:31.856 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b535e230-4bfe-4b80-dfa6-2471e3e370a2 127.0.0.1 -2022-09-15T21:42:31.856Z [INFO] agent.router: Initializing LAN area manager -42:31.856 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13979 network=udp -42:31.857 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13984 [Follower]" leader= -42:31.857 [INFO] TestAgent.server: Adding LAN server: server="Node-b535e230-4bfe-4b80-dfa6-2471e3e370a2 (Addr: tcp/127.0.0.1:13984) (DC: dc1)" -42:31.857 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b535e230-4bfe-4b80-dfa6-2471e3e370a2.dc1 area=wan -42:31.858 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13979 network=tcp -42:31.860 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13980 network=tcp -42:31.860 [INFO] TestAgent: started state syncer -42:31.860 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13985 network=tcp -42:31.898 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:31.898 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13984 [Candidate]" term=2 -42:31.898 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:31.898 [DEBUG] TestAgent.server.raft: vote granted: from=b535e230-4bfe-4b80-dfa6-2471e3e370a2 term=2 tally=1 -42:31.898 [INFO] TestAgent.server.raft: election won: tally=1 -42:31.898 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13984 [Leader]" -42:31.898 [INFO] TestAgent.server: cluster leadership acquired -42:31.899 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13984 -42:31.900 [INFO] TestAgent.server: New leader elected: payload=Node-b535e230-4bfe-4b80-dfa6-2471e3e370a2 -42:31.901 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:31.903 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:31.903 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:31.903 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:31.903 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:31.903 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:31.903 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b535e230-4bfe-4b80-dfa6-2471e3e370a2 -42:31.903 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b535e230-4bfe-4b80-dfa6-2471e3e370a2 -42:31.905 [INFO] TestAgent.server: federation state anti-entropy synced -42:32.019 [DEBUG] TestAgent.dns: request served from client: name=foo.bar.node.consul. type=ANY class=IN latency=266.84µs client=127.0.0.1:41393 client_network=udp -42:32.020 [INFO] TestAgent: Requesting shutdown -42:32.020 [INFO] TestAgent.server: shutting down server -42:32.020 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:32.020 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:32.020 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:32.020 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:32.020 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:32.020 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:32.020 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:32.020 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:32.020 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:32.020 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:32.020 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:32.020 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:32.021Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:32.021Z [INFO] agent.router.manager: shutting down -42:32.021 [INFO] TestAgent: consul server down -42:32.021 [INFO] TestAgent: shutdown complete -42:32.021 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13979 network=tcp -42:32.021 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13979 network=udp -42:32.021 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13980 network=tcp -42:32.522 [INFO] TestAgent: Waiting for endpoints to shut down -42:32.522 [INFO] TestAgent: Endpoints down -=== CONT TestDenylist -=== RUN TestDenylist/nothing_blocked_root ---- PASS: TestDNS_NodeLookup_PeriodName (0.80s) -=== RUN TestDenylist/nothing_blocked_path -=== RUN TestDenylist/exact_match_1 -=== RUN TestDenylist/exact_match_2 -=== RUN TestDenylist/subpath -=== RUN TestDenylist/longer_prefix -=== RUN TestDenylist/longer_subpath -=== RUN TestDenylist/partial_prefix -=== RUN TestDenylist/no_match ---- PASS: TestDenylist (0.00s) - --- PASS: TestDenylist/nothing_blocked_root (0.00s) - --- PASS: TestDenylist/nothing_blocked_path (0.00s) - --- PASS: TestDenylist/exact_match_1 (0.00s) - --- PASS: TestDenylist/exact_match_2 (0.00s) - --- PASS: TestDenylist/subpath (0.00s) - --- PASS: TestDenylist/longer_prefix (0.00s) - --- PASS: TestDenylist/longer_subpath (0.00s) - --- PASS: TestDenylist/partial_prefix (0.00s) - --- PASS: TestDenylist/no_match (0.00s) -=== CONT TestCoordinate_Update_ACLDeny -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDiscoveryChainRead/GET:_error_on_no_service_name -=== RUN TestDiscoveryChainRead/GET:_read_default_chain -=== RUN TestDiscoveryChainRead/GET:_read_default_chain;_evaluate_in_dc2 -=== RUN TestDiscoveryChainRead/GET:_read_default_chain_with_cache -=== RUN TestDiscoveryChainRead/POST:_error_on_no_service_name -=== RUN TestDiscoveryChainRead/POST:_read_default_chain -=== RUN TestDiscoveryChainRead/POST:_read_default_chain;_evaluate_in_dc2 -=== RUN TestDiscoveryChainRead/POST:_read_default_chain_with_cache -=== RUN TestDiscoveryChainRead/GET:_read_modified_chain -=== RUN TestDiscoveryChainRead/POST:_read_modified_chain_with_overrides_(camel_case) -=== RUN TestDiscoveryChainRead/POST:_read_modified_chain_with_overrides_(snake_case) -2022-09-15T21:42:31.992Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:32.047Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:32.054 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:dc3e1715-0698-f96e-4b2d-b8b5539465ee Address:127.0.0.1:14012}]" -42:32.055 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14012 [Follower]" leader= -42:32.055 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-dc3e1715-0698-f96e-4b2d-b8b5539465ee.dc1 127.0.0.1 -42:32.056 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-dc3e1715-0698-f96e-4b2d-b8b5539465ee 127.0.0.1 -2022-09-15T21:42:32.056Z [INFO] agent.router: Initializing LAN area manager -42:32.056 [INFO] TestAgent.server: Adding LAN server: server="Node-dc3e1715-0698-f96e-4b2d-b8b5539465ee (Addr: tcp/127.0.0.1:14012) (DC: dc1)" -42:32.056 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-dc3e1715-0698-f96e-4b2d-b8b5539465ee.dc1 area=wan -42:32.057 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14007 network=tcp -42:32.057 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14007 network=udp -42:32.058 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14008 network=tcp -42:32.058 [INFO] TestAgent: started state syncer -42:32.058 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14013 network=tcp -42:32.102 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:32.102 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14012 [Candidate]" term=2 -42:32.102 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:32.102 [DEBUG] TestAgent.server.raft: vote granted: from=dc3e1715-0698-f96e-4b2d-b8b5539465ee term=2 tally=1 -42:32.102 [INFO] TestAgent.server.raft: election won: tally=1 -42:32.102 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14012 [Leader]" -42:32.102 [INFO] TestAgent.server: cluster leadership acquired -42:32.102 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14012 -42:32.103 [INFO] TestAgent.server: New leader elected: payload=Node-dc3e1715-0698-f96e-4b2d-b8b5539465ee -42:32.104 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:32.107 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:32.107 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:32.107 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:32.107 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:32.107 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:32.107 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-dc3e1715-0698-f96e-4b2d-b8b5539465ee -42:32.107 [INFO] TestAgent.server: member joined, marking health alive: member=Node-dc3e1715-0698-f96e-4b2d-b8b5539465ee -42:32.108 [INFO] TestAgent.server: federation state anti-entropy synced -42:32.184 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:32.185 [INFO] TestAgent: Synced node info -42:32.194 [INFO] TestAgent: Requesting shutdown -42:32.195 [INFO] TestAgent.server: shutting down server -42:32.195 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:32.195 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:32.195 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:32.195 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:32.195 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:32.195 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:32.195 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:32.195 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -42:32.195 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:32.195 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:32.195 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -2022-09-15T21:42:32.196Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:32.196Z [INFO] agent.router.manager: shutting down -42:32.201 [INFO] TestAgent: consul server down -42:32.201 [INFO] TestAgent: shutdown complete -42:32.201 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14007 network=tcp -42:32.201 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14007 network=udp -42:32.201 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14008 network=tcp -42:32.703 [INFO] TestAgent: Waiting for endpoints to shut down -42:32.703 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_EmptyAltDomain (0.74s) -=== CONT TestCoordinate_Update -2022-09-15T21:42:31.789Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:31.868Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:31.881 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a6a975a6-2dfd-4e97-9370-d56f66e0b8c6 Address:127.0.0.1:13991}]" -42:31.882 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13991 [Follower]" leader= -42:31.883 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a6a975a6-2dfd-4e97-9370-d56f66e0b8c6.dc1 127.0.0.1 -42:31.886 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a6a975a6-2dfd-4e97-9370-d56f66e0b8c6 127.0.0.1 -2022-09-15T21:42:31.886Z [INFO] agent.router: Initializing LAN area manager -42:31.887 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13986 network=udp -42:31.887 [INFO] TestAgent.server: Adding LAN server: server="Node-a6a975a6-2dfd-4e97-9370-d56f66e0b8c6 (Addr: tcp/127.0.0.1:13991) (DC: dc1)" -42:31.888 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a6a975a6-2dfd-4e97-9370-d56f66e0b8c6.dc1 area=wan -42:31.888 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13986 network=tcp -42:31.889 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13987 network=tcp -42:31.889 [INFO] TestAgent: started state syncer -42:31.889 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13992 network=tcp -42:31.926 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:31.926 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13991 [Candidate]" term=2 -42:31.926 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:31.926 [DEBUG] TestAgent.server.raft: vote granted: from=a6a975a6-2dfd-4e97-9370-d56f66e0b8c6 term=2 tally=1 -42:31.926 [INFO] TestAgent.server.raft: election won: tally=1 -42:31.926 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13991 [Leader]" -42:31.926 [INFO] TestAgent.server: cluster leadership acquired -42:31.926 [INFO] TestAgent.server: New leader elected: payload=Node-a6a975a6-2dfd-4e97-9370-d56f66e0b8c6 -42:31.927 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13991 -42:31.928 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:31.931 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:31.931 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:31.931 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:31.931 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:31.931 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:31.931 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a6a975a6-2dfd-4e97-9370-d56f66e0b8c6 -42:31.931 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a6a975a6-2dfd-4e97-9370-d56f66e0b8c6 -42:31.932 [INFO] TestAgent.server: federation state anti-entropy synced -42:32.126 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:32.127 [INFO] TestAgent: Synced node info -42:32.127 [DEBUG] TestAgent: Node info in sync -42:32.231 [DEBUG] TestAgent.dns: request served from client: name=fOO.node.dc1.consul. type=ANY class=IN latency=168.52µs client=127.0.0.1:49520 client_network=udp -42:32.232 [INFO] TestAgent: Requesting shutdown -42:32.232 [INFO] TestAgent.server: shutting down server -42:32.232 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:32.232 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:32.232 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:32.232 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:32.232 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:32.232 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:32.232 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:32.232 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:32.232 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:32.232 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:32.232 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:32.232Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:32.232Z [INFO] agent.router.manager: shutting down -42:32.233 [INFO] TestAgent: consul server down -42:32.233 [INFO] TestAgent: shutdown complete -42:32.233 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13986 network=tcp -42:32.233 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13986 network=udp -42:32.233 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13987 network=tcp -42:32.733 [INFO] TestAgent: Waiting for endpoints to shut down -42:32.733 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_CaseInsensitiveNodeLookup (1.00s) -=== CONT TestCoordinate_Node -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:31.892Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:31.916Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:31.919 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7ac0bda7-6a0a-ca55-08a1-a4d9daae4621 Address:127.0.0.1:13998}]" -42:31.919 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13998 [Follower]" leader= -42:31.920 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7ac0bda7-6a0a-ca55-08a1-a4d9daae4621.dc1 127.0.0.1 -42:31.921 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7ac0bda7-6a0a-ca55-08a1-a4d9daae4621 127.0.0.1 -2022-09-15T21:42:31.921Z [INFO] agent.router: Initializing LAN area manager -42:31.922 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13993 network=udp -42:31.922 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7ac0bda7-6a0a-ca55-08a1-a4d9daae4621.dc1 area=wan -42:31.922 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13993 network=tcp -42:31.923 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13994 network=tcp -42:31.923 [INFO] TestAgent: started state syncer -42:31.926 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13999 network=tcp -42:31.929 [INFO] TestAgent.server: Adding LAN server: server="Node-7ac0bda7-6a0a-ca55-08a1-a4d9daae4621 (Addr: tcp/127.0.0.1:13998) (DC: dc1)" -42:31.974 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:31.974 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13998 [Candidate]" term=2 -42:31.974 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:31.974 [DEBUG] TestAgent.server.raft: vote granted: from=7ac0bda7-6a0a-ca55-08a1-a4d9daae4621 term=2 tally=1 -42:31.974 [INFO] TestAgent.server.raft: election won: tally=1 -42:31.974 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13998 [Leader]" -42:31.974 [INFO] TestAgent.server: cluster leadership acquired -42:31.975 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13998 -42:31.976 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:31.977 [INFO] TestAgent.server: New leader elected: payload=Node-7ac0bda7-6a0a-ca55-08a1-a4d9daae4621 -42:31.978 [INFO] TestAgent: Synced node info -42:31.985 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:31.985 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:31.985 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:31.985 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:31.985 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:31.985 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7ac0bda7-6a0a-ca55-08a1-a4d9daae4621 -42:31.985 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7ac0bda7-6a0a-ca55-08a1-a4d9daae4621 -42:31.995 [INFO] TestAgent.server: federation state anti-entropy synced -42:32.084 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:32.084 [DEBUG] TestAgent: Node info in sync -42:32.084 [DEBUG] TestAgent: Node info in sync -42:32.298 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=ANY class=IN latency=268.98µs client=127.0.0.1:35789 client_network=udp -42:32.299 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=A class=IN latency=174.4µs client=127.0.0.1:51847 client_network=udp -42:32.300 [DEBUG] TestAgent.dns: request served from client: name=foo.node.dc1.consul. type=ANY class=IN latency=168.22µs client=127.0.0.1:60164 client_network=udp -42:32.300 [DEBUG] TestAgent.dns: request served from client: name=nofoo.node.dc1.consul. type=ANY class=IN latency=158.04µs client=127.0.0.1:59294 client_network=udp -42:32.300 [INFO] TestAgent: Requesting shutdown -42:32.300 [INFO] TestAgent.server: shutting down server -42:32.300 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:32.300 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:32.300 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:32.300 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:32.300 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:32.301 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:32.301 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:32.301 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:32.301 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:32.301 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:32.301 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:32.301Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:32.301Z [INFO] agent.router.manager: shutting down -42:32.301 [INFO] TestAgent: consul server down -42:32.301 [INFO] TestAgent: shutdown complete -42:32.301 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13993 network=tcp -42:32.301 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13993 network=udp -42:32.301 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13994 network=tcp -42:32.801 [INFO] TestAgent: Waiting for endpoints to shut down -42:32.802 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_NodeLookup (0.98s) -=== CONT TestCoordinate_Nodes -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._tcp_(qType:=255)_compressed=false -=== RUN TestCoordinate_Update_ACLDeny/no_token -=== RUN TestCoordinate_Update_ACLDeny/valid_token -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:32.085Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:32.101Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:32.105 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ceaa48fd-e4ce-0d39-2463-3cca35aff4c6 Address:127.0.0.1:14019}]" -42:32.105 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ceaa48fd-e4ce-0d39-2463-3cca35aff4c6.dc1 127.0.0.1 -42:32.106 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14019 [Follower]" leader= -42:32.106 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ceaa48fd-e4ce-0d39-2463-3cca35aff4c6 127.0.0.1 -2022-09-15T21:42:32.106Z [INFO] agent.router: Initializing LAN area manager -42:32.107 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14014 network=udp -42:32.107 [INFO] TestAgent.server: Adding LAN server: server="Node-ceaa48fd-e4ce-0d39-2463-3cca35aff4c6 (Addr: tcp/127.0.0.1:14019) (DC: dc1)" -42:32.107 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ceaa48fd-e4ce-0d39-2463-3cca35aff4c6.dc1 area=wan -42:32.107 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14014 network=tcp -42:32.108 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14015 network=tcp -42:32.108 [INFO] TestAgent: started state syncer -42:32.108 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14020 network=tcp -42:32.158 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:32.158 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14019 [Candidate]" term=2 -42:32.158 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:32.158 [DEBUG] TestAgent.server.raft: vote granted: from=ceaa48fd-e4ce-0d39-2463-3cca35aff4c6 term=2 tally=1 -42:32.158 [INFO] TestAgent.server.raft: election won: tally=1 -42:32.158 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14019 [Leader]" -42:32.158 [INFO] TestAgent.server: cluster leadership acquired -42:32.159 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14019 -42:32.160 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:32.166 [INFO] TestAgent.server: New leader elected: payload=Node-ceaa48fd-e4ce-0d39-2463-3cca35aff4c6 -42:32.167 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:32.167 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:32.167 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:32.167 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:32.168 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:32.168 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ceaa48fd-e4ce-0d39-2463-3cca35aff4c6 -42:32.168 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ceaa48fd-e4ce-0d39-2463-3cca35aff4c6 -42:32.170 [INFO] TestAgent.server: federation state anti-entropy synced -42:32.336 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:32.337 [INFO] TestAgent: Synced node info -42:32.524 [DEBUG] TestAgent.dns: request served from client: name=foo.node.dc1.consul. type=ANY class=IN latency=326.78µs client=127.0.0.1:45060 client_network=tcp -42:32.524 [INFO] TestAgent: Requesting shutdown -42:32.525 [INFO] TestAgent.server: shutting down server -42:32.525 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:32.525 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:32.525 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:32.525 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:32.525 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:32.525 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:32.525 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:32.525 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:32.525 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:32.525 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:32.525 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:32.525Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:32.526Z [INFO] agent.router.manager: shutting down -42:32.526 [INFO] TestAgent: consul server down -42:32.526 [INFO] TestAgent: shutdown complete -42:32.526 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14014 network=tcp -42:32.526 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14014 network=udp -42:32.526 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14015 network=tcp -42:33.030 [INFO] TestAgent: Waiting for endpoints to shut down -42:33.030 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_Over_TCP (0.97s) -=== CONT TestCoordinate_Datacenters -2022-09-15T21:42:32.226Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:32.258Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:32.261 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:281a2a0c-3a70-f8c6-e2f4-d0be2a8dbf62 Address:127.0.0.1:14026}]" -42:32.261 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14026 [Follower]" leader= -42:32.262 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-281a2a0c-3a70-f8c6-e2f4-d0be2a8dbf62.dc1 127.0.0.1 -42:32.263 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-281a2a0c-3a70-f8c6-e2f4-d0be2a8dbf62 127.0.0.1 -2022-09-15T21:42:32.263Z [INFO] agent.router: Initializing LAN area manager -42:32.263 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14021 network=udp -42:32.264 [INFO] TestAgent.server: Adding LAN server: server="Node-281a2a0c-3a70-f8c6-e2f4-d0be2a8dbf62 (Addr: tcp/127.0.0.1:14026) (DC: dc1)" -42:32.264 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-281a2a0c-3a70-f8c6-e2f4-d0be2a8dbf62.dc1 area=wan -42:32.264 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14021 network=tcp -42:32.265 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14022 network=tcp -42:32.265 [INFO] TestAgent: started state syncer -42:32.265 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14027 network=tcp -42:32.311 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:32.312 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14026 [Candidate]" term=2 -42:32.312 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:32.312 [DEBUG] TestAgent.server.raft: vote granted: from=281a2a0c-3a70-f8c6-e2f4-d0be2a8dbf62 term=2 tally=1 -42:32.312 [INFO] TestAgent.server.raft: election won: tally=1 -42:32.312 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14026 [Leader]" -42:32.312 [INFO] TestAgent.server: cluster leadership acquired -42:32.312 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14026 -42:32.314 [INFO] TestAgent.server: New leader elected: payload=Node-281a2a0c-3a70-f8c6-e2f4-d0be2a8dbf62 -42:32.314 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:32.317 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:32.317 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:32.317 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:32.317 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:32.317 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:32.317 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-281a2a0c-3a70-f8c6-e2f4-d0be2a8dbf62 -42:32.317 [INFO] TestAgent.server: member joined, marking health alive: member=Node-281a2a0c-3a70-f8c6-e2f4-d0be2a8dbf62 -42:32.318 [INFO] TestAgent.server: federation state anti-entropy synced -42:32.612 [INFO] TestAgent: Requesting shutdown -42:32.612 [INFO] TestAgent.server: shutting down server -42:32.612 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:32.612 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:32.612 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:32.612 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:32.612 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:32.613 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:32.613 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:32.613 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:32.613 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:32.613 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:32.613 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:32.613 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:32.614Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:32.614Z [INFO] agent.router.manager: shutting down -42:32.614 [INFO] TestAgent: consul server down -42:32.614 [INFO] TestAgent: shutdown complete -42:32.614 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14021 network=tcp -42:32.614 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14021 network=udp -42:32.614 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14022 network=tcp -42:33.116 [INFO] TestAgent: Waiting for endpoints to shut down -42:33.116 [INFO] TestAgent: Endpoints down ---- PASS: TestDiscoveryChainRead (0.94s) - --- PASS: TestDiscoveryChainRead/GET:_error_on_no_service_name (0.00s) - --- PASS: TestDiscoveryChainRead/GET:_read_default_chain (0.00s) - --- PASS: TestDiscoveryChainRead/GET:_read_default_chain;_evaluate_in_dc2 (0.00s) - --- PASS: TestDiscoveryChainRead/GET:_read_default_chain_with_cache (0.00s) - --- PASS: TestDiscoveryChainRead/POST:_error_on_no_service_name (0.00s) - --- PASS: TestDiscoveryChainRead/POST:_read_default_chain (0.00s) - --- PASS: TestDiscoveryChainRead/POST:_read_default_chain;_evaluate_in_dc2 (0.00s) - --- PASS: TestDiscoveryChainRead/POST:_read_default_chain_with_cache (0.00s) - --- PASS: TestDiscoveryChainRead/GET:_read_modified_chain (0.03s) - --- PASS: TestDiscoveryChainRead/POST:_read_modified_chain_with_overrides_(camel_case) (0.00s) - --- PASS: TestDiscoveryChainRead/POST:_read_modified_chain_with_overrides_(snake_case) (0.00s) -=== CONT TestCoordinate_Disabled_Response -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestCoordinate_Disabled_Response/0 -=== RUN TestCoordinate_Disabled_Response/1 -=== RUN TestCoordinate_Disabled_Response/2 -=== RUN TestCoordinate_Disabled_Response/3 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._udp_(qType:=255)_compressed=true -2022-09-15T21:42:32.591Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:32.591Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:32.622Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:32.622Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:32.626 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:054e6702-afc8-a8e2-9090-51a8ce29d70c Address:127.0.0.1:14033}]" -42:32.628 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-054e6702-afc8-a8e2-9090-51a8ce29d70c.dc1 127.0.0.1 -42:32.632 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14033 [Follower]" leader= -42:32.633 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-054e6702-afc8-a8e2-9090-51a8ce29d70c 127.0.0.1 -2022-09-15T21:42:32.633Z [INFO] agent.router: Initializing LAN area manager -42:32.634 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14028 network=udp -42:32.635 [INFO] TestAgent.server: Adding LAN server: server="Node-054e6702-afc8-a8e2-9090-51a8ce29d70c (Addr: tcp/127.0.0.1:14033) (DC: dc1)" -42:32.636 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-054e6702-afc8-a8e2-9090-51a8ce29d70c.dc1 area=wan -42:32.637 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14028 network=tcp -42:32.639 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14029 network=tcp -42:32.640 [INFO] TestAgent: started state syncer -42:32.640 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14034 network=tcp -42:32.686 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:32.686 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14033 [Candidate]" term=2 -42:32.686 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:32.686 [DEBUG] TestAgent.server.raft: vote granted: from=054e6702-afc8-a8e2-9090-51a8ce29d70c term=2 tally=1 -42:32.686 [INFO] TestAgent.server.raft: election won: tally=1 -42:32.686 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14033 [Leader]" -42:32.686 [INFO] TestAgent.server: cluster leadership acquired -42:32.687 [INFO] TestAgent.server: initializing acls -42:32.687 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:32.687 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:32.688 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:32.689 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:32.689 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:32.689 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:32.690 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-054e6702-afc8-a8e2-9090-51a8ce29d70c -42:32.690 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-054e6702-afc8-a8e2-9090-51a8ce29d70c.dc1 -42:32.690 [INFO] TestAgent.server: New leader elected: payload=Node-054e6702-afc8-a8e2-9090-51a8ce29d70c -42:32.690 [INFO] TestAgent.server: Updating LAN server: server="Node-054e6702-afc8-a8e2-9090-51a8ce29d70c (Addr: tcp/127.0.0.1:14033) (DC: dc1)" -42:32.691 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-054e6702-afc8-a8e2-9090-51a8ce29d70c.dc1 area=wan -42:32.691 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:32.696 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:32.696 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:32.696 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:32.696 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:32.696 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:32.696 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-054e6702-afc8-a8e2-9090-51a8ce29d70c -42:32.697 [INFO] TestAgent.server: member joined, marking health alive: member=Node-054e6702-afc8-a8e2-9090-51a8ce29d70c -42:32.698 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-054e6702-afc8-a8e2-9090-51a8ce29d70c -42:32.699 [INFO] TestAgent.server: federation state anti-entropy synced -42:32.959 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-054e6702-afc8-a8e2-9090-51a8ce29d70c -42:32.959 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-054e6702-afc8-a8e2-9090-51a8ce29d70c -42:32.961 [INFO] TestAgent: Requesting shutdown -42:32.961 [INFO] TestAgent.server: shutting down server -42:32.961 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:32.961 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:32.961 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:32.961 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:32.961 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:32.961 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:32.961 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:32.961 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:32.961 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:32.961 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:32.961 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:32.961 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:32.961 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:32.961 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:32.961 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:32.961Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:32.961Z [INFO] agent.router.manager: shutting down -42:32.962 [INFO] TestAgent: consul server down -42:32.962 [INFO] TestAgent: shutdown complete -42:32.962 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14028 network=tcp -42:32.962 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14028 network=udp -42:32.962 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14029 network=tcp -42:33.462 [INFO] TestAgent: Waiting for endpoints to shut down -42:33.462 [INFO] TestAgent: Endpoints down ---- PASS: TestCoordinate_Update_ACLDeny (0.94s) - --- PASS: TestCoordinate_Update_ACLDeny/no_token (0.00s) - --- PASS: TestCoordinate_Update_ACLDeny/valid_token (0.00s) -=== CONT TestConnectCAConfig -=== RUN TestConnectCAConfig/basic -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._udp_(qType:=255)_compressed=false -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=1)_compressed=true -2022-09-15T21:42:32.755Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:32.836Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:32.840 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d089b9f0-54bc-a983-e41f-a9a5332be55d Address:127.0.0.1:14040}]" -42:32.841 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d089b9f0-54bc-a983-e41f-a9a5332be55d.dc1 127.0.0.1 -42:32.842 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d089b9f0-54bc-a983-e41f-a9a5332be55d 127.0.0.1 -2022-09-15T21:42:32.843Z [INFO] agent.router: Initializing LAN area manager -42:32.843 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14035 network=udp -42:32.843 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14040 [Follower]" leader= -42:32.845 [INFO] TestAgent.server: Adding LAN server: server="Node-d089b9f0-54bc-a983-e41f-a9a5332be55d (Addr: tcp/127.0.0.1:14040) (DC: dc1)" -42:32.845 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d089b9f0-54bc-a983-e41f-a9a5332be55d.dc1 area=wan -42:32.847 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14035 network=tcp -42:32.849 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14036 network=tcp -42:32.849 [INFO] TestAgent: started state syncer -42:32.850 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14041 network=tcp -42:32.881 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:32.881 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14040 [Candidate]" term=2 -42:32.881 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:32.881 [DEBUG] TestAgent.server.raft: vote granted: from=d089b9f0-54bc-a983-e41f-a9a5332be55d term=2 tally=1 -42:32.881 [INFO] TestAgent.server.raft: election won: tally=1 -42:32.881 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14040 [Leader]" -42:32.881 [INFO] TestAgent.server: cluster leadership acquired -42:32.881 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14040 -42:32.883 [INFO] TestAgent.server: New leader elected: payload=Node-d089b9f0-54bc-a983-e41f-a9a5332be55d -42:32.883 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:32.887 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:32.887 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:32.887 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:32.887 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:32.887 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:32.887 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d089b9f0-54bc-a983-e41f-a9a5332be55d -42:32.887 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d089b9f0-54bc-a983-e41f-a9a5332be55d -42:32.889 [INFO] TestAgent.server: federation state anti-entropy synced -42:33.042 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:33.043 [INFO] TestAgent: Synced node info -42:33.244 [INFO] TestAgent: Requesting shutdown -42:33.245 [INFO] TestAgent.server: shutting down server -42:33.245 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:33.245 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:33.245 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:33.245 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:33.245 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:33.246 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:33.246 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:33.246 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:33.246 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:33.246 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:33.249 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:33.249Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:33.249Z [INFO] agent.router.manager: shutting down -42:33.254 [INFO] TestAgent: consul server down -42:33.254 [INFO] TestAgent: shutdown complete -42:33.254 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14035 network=tcp -42:33.254 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14035 network=udp -42:33.254 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14036 network=tcp -42:33.756 [INFO] TestAgent: Waiting for endpoints to shut down -42:33.756 [INFO] TestAgent: Endpoints down ---- PASS: TestCoordinate_Update (1.05s) -=== CONT TestConnectCARoots_list -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:32.845Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:32.874Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:32.878 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6f1fa895-c463-f29a-be39-ff5433fe5793 Address:127.0.0.1:14054}]" -42:32.878 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14054 [Follower]" leader= -42:32.879 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6f1fa895-c463-f29a-be39-ff5433fe5793.dc1 127.0.0.1 -42:32.880 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6f1fa895-c463-f29a-be39-ff5433fe5793 127.0.0.1 -2022-09-15T21:42:32.880Z [INFO] agent.router: Initializing LAN area manager -42:32.881 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6f1fa895-c463-f29a-be39-ff5433fe5793.dc1 area=wan -42:32.881 [INFO] TestAgent.server: Adding LAN server: server="Node-6f1fa895-c463-f29a-be39-ff5433fe5793 (Addr: tcp/127.0.0.1:14054) (DC: dc1)" -42:32.881 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14049 network=udp -42:32.881 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14049 network=tcp -42:32.882 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14050 network=tcp -42:32.883 [INFO] TestAgent: started state syncer -42:32.883 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14055 network=tcp -42:32.925 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:32.925 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14054 [Candidate]" term=2 -42:32.925 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:32.925 [DEBUG] TestAgent.server.raft: vote granted: from=6f1fa895-c463-f29a-be39-ff5433fe5793 term=2 tally=1 -42:32.925 [INFO] TestAgent.server.raft: election won: tally=1 -42:32.925 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14054 [Leader]" -42:32.926 [INFO] TestAgent.server: cluster leadership acquired -42:32.926 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14054 -42:32.926 [INFO] TestAgent.server: New leader elected: payload=Node-6f1fa895-c463-f29a-be39-ff5433fe5793 -42:32.927 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:32.930 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:32.930 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:32.930 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:32.930 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:32.930 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:32.930 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6f1fa895-c463-f29a-be39-ff5433fe5793 -42:32.930 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6f1fa895-c463-f29a-be39-ff5433fe5793 -42:32.932 [INFO] TestAgent.server: federation state anti-entropy synced -42:32.961 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:32.962 [INFO] TestAgent: Synced node info -42:32.962 [DEBUG] TestAgent: Node info in sync -42:33.304 [DEBUG] TestAgent.dns: request served from client: name=nofoo.node.dc1.consul. type=ANY class=IN latency=317.52µs client=127.0.0.1:35039 client_network=udp -42:33.304 [INFO] TestAgent: Requesting shutdown -42:33.304 [INFO] TestAgent.server: shutting down server -42:33.304 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:33.304 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:33.304 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:33.304 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:33.304 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:33.305 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:33.305 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:33.305 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:33.305 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:33.305 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:33.305 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:33.305Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:33.305Z [INFO] agent.router.manager: shutting down -42:33.306 [INFO] TestAgent: consul server down -42:33.306 [INFO] TestAgent: shutdown complete -42:33.306 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14049 network=tcp -42:33.306 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14049 network=udp -42:33.306 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14050 network=tcp -42:33.811 [INFO] TestAgent: Waiting for endpoints to shut down -42:33.811 [INFO] TestAgent: Endpoints down -2022-09-15T21:42:31.967Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:32.016Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:32.024 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8f8dbb4e-fce9-c414-7c39-5b00ef4bdf4c Address:127.0.0.1:14005}]" -42:32.025 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14005 [Follower]" leader= -42:32.025 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8f8dbb4e-fce9-c414-7c39-5b00ef4bdf4c.dc1 127.0.0.1 -42:32.026 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8f8dbb4e-fce9-c414-7c39-5b00ef4bdf4c 127.0.0.1 -2022-09-15T21:42:32.026Z [INFO] agent.router: Initializing LAN area manager -42:32.027 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14000 network=udp -42:32.027 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8f8dbb4e-fce9-c414-7c39-5b00ef4bdf4c.dc1 area=wan -42:32.027 [INFO] TestAgent.server: Adding LAN server: server="Node-8f8dbb4e-fce9-c414-7c39-5b00ef4bdf4c (Addr: tcp/127.0.0.1:14005) (DC: dc1)" -42:32.027 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14000 network=tcp -42:32.029 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14001 network=tcp -42:32.029 [INFO] TestAgent: started state syncer -42:32.029 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14006 network=tcp -42:32.062 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:32.062 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14005 [Candidate]" term=2 -42:32.062 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:32.062 [DEBUG] TestAgent.server.raft: vote granted: from=8f8dbb4e-fce9-c414-7c39-5b00ef4bdf4c term=2 tally=1 -42:32.062 [INFO] TestAgent.server.raft: election won: tally=1 -42:32.062 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14005 [Leader]" -42:32.063 [INFO] TestAgent.server: cluster leadership acquired -42:32.063 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14005 -42:32.063 [INFO] TestAgent.server: New leader elected: payload=Node-8f8dbb4e-fce9-c414-7c39-5b00ef4bdf4c -42:32.066 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:32.069 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:32.069 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:32.069 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:32.069 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:32.069 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:32.069 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8f8dbb4e-fce9-c414-7c39-5b00ef4bdf4c -42:32.069 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8f8dbb4e-fce9-c414-7c39-5b00ef4bdf4c -42:32.070 [INFO] TestAgent.server: federation state anti-entropy synced -42:32.106 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:32.107 [INFO] TestAgent: Synced node info -42:32.282 [DEBUG] TestAgent.dns: request served from client: name=nofoo.node.dc1.consul. type=ANY class=IN latency=218.24µs client=127.0.0.1:40494 client_network=udp -42:32.283 [INFO] TestAgent: Requesting shutdown -42:32.283 [INFO] TestAgent.server: shutting down server -42:32.283 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:32.283 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:32.283 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:32.283 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:32.283 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:32.283 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:32.283 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:32.283 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:32.283 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:32.283 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:32.283 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:32.283Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:32.283Z [INFO] agent.router.manager: shutting down -42:32.284 [INFO] TestAgent: consul server down -42:32.284 [INFO] TestAgent: shutdown complete -42:32.284 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14000 network=tcp -42:32.284 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14000 network=udp -42:32.284 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14001 network=tcp -42:32.784 [INFO] TestAgent: Waiting for endpoints to shut down -42:32.784 [INFO] TestAgent: Endpoints down -2022-09-15T21:42:30.955Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:31.054Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:31.068 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:57c721be-ae45-fbe9-409a-97c32c18b11e Address:127.0.0.1:13956}]" -42:31.070 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-57c721be-ae45-fbe9-409a-97c32c18b11e.dc1 127.0.0.1 -42:31.071 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-57c721be-ae45-fbe9-409a-97c32c18b11e 127.0.0.1 -2022-09-15T21:42:31.071Z [INFO] agent.router: Initializing LAN area manager -42:31.072 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13951 network=udp -42:31.072 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13956 [Follower]" leader= -42:31.072 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-57c721be-ae45-fbe9-409a-97c32c18b11e.dc1 area=wan -42:31.073 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13951 network=tcp -42:31.073 [INFO] TestAgent.server: Adding LAN server: server="Node-57c721be-ae45-fbe9-409a-97c32c18b11e (Addr: tcp/127.0.0.1:13956) (DC: dc1)" -42:31.075 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13952 network=tcp -42:31.075 [INFO] TestAgent: started state syncer -42:31.075 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13957 network=tcp -42:31.134 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:31.134 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13956 [Candidate]" term=2 -42:31.134 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:31.134 [DEBUG] TestAgent.server.raft: vote granted: from=57c721be-ae45-fbe9-409a-97c32c18b11e term=2 tally=1 -42:31.134 [INFO] TestAgent.server.raft: election won: tally=1 -42:31.134 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13956 [Leader]" -42:31.134 [INFO] TestAgent.server: cluster leadership acquired -42:31.150 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13956 -42:31.150 [INFO] TestAgent.server: New leader elected: payload=Node-57c721be-ae45-fbe9-409a-97c32c18b11e -42:31.154 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:31.215 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:31.215 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:31.215 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:31.215 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:31.215 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:31.215 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-57c721be-ae45-fbe9-409a-97c32c18b11e -42:31.215 [INFO] TestAgent.server: member joined, marking health alive: member=Node-57c721be-ae45-fbe9-409a-97c32c18b11e -42:31.216 [INFO] TestAgent.server: federation state anti-entropy synced -42:31.342 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:31.343 [INFO] TestAgent: Synced node info -42:31.439 [DEBUG] TestAgent.dns: request served from client: name=nofoo.node.dc1.consul. type=ANY class=IN latency=243.7µs client=127.0.0.1:51062 client_network=udp -42:31.440 [INFO] TestAgent: Requesting shutdown -42:31.440 [INFO] TestAgent.server: shutting down server -42:31.440 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:31.440 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:31.440 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:31.440 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:31.440 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:31.440 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:31.440 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:31.440 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:31.440 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:31.440 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:31.440 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:31.440Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:31.440Z [INFO] agent.router.manager: shutting down -42:31.441 [INFO] TestAgent: consul server down -42:31.441 [INFO] TestAgent: shutdown complete -42:31.441 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13951 network=tcp -42:31.441 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13951 network=udp -42:31.441 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13952 network=tcp -42:31.941 [INFO] TestAgent: Waiting for endpoints to shut down -42:31.941 [INFO] TestAgent: Endpoints down -2022-09-15T21:42:29.904Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:29.920Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:29.923 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bec56c24-cc95-8ed6-e973-9dcbfecf1620 Address:127.0.0.1:13886}]" -42:29.924 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bec56c24-cc95-8ed6-e973-9dcbfecf1620.dc1 127.0.0.1 -42:29.925 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bec56c24-cc95-8ed6-e973-9dcbfecf1620 127.0.0.1 -2022-09-15T21:42:29.925Z [INFO] agent.router: Initializing LAN area manager -42:29.926 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13886 [Follower]" leader= -42:29.926 [INFO] TestAgent.server: Adding LAN server: server="Node-bec56c24-cc95-8ed6-e973-9dcbfecf1620 (Addr: tcp/127.0.0.1:13886) (DC: dc1)" -42:29.926 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bec56c24-cc95-8ed6-e973-9dcbfecf1620.dc1 area=wan -42:29.926 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13895 network=udp -42:29.926 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13895 network=tcp -42:29.927 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13896 network=tcp -42:29.927 [INFO] TestAgent: started state syncer -42:29.927 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13887 network=tcp -42:29.998 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:29.998 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13886 [Candidate]" term=2 -42:29.998 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:29.998 [DEBUG] TestAgent.server.raft: vote granted: from=bec56c24-cc95-8ed6-e973-9dcbfecf1620 term=2 tally=1 -42:29.998 [INFO] TestAgent.server.raft: election won: tally=1 -42:29.998 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13886 [Leader]" -42:29.998 [INFO] TestAgent.server: cluster leadership acquired -42:29.998 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13886 -42:30.000 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:30.002 [INFO] TestAgent.server: New leader elected: payload=Node-bec56c24-cc95-8ed6-e973-9dcbfecf1620 -42:30.003 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:30.003 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:30.003 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:30.003 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:30.003 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:30.003 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bec56c24-cc95-8ed6-e973-9dcbfecf1620 -42:30.003 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bec56c24-cc95-8ed6-e973-9dcbfecf1620 -42:30.004 [INFO] TestAgent.server: federation state anti-entropy synced -42:30.222 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:30.222 [INFO] TestAgent: Synced node info -42:30.370 [DEBUG] TestAgent.dns: request served from client: name=nofoo.node.dc1.consul. type=ANY class=IN latency=215.52µs client=127.0.0.1:55077 client_network=udp -42:30.370 [INFO] TestAgent: Requesting shutdown -42:30.370 [INFO] TestAgent.server: shutting down server -42:30.370 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:30.370 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:30.370 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:30.370 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:30.370 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:30.370 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:30.370 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:30.370 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:30.371 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:30.371 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:30.371 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:30.371Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:30.371Z [INFO] agent.router.manager: shutting down -42:30.371 [INFO] TestAgent: consul server down -42:30.371 [INFO] TestAgent: shutdown complete -42:30.371 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13895 network=tcp -42:30.371 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13895 network=udp -42:30.371 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13896 network=tcp -42:30.877 [INFO] TestAgent: Waiting for endpoints to shut down -42:30.877 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_SOA_Settings (3.94s) -=== CONT TestConnectCARoots_empty -2022-09-15T21:42:32.862Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:32.900Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:32.904 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:24919e11-88ca-9988-175f-a42371638a05 Address:127.0.0.1:14061}]" -42:32.904 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14061 [Follower]" leader= -42:32.905 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-24919e11-88ca-9988-175f-a42371638a05.dc1 127.0.0.1 -42:32.905 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-24919e11-88ca-9988-175f-a42371638a05 127.0.0.1 -2022-09-15T21:42:32.905Z [INFO] agent.router: Initializing LAN area manager -42:32.906 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14056 network=udp -42:32.906 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-24919e11-88ca-9988-175f-a42371638a05.dc1 area=wan -42:32.906 [INFO] TestAgent.server: Adding LAN server: server="Node-24919e11-88ca-9988-175f-a42371638a05 (Addr: tcp/127.0.0.1:14061) (DC: dc1)" -42:32.906 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14056 network=tcp -42:32.907 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14057 network=tcp -42:32.907 [INFO] TestAgent: started state syncer -42:32.907 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14062 network=tcp -42:32.964 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:32.964 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14061 [Candidate]" term=2 -42:32.964 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:32.964 [DEBUG] TestAgent.server.raft: vote granted: from=24919e11-88ca-9988-175f-a42371638a05 term=2 tally=1 -42:32.964 [INFO] TestAgent.server.raft: election won: tally=1 -42:32.964 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14061 [Leader]" -42:32.964 [INFO] TestAgent.server: cluster leadership acquired -42:32.965 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14061 -42:32.965 [INFO] TestAgent.server: New leader elected: payload=Node-24919e11-88ca-9988-175f-a42371638a05 -42:32.966 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:32.969 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:32.969 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:32.969 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:32.969 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:32.969 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:32.969 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-24919e11-88ca-9988-175f-a42371638a05 -42:32.969 [INFO] TestAgent.server: member joined, marking health alive: member=Node-24919e11-88ca-9988-175f-a42371638a05 -42:32.970 [INFO] TestAgent.server: federation state anti-entropy synced -42:33.107 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:33.114 [INFO] TestAgent: Synced node info -42:33.114 [DEBUG] TestAgent: Node info in sync -42:33.308 [INFO] TestAgent: Requesting shutdown -42:33.308 [INFO] TestAgent.server: shutting down server -42:33.308 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:33.308 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:33.308 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:33.308 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:33.308 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:33.308 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:33.308 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:33.308 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:33.308 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:33.308 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:33.309 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:33.309Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:33.309Z [INFO] agent.router.manager: shutting down -42:33.309 [INFO] TestAgent: consul server down -42:33.309 [INFO] TestAgent: shutdown complete -42:33.309 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14056 network=tcp -42:33.314 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14056 network=udp -42:33.314 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14057 network=tcp -42:33.834 [INFO] TestAgent: Waiting for endpoints to shut down -42:33.834 [INFO] TestAgent: Endpoints down ---- PASS: TestCoordinate_Nodes (1.03s) -=== CONT TestConfig_Apply_ProxyDefaultsExpose -2022-09-15T21:42:32.785Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:32.861Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:32.874 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f0ec9d71-5963-a68c-4bec-03d23e746ab5 Address:127.0.0.1:14047}]" -42:32.875 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f0ec9d71-5963-a68c-4bec-03d23e746ab5.dc1 127.0.0.1 -42:32.876 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f0ec9d71-5963-a68c-4bec-03d23e746ab5 127.0.0.1 -2022-09-15T21:42:32.876Z [INFO] agent.router: Initializing LAN area manager -42:32.877 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14042 network=udp -42:32.877 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14047 [Follower]" leader= -42:32.877 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f0ec9d71-5963-a68c-4bec-03d23e746ab5.dc1 area=wan -42:32.877 [INFO] TestAgent.server: Adding LAN server: server="Node-f0ec9d71-5963-a68c-4bec-03d23e746ab5 (Addr: tcp/127.0.0.1:14047) (DC: dc1)" -42:32.877 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14042 network=tcp -42:32.880 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14043 network=tcp -42:32.880 [INFO] TestAgent: started state syncer -42:32.880 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14048 network=tcp -42:32.937 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:32.937 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14047 [Candidate]" term=2 -42:32.937 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:32.937 [DEBUG] TestAgent.server.raft: vote granted: from=f0ec9d71-5963-a68c-4bec-03d23e746ab5 term=2 tally=1 -42:32.937 [INFO] TestAgent.server.raft: election won: tally=1 -42:32.937 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14047 [Leader]" -42:32.937 [INFO] TestAgent.server: cluster leadership acquired -42:32.937 [INFO] TestAgent.server: New leader elected: payload=Node-f0ec9d71-5963-a68c-4bec-03d23e746ab5 -42:32.937 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14047 -42:32.939 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:32.941 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:32.941 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:32.941 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:32.941 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:32.941 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:32.941 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f0ec9d71-5963-a68c-4bec-03d23e746ab5 -42:32.941 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f0ec9d71-5963-a68c-4bec-03d23e746ab5 -42:32.942 [INFO] TestAgent.server: federation state anti-entropy synced -42:32.949 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:32.950 [INFO] TestAgent: Synced node info -42:33.434 [INFO] TestAgent: Requesting shutdown -42:33.435 [INFO] TestAgent.server: shutting down server -42:33.435 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:33.435 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:33.435 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:33.435 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:33.435 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:33.435 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:33.435 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:33.435 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:33.435 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:33.435 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:33.435 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:33.435Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:33.435Z [INFO] agent.router.manager: shutting down -42:33.435 [INFO] TestAgent: consul server down -42:33.435 [INFO] TestAgent: shutdown complete -42:33.435 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14042 network=tcp -42:33.436 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14042 network=udp -42:33.436 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14043 network=tcp -42:33.938 [INFO] TestAgent: Waiting for endpoints to shut down -42:33.938 [INFO] TestAgent: Endpoints down ---- PASS: TestCoordinate_Node (1.20s) -=== CONT TestConfig_Apply_Decoding -2022-09-15T21:42:33.148Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:33.208Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:33.219 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:87a13f1c-809c-5a89-0bb2-96b1a85d4513 Address:127.0.0.1:14075}]" -42:33.220 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14075 [Follower]" leader= -42:33.224 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-87a13f1c-809c-5a89-0bb2-96b1a85d4513.dc1 127.0.0.1 -42:33.226 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-87a13f1c-809c-5a89-0bb2-96b1a85d4513 127.0.0.1 -2022-09-15T21:42:33.226Z [INFO] agent.router: Initializing LAN area manager -42:33.228 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14070 network=udp -42:33.228 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-87a13f1c-809c-5a89-0bb2-96b1a85d4513.dc1 area=wan -42:33.229 [INFO] TestAgent.server: Adding LAN server: server="Node-87a13f1c-809c-5a89-0bb2-96b1a85d4513 (Addr: tcp/127.0.0.1:14075) (DC: dc1)" -42:33.230 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14070 network=tcp -42:33.231 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14071 network=tcp -42:33.231 [INFO] TestAgent: started state syncer -42:33.232 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14076 network=tcp -42:33.294 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:33.294 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14075 [Candidate]" term=2 -42:33.294 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:33.294 [DEBUG] TestAgent.server.raft: vote granted: from=87a13f1c-809c-5a89-0bb2-96b1a85d4513 term=2 tally=1 -42:33.294 [INFO] TestAgent.server.raft: election won: tally=1 -42:33.295 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14075 [Leader]" -42:33.295 [INFO] TestAgent.server: cluster leadership acquired -42:33.295 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14075 -42:33.298 [INFO] TestAgent.server: New leader elected: payload=Node-87a13f1c-809c-5a89-0bb2-96b1a85d4513 -42:33.310 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:33.315 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:33.315 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:33.315 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:33.315 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:33.315 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:33.315 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-87a13f1c-809c-5a89-0bb2-96b1a85d4513 -42:33.315 [INFO] TestAgent.server: member joined, marking health alive: member=Node-87a13f1c-809c-5a89-0bb2-96b1a85d4513 -42:33.319 [INFO] TestAgent.server: federation state anti-entropy synced -42:33.435 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:33.437 [INFO] TestAgent: Requesting shutdown -42:33.437 [INFO] TestAgent.server: shutting down server -42:33.437 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:33.437 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:33.437 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:33.437 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:33.437 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:33.437 [INFO] TestAgent: Synced node info -42:33.437 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:33.437 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:33.437 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -42:33.437 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:33.437 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:33.437 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -2022-09-15T21:42:33.438Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:33.438Z [INFO] agent.router.manager: shutting down -42:33.438 [INFO] TestAgent: consul server down -42:33.438 [INFO] TestAgent: shutdown complete -42:33.438 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14070 network=tcp -42:33.438 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14070 network=udp -42:33.438 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14071 network=tcp -42:33.942 [INFO] TestAgent: Waiting for endpoints to shut down -42:33.942 [INFO] TestAgent: Endpoints down ---- PASS: TestCoordinate_Disabled_Response (0.83s) - --- PASS: TestCoordinate_Disabled_Response/0 (0.00s) - --- PASS: TestCoordinate_Disabled_Response/1 (0.00s) - --- PASS: TestCoordinate_Disabled_Response/2 (0.00s) - --- PASS: TestCoordinate_Disabled_Response/3 (0.00s) -=== CONT TestConfig_Apply_CAS -[WARN] freeport: 4 out of 53 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:33.061Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:33.123Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:33.127 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7b0995b0-8f39-061e-29dd-1273dc1291e0 Address:127.0.0.1:14068}]" -42:33.128 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7b0995b0-8f39-061e-29dd-1273dc1291e0.dc1 127.0.0.1 -42:33.129 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14068 [Follower]" leader= -42:33.133 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7b0995b0-8f39-061e-29dd-1273dc1291e0 127.0.0.1 -2022-09-15T21:42:33.133Z [INFO] agent.router: Initializing LAN area manager -42:33.133 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7b0995b0-8f39-061e-29dd-1273dc1291e0.dc1 area=wan -42:33.133 [INFO] TestAgent.server: Adding LAN server: server="Node-7b0995b0-8f39-061e-29dd-1273dc1291e0 (Addr: tcp/127.0.0.1:14068) (DC: dc1)" -42:33.137 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14063 network=tcp -42:33.137 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14063 network=udp -42:33.138 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14064 network=tcp -42:33.146 [INFO] TestAgent: started state syncer -42:33.146 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14069 network=tcp -42:33.189 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:33.189 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14068 [Candidate]" term=2 -42:33.191 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:33.191 [DEBUG] TestAgent.server.raft: vote granted: from=7b0995b0-8f39-061e-29dd-1273dc1291e0 term=2 tally=1 -42:33.191 [INFO] TestAgent.server.raft: election won: tally=1 -42:33.192 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14068 [Leader]" -42:33.193 [INFO] TestAgent.server: cluster leadership acquired -42:33.194 [INFO] TestAgent.server: New leader elected: payload=Node-7b0995b0-8f39-061e-29dd-1273dc1291e0 -42:33.194 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14068 -42:33.195 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:33.198 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:33.198 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:33.198 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:33.198 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:33.198 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:33.199 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7b0995b0-8f39-061e-29dd-1273dc1291e0 -42:33.199 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7b0995b0-8f39-061e-29dd-1273dc1291e0 -42:33.200 [INFO] TestAgent.server: federation state anti-entropy synced -42:33.245 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:33.250 [INFO] TestAgent: Synced node info -42:33.590 [INFO] TestAgent: Requesting shutdown -42:33.590 [INFO] TestAgent.server: shutting down server -42:33.590 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:33.590 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:33.590 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:33.590 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:33.590 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:33.590 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:33.591 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:33.591 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:33.591 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:33.591 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:33.591 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:33.591Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:33.591Z [INFO] agent.router.manager: shutting down -42:33.591 [INFO] TestAgent: consul server down -42:33.591 [INFO] TestAgent: shutdown complete -42:33.591 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14063 network=tcp -42:33.591 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14063 network=udp -42:33.591 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14064 network=tcp -42:34.098 [INFO] TestAgent: Waiting for endpoints to shut down -42:34.098 [INFO] TestAgent: Endpoints down ---- PASS: TestCoordinate_Datacenters (1.07s) -=== CONT TestConfig_Apply_ProxyDefaultsMeshGateway -=== RUN TestConfig_Apply_Decoding/No_Kind -=== RUN TestConfig_Apply_Decoding/Kind_Not_String -=== RUN TestConfig_Apply_Decoding/Lowercase_kind -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=1)_compressed=false -2022-09-15T21:42:33.481Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:33.500Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:33.503 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f89c8c03-61bb-8193-8537-26758580c6d7 Address:127.0.0.1:14082}]" -42:33.503 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14082 [Follower]" leader= -42:33.504 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f89c8c03-61bb-8193-8537-26758580c6d7.dc1 127.0.0.1 -42:33.505 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f89c8c03-61bb-8193-8537-26758580c6d7 127.0.0.1 -2022-09-15T21:42:33.505Z [INFO] agent.router: Initializing LAN area manager -42:33.505 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14077 network=udp -42:33.505 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f89c8c03-61bb-8193-8537-26758580c6d7.dc1 area=wan -42:33.505 [INFO] TestAgent.server: Adding LAN server: server="Node-f89c8c03-61bb-8193-8537-26758580c6d7 (Addr: tcp/127.0.0.1:14082) (DC: dc1)" -42:33.506 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14077 network=tcp -42:33.507 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14078 network=tcp -42:33.507 [INFO] TestAgent: started state syncer -42:33.507 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14083 network=tcp -42:33.567 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:33.567 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14082 [Candidate]" term=2 -42:33.567 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:33.567 [DEBUG] TestAgent.server.raft: vote granted: from=f89c8c03-61bb-8193-8537-26758580c6d7 term=2 tally=1 -42:33.567 [INFO] TestAgent.server.raft: election won: tally=1 -42:33.567 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14082 [Leader]" -42:33.567 [INFO] TestAgent.server: cluster leadership acquired -42:33.568 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14082 -42:33.569 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:33.570 [INFO] TestAgent.server: New leader elected: payload=Node-f89c8c03-61bb-8193-8537-26758580c6d7 -42:33.581 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:33.581 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:33.581 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:33.581 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:33.581 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:33.582 [INFO] TestAgent.server: federation state anti-entropy synced -42:33.590 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f89c8c03-61bb-8193-8537-26758580c6d7 -42:33.590 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f89c8c03-61bb-8193-8537-26758580c6d7 -42:33.864 [INFO] TestAgent.server.connect: CA provider config updated -42:33.864 [INFO] TestAgent: Requesting shutdown -42:33.864 [INFO] TestAgent.server: shutting down server -42:33.864 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:33.864 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:33.864 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:33.864 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:33.864 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:33.864 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:33.864 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:33.864 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:33.864 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:33.865 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:33.865 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:33.865 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:33.865Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:33.865Z [INFO] agent.router.manager: shutting down -42:33.865 [INFO] TestAgent: consul server down -42:33.865 [INFO] TestAgent: shutdown complete -42:33.865 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14077 network=tcp -42:33.865 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14077 network=udp -42:33.865 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14078 network=tcp -42:34.366 [INFO] TestAgent: Waiting for endpoints to shut down -42:34.366 [INFO] TestAgent: Endpoints down -=== RUN TestConnectCAConfig/basic_with_IntermediateCertTTL -2022-09-15T21:42:33.774Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:33.794Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:33.797 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:537440bd-5acc-c455-9254-1ce55e126305 Address:127.0.0.1:13069}]" -42:33.797 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13069 [Follower]" leader= -42:33.798 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-537440bd-5acc-c455-9254-1ce55e126305.dc1 127.0.0.1 -42:33.799 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-537440bd-5acc-c455-9254-1ce55e126305 127.0.0.1 -2022-09-15T21:42:33.799Z [INFO] agent.router: Initializing LAN area manager -42:33.799 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13064 network=udp -42:33.799 [INFO] TestAgent.server: Adding LAN server: server="Node-537440bd-5acc-c455-9254-1ce55e126305 (Addr: tcp/127.0.0.1:13069) (DC: dc1)" -42:33.799 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-537440bd-5acc-c455-9254-1ce55e126305.dc1 area=wan -42:33.800 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13064 network=tcp -42:33.800 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13065 network=tcp -42:33.801 [INFO] TestAgent: started state syncer -42:33.801 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13070 network=tcp -42:33.842 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:33.842 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13069 [Candidate]" term=2 -42:33.842 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:33.842 [DEBUG] TestAgent.server.raft: vote granted: from=537440bd-5acc-c455-9254-1ce55e126305 term=2 tally=1 -42:33.842 [INFO] TestAgent.server.raft: election won: tally=1 -42:33.842 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13069 [Leader]" -42:33.842 [INFO] TestAgent.server: cluster leadership acquired -42:33.843 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13069 -42:33.844 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:33.856 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:33.856 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:33.856 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:33.856 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:33.856 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:33.856 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-537440bd-5acc-c455-9254-1ce55e126305 -42:33.856 [INFO] TestAgent.server: member joined, marking health alive: member=Node-537440bd-5acc-c455-9254-1ce55e126305 -42:33.862 [INFO] TestAgent.server: New leader elected: payload=Node-537440bd-5acc-c455-9254-1ce55e126305 -42:33.862 [INFO] TestAgent.server: federation state anti-entropy synced -42:33.935 [DEBUG] connect.ca.consul: consul CA provider configured: id=9e:b6:b9:6c:75:46:8d:b6:69:60:9e:37:ca:d1:e5:a2:cc:78:35:a5:30:84:be:8d:cf:27:cf:c5:59:16:f0:3a is_primary=true -42:33.944 [INFO] TestAgent.server.connect: CA rotated to new root under provider: provider=consul -42:33.944 [INFO] TestAgent: Requesting shutdown -42:33.945 [INFO] TestAgent.server: shutting down server -42:33.945 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:33.945 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:33.945 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:33.945 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:33.945 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:33.946 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:33.946 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:33.946 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:33.946 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:33.946 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:33.946 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:33.946 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:33.946Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:33.946Z [INFO] agent.router.manager: shutting down -42:33.947 [INFO] TestAgent: consul server down -42:33.947 [INFO] TestAgent: shutdown complete -42:33.947 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13064 network=tcp -42:33.947 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13064 network=udp -42:33.947 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13065 network=tcp -42:34.450 [INFO] TestAgent: Waiting for endpoints to shut down -42:34.450 [INFO] TestAgent: Endpoints down ---- PASS: TestConnectCARoots_list (0.69s) -=== CONT TestAgent_RegisterCheck -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:33.893Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:33.923Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:33.952 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8b050452-1227-ba7d-5d0e-ce7b48f035fc Address:127.0.0.1:13118}]" -42:33.955 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8b050452-1227-ba7d-5d0e-ce7b48f035fc.dc1 127.0.0.1 -42:33.962 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8b050452-1227-ba7d-5d0e-ce7b48f035fc 127.0.0.1 -2022-09-15T21:42:33.962Z [INFO] agent.router: Initializing LAN area manager -42:33.963 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13118 [Follower]" leader= -42:33.963 [INFO] TestAgent.server: Adding LAN server: server="Node-8b050452-1227-ba7d-5d0e-ce7b48f035fc (Addr: tcp/127.0.0.1:13118) (DC: dc1)" -42:33.964 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8b050452-1227-ba7d-5d0e-ce7b48f035fc.dc1 area=wan -42:33.970 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13071 network=udp -42:33.970 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13071 network=tcp -42:33.974 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13072 network=tcp -42:33.974 [INFO] TestAgent: started state syncer -42:33.975 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13119 network=tcp -42:34.005 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:34.005 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13118 [Candidate]" term=2 -42:34.005 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:34.005 [DEBUG] TestAgent.server.raft: vote granted: from=8b050452-1227-ba7d-5d0e-ce7b48f035fc term=2 tally=1 -42:34.005 [INFO] TestAgent.server.raft: election won: tally=1 -42:34.005 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13118 [Leader]" -42:34.005 [INFO] TestAgent.server: cluster leadership acquired -42:34.014 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13118 -42:34.014 [INFO] TestAgent.server: New leader elected: payload=Node-8b050452-1227-ba7d-5d0e-ce7b48f035fc -42:34.016 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:34.019 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:34.019 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:34.019 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:34.019 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:34.019 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:34.019 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8b050452-1227-ba7d-5d0e-ce7b48f035fc -42:34.019 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8b050452-1227-ba7d-5d0e-ce7b48f035fc -42:34.020 [INFO] TestAgent.server: federation state anti-entropy synced -42:34.026 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:34.026 [INFO] TestAgent: Synced node info -42:34.031 [INFO] TestAgent: Requesting shutdown -42:34.031 [INFO] TestAgent.server: shutting down server -42:34.031 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:34.031 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:34.031 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:34.031 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:34.031 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:34.031 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:34.031 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:34.031 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:34.031 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:34.031 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:34.031 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:34.031Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:34.032Z [INFO] agent.router.manager: shutting down -42:34.032 [INFO] TestAgent: consul server down -42:34.032 [INFO] TestAgent: shutdown complete -42:34.032 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13071 network=tcp -42:34.032 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13071 network=udp -42:34.032 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13072 network=tcp -42:34.532 [INFO] TestAgent: Waiting for endpoints to shut down -42:34.533 [INFO] TestAgent: Endpoints down ---- PASS: TestConfig_Apply_ProxyDefaultsExpose (0.70s) -=== CONT TestRPC_HTTPSMaxConnsPerClient -=== RUN TestRPC_HTTPSMaxConnsPerClient/HTTP -2022-09-15T21:42:33.999Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:34.028Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:34.031 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:81459c7f-2565-03ca-b547-00b95524e9e5 Address:127.0.0.1:13125}]" -42:34.032 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13125 [Follower]" leader= -42:34.032 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-81459c7f-2565-03ca-b547-00b95524e9e5.dc1 127.0.0.1 -42:34.033 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-81459c7f-2565-03ca-b547-00b95524e9e5 127.0.0.1 -2022-09-15T21:42:34.033Z [INFO] agent.router: Initializing LAN area manager -42:34.033 [INFO] TestAgent.server: Adding LAN server: server="Node-81459c7f-2565-03ca-b547-00b95524e9e5 (Addr: tcp/127.0.0.1:13125) (DC: dc1)" -42:34.034 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-81459c7f-2565-03ca-b547-00b95524e9e5.dc1 area=wan -42:34.034 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13127 network=tcp -42:34.034 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13127 network=udp -42:34.035 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13128 network=tcp -42:34.035 [INFO] TestAgent: started state syncer -42:34.035 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13126 network=tcp -42:34.075 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:34.076 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13125 [Candidate]" term=2 -42:34.076 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:34.076 [DEBUG] TestAgent.server.raft: vote granted: from=81459c7f-2565-03ca-b547-00b95524e9e5 term=2 tally=1 -42:34.076 [INFO] TestAgent.server.raft: election won: tally=1 -42:34.076 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13125 [Leader]" -42:34.078 [INFO] TestAgent.server: cluster leadership acquired -42:34.080 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13125 -42:34.082 [INFO] TestAgent.server: New leader elected: payload=Node-81459c7f-2565-03ca-b547-00b95524e9e5 -42:34.087 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:34.098 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:34.098 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:34.098 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:34.098 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:34.098 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:34.100 [INFO] TestAgent.server: federation state anti-entropy synced -42:34.101 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-81459c7f-2565-03ca-b547-00b95524e9e5 -42:34.103 [INFO] TestAgent.server: member joined, marking health alive: member=Node-81459c7f-2565-03ca-b547-00b95524e9e5 -42:34.143 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:34.145 [INFO] TestAgent: Requesting shutdown -42:34.145 [INFO] TestAgent.server: shutting down server -42:34.145 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:34.145 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:34.145 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:34.145 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:34.145 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:34.148 [INFO] TestAgent: Synced node info -42:34.148 [DEBUG] TestAgent: Node info in sync -42:34.148 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:34.148 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:34.148 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:34.149 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:34.149Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:34.149Z [INFO] agent.router.manager: shutting down -42:34.149 [INFO] TestAgent: consul server down -42:34.149 [INFO] TestAgent: shutdown complete -42:34.149 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13127 network=tcp -42:34.150 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:34.150 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:34.155 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13127 network=udp -42:34.155 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13128 network=tcp -42:34.660 [INFO] TestAgent: Waiting for endpoints to shut down -42:34.660 [INFO] TestAgent: Endpoints down ---- PASS: TestConfig_Apply_Decoding (0.73s) - --- PASS: TestConfig_Apply_Decoding/No_Kind (0.00s) - --- PASS: TestConfig_Apply_Decoding/Kind_Not_String (0.00s) - --- PASS: TestConfig_Apply_Decoding/Lowercase_kind (0.00s) -=== CONT TestIntentionsCreate_good -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:33.888Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:33.916Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:33.920 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:01d6bde4-607f-5fa3-c6a9-e5a4b09f4418 Address:127.0.0.1:13083}]" -42:33.921 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13083 [Follower]" leader= -42:33.921 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-01d6bde4-607f-5fa3-c6a9-e5a4b09f4418.dc1 127.0.0.1 -42:33.924 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-01d6bde4-607f-5fa3-c6a9-e5a4b09f4418 127.0.0.1 -2022-09-15T21:42:33.924Z [INFO] agent.router: Initializing LAN area manager -42:33.924 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13078 network=udp -42:33.924 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13078 network=tcp -42:33.925 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-01d6bde4-607f-5fa3-c6a9-e5a4b09f4418.dc1 area=wan -42:33.931 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13079 network=tcp -42:33.931 [INFO] TestAgent.server: Adding LAN server: server="Node-01d6bde4-607f-5fa3-c6a9-e5a4b09f4418 (Addr: tcp/127.0.0.1:13083) (DC: dc1)" -42:33.931 [INFO] TestAgent: started state syncer -42:33.931 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13084 network=tcp -42:33.964 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:33.964 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13083 [Candidate]" term=2 -42:33.965 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:33.965 [DEBUG] TestAgent.server.raft: vote granted: from=01d6bde4-607f-5fa3-c6a9-e5a4b09f4418 term=2 tally=1 -42:33.965 [INFO] TestAgent.server.raft: election won: tally=1 -42:33.965 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13083 [Leader]" -42:33.965 [INFO] TestAgent.server: cluster leadership acquired -42:33.965 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13083 -42:33.966 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:33.966 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:33.966 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-01d6bde4-607f-5fa3-c6a9-e5a4b09f4418 -42:33.966 [INFO] TestAgent.server: member joined, marking health alive: member=Node-01d6bde4-607f-5fa3-c6a9-e5a4b09f4418 -42:33.968 [INFO] TestAgent.server: New leader elected: payload=Node-01d6bde4-607f-5fa3-c6a9-e5a4b09f4418 -42:33.969 [INFO] TestAgent.server: federation state anti-entropy synced -42:34.364 [INFO] TestAgent: Requesting shutdown -42:34.364 [INFO] TestAgent.server: shutting down server -42:34.364 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:34.364 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:34.364 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:34.365 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:34.365 [INFO] TestAgent: Synced node info -42:34.365 [DEBUG] TestAgent: Node info in sync -42:34.365 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:34.366 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -42:34.366 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:34.366 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -2022-09-15T21:42:34.366Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:34.366Z [INFO] agent.router.manager: shutting down -42:34.366 [INFO] TestAgent: consul server down -42:34.366 [INFO] TestAgent: shutdown complete -42:34.366 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13078 network=tcp -42:34.366 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13078 network=udp -42:34.366 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13079 network=tcp -42:34.866 [INFO] TestAgent: Waiting for endpoints to shut down -42:34.866 [INFO] TestAgent: Endpoints down ---- PASS: TestConnectCARoots_empty (1.05s) -=== CONT TestIntentionsCheck_noDestination -2022-09-15T21:42:34.015Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:34.041Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:34.063 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:33dd665a-2387-2b7c-406c-c4b487e2a4e8 Address:127.0.0.1:13097}]" -42:34.064 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-33dd665a-2387-2b7c-406c-c4b487e2a4e8.dc1 127.0.0.1 -42:34.065 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-33dd665a-2387-2b7c-406c-c4b487e2a4e8 127.0.0.1 -2022-09-15T21:42:34.065Z [INFO] agent.router: Initializing LAN area manager -42:34.066 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13097 [Follower]" leader= -42:34.067 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13113 network=udp -42:34.067 [INFO] TestAgent.server: Adding LAN server: server="Node-33dd665a-2387-2b7c-406c-c4b487e2a4e8 (Addr: tcp/127.0.0.1:13097) (DC: dc1)" -42:34.067 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-33dd665a-2387-2b7c-406c-c4b487e2a4e8.dc1 area=wan -42:34.067 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13113 network=tcp -42:34.069 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13114 network=tcp -42:34.069 [INFO] TestAgent: started state syncer -42:34.069 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13098 network=tcp -42:34.110 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:34.110 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13097 [Candidate]" term=2 -42:34.110 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:34.110 [DEBUG] TestAgent.server.raft: vote granted: from=33dd665a-2387-2b7c-406c-c4b487e2a4e8 term=2 tally=1 -42:34.110 [INFO] TestAgent.server.raft: election won: tally=1 -42:34.110 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13097 [Leader]" -42:34.110 [INFO] TestAgent.server: cluster leadership acquired -42:34.111 [INFO] TestAgent.server: New leader elected: payload=Node-33dd665a-2387-2b7c-406c-c4b487e2a4e8 -42:34.111 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13097 -42:34.112 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:34.116 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:34.116 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:34.116 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:34.116 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:34.116 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:34.117 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-33dd665a-2387-2b7c-406c-c4b487e2a4e8 -42:34.117 [INFO] TestAgent.server: member joined, marking health alive: member=Node-33dd665a-2387-2b7c-406c-c4b487e2a4e8 -42:34.118 [INFO] TestAgent.server: federation state anti-entropy synced -42:34.345 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:34.346 [INFO] TestAgent: Synced node info -42:34.346 [DEBUG] TestAgent: Node info in sync -42:34.402 [INFO] TestAgent: Requesting shutdown -42:34.402 [INFO] TestAgent.server: shutting down server -42:34.402 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:34.402 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:34.402 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:34.402 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:34.402 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:34.402 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:34.402 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:34.402 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:34.402 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:34.402 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:34.403 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:34.403Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:34.403Z [INFO] agent.router.manager: shutting down -42:34.403 [INFO] TestAgent: consul server down -42:34.403 [INFO] TestAgent: shutdown complete -42:34.403 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13113 network=tcp -42:34.403 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13113 network=udp -42:34.403 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13114 network=tcp -42:34.905 [INFO] TestAgent: Waiting for endpoints to shut down -42:34.905 [INFO] TestAgent: Endpoints down ---- PASS: TestConfig_Apply_CAS (0.97s) -=== CONT TestIntentionsCheck_noSource -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=1)_compressed=true -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=1)_compressed=false -2022-09-15T21:42:34.388Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:34.407Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:34.411 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9b3b1639-173b-c4b2-1213-41963e0a28a8 Address:127.0.0.1:13111}]" -42:34.411 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13111 [Follower]" leader= -42:34.412 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9b3b1639-173b-c4b2-1213-41963e0a28a8.dc1 127.0.0.1 -42:34.413 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9b3b1639-173b-c4b2-1213-41963e0a28a8 127.0.0.1 -2022-09-15T21:42:34.413Z [INFO] agent.router: Initializing LAN area manager -42:34.414 [INFO] TestAgent.server: Adding LAN server: server="Node-9b3b1639-173b-c4b2-1213-41963e0a28a8 (Addr: tcp/127.0.0.1:13111) (DC: dc1)" -42:34.414 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9b3b1639-173b-c4b2-1213-41963e0a28a8.dc1 area=wan -42:34.414 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13134 network=udp -42:34.414 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13134 network=tcp -42:34.415 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13135 network=tcp -42:34.416 [INFO] TestAgent: started state syncer -42:34.416 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13112 network=tcp -42:34.450 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:34.450 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13111 [Candidate]" term=2 -42:34.450 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:34.450 [DEBUG] TestAgent.server.raft: vote granted: from=9b3b1639-173b-c4b2-1213-41963e0a28a8 term=2 tally=1 -42:34.450 [INFO] TestAgent.server.raft: election won: tally=1 -42:34.450 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13111 [Leader]" -42:34.450 [INFO] TestAgent.server: cluster leadership acquired -42:34.451 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13111 -42:34.452 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:34.457 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:34.457 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:34.457 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:34.457 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:34.457 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:34.457 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9b3b1639-173b-c4b2-1213-41963e0a28a8 -42:34.457 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9b3b1639-173b-c4b2-1213-41963e0a28a8 -42:34.459 [INFO] TestAgent.server: New leader elected: payload=Node-9b3b1639-173b-c4b2-1213-41963e0a28a8 -42:34.459 [INFO] TestAgent.server: federation state anti-entropy synced -42:34.672 [INFO] TestAgent.server.connect: CA provider config updated -42:34.672 [INFO] TestAgent: Requesting shutdown -42:34.672 [INFO] TestAgent.server: shutting down server -42:34.672 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:34.672 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:34.672 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:34.672 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:34.672 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:34.673 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:34.673 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:34.673 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -42:34.673 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:34.673 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:34.673 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:34.673 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2022-09-15T21:42:34.673Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:34.673Z [INFO] agent.router.manager: shutting down -42:34.673 [INFO] TestAgent: consul server down -42:34.673 [INFO] TestAgent: shutdown complete -42:34.673 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13134 network=tcp -42:34.673 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13134 network=udp -42:34.673 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13135 network=tcp -42:35.174 [INFO] TestAgent: Waiting for endpoints to shut down -42:35.174 [INFO] TestAgent: Endpoints down -=== RUN TestConnectCAConfig/force_without_cross_sign_CamelCase -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._tcp_(qType:=1)_compressed=true -2022-09-15T21:42:34.178Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:34.211Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:34.220 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f72db55d-a90f-63b4-38d1-65f78719252d Address:127.0.0.1:13132}]" -42:34.226 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f72db55d-a90f-63b4-38d1-65f78719252d.dc1 127.0.0.1 -42:34.227 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f72db55d-a90f-63b4-38d1-65f78719252d 127.0.0.1 -2022-09-15T21:42:34.227Z [INFO] agent.router: Initializing LAN area manager -42:34.228 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13132 [Follower]" leader= -42:34.228 [INFO] TestAgent.server: Adding LAN server: server="Node-f72db55d-a90f-63b4-38d1-65f78719252d (Addr: tcp/127.0.0.1:13132) (DC: dc1)" -42:34.228 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f72db55d-a90f-63b4-38d1-65f78719252d.dc1 area=wan -42:34.229 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13099 network=tcp -42:34.229 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13099 network=udp -42:34.231 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13100 network=tcp -42:34.231 [INFO] TestAgent: started state syncer -42:34.231 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13133 network=tcp -42:34.298 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:34.298 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13132 [Candidate]" term=2 -42:34.298 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:34.298 [DEBUG] TestAgent.server.raft: vote granted: from=f72db55d-a90f-63b4-38d1-65f78719252d term=2 tally=1 -42:34.298 [INFO] TestAgent.server.raft: election won: tally=1 -42:34.298 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13132 [Leader]" -42:34.299 [INFO] TestAgent.server: cluster leadership acquired -42:34.299 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13132 -42:34.301 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:34.302 [INFO] TestAgent.server: New leader elected: payload=Node-f72db55d-a90f-63b4-38d1-65f78719252d -42:34.304 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:34.304 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:34.304 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:34.304 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:34.304 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:34.304 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f72db55d-a90f-63b4-38d1-65f78719252d -42:34.304 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f72db55d-a90f-63b4-38d1-65f78719252d -42:34.314 [INFO] TestAgent.server: federation state anti-entropy synced -42:34.642 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:34.643 [INFO] TestAgent: Synced node info -42:34.643 [DEBUG] TestAgent: Node info in sync -42:34.729 [INFO] TestAgent: Requesting shutdown -42:34.729 [INFO] TestAgent.server: shutting down server -42:34.729 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:34.729 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:34.729 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:34.729 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:34.729 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:34.729 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:34.729 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:34.735 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -42:34.737 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:34.737 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -2022-09-15T21:42:34.738Z [INFO] agent.router.manager: shutting down -42:34.743 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -2022-09-15T21:42:34.743Z [INFO] agent.router.manager: shutting down -42:34.745 [INFO] TestAgent: consul server down -42:34.745 [INFO] TestAgent: shutdown complete -42:34.745 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13099 network=tcp -42:34.745 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13099 network=udp -42:34.745 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13100 network=tcp -42:35.245 [INFO] TestAgent: Waiting for endpoints to shut down -42:35.245 [INFO] TestAgent: Endpoints down ---- PASS: TestConfig_Apply_ProxyDefaultsMeshGateway (1.14s) -=== CONT TestIntentionsCheck_basic -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:34.498Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:34.523Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:34.534 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:762a7fd7-c027-da77-0245-b669c4026dfa Address:127.0.0.1:13090}]" -42:34.536 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-762a7fd7-c027-da77-0245-b669c4026dfa.dc1 127.0.0.1 -42:34.537 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-762a7fd7-c027-da77-0245-b669c4026dfa 127.0.0.1 -2022-09-15T21:42:34.537Z [INFO] agent.router: Initializing LAN area manager -42:34.537 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13120 network=udp -42:34.537 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13090 [Follower]" leader= -42:34.540 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-762a7fd7-c027-da77-0245-b669c4026dfa.dc1 area=wan -42:34.541 [INFO] TestAgent.server: Adding LAN server: server="Node-762a7fd7-c027-da77-0245-b669c4026dfa (Addr: tcp/127.0.0.1:13090) (DC: dc1)" -42:34.542 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13120 network=tcp -42:34.549 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13121 network=tcp -42:34.549 [INFO] TestAgent: started state syncer -42:34.549 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13091 network=tcp -42:34.596 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:34.596 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13090 [Candidate]" term=2 -42:34.596 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:34.596 [DEBUG] TestAgent.server.raft: vote granted: from=762a7fd7-c027-da77-0245-b669c4026dfa term=2 tally=1 -42:34.596 [INFO] TestAgent.server.raft: election won: tally=1 -42:34.596 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13090 [Leader]" -42:34.596 [INFO] TestAgent.server: cluster leadership acquired -42:34.597 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13090 -42:34.598 [INFO] TestAgent.server: New leader elected: payload=Node-762a7fd7-c027-da77-0245-b669c4026dfa -42:34.598 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:34.605 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:34.605 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:34.605 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:34.605 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:34.605 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:34.605 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-762a7fd7-c027-da77-0245-b669c4026dfa -42:34.605 [INFO] TestAgent.server: member joined, marking health alive: member=Node-762a7fd7-c027-da77-0245-b669c4026dfa -42:34.608 [INFO] TestAgent.server: federation state anti-entropy synced -42:34.618 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:34.619 [INFO] TestAgent: Synced node info -42:34.819 [DEBUG] TestAgent: Node info in sync -42:34.820 [INFO] TestAgent: Synced check: check=test -42:34.820 [INFO] TestAgent: Requesting shutdown -42:34.821 [INFO] TestAgent.server: shutting down server -42:34.821 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:34.821 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:34.821 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:34.821 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:34.821 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:34.821 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:34.821 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:34.821 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:34.821 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:34.821 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:34.821 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:34.822Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:34.822Z [INFO] agent.router.manager: shutting down -42:34.822 [INFO] TestAgent: consul server down -42:34.822 [INFO] TestAgent: shutdown complete -42:34.822 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13120 network=tcp -42:34.822 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13120 network=udp -42:34.822 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13121 network=tcp -42:35.326 [INFO] TestAgent: Waiting for endpoints to shut down -42:35.326 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_RegisterCheck (0.88s) -=== CONT TestIntentionsMatch_noName -2022-09-15T21:42:34.602Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:34.680Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:34.683 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:be27c509-7ce3-c186-766d-f46bb6b3b15a Address:127.0.0.1:13104}]" -42:34.684 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-be27c509-7ce3-c186-766d-f46bb6b3b15a.dc1 127.0.0.1 -42:34.685 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-be27c509-7ce3-c186-766d-f46bb6b3b15a 127.0.0.1 -2022-09-15T21:42:34.685Z [INFO] agent.router: Initializing LAN area manager -42:34.686 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13104 [Follower]" leader= -42:34.686 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13092 network=udp -42:34.686 [INFO] TestAgent.server: Adding LAN server: server="Node-be27c509-7ce3-c186-766d-f46bb6b3b15a (Addr: tcp/127.0.0.1:13104) (DC: dc1)" -42:34.686 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13092 network=tcp -42:34.686 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-be27c509-7ce3-c186-766d-f46bb6b3b15a.dc1 area=wan -42:34.687 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13093 network=tcp -42:34.687 [INFO] TestAgent: started state syncer -42:34.688 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13105 network=tcp -42:34.756 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:34.756 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13104 [Candidate]" term=2 -42:34.756 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:34.756 [DEBUG] TestAgent.server.raft: vote granted: from=be27c509-7ce3-c186-766d-f46bb6b3b15a term=2 tally=1 -42:34.756 [INFO] TestAgent.server.raft: election won: tally=1 -42:34.756 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13104 [Leader]" -42:34.757 [INFO] TestAgent.server: cluster leadership acquired -42:34.758 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13104 -42:34.768 [INFO] TestAgent.server: New leader elected: payload=Node-be27c509-7ce3-c186-766d-f46bb6b3b15a -42:34.770 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:34.773 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:34.773 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:34.773 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:34.773 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:34.773 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:34.773 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-be27c509-7ce3-c186-766d-f46bb6b3b15a -42:34.773 [INFO] TestAgent.server: member joined, marking health alive: member=Node-be27c509-7ce3-c186-766d-f46bb6b3b15a -42:34.775 [INFO] TestAgent.server: federation state anti-entropy synced -42:34.872 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:34.873 [INFO] TestAgent: Synced node info -42:34.902 [INFO] TestAgent: Requesting shutdown -42:34.903 [INFO] TestAgent.server: shutting down server -42:34.903 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:34.903 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:34.903 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:34.904Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:34.904Z [INFO] agent.router.manager: shutting down -42:34.905 [INFO] TestAgent: consul server down -42:34.905 [INFO] TestAgent: shutdown complete -42:34.905 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13092 network=tcp -42:34.905 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13092 network=udp -42:34.905 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13093 network=tcp -42:35.405 [INFO] TestAgent: Waiting for endpoints to shut down -42:35.405 [INFO] TestAgent: Endpoints down -=== RUN TestRPC_HTTPSMaxConnsPerClient/HTTPS -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:34.773Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:34.848Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:34.851 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4f1e154a-216b-5003-396e-a1ba631a5cd0 Address:127.0.0.1:13076}]" -42:34.852 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4f1e154a-216b-5003-396e-a1ba631a5cd0.dc1 127.0.0.1 -42:34.853 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4f1e154a-216b-5003-396e-a1ba631a5cd0 127.0.0.1 -2022-09-15T21:42:34.853Z [INFO] agent.router: Initializing LAN area manager -42:34.854 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13076 [Follower]" leader= -42:34.854 [INFO] TestAgent.server: Adding LAN server: server="Node-4f1e154a-216b-5003-396e-a1ba631a5cd0 (Addr: tcp/127.0.0.1:13076) (DC: dc1)" -42:34.854 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4f1e154a-216b-5003-396e-a1ba631a5cd0.dc1 area=wan -42:34.855 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13106 network=tcp -42:34.857 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13106 network=udp -42:34.859 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13107 network=tcp -42:34.859 [INFO] TestAgent: started state syncer -42:34.859 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13077 network=tcp -42:34.895 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:34.895 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13076 [Candidate]" term=2 -42:34.895 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:34.895 [DEBUG] TestAgent.server.raft: vote granted: from=4f1e154a-216b-5003-396e-a1ba631a5cd0 term=2 tally=1 -42:34.896 [INFO] TestAgent.server.raft: election won: tally=1 -42:34.896 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13076 [Leader]" -42:34.896 [INFO] TestAgent.server: cluster leadership acquired -42:34.897 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13076 -42:34.902 [INFO] TestAgent.server: New leader elected: payload=Node-4f1e154a-216b-5003-396e-a1ba631a5cd0 -42:34.908 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:34.917 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:34.917 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:34.917 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:34.917 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:34.917 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:34.918 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4f1e154a-216b-5003-396e-a1ba631a5cd0 -42:34.922 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4f1e154a-216b-5003-396e-a1ba631a5cd0 -42:34.950 [INFO] TestAgent.server: federation state anti-entropy synced -42:35.099 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:35.100 [INFO] TestAgent: Synced node info -42:35.162 [INFO] TestAgent: Requesting shutdown -42:35.162 [INFO] TestAgent.server: shutting down server -42:35.162 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:35.162 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:35.162 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:35.162 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:35.162 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:35.163 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:35.163 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:35.163 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:35.163 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:35.163 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:35.163 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:35.163Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:35.163Z [INFO] agent.router.manager: shutting down -42:35.163 [INFO] TestAgent: consul server down -42:35.163 [INFO] TestAgent: shutdown complete -42:35.163 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13106 network=tcp -42:35.163 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13106 network=udp -42:35.163 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13107 network=tcp -42:35.666 [INFO] TestAgent: Waiting for endpoints to shut down -42:35.666 [INFO] TestAgent: Endpoints down ---- PASS: TestIntentionsCreate_good (1.00s) -=== CONT TestIntentionsMatch_byInvalid -2022-09-15T21:42:34.946Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:34.988Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:34.997 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4b3dbba7-421c-580a-85f4-899d9c1e1b69 Address:127.0.0.1:13139}]" -42:34.997 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13139 [Follower]" leader= -42:34.998 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4b3dbba7-421c-580a-85f4-899d9c1e1b69.dc1 127.0.0.1 -42:34.999 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4b3dbba7-421c-580a-85f4-899d9c1e1b69 127.0.0.1 -2022-09-15T21:42:34.999Z [INFO] agent.router: Initializing LAN area manager -42:34.999 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13085 network=udp -42:34.999 [INFO] TestAgent.server: Adding LAN server: server="Node-4b3dbba7-421c-580a-85f4-899d9c1e1b69 (Addr: tcp/127.0.0.1:13139) (DC: dc1)" -42:34.999 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4b3dbba7-421c-580a-85f4-899d9c1e1b69.dc1 area=wan -42:34.999 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13085 network=tcp -42:35.000 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13086 network=tcp -42:35.000 [INFO] TestAgent: started state syncer -42:35.001 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13140 network=tcp -42:35.054 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:35.055 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13139 [Candidate]" term=2 -42:35.055 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:35.055 [DEBUG] TestAgent.server.raft: vote granted: from=4b3dbba7-421c-580a-85f4-899d9c1e1b69 term=2 tally=1 -42:35.055 [INFO] TestAgent.server.raft: election won: tally=1 -42:35.055 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13139 [Leader]" -42:35.055 [INFO] TestAgent.server: cluster leadership acquired -42:35.055 [INFO] TestAgent.server: New leader elected: payload=Node-4b3dbba7-421c-580a-85f4-899d9c1e1b69 -42:35.055 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13139 -42:35.057 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:35.059 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:35.059 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:35.059 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:35.059 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:35.059 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:35.060 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4b3dbba7-421c-580a-85f4-899d9c1e1b69 -42:35.060 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4b3dbba7-421c-580a-85f4-899d9c1e1b69 -42:35.061 [INFO] TestAgent.server: federation state anti-entropy synced -42:35.061 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:35.061 [INFO] TestAgent: Synced node info -42:35.163 [INFO] TestAgent: Requesting shutdown -42:35.163 [INFO] TestAgent.server: shutting down server -42:35.163 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:35.163 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:35.163 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:35.163 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:35.163 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:35.163 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:35.163 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:35.163 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:35.163 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:35.164 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:35.164 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:35.164Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:35.164Z [INFO] agent.router.manager: shutting down -42:35.164 [INFO] TestAgent: consul server down -42:35.164 [INFO] TestAgent: shutdown complete -42:35.164 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13085 network=tcp -42:35.164 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13085 network=udp -42:35.164 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13086 network=tcp -42:35.670 [INFO] TestAgent: Waiting for endpoints to shut down -42:35.670 [INFO] TestAgent: Endpoints down ---- PASS: TestIntentionsCheck_noDestination (0.80s) -=== CONT TestIntentionsMatch_noBy -2022-09-15T21:42:34.971Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:35.003Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:35.006 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c3443aec-f859-7d70-69b7-f8ad31bc1487 Address:127.0.0.1:13160}]" -42:35.006 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13160 [Follower]" leader= -42:35.007 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c3443aec-f859-7d70-69b7-f8ad31bc1487.dc1 127.0.0.1 -42:35.008 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c3443aec-f859-7d70-69b7-f8ad31bc1487 127.0.0.1 -2022-09-15T21:42:35.008Z [INFO] agent.router: Initializing LAN area manager -42:35.008 [INFO] TestAgent.server: Adding LAN server: server="Node-c3443aec-f859-7d70-69b7-f8ad31bc1487 (Addr: tcp/127.0.0.1:13160) (DC: dc1)" -42:35.008 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13148 network=udp -42:35.008 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c3443aec-f859-7d70-69b7-f8ad31bc1487.dc1 area=wan -42:35.009 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13148 network=tcp -42:35.009 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13149 network=tcp -42:35.010 [INFO] TestAgent: started state syncer -42:35.010 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13161 network=tcp -42:35.071 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:35.071 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13160 [Candidate]" term=2 -42:35.071 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:35.071 [DEBUG] TestAgent.server.raft: vote granted: from=c3443aec-f859-7d70-69b7-f8ad31bc1487 term=2 tally=1 -42:35.071 [INFO] TestAgent.server.raft: election won: tally=1 -42:35.071 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13160 [Leader]" -42:35.071 [INFO] TestAgent.server: cluster leadership acquired -42:35.071 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13160 -42:35.072 [INFO] TestAgent.server: New leader elected: payload=Node-c3443aec-f859-7d70-69b7-f8ad31bc1487 -42:35.073 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:35.075 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:35.075 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:35.075 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:35.075 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:35.075 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:35.076 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c3443aec-f859-7d70-69b7-f8ad31bc1487 -42:35.076 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c3443aec-f859-7d70-69b7-f8ad31bc1487 -42:35.077 [INFO] TestAgent.server: federation state anti-entropy synced -42:35.188 [INFO] TestAgent: Requesting shutdown -42:35.188 [INFO] TestAgent.server: shutting down server -42:35.188 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:35.188 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:35.188 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:35.188 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:35.188 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:35.188 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:35.188 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:35.188 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:35.188 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:35.188 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:35.188 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:35.188 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:35.188Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:35.189Z [INFO] agent.router.manager: shutting down -42:35.189 [INFO] TestAgent: consul server down -42:35.189 [INFO] TestAgent: shutdown complete -42:35.189 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13148 network=tcp -42:35.189 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13148 network=udp -42:35.189 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13149 network=tcp -42:35.692 [INFO] TestAgent: Waiting for endpoints to shut down -42:35.692 [INFO] TestAgent: Endpoints down ---- PASS: TestIntentionsCheck_noSource (0.79s) -=== CONT TestIntentionsMatch_basic -[WARN] freeport: 4 out of 25 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022/09/15 21:42:35 http: TLS handshake error from 127.0.0.1:44902: read tcp 127.0.0.1:13164->127.0.0.1:44902: use of closed network connection -2022/09/15 21:42:35 http: TLS handshake error from 127.0.0.1:44900: EOF -2022/09/15 21:42:35 http: TLS handshake error from 127.0.0.1:44898: EOF -2022/09/15 21:42:35 http: TLS handshake error from 127.0.0.1:44904: EOF -2022-09-15T21:42:35.193Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:35.211Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:35.214 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:212fd10c-4523-7b31-7cae-b738a46408ee Address:127.0.0.1:13174}]" -42:35.215 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13174 [Follower]" leader= -42:35.216 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-212fd10c-4523-7b31-7cae-b738a46408ee.dc1 127.0.0.1 -42:35.216 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-212fd10c-4523-7b31-7cae-b738a46408ee 127.0.0.1 -2022-09-15T21:42:35.216Z [INFO] agent.router: Initializing LAN area manager -42:35.217 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13176 network=udp -42:35.217 [INFO] TestAgent.server: Adding LAN server: server="Node-212fd10c-4523-7b31-7cae-b738a46408ee (Addr: tcp/127.0.0.1:13174) (DC: dc1)" -42:35.217 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-212fd10c-4523-7b31-7cae-b738a46408ee.dc1 area=wan -42:35.217 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13176 network=tcp -42:35.219 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13177 network=tcp -42:35.219 [INFO] TestAgent: started state syncer -42:35.219 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13175 network=tcp -42:35.264 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:35.264 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13174 [Candidate]" term=2 -42:35.264 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:35.264 [DEBUG] TestAgent.server.raft: vote granted: from=212fd10c-4523-7b31-7cae-b738a46408ee term=2 tally=1 -42:35.264 [INFO] TestAgent.server.raft: election won: tally=1 -42:35.264 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13174 [Leader]" -42:35.264 [INFO] TestAgent.server: cluster leadership acquired -42:35.265 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13174 -42:35.266 [INFO] TestAgent.server: New leader elected: payload=Node-212fd10c-4523-7b31-7cae-b738a46408ee -42:35.266 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:35.269 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:35.269 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:35.269 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:35.269 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:35.269 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:35.269 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-212fd10c-4523-7b31-7cae-b738a46408ee -42:35.269 [INFO] TestAgent.server: member joined, marking health alive: member=Node-212fd10c-4523-7b31-7cae-b738a46408ee -42:35.271 [INFO] TestAgent.server: federation state anti-entropy synced -42:35.287 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:35.288 [INFO] TestAgent: Synced node info -42:35.288 [DEBUG] TestAgent: Node info in sync -42:35.494 [INFO] TestAgent.server.connect: CA provider config updated -42:35.494 [INFO] TestAgent: Requesting shutdown -42:35.494 [INFO] TestAgent.server: shutting down server -42:35.494 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:35.494 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:35.494 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:35.494 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:35.494 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:35.495 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:35.495 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:35.495 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:35.495 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:35.495 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:35.495 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:35.495Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:35.495Z [INFO] agent.router.manager: shutting down -42:35.500 [INFO] TestAgent: consul server down -42:35.500 [INFO] TestAgent: shutdown complete -42:35.500 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13176 network=tcp -42:35.500 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13176 network=udp -42:35.500 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13177 network=tcp -42:36.003 [INFO] TestAgent: Waiting for endpoints to shut down -42:36.003 [INFO] TestAgent: Endpoints down -=== RUN TestConnectCAConfig/force_without_cross_sign_snake_case -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._tcp_(qType:=1)_compressed=false -2022-09-15T21:42:35.264Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:35.281Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:35.284 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:db8556b9-7579-4fc0-2827-90134b7b0e22 Address:127.0.0.1:13181}]" -42:35.284 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13181 [Follower]" leader= -42:35.285 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-db8556b9-7579-4fc0-2827-90134b7b0e22.dc1 127.0.0.1 -42:35.286 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-db8556b9-7579-4fc0-2827-90134b7b0e22 127.0.0.1 -2022-09-15T21:42:35.286Z [INFO] agent.router: Initializing LAN area manager -42:35.287 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13190 network=udp -42:35.287 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-db8556b9-7579-4fc0-2827-90134b7b0e22.dc1 area=wan -42:35.287 [INFO] TestAgent.server: Adding LAN server: server="Node-db8556b9-7579-4fc0-2827-90134b7b0e22 (Addr: tcp/127.0.0.1:13181) (DC: dc1)" -42:35.287 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13190 network=tcp -42:35.288 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13191 network=tcp -42:35.288 [INFO] TestAgent: started state syncer -42:35.288 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13182 network=tcp -42:35.345 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:35.345 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13181 [Candidate]" term=2 -42:35.358 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:35.358 [DEBUG] TestAgent.server.raft: vote granted: from=db8556b9-7579-4fc0-2827-90134b7b0e22 term=2 tally=1 -42:35.358 [INFO] TestAgent.server.raft: election won: tally=1 -42:35.358 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13181 [Leader]" -42:35.358 [INFO] TestAgent.server: cluster leadership acquired -42:35.358 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13181 -42:35.360 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:35.361 [INFO] TestAgent.server: New leader elected: payload=Node-db8556b9-7579-4fc0-2827-90134b7b0e22 -42:35.364 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:35.364 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:35.364 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:35.364 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:35.364 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:35.364 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-db8556b9-7579-4fc0-2827-90134b7b0e22 -42:35.364 [INFO] TestAgent.server: member joined, marking health alive: member=Node-db8556b9-7579-4fc0-2827-90134b7b0e22 -42:35.366 [INFO] TestAgent.server: federation state anti-entropy synced -42:35.650 [INFO] TestAgent: Requesting shutdown -42:35.650 [INFO] TestAgent.server: shutting down server -42:35.650 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:35.650 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:35.651 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:35.651 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:35.651 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:35.651 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:35.651 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:35.651 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:35.651 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:35.651 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:35.651 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:35.651 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:35.651Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:35.651Z [INFO] agent.router.manager: shutting down -42:35.652 [INFO] TestAgent: consul server down -42:35.652 [INFO] TestAgent: shutdown complete -42:35.652 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13190 network=tcp -42:35.652 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13190 network=udp -42:35.652 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13191 network=tcp -42:36.158 [INFO] TestAgent: Waiting for endpoints to shut down -42:36.158 [INFO] TestAgent: Endpoints down ---- PASS: TestIntentionsCheck_basic (0.92s) -=== CONT TestIntentionsList_values -2022-09-15T21:42:35.398Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:35.425Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:35.428 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a9dcbf37-3acb-03c8-8eb4-4609adf1161d Address:127.0.0.1:13146}]" -42:35.429 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13146 [Follower]" leader= -42:35.430 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a9dcbf37-3acb-03c8-8eb4-4609adf1161d.dc1 127.0.0.1 -42:35.432 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a9dcbf37-3acb-03c8-8eb4-4609adf1161d 127.0.0.1 -2022-09-15T21:42:35.432Z [INFO] agent.router: Initializing LAN area manager -42:35.433 [INFO] TestAgent.server: Adding LAN server: server="Node-a9dcbf37-3acb-03c8-8eb4-4609adf1161d (Addr: tcp/127.0.0.1:13146) (DC: dc1)" -42:35.433 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a9dcbf37-3acb-03c8-8eb4-4609adf1161d.dc1 area=wan -42:35.435 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13169 network=udp -42:35.435 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13169 network=tcp -42:35.438 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13170 network=tcp -42:35.438 [INFO] TestAgent: started state syncer -42:35.438 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13147 network=tcp -42:35.495 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:35.495 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13146 [Candidate]" term=2 -42:35.496 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:35.496 [DEBUG] TestAgent.server.raft: vote granted: from=a9dcbf37-3acb-03c8-8eb4-4609adf1161d term=2 tally=1 -42:35.496 [INFO] TestAgent.server.raft: election won: tally=1 -42:35.496 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13146 [Leader]" -42:35.496 [INFO] TestAgent.server: cluster leadership acquired -42:35.496 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13146 -42:35.498 [INFO] TestAgent.server: New leader elected: payload=Node-a9dcbf37-3acb-03c8-8eb4-4609adf1161d -42:35.498 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:35.515 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:35.515 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:35.515 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:35.515 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:35.515 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:35.515 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a9dcbf37-3acb-03c8-8eb4-4609adf1161d -42:35.515 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a9dcbf37-3acb-03c8-8eb4-4609adf1161d -42:35.517 [INFO] TestAgent.server: federation state anti-entropy synced -42:35.716 [INFO] TestAgent: Requesting shutdown -42:35.717 [INFO] TestAgent.server: shutting down server -42:35.717 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:35.717 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:35.717 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:35.717 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:35.717 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:35.717 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:35.717 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:35.717 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:35.717 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:35.717 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:35.717 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:35.717 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:35.721Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:35.721Z [INFO] agent.router.manager: shutting down -42:35.721 [INFO] TestAgent: consul server down -42:35.721 [INFO] TestAgent: shutdown complete -42:35.721 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13169 network=tcp -42:35.721 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13169 network=udp -42:35.722 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13170 network=tcp -42:36.222 [INFO] TestAgent: Waiting for endpoints to shut down -42:36.222 [INFO] TestAgent: Endpoints down ---- PASS: TestIntentionsMatch_noName (0.89s) -=== CONT TestIntentionsList_empty -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:35.795Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:35.859Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:35.865 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:09520d3c-cfb7-9be2-1eea-13bd4cb23b56 Address:127.0.0.1:13153}]" -42:35.866 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13153 [Follower]" leader= -42:35.868 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-09520d3c-cfb7-9be2-1eea-13bd4cb23b56.dc1 127.0.0.1 -42:35.869 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-09520d3c-cfb7-9be2-1eea-13bd4cb23b56 127.0.0.1 -2022-09-15T21:42:35.869Z [INFO] agent.router: Initializing LAN area manager -42:35.869 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13155 network=udp -42:35.870 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-09520d3c-cfb7-9be2-1eea-13bd4cb23b56.dc1 area=wan -42:35.870 [INFO] TestAgent.server: Adding LAN server: server="Node-09520d3c-cfb7-9be2-1eea-13bd4cb23b56 (Addr: tcp/127.0.0.1:13153) (DC: dc1)" -42:35.870 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13155 network=tcp -42:35.888 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13156 network=tcp -42:35.888 [INFO] TestAgent: started state syncer -42:35.892 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13154 network=tcp -42:35.914 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:35.914 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13153 [Candidate]" term=2 -42:35.915 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:35.915 [DEBUG] TestAgent.server.raft: vote granted: from=09520d3c-cfb7-9be2-1eea-13bd4cb23b56 term=2 tally=1 -42:35.915 [INFO] TestAgent.server.raft: election won: tally=1 -42:35.915 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13153 [Leader]" -42:35.918 [INFO] TestAgent.server: cluster leadership acquired -42:35.918 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13153 -42:35.921 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:35.922 [INFO] TestAgent.server: New leader elected: payload=Node-09520d3c-cfb7-9be2-1eea-13bd4cb23b56 -42:35.936 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:35.936 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:35.936 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:35.936 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:35.937 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:35.937 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-09520d3c-cfb7-9be2-1eea-13bd4cb23b56 -42:35.937 [INFO] TestAgent.server: member joined, marking health alive: member=Node-09520d3c-cfb7-9be2-1eea-13bd4cb23b56 -42:35.946 [INFO] TestAgent: Requesting shutdown -42:35.947 [INFO] TestAgent.server: shutting down server -42:35.947 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:35.947 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:35.947 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:35.947 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:35.947 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:35.947 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:35.947 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:35.947 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:35.947 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:35.949 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:35.949Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:35.949Z [INFO] agent.router.manager: shutting down -42:35.950 [INFO] TestAgent.server: federation state anti-entropy synced -42:35.950 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:35.950 [INFO] TestAgent: consul server down -42:35.950 [INFO] TestAgent: shutdown complete -42:35.950 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13155 network=tcp -42:35.950 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13155 network=udp -42:35.950 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13156 network=tcp -42:36.450 [INFO] TestAgent: Waiting for endpoints to shut down -42:36.450 [INFO] TestAgent: Endpoints down ---- PASS: TestIntentionsMatch_byInvalid (0.78s) -=== CONT TestAgentConnectAuthorize_badBody -2022-09-15T21:42:35.424Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:35.444Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:35.475 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:52eebe25-b159-a986-d6be-c74fa52d6f5f Address:127.0.0.1:13167}]" -42:35.477 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-52eebe25-b159-a986-d6be-c74fa52d6f5f.dc1 127.0.0.1 -42:35.478 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13167 [Follower]" leader= -42:35.481 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-52eebe25-b159-a986-d6be-c74fa52d6f5f 127.0.0.1 -2022-09-15T21:42:35.481Z [INFO] agent.router: Initializing LAN area manager -42:35.482 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13162 network=udp -42:35.482 [INFO] TestAgent.server: Adding LAN server: server="Node-52eebe25-b159-a986-d6be-c74fa52d6f5f (Addr: tcp/127.0.0.1:13167) (DC: dc1)" -42:35.482 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-52eebe25-b159-a986-d6be-c74fa52d6f5f.dc1 area=wan -42:35.482 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13162 network=tcp -42:35.491 [INFO] TestAgent: Started HTTPS server: address=127.0.0.1:13164 network=tcp -42:35.491 [INFO] TestAgent: started state syncer -42:35.492 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13168 network=tcp -42:35.524 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:35.524 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13167 [Candidate]" term=2 -42:35.524 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:35.524 [DEBUG] TestAgent.server.raft: vote granted: from=52eebe25-b159-a986-d6be-c74fa52d6f5f term=2 tally=1 -42:35.524 [INFO] TestAgent.server.raft: election won: tally=1 -42:35.524 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13167 [Leader]" -42:35.525 [INFO] TestAgent.server: cluster leadership acquired -42:35.525 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13167 -42:35.525 [INFO] TestAgent.server: New leader elected: payload=Node-52eebe25-b159-a986-d6be-c74fa52d6f5f -42:35.535 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:35.539 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:35.539 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:35.539 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:35.539 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:35.539 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:35.539 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-52eebe25-b159-a986-d6be-c74fa52d6f5f -42:35.539 [INFO] TestAgent.server: member joined, marking health alive: member=Node-52eebe25-b159-a986-d6be-c74fa52d6f5f -42:35.541 [INFO] TestAgent.server: federation state anti-entropy synced -42:35.614 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:35.622 [INFO] TestAgent: Synced node info -42:35.622 [DEBUG] TestAgent: Node info in sync -42:35.962 [INFO] TestAgent: Requesting shutdown -42:35.962 [INFO] TestAgent.server: shutting down server -42:35.962 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:35.962 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:35.962 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:35.963Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:35.963Z [INFO] agent.router.manager: shutting down -42:35.963 [INFO] TestAgent: consul server down -42:35.963 [INFO] TestAgent: shutdown complete -42:35.963 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13162 network=tcp -42:35.963 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13162 network=udp -42:35.963 [INFO] TestAgent: Stopping server: protocol=HTTPS address=127.0.0.1:13164 network=tcp -42:36.464 [INFO] TestAgent: Waiting for endpoints to shut down -42:36.464 [INFO] TestAgent: Endpoints down ---- PASS: TestRPC_HTTPSMaxConnsPerClient (1.93s) - --- PASS: TestRPC_HTTPSMaxConnsPerClient/HTTP (0.87s) - --- PASS: TestRPC_HTTPSMaxConnsPerClient/HTTPS (1.06s) -=== CONT TestConfig_Apply_TerminatingGateway -2022-09-15T21:42:35.727Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:35.773Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:35.806 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d89c242b-4efc-8d09-2523-d09f9e452075 Address:127.0.0.1:13230}]" -42:35.806 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13230 [Follower]" leader= -42:35.810 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d89c242b-4efc-8d09-2523-d09f9e452075.dc1 127.0.0.1 -42:35.811 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d89c242b-4efc-8d09-2523-d09f9e452075 127.0.0.1 -2022-09-15T21:42:35.811Z [INFO] agent.router: Initializing LAN area manager -42:35.812 [INFO] TestAgent.server: Adding LAN server: server="Node-d89c242b-4efc-8d09-2523-d09f9e452075 (Addr: tcp/127.0.0.1:13230) (DC: dc1)" -42:35.812 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13141 network=udp -42:35.812 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d89c242b-4efc-8d09-2523-d09f9e452075.dc1 area=wan -42:35.812 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13141 network=tcp -42:35.814 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13142 network=tcp -42:35.814 [INFO] TestAgent: started state syncer -42:35.814 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13231 network=tcp -42:35.870 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:35.870 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13230 [Candidate]" term=2 -42:35.870 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:35.870 [DEBUG] TestAgent.server.raft: vote granted: from=d89c242b-4efc-8d09-2523-d09f9e452075 term=2 tally=1 -42:35.870 [INFO] TestAgent.server.raft: election won: tally=1 -42:35.870 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13230 [Leader]" -42:35.870 [INFO] TestAgent.server: cluster leadership acquired -42:35.871 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13230 -42:35.872 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:35.874 [INFO] TestAgent.server: New leader elected: payload=Node-d89c242b-4efc-8d09-2523-d09f9e452075 -42:35.875 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:35.875 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:35.875 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:35.875 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:35.875 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:35.875 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d89c242b-4efc-8d09-2523-d09f9e452075 -42:35.875 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d89c242b-4efc-8d09-2523-d09f9e452075 -42:35.877 [INFO] TestAgent.server: federation state anti-entropy synced -42:35.982 [INFO] TestAgent: Requesting shutdown -42:35.982 [INFO] TestAgent.server: shutting down server -42:35.982 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:35.982 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:35.982 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:35.982 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:35.982 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:35.982 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:35.982 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:35.982 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:35.982 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:35.982 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:35.982 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:35.982 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:35.983Z [INFO] agent.router.manager: shutting down -42:35.983 [INFO] TestAgent: consul server down -42:35.983 [INFO] TestAgent: shutdown complete -42:35.983 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13141 network=tcp -42:35.983 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13141 network=udp -42:35.983 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13142 network=tcp -2022-09-15T21:42:35.990Z [INFO] agent.router.manager: shutting down -42:36.486 [INFO] TestAgent: Waiting for endpoints to shut down -42:36.486 [INFO] TestAgent: Endpoints down ---- PASS: TestIntentionsMatch_noBy (0.82s) -=== CONT TestConfig_Apply -2022-09-15T21:42:35.784Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:35.824Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:35.828 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8ef6d68c-5b16-c9af-20bc-f76cb8a11348 Address:127.0.0.1:13209}]" -42:35.829 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8ef6d68c-5b16-c9af-20bc-f76cb8a11348.dc1 127.0.0.1 -42:35.830 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13209 [Follower]" leader= -42:35.833 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8ef6d68c-5b16-c9af-20bc-f76cb8a11348 127.0.0.1 -2022-09-15T21:42:35.833Z [INFO] agent.router: Initializing LAN area manager -42:35.836 [INFO] TestAgent.server: Adding LAN server: server="Node-8ef6d68c-5b16-c9af-20bc-f76cb8a11348 (Addr: tcp/127.0.0.1:13209) (DC: dc1)" -42:35.836 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8ef6d68c-5b16-c9af-20bc-f76cb8a11348.dc1 area=wan -42:35.840 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13232 network=tcp -42:35.842 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13232 network=udp -42:35.863 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13233 network=tcp -42:35.863 [INFO] TestAgent: started state syncer -42:35.863 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13210 network=tcp -42:35.867 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:35.867 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13209 [Candidate]" term=2 -42:35.867 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:35.867 [DEBUG] TestAgent.server.raft: vote granted: from=8ef6d68c-5b16-c9af-20bc-f76cb8a11348 term=2 tally=1 -42:35.867 [INFO] TestAgent.server.raft: election won: tally=1 -42:35.867 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13209 [Leader]" -42:35.867 [INFO] TestAgent.server: cluster leadership acquired -42:35.868 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13209 -42:35.869 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:35.870 [INFO] TestAgent.server: New leader elected: payload=Node-8ef6d68c-5b16-c9af-20bc-f76cb8a11348 -42:35.872 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:35.872 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:35.872 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:35.872 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:35.872 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:35.872 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8ef6d68c-5b16-c9af-20bc-f76cb8a11348 -42:35.873 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8ef6d68c-5b16-c9af-20bc-f76cb8a11348 -42:35.877 [INFO] TestAgent.server: federation state anti-entropy synced -42:35.966 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:35.967 [INFO] TestAgent: Synced node info -42:35.988 [INFO] TestAgent: Requesting shutdown -42:35.988 [INFO] TestAgent.server: shutting down server -42:35.988 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:35.988 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:35.988 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:35.988 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:35.988 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:35.988 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:35.988 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:35.989 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:35.989 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:35.989 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:35.989 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:35.989Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:35.989Z [INFO] agent.router.manager: shutting down -42:35.989 [INFO] TestAgent: consul server down -42:35.989 [INFO] TestAgent: shutdown complete -42:35.989 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13232 network=tcp -42:35.989 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13232 network=udp -42:35.989 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13233 network=tcp -42:36.490 [INFO] TestAgent: Waiting for endpoints to shut down -42:36.490 [INFO] TestAgent: Endpoints down ---- PASS: TestIntentionsMatch_basic (0.79s) -=== CONT TestConfig_Delete -[WARN] freeport: 4 out of 32 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:36.044Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:36.108Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:36.113 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2ea1497b-79dd-6e03-8cf0-ba77bbf862b0 Address:127.0.0.1:13202}]" -42:36.114 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13202 [Follower]" leader= -42:36.115 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2ea1497b-79dd-6e03-8cf0-ba77bbf862b0.dc1 127.0.0.1 -42:36.116 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2ea1497b-79dd-6e03-8cf0-ba77bbf862b0 127.0.0.1 -2022-09-15T21:42:36.116Z [INFO] agent.router: Initializing LAN area manager -42:36.117 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13197 network=udp -42:36.117 [INFO] TestAgent.server: Adding LAN server: server="Node-2ea1497b-79dd-6e03-8cf0-ba77bbf862b0 (Addr: tcp/127.0.0.1:13202) (DC: dc1)" -42:36.117 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2ea1497b-79dd-6e03-8cf0-ba77bbf862b0.dc1 area=wan -42:36.117 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13197 network=tcp -42:36.119 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13198 network=tcp -42:36.120 [INFO] TestAgent: started state syncer -42:36.120 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13203 network=tcp -42:36.183 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:36.183 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13202 [Candidate]" term=2 -42:36.184 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:36.184 [DEBUG] TestAgent.server.raft: vote granted: from=2ea1497b-79dd-6e03-8cf0-ba77bbf862b0 term=2 tally=1 -42:36.184 [INFO] TestAgent.server.raft: election won: tally=1 -42:36.184 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13202 [Leader]" -42:36.184 [INFO] TestAgent.server: cluster leadership acquired -42:36.185 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13202 -42:36.186 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:36.189 [INFO] TestAgent.server: New leader elected: payload=Node-2ea1497b-79dd-6e03-8cf0-ba77bbf862b0 -42:36.193 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:36.193 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:36.193 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:36.193 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:36.193 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:36.193 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2ea1497b-79dd-6e03-8cf0-ba77bbf862b0 -42:36.193 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2ea1497b-79dd-6e03-8cf0-ba77bbf862b0 -42:36.194 [INFO] TestAgent.server: federation state anti-entropy synced -42:36.278 [INFO] TestAgent.server.connect: CA provider config updated -42:36.278 [INFO] TestAgent: Requesting shutdown -42:36.278 [INFO] TestAgent.server: shutting down server -42:36.278 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:36.278 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:36.278 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:36.278 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:36.278 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:36.278 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:36.278 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:36.278 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:36.278 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:36.278 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:36.278 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:36.278 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:36.278Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:36.279Z [INFO] agent.router.manager: shutting down -42:36.279 [INFO] TestAgent: consul server down -42:36.279 [INFO] TestAgent: shutdown complete -42:36.279 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13197 network=tcp -42:36.279 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13197 network=udp -42:36.279 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13198 network=tcp -42:36.782 [INFO] TestAgent: Waiting for endpoints to shut down -42:36.783 [INFO] TestAgent: Endpoints down -=== RUN TestConnectCAConfig/setting_state_fails -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:36.224Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:36.255Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:36.258 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8d4674c5-33b8-fa89-e9a4-f1570b631665 Address:127.0.0.1:13237}]" -42:36.259 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13237 [Follower]" leader= -42:36.260 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8d4674c5-33b8-fa89-e9a4-f1570b631665.dc1 127.0.0.1 -42:36.261 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8d4674c5-33b8-fa89-e9a4-f1570b631665 127.0.0.1 -2022-09-15T21:42:36.261Z [INFO] agent.router: Initializing LAN area manager -42:36.261 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13218 network=udp -42:36.261 [INFO] TestAgent.server: Adding LAN server: server="Node-8d4674c5-33b8-fa89-e9a4-f1570b631665 (Addr: tcp/127.0.0.1:13237) (DC: dc1)" -42:36.261 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8d4674c5-33b8-fa89-e9a4-f1570b631665.dc1 area=wan -42:36.261 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13218 network=tcp -42:36.263 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13219 network=tcp -42:36.263 [INFO] TestAgent: started state syncer -42:36.263 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13238 network=tcp -42:36.327 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:36.327 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13237 [Candidate]" term=2 -42:36.327 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:36.327 [DEBUG] TestAgent.server.raft: vote granted: from=8d4674c5-33b8-fa89-e9a4-f1570b631665 term=2 tally=1 -42:36.327 [INFO] TestAgent.server.raft: election won: tally=1 -42:36.327 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13237 [Leader]" -42:36.328 [INFO] TestAgent.server: cluster leadership acquired -42:36.329 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13237 -42:36.331 [INFO] TestAgent.server: New leader elected: payload=Node-8d4674c5-33b8-fa89-e9a4-f1570b631665 -42:36.333 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:36.337 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:36.337 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:36.337 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:36.337 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:36.337 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:36.337 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8d4674c5-33b8-fa89-e9a4-f1570b631665 -42:36.337 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8d4674c5-33b8-fa89-e9a4-f1570b631665 -42:36.339 [INFO] TestAgent.server: federation state anti-entropy synced -42:36.340 [INFO] TestAgent: Requesting shutdown -42:36.340 [INFO] TestAgent.server: shutting down server -42:36.340 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:36.340 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:36.340 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:36.340 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:36.340 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:36.340 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:36.341 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:36.341 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:36.341 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:36.341 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:36.341 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -42:36.341 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2022-09-15T21:42:36.341Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:36.341Z [INFO] agent.router.manager: shutting down -42:36.345 [INFO] TestAgent: consul server down -42:36.345 [INFO] TestAgent: shutdown complete -42:36.345 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13218 network=tcp -42:36.346 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13218 network=udp -42:36.346 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13219 network=tcp -42:36.847 [INFO] TestAgent: Waiting for endpoints to shut down -42:36.848 [INFO] TestAgent: Endpoints down ---- PASS: TestIntentionsList_values (0.69s) -=== CONT TestConfig_Get -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._udp_(qType:=1)_compressed=true -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestConfig_Get/get_a_single_service_entry -=== RUN TestConfig_Get/list_both_service_entries -=== RUN TestConfig_Get/get_global_proxy_config -=== RUN TestConfig_Get/error_on_no_arguments -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._udp_(qType:=1)_compressed=false -2022-09-15T21:42:36.244Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:36.261Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:36.264 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:365b7d11-7b8f-9d3e-6937-102887e4a2d3 Address:127.0.0.1:13216}]" -42:36.264 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13216 [Follower]" leader= -42:36.265 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-365b7d11-7b8f-9d3e-6937-102887e4a2d3.dc1 127.0.0.1 -42:36.266 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-365b7d11-7b8f-9d3e-6937-102887e4a2d3 127.0.0.1 -2022-09-15T21:42:36.266Z [INFO] agent.router: Initializing LAN area manager -42:36.267 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13239 network=udp -42:36.267 [INFO] TestAgent.server: Adding LAN server: server="Node-365b7d11-7b8f-9d3e-6937-102887e4a2d3 (Addr: tcp/127.0.0.1:13216) (DC: dc1)" -42:36.267 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-365b7d11-7b8f-9d3e-6937-102887e4a2d3.dc1 area=wan -42:36.267 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13239 network=tcp -42:36.268 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13240 network=tcp -42:36.268 [INFO] TestAgent: started state syncer -42:36.268 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13217 network=tcp -42:36.327 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:36.327 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13216 [Candidate]" term=2 -42:36.328 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:36.328 [DEBUG] TestAgent.server.raft: vote granted: from=365b7d11-7b8f-9d3e-6937-102887e4a2d3 term=2 tally=1 -42:36.328 [INFO] TestAgent.server.raft: election won: tally=1 -42:36.328 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13216 [Leader]" -42:36.330 [INFO] TestAgent.server: cluster leadership acquired -42:36.331 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13216 -42:36.331 [INFO] TestAgent.server: New leader elected: payload=Node-365b7d11-7b8f-9d3e-6937-102887e4a2d3 -42:36.333 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:36.349 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:36.349 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:36.349 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:36.349 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:36.349 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:36.349 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-365b7d11-7b8f-9d3e-6937-102887e4a2d3 -42:36.349 [INFO] TestAgent.server: member joined, marking health alive: member=Node-365b7d11-7b8f-9d3e-6937-102887e4a2d3 -42:36.355 [INFO] TestAgent.server: federation state anti-entropy synced -42:36.502 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:36.503 [INFO] TestAgent: Synced node info -42:36.503 [DEBUG] TestAgent: Node info in sync -42:36.647 [INFO] TestAgent: Requesting shutdown -42:36.647 [INFO] TestAgent.server: shutting down server -42:36.647 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:36.647 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:36.647 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:36.647 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:36.647 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:36.647 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:36.647 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:36.647 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:36.647 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:36.647 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:36.647 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:36.648Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:36.648Z [INFO] agent.router.manager: shutting down -42:36.648 [INFO] TestAgent: consul server down -42:36.648 [INFO] TestAgent: shutdown complete -42:36.648 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13239 network=tcp -42:36.648 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13239 network=udp -42:36.648 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13240 network=tcp -42:37.148 [INFO] TestAgent: Waiting for endpoints to shut down -42:37.148 [INFO] TestAgent: Endpoints down ---- PASS: TestIntentionsList_empty (0.93s) -=== CONT TestCatalog_GatewayServices_Ingress -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=33)_compressed=true -2022-09-15T21:42:36.551Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:36.588Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:36.603 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e6c76d48-76f1-7976-2a79-56a920646d86 Address:127.0.0.1:13195}]" -42:36.604 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13195 [Follower]" leader= -42:36.605 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e6c76d48-76f1-7976-2a79-56a920646d86.dc1 127.0.0.1 -42:36.610 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e6c76d48-76f1-7976-2a79-56a920646d86 127.0.0.1 -2022-09-15T21:42:36.610Z [INFO] agent.router: Initializing LAN area manager -42:36.611 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13204 network=udp -42:36.611 [INFO] TestAgent.server: Adding LAN server: server="Node-e6c76d48-76f1-7976-2a79-56a920646d86 (Addr: tcp/127.0.0.1:13195) (DC: dc1)" -42:36.611 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e6c76d48-76f1-7976-2a79-56a920646d86.dc1 area=wan -42:36.611 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13204 network=tcp -42:36.613 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13205 network=tcp -42:36.613 [INFO] TestAgent: started state syncer -42:36.613 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13196 network=tcp -42:36.657 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:36.658 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13195 [Candidate]" term=2 -42:36.658 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:36.658 [DEBUG] TestAgent.server.raft: vote granted: from=e6c76d48-76f1-7976-2a79-56a920646d86 term=2 tally=1 -42:36.658 [INFO] TestAgent.server.raft: election won: tally=1 -42:36.658 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13195 [Leader]" -42:36.658 [INFO] TestAgent.server: cluster leadership acquired -42:36.659 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13195 -42:36.659 [INFO] TestAgent.server: New leader elected: payload=Node-e6c76d48-76f1-7976-2a79-56a920646d86 -42:36.660 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:36.666 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:36.666 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:36.666 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:36.666 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:36.666 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:36.667 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e6c76d48-76f1-7976-2a79-56a920646d86 -42:36.667 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e6c76d48-76f1-7976-2a79-56a920646d86 -42:36.676 [INFO] TestAgent.server: federation state anti-entropy synced -42:36.740 [INFO] TestAgent: Requesting shutdown -42:36.740 [INFO] TestAgent.server: shutting down server -42:36.740 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:36.740 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:36.740 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:36.740 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:36.740 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:36.740 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:36.740 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:36.740 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:36.740 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:36.741 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:36.741 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:36.741 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:36.741Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:36.741Z [INFO] agent.router.manager: shutting down -42:36.741 [INFO] TestAgent: consul server down -42:36.741 [INFO] TestAgent: shutdown complete -42:36.741 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13204 network=tcp -42:36.741 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13204 network=udp -42:36.741 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13205 network=tcp -42:37.242 [INFO] TestAgent: Waiting for endpoints to shut down -42:37.242 [INFO] TestAgent: Endpoints down ---- PASS: TestConfig_Delete (0.76s) -=== CONT TestCatalog_GatewayServices_Terminating -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:36.525Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:36.601Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:36.622 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:df5ba157-ae00-498e-1b49-0affec0f577d Address:127.0.0.1:13244}]" -42:36.624 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-df5ba157-ae00-498e-1b49-0affec0f577d.dc1 127.0.0.1 -42:36.625 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13244 [Follower]" leader= -42:36.625 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-df5ba157-ae00-498e-1b49-0affec0f577d 127.0.0.1 -2022-09-15T21:42:36.625Z [INFO] agent.router: Initializing LAN area manager -42:36.632 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13211 network=udp -42:36.636 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13211 network=tcp -42:36.639 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13212 network=tcp -42:36.639 [INFO] TestAgent: started state syncer -42:36.640 [INFO] TestAgent.server: Adding LAN server: server="Node-df5ba157-ae00-498e-1b49-0affec0f577d (Addr: tcp/127.0.0.1:13244) (DC: dc1)" -42:36.640 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-df5ba157-ae00-498e-1b49-0affec0f577d.dc1 area=wan -42:36.641 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13245 network=tcp -42:36.670 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:36.670 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13244 [Candidate]" term=2 -42:36.670 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:36.670 [DEBUG] TestAgent.server.raft: vote granted: from=df5ba157-ae00-498e-1b49-0affec0f577d term=2 tally=1 -42:36.670 [INFO] TestAgent.server.raft: election won: tally=1 -42:36.670 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13244 [Leader]" -42:36.671 [INFO] TestAgent.server: cluster leadership acquired -42:36.671 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13244 -42:36.671 [INFO] TestAgent.server: New leader elected: payload=Node-df5ba157-ae00-498e-1b49-0affec0f577d -42:36.673 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:36.681 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:36.681 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:36.681 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:36.681 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:36.681 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:36.681 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-df5ba157-ae00-498e-1b49-0affec0f577d -42:36.681 [INFO] TestAgent.server: member joined, marking health alive: member=Node-df5ba157-ae00-498e-1b49-0affec0f577d -42:36.687 [INFO] TestAgent.server: federation state anti-entropy synced -42:36.844 [INFO] TestAgent: Requesting shutdown -42:36.844 [INFO] TestAgent.server: shutting down server -42:36.844 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:36.844 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:36.844 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:36.844 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:36.844 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:36.846 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:36.846 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:36.846 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:36.846 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:36.846 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:36.846 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:36.846 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:36.847Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:36.847Z [INFO] agent.router.manager: shutting down -42:36.847 [INFO] TestAgent: consul server down -42:36.847 [INFO] TestAgent: shutdown complete -42:36.847 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13211 network=tcp -42:36.847 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13211 network=udp -42:36.847 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13212 network=tcp -42:37.348 [INFO] TestAgent: Waiting for endpoints to shut down -42:37.348 [INFO] TestAgent: Endpoints down ---- PASS: TestConfig_Apply_TerminatingGateway (0.88s) -=== CONT TestCatalogNodeServices_WanTranslation -2022-09-15T21:42:36.497Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:36.548Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:36.552 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9ac93bd9-4f02-9af7-ccb8-f3c98095f9a1 Address:127.0.0.1:13223}]" -42:36.553 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9ac93bd9-4f02-9af7-ccb8-f3c98095f9a1.dc1 127.0.0.1 -42:36.554 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13223 [Follower]" leader= -42:36.554 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9ac93bd9-4f02-9af7-ccb8-f3c98095f9a1 127.0.0.1 -2022-09-15T21:42:36.554Z [INFO] agent.router: Initializing LAN area manager -42:36.555 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13225 network=udp -42:36.555 [INFO] TestAgent.server: Adding LAN server: server="Node-9ac93bd9-4f02-9af7-ccb8-f3c98095f9a1 (Addr: tcp/127.0.0.1:13223) (DC: dc1)" -42:36.555 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9ac93bd9-4f02-9af7-ccb8-f3c98095f9a1.dc1 area=wan -42:36.556 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13225 network=tcp -42:36.557 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13226 network=tcp -42:36.557 [INFO] TestAgent: started state syncer -42:36.557 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13224 network=tcp -42:36.613 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:36.613 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13223 [Candidate]" term=2 -42:36.626 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:36.626 [DEBUG] TestAgent.server.raft: vote granted: from=9ac93bd9-4f02-9af7-ccb8-f3c98095f9a1 term=2 tally=1 -42:36.626 [INFO] TestAgent.server.raft: election won: tally=1 -42:36.626 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13223 [Leader]" -42:36.626 [INFO] TestAgent.server: cluster leadership acquired -42:36.627 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13223 -42:36.629 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:36.631 [INFO] TestAgent.server: New leader elected: payload=Node-9ac93bd9-4f02-9af7-ccb8-f3c98095f9a1 -42:36.637 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:36.637 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:36.637 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:36.637 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:36.637 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:36.637 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9ac93bd9-4f02-9af7-ccb8-f3c98095f9a1 -42:36.637 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9ac93bd9-4f02-9af7-ccb8-f3c98095f9a1 -42:36.646 [INFO] TestAgent.server: federation state anti-entropy synced -42:36.963 [INFO] TestAgent: Requesting shutdown -42:36.963 [INFO] TestAgent.server: shutting down server -42:36.963 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:36.963 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:36.963 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:36.963 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:36.963 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:36.963 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:36.963 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:36.963 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:36.964 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:36.964 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:36.964 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:36.964 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:36.964Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:36.964Z [INFO] agent.router.manager: shutting down -42:36.964 [INFO] TestAgent: consul server down -42:36.964 [INFO] TestAgent: shutdown complete -42:36.964 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13225 network=tcp -42:36.964 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13225 network=udp -42:36.964 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13226 network=tcp -42:37.465 [INFO] TestAgent: Waiting for endpoints to shut down -42:37.465 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentConnectAuthorize_badBody (1.03s) -=== CONT TestCatalogNodeServices_ConnectProxy -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:36.579Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:36.629Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:36.637 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c1938b95-ac51-200c-5a0d-84db3c192139 Address:127.0.0.1:13188}]" -42:36.640 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c1938b95-ac51-200c-5a0d-84db3c192139.dc1 127.0.0.1 -42:36.643 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c1938b95-ac51-200c-5a0d-84db3c192139 127.0.0.1 -2022-09-15T21:42:36.643Z [INFO] agent.router: Initializing LAN area manager -42:36.645 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13188 [Follower]" leader= -42:36.646 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13246 network=udp -42:36.647 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13246 network=tcp -42:36.647 [INFO] TestAgent.server: Adding LAN server: server="Node-c1938b95-ac51-200c-5a0d-84db3c192139 (Addr: tcp/127.0.0.1:13188) (DC: dc1)" -42:36.648 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13247 network=tcp -42:36.648 [INFO] TestAgent: started state syncer -42:36.648 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13189 network=tcp -42:36.647 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c1938b95-ac51-200c-5a0d-84db3c192139.dc1 area=wan -42:36.710 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:36.710 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13188 [Candidate]" term=2 -42:36.710 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:36.710 [DEBUG] TestAgent.server.raft: vote granted: from=c1938b95-ac51-200c-5a0d-84db3c192139 term=2 tally=1 -42:36.710 [INFO] TestAgent.server.raft: election won: tally=1 -42:36.710 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13188 [Leader]" -42:36.711 [INFO] TestAgent.server: cluster leadership acquired -42:36.712 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13188 -42:36.713 [INFO] TestAgent.server: New leader elected: payload=Node-c1938b95-ac51-200c-5a0d-84db3c192139 -42:36.717 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:36.726 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:36.726 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:36.726 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:36.726 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:36.726 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:36.726 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c1938b95-ac51-200c-5a0d-84db3c192139 -42:36.726 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c1938b95-ac51-200c-5a0d-84db3c192139 -42:36.727 [INFO] TestAgent.server: federation state anti-entropy synced -42:36.747 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:36.748 [INFO] TestAgent: Synced node info -42:37.036 [INFO] TestAgent: Requesting shutdown -42:37.036 [INFO] TestAgent.server: shutting down server -42:37.036 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:37.036 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:37.036 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:37.036 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:37.036 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:37.036 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:37.036 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:37.036 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:37.036 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:37.036 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:37.036 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:37.036Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:37.036Z [INFO] agent.router.manager: shutting down -42:37.037 [INFO] TestAgent: consul server down -42:37.037 [INFO] TestAgent: shutdown complete -42:37.037 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13246 network=tcp -42:37.037 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13246 network=udp -42:37.037 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13247 network=tcp -42:37.538 [INFO] TestAgent: Waiting for endpoints to shut down -42:37.538 [INFO] TestAgent: Endpoints down ---- PASS: TestConfig_Apply (1.05s) -=== CONT TestCatalogNodeServices_Filter -2022-09-15T21:42:36.832Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:36.894Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:36.903 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f1f5b611-0306-706a-11b0-03fcd05a6a4a Address:127.0.0.1:13251}]" -42:36.903 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13251 [Follower]" leader= -42:36.904 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f1f5b611-0306-706a-11b0-03fcd05a6a4a.dc1 127.0.0.1 -42:36.905 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f1f5b611-0306-706a-11b0-03fcd05a6a4a 127.0.0.1 -2022-09-15T21:42:36.905Z [INFO] agent.router: Initializing LAN area manager -42:36.905 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13183 network=udp -42:36.905 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f1f5b611-0306-706a-11b0-03fcd05a6a4a.dc1 area=wan -42:36.905 [INFO] TestAgent.server: Adding LAN server: server="Node-f1f5b611-0306-706a-11b0-03fcd05a6a4a (Addr: tcp/127.0.0.1:13251) (DC: dc1)" -42:36.905 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13183 network=tcp -42:36.911 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13184 network=tcp -42:36.911 [INFO] TestAgent: started state syncer -42:36.911 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13252 network=tcp -42:36.974 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:36.974 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13251 [Candidate]" term=2 -42:36.974 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:36.974 [DEBUG] TestAgent.server.raft: vote granted: from=f1f5b611-0306-706a-11b0-03fcd05a6a4a term=2 tally=1 -42:36.974 [INFO] TestAgent.server.raft: election won: tally=1 -42:36.975 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13251 [Leader]" -42:36.975 [INFO] TestAgent.server: cluster leadership acquired -42:36.975 [INFO] TestAgent.server: New leader elected: payload=Node-f1f5b611-0306-706a-11b0-03fcd05a6a4a -42:36.977 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13251 -42:36.980 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:36.984 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:36.984 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:36.984 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:36.984 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:36.984 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:36.984 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f1f5b611-0306-706a-11b0-03fcd05a6a4a -42:36.984 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f1f5b611-0306-706a-11b0-03fcd05a6a4a -42:36.989 [INFO] TestAgent.server: federation state anti-entropy synced -42:37.016 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:37.017 [INFO] TestAgent: Synced node info -42:37.017 [DEBUG] TestAgent: Node info in sync -42:37.052 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:37.052 [DEBUG] TestAgent: Node info in sync -42:37.066 [INFO] TestAgent: Requesting shutdown -42:37.066 [INFO] TestAgent.server: shutting down server -42:37.066 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:37.066 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:37.066 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:37.066 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:37.066 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:37.067 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:37.067 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:37.067 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:37.067 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:37.067 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:37.067 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:37.067Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:37.067Z [INFO] agent.router.manager: shutting down -42:37.067 [INFO] TestAgent: consul server down -42:37.067 [INFO] TestAgent: shutdown complete -42:37.067 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13183 network=tcp -42:37.067 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13183 network=udp -42:37.067 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13184 network=tcp -42:37.572 [INFO] TestAgent: Waiting for endpoints to shut down -42:37.572 [INFO] TestAgent: Endpoints down -=== RUN TestConnectCAConfig/updating_config_with_same_state -2022-09-15T21:42:36.898Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:36.958Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:36.962 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5bd5af84-a969-73fc-4ff4-d5e1f68868d4 Address:127.0.0.1:13265}]" -42:36.962 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13265 [Follower]" leader= -42:36.971 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5bd5af84-a969-73fc-4ff4-d5e1f68868d4.dc1 127.0.0.1 -42:36.972 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5bd5af84-a969-73fc-4ff4-d5e1f68868d4 127.0.0.1 -2022-09-15T21:42:36.972Z [INFO] agent.router: Initializing LAN area manager -42:36.973 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13253 network=udp -42:36.973 [INFO] TestAgent.server: Adding LAN server: server="Node-5bd5af84-a969-73fc-4ff4-d5e1f68868d4 (Addr: tcp/127.0.0.1:13265) (DC: dc1)" -42:36.974 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13253 network=tcp -42:36.974 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5bd5af84-a969-73fc-4ff4-d5e1f68868d4.dc1 area=wan -42:36.975 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13254 network=tcp -42:36.976 [INFO] TestAgent: started state syncer -42:36.976 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13266 network=tcp -42:37.001 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:37.001 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13265 [Candidate]" term=2 -42:37.001 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:37.001 [DEBUG] TestAgent.server.raft: vote granted: from=5bd5af84-a969-73fc-4ff4-d5e1f68868d4 term=2 tally=1 -42:37.001 [INFO] TestAgent.server.raft: election won: tally=1 -42:37.001 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13265 [Leader]" -42:37.001 [INFO] TestAgent.server: cluster leadership acquired -42:37.002 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13265 -42:37.003 [INFO] TestAgent.server: New leader elected: payload=Node-5bd5af84-a969-73fc-4ff4-d5e1f68868d4 -42:37.003 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:37.006 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:37.006 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:37.006 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:37.006 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:37.006 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:37.006 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5bd5af84-a969-73fc-4ff4-d5e1f68868d4 -42:37.006 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5bd5af84-a969-73fc-4ff4-d5e1f68868d4 -42:37.007 [INFO] TestAgent.server: federation state anti-entropy synced -42:37.080 [INFO] TestAgent: Requesting shutdown -42:37.080 [INFO] TestAgent.server: shutting down server -42:37.080 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:37.080 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:37.080 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:37.080 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:37.080 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:37.080 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:37.080 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:37.080 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:37.080 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:37.080 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:37.080 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:37.080 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:37.081Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:37.081Z [INFO] agent.router.manager: shutting down -42:37.081 [INFO] TestAgent: consul server down -42:37.081 [INFO] TestAgent: shutdown complete -42:37.081 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13253 network=tcp -42:37.081 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13253 network=udp -42:37.081 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13254 network=tcp -42:37.582 [INFO] TestAgent: Waiting for endpoints to shut down -42:37.582 [INFO] TestAgent: Endpoints down ---- PASS: TestConfig_Get (0.75s) - --- PASS: TestConfig_Get/get_a_single_service_entry (0.00s) - --- PASS: TestConfig_Get/list_both_service_entries (0.00s) - --- PASS: TestConfig_Get/get_global_proxy_config (0.00s) - --- PASS: TestConfig_Get/error_on_no_arguments (0.00s) -=== CONT TestCatalogNodeServiceList -[WARN] freeport: 4 out of 25 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:37.299Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:37.317Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:37.321 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6be96669-37f0-05ed-193e-8ff925aa5e39 Address:127.0.0.1:13293}]" -42:37.322 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13293 [Follower]" leader= -42:37.322 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6be96669-37f0-05ed-193e-8ff925aa5e39.dc1 127.0.0.1 -42:37.323 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6be96669-37f0-05ed-193e-8ff925aa5e39 127.0.0.1 -2022-09-15T21:42:37.323Z [INFO] agent.router: Initializing LAN area manager -42:37.324 [INFO] TestAgent.server: Adding LAN server: server="Node-6be96669-37f0-05ed-193e-8ff925aa5e39 (Addr: tcp/127.0.0.1:13293) (DC: dc1)" -42:37.325 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6be96669-37f0-05ed-193e-8ff925aa5e39.dc1 area=wan -42:37.325 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13267 network=udp -42:37.325 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13267 network=tcp -42:37.326 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13268 network=tcp -42:37.326 [INFO] TestAgent: started state syncer -42:37.326 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13294 network=tcp -42:37.362 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:37.362 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13293 [Candidate]" term=2 -42:37.362 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:37.362 [DEBUG] TestAgent.server.raft: vote granted: from=6be96669-37f0-05ed-193e-8ff925aa5e39 term=2 tally=1 -42:37.362 [INFO] TestAgent.server.raft: election won: tally=1 -42:37.362 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13293 [Leader]" -42:37.362 [INFO] TestAgent.server: cluster leadership acquired -42:37.363 [INFO] TestAgent.server: New leader elected: payload=Node-6be96669-37f0-05ed-193e-8ff925aa5e39 -42:37.363 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13293 -42:37.364 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:37.367 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:37.367 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:37.367 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:37.367 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:37.367 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:37.367 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6be96669-37f0-05ed-193e-8ff925aa5e39 -42:37.367 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6be96669-37f0-05ed-193e-8ff925aa5e39 -42:37.368 [INFO] TestAgent.server: federation state anti-entropy synced -42:37.444 [INFO] TestAgent: Requesting shutdown -42:37.444 [INFO] TestAgent.server: shutting down server -42:37.444 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:37.444 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:37.444 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:37.444 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:37.444 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:37.444 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:37.444 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:37.444 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:37.444 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:37.444 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:37.444 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:37.445 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:37.445Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:37.445Z [INFO] agent.router.manager: shutting down -42:37.445 [INFO] TestAgent: consul server down -42:37.445 [INFO] TestAgent: shutdown complete -42:37.445 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13267 network=tcp -42:37.445 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13267 network=udp -42:37.445 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13268 network=tcp -42:37.946 [INFO] TestAgent: Waiting for endpoints to shut down -42:37.951 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalog_GatewayServices_Terminating (0.70s) -=== CONT TestCatalogNodeServices -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=33)_compressed=false -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:37.526Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:37.553Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:37.557 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f678d606-1cd3-e48d-72f2-f420f709c3e9 Address:127.0.0.1:13300}]" -42:37.562 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13300 [Follower]" leader= -42:37.571 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f678d606-1cd3-e48d-72f2-f420f709c3e9.dc1 127.0.0.1 -42:37.572 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f678d606-1cd3-e48d-72f2-f420f709c3e9 127.0.0.1 -2022-09-15T21:42:37.572Z [INFO] agent.router: Initializing LAN area manager -42:37.575 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f678d606-1cd3-e48d-72f2-f420f709c3e9.dc1 area=wan -42:37.579 [INFO] TestAgent.server: Adding LAN server: server="Node-f678d606-1cd3-e48d-72f2-f420f709c3e9 (Addr: tcp/127.0.0.1:13300) (DC: dc1)" -42:37.581 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13281 network=tcp -42:37.593 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13281 network=udp -42:37.594 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13282 network=tcp -42:37.594 [INFO] TestAgent: started state syncer -42:37.595 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13301 network=tcp -42:37.610 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:37.610 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13300 [Candidate]" term=2 -42:37.610 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:37.610 [DEBUG] TestAgent.server.raft: vote granted: from=f678d606-1cd3-e48d-72f2-f420f709c3e9 term=2 tally=1 -42:37.610 [INFO] TestAgent.server.raft: election won: tally=1 -42:37.610 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13300 [Leader]" -42:37.611 [INFO] TestAgent.server: cluster leadership acquired -42:37.611 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13300 -42:37.613 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:37.616 [INFO] TestAgent.server: New leader elected: payload=Node-f678d606-1cd3-e48d-72f2-f420f709c3e9 -42:37.617 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:37.618 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:37.618 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:37.618 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:37.618 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:37.618 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f678d606-1cd3-e48d-72f2-f420f709c3e9 -42:37.618 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f678d606-1cd3-e48d-72f2-f420f709c3e9 -42:37.626 [INFO] TestAgent.server: federation state anti-entropy synced -42:37.654 [INFO] TestAgent: Requesting shutdown -42:37.654 [INFO] TestAgent.server: shutting down server -42:37.654 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:37.654 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:37.654 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:37.654 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:37.654 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:37.655 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:37.655 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:37.655 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:37.655 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:37.655 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:37.655 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:37.655 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:37.655Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:37.655Z [INFO] agent.router.manager: shutting down -42:37.655 [INFO] TestAgent: consul server down -42:37.655 [INFO] TestAgent: shutdown complete -42:37.655 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13281 network=tcp -42:37.655 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13281 network=udp -42:37.655 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13282 network=tcp -42:38.156 [INFO] TestAgent: Waiting for endpoints to shut down -42:38.156 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogNodeServices_ConnectProxy (0.68s) -=== CONT TestCatalogConnectServiceNodes_good -2022-09-15T21:42:37.177Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:37.194Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:37.197 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e177c362-5fe9-6386-e9e5-c3355c9f25d9 Address:127.0.0.1:13279}]" -42:37.198 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13279 [Follower]" leader= -42:37.199 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e177c362-5fe9-6386-e9e5-c3355c9f25d9.dc1 127.0.0.1 -42:37.199 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e177c362-5fe9-6386-e9e5-c3355c9f25d9 127.0.0.1 -2022-09-15T21:42:37.199Z [INFO] agent.router: Initializing LAN area manager -42:37.200 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13274 network=udp -42:37.200 [INFO] TestAgent.server: Adding LAN server: server="Node-e177c362-5fe9-6386-e9e5-c3355c9f25d9 (Addr: tcp/127.0.0.1:13279) (DC: dc1)" -42:37.200 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e177c362-5fe9-6386-e9e5-c3355c9f25d9.dc1 area=wan -42:37.201 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13274 network=tcp -42:37.201 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13275 network=tcp -42:37.202 [INFO] TestAgent: started state syncer -42:37.202 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13280 network=tcp -42:37.258 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:37.258 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13279 [Candidate]" term=2 -42:37.258 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:37.258 [DEBUG] TestAgent.server.raft: vote granted: from=e177c362-5fe9-6386-e9e5-c3355c9f25d9 term=2 tally=1 -42:37.258 [INFO] TestAgent.server.raft: election won: tally=1 -42:37.258 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13279 [Leader]" -42:37.258 [INFO] TestAgent.server: cluster leadership acquired -42:37.259 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13279 -42:37.260 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:37.263 [INFO] TestAgent.server: New leader elected: payload=Node-e177c362-5fe9-6386-e9e5-c3355c9f25d9 -42:37.272 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:37.272 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:37.272 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:37.272 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:37.272 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:37.272 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e177c362-5fe9-6386-e9e5-c3355c9f25d9 -42:37.272 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e177c362-5fe9-6386-e9e5-c3355c9f25d9 -42:37.278 [INFO] TestAgent.server: federation state anti-entropy synced -42:37.554 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:37.555 [INFO] TestAgent: Synced node info -42:37.555 [DEBUG] TestAgent: Node info in sync -42:37.662 [INFO] TestAgent: Requesting shutdown -42:37.662 [INFO] TestAgent.server: shutting down server -42:37.662 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:37.662 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:37.662 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:37.662 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:37.662 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:37.662 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:37.662 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:37.662 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:37.662 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:37.662 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:37.662 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:37.663Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:37.663Z [INFO] agent.router.manager: shutting down -42:37.663 [INFO] TestAgent: consul server down -42:37.663 [INFO] TestAgent: shutdown complete -42:37.663 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13274 network=tcp -42:37.663 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13274 network=udp -42:37.663 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13275 network=tcp -42:38.170 [INFO] TestAgent: Waiting for endpoints to shut down -42:38.170 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalog_GatewayServices_Ingress (1.02s) -=== CONT TestCatalogServiceNodes_ConnectProxy -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:37.599Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:37.675Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:37.683 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6a4eb844-f409-68dd-2af3-20db26ecb051 Address:127.0.0.1:13286}]" -42:37.684 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13286 [Follower]" leader= -42:37.696 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6a4eb844-f409-68dd-2af3-20db26ecb051.dc1 127.0.0.1 -42:37.697 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6a4eb844-f409-68dd-2af3-20db26ecb051 127.0.0.1 -2022-09-15T21:42:37.697Z [INFO] agent.router: Initializing LAN area manager -42:37.698 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13302 network=udp -42:37.699 [INFO] TestAgent.server: Adding LAN server: server="Node-6a4eb844-f409-68dd-2af3-20db26ecb051 (Addr: tcp/127.0.0.1:13286) (DC: dc1)" -42:37.699 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6a4eb844-f409-68dd-2af3-20db26ecb051.dc1 area=wan -42:37.700 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13302 network=tcp -42:37.708 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13303 network=tcp -42:37.708 [INFO] TestAgent: started state syncer -42:37.709 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13287 network=tcp -42:37.722 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:37.722 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13286 [Candidate]" term=2 -42:37.722 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:37.722 [DEBUG] TestAgent.server.raft: vote granted: from=6a4eb844-f409-68dd-2af3-20db26ecb051 term=2 tally=1 -42:37.722 [INFO] TestAgent.server.raft: election won: tally=1 -42:37.722 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13286 [Leader]" -42:37.722 [INFO] TestAgent.server: cluster leadership acquired -42:37.723 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13286 -42:37.725 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:37.726 [INFO] TestAgent.server: New leader elected: payload=Node-6a4eb844-f409-68dd-2af3-20db26ecb051 -42:37.739 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:37.739 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:37.739 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:37.739 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:37.739 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:37.739 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6a4eb844-f409-68dd-2af3-20db26ecb051 -42:37.746 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6a4eb844-f409-68dd-2af3-20db26ecb051 -42:37.749 [INFO] TestAgent.server: federation state anti-entropy synced -42:37.788 [INFO] TestAgent: Requesting shutdown -42:37.789 [INFO] TestAgent.server: shutting down server -42:37.789 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:37.789 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:37.789 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:37.789 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:37.789 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:37.789 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:37.789 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:37.789 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:37.789 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:37.789 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:37.789 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:37.789 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:37.802Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:37.802Z [INFO] agent.router.manager: shutting down -42:37.802 [INFO] TestAgent: consul server down -42:37.802 [INFO] TestAgent: shutdown complete -42:37.802 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13302 network=tcp -42:37.802 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13302 network=udp -42:37.803 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13303 network=tcp -42:38.303 [INFO] TestAgent: Waiting for endpoints to shut down -42:38.304 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogNodeServices_Filter (0.77s) -=== CONT TestCatalogServiceNodes_DistanceSort -2022-09-15T21:42:37.691Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:37.768Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:37.774 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e092a387-7127-8266-597d-31d954ba73c1 Address:127.0.0.1:13321}]" -42:37.776 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e092a387-7127-8266-597d-31d954ba73c1.dc1 127.0.0.1 -42:37.776 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13321 [Follower]" leader= -42:37.777 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e092a387-7127-8266-597d-31d954ba73c1 127.0.0.1 -2022-09-15T21:42:37.777Z [INFO] agent.router: Initializing LAN area manager -42:37.777 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13316 network=udp -42:37.777 [INFO] TestAgent.server: Adding LAN server: server="Node-e092a387-7127-8266-597d-31d954ba73c1 (Addr: tcp/127.0.0.1:13321) (DC: dc1)" -42:37.777 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e092a387-7127-8266-597d-31d954ba73c1.dc1 area=wan -42:37.778 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13316 network=tcp -42:37.779 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13317 network=tcp -42:37.779 [INFO] TestAgent: started state syncer -42:37.779 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13322 network=tcp -42:37.834 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:37.834 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13321 [Candidate]" term=2 -42:37.834 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:37.834 [DEBUG] TestAgent.server.raft: vote granted: from=e092a387-7127-8266-597d-31d954ba73c1 term=2 tally=1 -42:37.834 [INFO] TestAgent.server.raft: election won: tally=1 -42:37.834 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13321 [Leader]" -42:37.834 [INFO] TestAgent.server: cluster leadership acquired -42:37.835 [INFO] TestAgent.server: New leader elected: payload=Node-e092a387-7127-8266-597d-31d954ba73c1 -42:37.835 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13321 -42:37.837 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:37.865 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:37.865 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:37.865 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:37.865 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:37.865 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:37.865 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e092a387-7127-8266-597d-31d954ba73c1 -42:37.865 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e092a387-7127-8266-597d-31d954ba73c1 -42:37.867 [INFO] TestAgent.server: federation state anti-entropy synced -42:37.918 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:37.919 [INFO] TestAgent: Synced node info -42:37.956 [INFO] TestAgent: Requesting shutdown -42:37.956 [INFO] TestAgent.server: shutting down server -42:37.956 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:37.956 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:37.956 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:37.956 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:37.956 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:37.956 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:37.956 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:37.956 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:37.957 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:37.957 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:37.957 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:37.957Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:37.957Z [INFO] agent.router.manager: shutting down -42:37.957 [INFO] TestAgent: consul server down -42:37.957 [INFO] TestAgent: shutdown complete -42:37.957 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13316 network=tcp -42:37.957 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13316 network=udp -42:37.957 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13317 network=tcp -42:38.458 [INFO] TestAgent: Waiting for endpoints to shut down -42:38.458 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogNodeServiceList (0.86s) -=== CONT TestCatalogServiceNodes_WanTranslation -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=33)_compressed=true -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:37.637Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:37.734Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:37.751 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a9526fe1-2d19-922d-07a1-dd8ae6469ba1 Address:127.0.0.1:13314}]" -42:37.752 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13314 [Follower]" leader= -42:37.757 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a9526fe1-2d19-922d-07a1-dd8ae6469ba1.dc1 127.0.0.1 -42:37.758 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a9526fe1-2d19-922d-07a1-dd8ae6469ba1 127.0.0.1 -2022-09-15T21:42:37.758Z [INFO] agent.router: Initializing LAN area manager -42:37.761 [INFO] TestAgent.server: Adding LAN server: server="Node-a9526fe1-2d19-922d-07a1-dd8ae6469ba1 (Addr: tcp/127.0.0.1:13314) (DC: dc1)" -42:37.761 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a9526fe1-2d19-922d-07a1-dd8ae6469ba1.dc1 area=wan -42:37.762 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13288 network=udp -42:37.762 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13288 network=tcp -42:37.763 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13289 network=tcp -42:37.763 [INFO] TestAgent: started state syncer -42:37.763 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13315 network=tcp -42:37.803 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:37.803 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13314 [Candidate]" term=2 -42:37.803 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:37.803 [DEBUG] TestAgent.server.raft: vote granted: from=a9526fe1-2d19-922d-07a1-dd8ae6469ba1 term=2 tally=1 -42:37.803 [INFO] TestAgent.server.raft: election won: tally=1 -42:37.803 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13314 [Leader]" -42:37.804 [INFO] TestAgent.server: cluster leadership acquired -42:37.804 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13314 -42:37.805 [INFO] TestAgent.server: New leader elected: payload=Node-a9526fe1-2d19-922d-07a1-dd8ae6469ba1 -42:37.823 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:37.828 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:37.828 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:37.828 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:37.828 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:37.828 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:37.828 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a9526fe1-2d19-922d-07a1-dd8ae6469ba1 -42:37.828 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a9526fe1-2d19-922d-07a1-dd8ae6469ba1 -42:37.834 [INFO] TestAgent.server: federation state anti-entropy synced -42:37.862 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:37.863 [INFO] TestAgent: Synced node info -42:37.863 [DEBUG] TestAgent: Node info in sync -42:38.141 [INFO] TestAgent.server.connect: CA provider config updated -42:38.142 [INFO] TestAgent: Requesting shutdown -42:38.142 [INFO] TestAgent.server: shutting down server -42:38.142 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:38.142 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:38.142 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:38.142 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:38.142 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:38.142 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:38.142 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:38.142 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:38.142 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:38.142 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:38.142 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:38.143Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:38.143Z [INFO] agent.router.manager: shutting down -42:38.143 [INFO] TestAgent: consul server down -42:38.143 [INFO] TestAgent: shutdown complete -42:38.143 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13288 network=tcp -42:38.143 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13288 network=udp -42:38.143 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13289 network=tcp -42:38.646 [INFO] TestAgent: Waiting for endpoints to shut down -42:38.646 [INFO] TestAgent: Endpoints down ---- PASS: TestConnectCAConfig (5.18s) - --- PASS: TestConnectCAConfig/basic (0.90s) - --- PASS: TestConnectCAConfig/basic_with_IntermediateCertTTL (0.81s) - --- PASS: TestConnectCAConfig/force_without_cross_sign_CamelCase (0.83s) - --- PASS: TestConnectCAConfig/force_without_cross_sign_snake_case (0.78s) - --- PASS: TestConnectCAConfig/setting_state_fails (0.79s) - --- PASS: TestConnectCAConfig/updating_config_with_same_state (1.07s) -=== CONT TestCatalogServiceNodes_Filter -2022-09-15T21:42:37.976Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:37.995Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:37.998 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1b024ffb-3f5a-5282-3f43-6f01e7822b7a Address:127.0.0.1:13258}]" -42:37.998 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13258 [Follower]" leader= -42:37.999 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1b024ffb-3f5a-5282-3f43-6f01e7822b7a.dc1 127.0.0.1 -42:38.000 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1b024ffb-3f5a-5282-3f43-6f01e7822b7a 127.0.0.1 -2022-09-15T21:42:38.000Z [INFO] agent.router: Initializing LAN area manager -42:38.001 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13309 network=udp -42:38.001 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1b024ffb-3f5a-5282-3f43-6f01e7822b7a.dc1 area=wan -42:38.001 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13309 network=tcp -42:38.002 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13310 network=tcp -42:38.002 [INFO] TestAgent: started state syncer -42:38.002 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13259 network=tcp -42:38.002 [INFO] TestAgent.server: Adding LAN server: server="Node-1b024ffb-3f5a-5282-3f43-6f01e7822b7a (Addr: tcp/127.0.0.1:13258) (DC: dc1)" -42:38.052 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:38.052 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13258 [Candidate]" term=2 -42:38.052 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:38.053 [DEBUG] TestAgent.server.raft: vote granted: from=1b024ffb-3f5a-5282-3f43-6f01e7822b7a term=2 tally=1 -42:38.053 [INFO] TestAgent.server.raft: election won: tally=1 -42:38.053 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13258 [Leader]" -42:38.053 [INFO] TestAgent.server: cluster leadership acquired -42:38.053 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13258 -42:38.054 [INFO] TestAgent.server: New leader elected: payload=Node-1b024ffb-3f5a-5282-3f43-6f01e7822b7a -42:38.054 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:38.057 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:38.057 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:38.057 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:38.057 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:38.057 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:38.057 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1b024ffb-3f5a-5282-3f43-6f01e7822b7a -42:38.057 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1b024ffb-3f5a-5282-3f43-6f01e7822b7a -42:38.058 [INFO] TestAgent.server: federation state anti-entropy synced -42:38.183 [INFO] TestAgent: Requesting shutdown -42:38.183 [INFO] TestAgent.server: shutting down server -42:38.183 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:38.183 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:38.183 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:38.183 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:38.183 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:38.184 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:38.184 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:38.184 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:38.184 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:38.184 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:38.184 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:38.184 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:38.184Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:38.184Z [INFO] agent.router.manager: shutting down -42:38.185 [INFO] TestAgent: consul server down -42:38.185 [INFO] TestAgent: shutdown complete -42:38.185 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13309 network=tcp -42:38.185 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13309 network=udp -42:38.185 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13310 network=tcp -42:38.686 [INFO] TestAgent: Waiting for endpoints to shut down -42:38.686 [INFO] TestAgent: Endpoints down -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=33)_compressed=false ---- PASS: TestCatalogNodeServices (0.74s) === CONT TestCatalogServiceNodes_NodeMetaFilter -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._tcp_(qType:=33)_compressed=true -2022-09-15T21:42:37.665Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:37.665Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:37.768Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:37.768Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:37.772 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d4997c75-fe90-2ef8-6f2e-643211ccd763 Address:127.0.0.1:13307}]" -42:37.773 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13307 [Follower]" leader= -42:37.777 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d4997c75-fe90-2ef8-6f2e-643211ccd763.dc2 127.0.0.1 -42:37.778 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d4997c75-fe90-2ef8-6f2e-643211ccd763 127.0.0.1 -2022-09-15T21:42:37.778Z [INFO] agent.router: Initializing LAN area manager -42:37.778 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13323 network=udp -42:37.778 [INFO] TestAgent.server: Adding LAN server: server="Node-d4997c75-fe90-2ef8-6f2e-643211ccd763 (Addr: tcp/127.0.0.1:13307) (DC: dc2)" -42:37.778 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d4997c75-fe90-2ef8-6f2e-643211ccd763.dc2 area=wan -42:37.778 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13323 network=tcp -42:37.779 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13324 network=tcp -42:37.779 [INFO] TestAgent: started state syncer -42:37.780 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13308 network=tcp -42:37.834 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -42:37.843 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:37.843 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13307 [Candidate]" term=2 -42:37.844 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:37.844 [DEBUG] TestAgent.server.raft: vote granted: from=d4997c75-fe90-2ef8-6f2e-643211ccd763 term=2 tally=1 -42:37.844 [INFO] TestAgent.server.raft: election won: tally=1 -42:37.844 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13307 [Leader]" -42:37.845 [INFO] TestAgent.server: cluster leadership acquired -42:37.845 [INFO] TestAgent.server: initializing acls -42:37.846 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:37.847 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:37.847 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:37.847 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:37.848 [INFO] TestAgent.server: New leader elected: payload=Node-d4997c75-fe90-2ef8-6f2e-643211ccd763 -42:37.848 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-d4997c75-fe90-2ef8-6f2e-643211ccd763 -42:37.848 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-d4997c75-fe90-2ef8-6f2e-643211ccd763.dc2 -42:37.855 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-d4997c75-fe90-2ef8-6f2e-643211ccd763.dc2 area=wan -42:37.858 [INFO] TestAgent.server: Updating LAN server: server="Node-d4997c75-fe90-2ef8-6f2e-643211ccd763 (Addr: tcp/127.0.0.1:13307) (DC: dc2)" -42:37.858 [INFO] TestAgent: Synced node info -42:37.858 [DEBUG] TestAgent: Node info in sync -42:37.864 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:37.867 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:37.867 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:37.867 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:37.867 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:37.867 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:37.867 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d4997c75-fe90-2ef8-6f2e-643211ccd763 -42:37.867 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d4997c75-fe90-2ef8-6f2e-643211ccd763 -42:37.868 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d4997c75-fe90-2ef8-6f2e-643211ccd763 -42:37.869 [INFO] TestAgent.server: federation state anti-entropy synced -42:37.922 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:13271] -42:37.923 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:13271 -42:37.925 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0d21b922-b29a-f559-88a1-de1c5ecc374e.dc1 127.0.0.1 -42:37.925 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 -42:37.939 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0d21b922-b29a-f559-88a1-de1c5ecc374e.dc1 area=wan -42:37.955 [INFO] TestAgent: Requesting shutdown -42:37.955 [INFO] TestAgent.server: shutting down server -42:37.955 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:37.955 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:37.955 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:37.955 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:37.955 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:37.955 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:37.955 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:37.956 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:37.956 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:37.956 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:37.956 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:37.956 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:37.956 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -42:37.956 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:37.956 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2022-09-15T21:42:37.956Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:37.956Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:37.956Z [INFO] agent.router.manager: shutting down -42:37.956 [INFO] TestAgent: consul server down -42:37.956 [INFO] TestAgent: shutdown complete -42:37.956 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13323 network=tcp -42:37.956 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13323 network=udp -42:37.956 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13324 network=tcp -42:38.459 [INFO] TestAgent: Waiting for endpoints to shut down -42:38.459 [INFO] TestAgent: Endpoints down -2022-09-15T21:42:37.370Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:37.370Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:37.391Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:37.391Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:37.395 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0d21b922-b29a-f559-88a1-de1c5ecc374e Address:127.0.0.1:13272}]" -42:37.396 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13272 [Follower]" leader= -42:37.397 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0d21b922-b29a-f559-88a1-de1c5ecc374e.dc1 127.0.0.1 -42:37.398 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0d21b922-b29a-f559-88a1-de1c5ecc374e 127.0.0.1 -2022-09-15T21:42:37.398Z [INFO] agent.router: Initializing LAN area manager -42:37.398 [INFO] TestAgent.server: Adding LAN server: server="Node-0d21b922-b29a-f559-88a1-de1c5ecc374e (Addr: tcp/127.0.0.1:13272) (DC: dc1)" -42:37.398 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0d21b922-b29a-f559-88a1-de1c5ecc374e.dc1 area=wan -42:37.398 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13295 network=tcp -42:37.399 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13295 network=udp -42:37.414 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13296 network=tcp -42:37.415 [INFO] TestAgent: started state syncer -42:37.415 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13273 network=tcp -42:37.439 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:37.439 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13272 [Candidate]" term=2 -42:37.440 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:37.440 [DEBUG] TestAgent.server.raft: vote granted: from=0d21b922-b29a-f559-88a1-de1c5ecc374e term=2 tally=1 -42:37.440 [INFO] TestAgent.server.raft: election won: tally=1 -42:37.440 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13272 [Leader]" -42:37.441 [INFO] TestAgent.server: cluster leadership acquired -42:37.442 [INFO] TestAgent.server: New leader elected: payload=Node-0d21b922-b29a-f559-88a1-de1c5ecc374e -42:37.442 [INFO] TestAgent.server: initializing acls -42:37.442 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:37.443 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:37.443 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:37.443 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:37.443 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-0d21b922-b29a-f559-88a1-de1c5ecc374e -42:37.443 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-0d21b922-b29a-f559-88a1-de1c5ecc374e.dc1 -42:37.445 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:37.445 [INFO] TestAgent.server: Updating LAN server: server="Node-0d21b922-b29a-f559-88a1-de1c5ecc374e (Addr: tcp/127.0.0.1:13272) (DC: dc1)" -42:37.445 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-0d21b922-b29a-f559-88a1-de1c5ecc374e.dc1 area=wan -42:37.448 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:37.448 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:37.448 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:37.448 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:37.448 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:37.448 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0d21b922-b29a-f559-88a1-de1c5ecc374e -42:37.448 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0d21b922-b29a-f559-88a1-de1c5ecc374e -42:37.449 [INFO] TestAgent.server: federation state anti-entropy synced -42:37.449 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0d21b922-b29a-f559-88a1-de1c5ecc374e -42:37.694 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:37.696 [INFO] TestAgent: Synced node info -42:37.923 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:60854 -42:37.924 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d4997c75-fe90-2ef8-6f2e-643211ccd763.dc2 127.0.0.1 -42:37.925 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d4997c75-fe90-2ef8-6f2e-643211ccd763.dc2 area=wan -42:38.459 [INFO] TestAgent: Requesting shutdown -42:38.459 [INFO] TestAgent.server: shutting down server -42:38.459 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:38.459 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:38.459 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:38.459 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:38.459 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:38.459 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:38.459 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:38.459 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:38.459 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:38.459 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:38.459 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:38.459 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:38.459 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:38.459 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:38.459 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:38.460Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:38.460Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:38.460Z [INFO] agent.router.manager: shutting down -42:38.460 [INFO] TestAgent: consul server down -42:38.460 [INFO] TestAgent: shutdown complete -42:38.460 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13295 network=tcp -42:38.460 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13295 network=udp -42:38.460 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13296 network=tcp -42:38.968 [INFO] TestAgent: Waiting for endpoints to shut down -42:38.968 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogNodeServices_WanTranslation (1.62s) +2021-08-13T15:33:57.390Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:57.435Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:57.439 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:20584aa5-8a6b-1f09-7593-7f6c2505ea37 Address:127.0.0.1:24365}]" +33:57.440 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24365 [Follower]" leader= +33:57.441 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-20584aa5-8a6b-1f09-7593-7f6c2505ea37.dc1 127.0.0.1 +33:57.442 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-20584aa5-8a6b-1f09-7593-7f6c2505ea37 127.0.0.1 +2021-08-13T15:33:57.442Z [INFO] agent.router: Initializing LAN area manager +33:57.442 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-20584aa5-8a6b-1f09-7593-7f6c2505ea37.dc1 area=wan +33:57.442 [INFO] TestAgent.server: Adding LAN server: server="Node-20584aa5-8a6b-1f09-7593-7f6c2505ea37 (Addr: tcp/127.0.0.1:24365) (DC: dc1)" +33:57.442 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24346 network=udp +33:57.442 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24346 network=tcp +33:57.443 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24347 network=tcp +33:57.443 [INFO] TestAgent: started state syncer +33:57.443 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24366 network=tcp +33:57.506 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:57.506 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24365 [Candidate]" term=2 +33:57.506 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:57.506 [DEBUG] TestAgent.server.raft: vote granted: from=20584aa5-8a6b-1f09-7593-7f6c2505ea37 term=2 tally=1 +33:57.506 [INFO] TestAgent.server.raft: election won: tally=1 +33:57.506 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24365 [Leader]" +33:57.506 [INFO] TestAgent.server: cluster leadership acquired +33:57.507 [INFO] TestAgent.server: New leader elected: payload=Node-20584aa5-8a6b-1f09-7593-7f6c2505ea37 +33:57.507 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24365 +33:57.508 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:57.510 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:57.510 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:57.510 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:57.510 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:57.510 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:57.510 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-20584aa5-8a6b-1f09-7593-7f6c2505ea37 +33:57.510 [INFO] TestAgent.server: member joined, marking health alive: member=Node-20584aa5-8a6b-1f09-7593-7f6c2505ea37 +33:57.511 [INFO] TestAgent.server: federation state anti-entropy synced +33:57.511 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:57.512 [INFO] TestAgent: Synced node info +33:57.652 [DEBUG] TestAgent.server: User event: event=deploy +33:57.652 [DEBUG] TestAgent.server: User event: event=deploy +33:57.652 [DEBUG] TestAgent: new event: event_name=deploy event_id=28d19e58-89c9-5a3c-4e1f-c377c98017a0 +33:57.652 [INFO] TestAgent: Requesting shutdown +33:57.652 [INFO] TestAgent.server: shutting down server +33:57.652 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:57.652 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:57.652 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:57.652 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:57.652 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:57.652 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +33:57.653 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:57.653 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:57.653 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:57.653 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:57.653 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2021-08-13T15:33:57.653Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:57.653Z [INFO] agent.router.manager: shutting down +33:57.653 [INFO] TestAgent: consul server down +33:57.653 [INFO] TestAgent: shutdown complete +33:57.653 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24346 network=tcp +33:57.653 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24346 network=udp +33:57.653 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24347 network=tcp +33:58.154 [INFO] TestAgent: Waiting for endpoints to shut down +33:58.154 [INFO] TestAgent: Endpoints down +2021-08-13T15:33:57.336Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:57.389Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:57.393 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bff87e48-0a27-5ccc-96fb-91b1760e9926 Address:127.0.0.1:24316}]" +33:57.394 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bff87e48-0a27-5ccc-96fb-91b1760e9926.dc1 127.0.0.1 +33:57.395 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bff87e48-0a27-5ccc-96fb-91b1760e9926 127.0.0.1 +2021-08-13T15:33:57.395Z [INFO] agent.router: Initializing LAN area manager +33:57.395 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24316 [Follower]" leader= +33:57.395 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24311 network=udp +33:57.395 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bff87e48-0a27-5ccc-96fb-91b1760e9926.dc1 area=wan +33:57.395 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24311 network=tcp +33:57.396 [INFO] TestAgent.server: Adding LAN server: server="Node-bff87e48-0a27-5ccc-96fb-91b1760e9926 (Addr: tcp/127.0.0.1:24316) (DC: dc1)" +33:57.396 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24312 network=tcp +33:57.396 [INFO] TestAgent: started state syncer +33:57.396 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24317 network=tcp +33:57.458 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:57.458 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24316 [Candidate]" term=2 +33:57.458 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:57.458 [DEBUG] TestAgent.server.raft: vote granted: from=bff87e48-0a27-5ccc-96fb-91b1760e9926 term=2 tally=1 +33:57.458 [INFO] TestAgent.server.raft: election won: tally=1 +33:57.458 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24316 [Leader]" +33:57.458 [INFO] TestAgent.server: cluster leadership acquired +33:57.459 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24316 +33:57.459 [INFO] TestAgent.server: New leader elected: payload=Node-bff87e48-0a27-5ccc-96fb-91b1760e9926 +33:57.460 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:57.462 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:57.462 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:57.462 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:57.462 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:57.462 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:57.462 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bff87e48-0a27-5ccc-96fb-91b1760e9926 +33:57.462 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bff87e48-0a27-5ccc-96fb-91b1760e9926 +33:57.463 [INFO] TestAgent.server: federation state anti-entropy synced +33:57.653 [INFO] TestAgent: Requesting shutdown +33:57.653 [INFO] TestAgent.server: shutting down server +33:57.653 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:57.653 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:57.653 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:57.653 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:57.653 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:57.653 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:57.653 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:57.653 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:57.653 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:57.653 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:57.653 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:57.653 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:57.653Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:57.653Z [INFO] agent.router.manager: shutting down +33:57.653 [INFO] TestAgent: consul server down +33:57.653 [INFO] TestAgent: shutdown complete +33:57.653 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24311 network=tcp +33:57.653 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24311 network=udp +33:57.654 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24312 network=tcp +33:58.154 [INFO] TestAgent: Waiting for endpoints to shut down +33:58.154 [INFO] TestAgent: Endpoints down +--- PASS: TestFireReceiveEvent (0.84s) +--- PASS: TestUiNodes_Filter (0.84s) +2021-08-13T15:33:57.382Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:57.431Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:57.435 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7fe37870-6d3e-5fe0-4594-a8f5a8025a7c Address:127.0.0.1:24344}]" +33:57.435 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24344 [Follower]" leader= +33:57.436 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7fe37870-6d3e-5fe0-4594-a8f5a8025a7c.dc1 127.0.0.1 +33:57.437 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7fe37870-6d3e-5fe0-4594-a8f5a8025a7c 127.0.0.1 +2021-08-13T15:33:57.437Z [INFO] agent.router: Initializing LAN area manager +33:57.437 [INFO] TestAgent.server: Adding LAN server: server="Node-7fe37870-6d3e-5fe0-4594-a8f5a8025a7c (Addr: tcp/127.0.0.1:24344) (DC: dc1)" +33:57.437 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7fe37870-6d3e-5fe0-4594-a8f5a8025a7c.dc1 area=wan +33:57.438 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24353 network=udp +33:57.438 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24353 network=tcp +33:57.439 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24354 network=tcp +33:57.440 [INFO] TestAgent: started state syncer +33:57.440 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24345 network=tcp +33:57.487 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:57.487 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24344 [Candidate]" term=2 +33:57.487 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:57.487 [DEBUG] TestAgent.server.raft: vote granted: from=7fe37870-6d3e-5fe0-4594-a8f5a8025a7c term=2 tally=1 +33:57.487 [INFO] TestAgent.server.raft: election won: tally=1 +33:57.487 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24344 [Leader]" +33:57.487 [INFO] TestAgent.server: cluster leadership acquired +33:57.488 [INFO] TestAgent.server: New leader elected: payload=Node-7fe37870-6d3e-5fe0-4594-a8f5a8025a7c +33:57.488 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24344 +33:57.489 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:57.491 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:57.491 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:57.491 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:57.491 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:57.491 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:57.492 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7fe37870-6d3e-5fe0-4594-a8f5a8025a7c +33:57.492 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7fe37870-6d3e-5fe0-4594-a8f5a8025a7c +33:57.492 [INFO] TestAgent.server: federation state anti-entropy synced +33:57.651 [INFO] TestAgent: Requesting shutdown +33:57.652 [INFO] TestAgent.server: shutting down server +33:57.652 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:57.652 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:57.652 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:57.652 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:57.652 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:57.652 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:57.652 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:57.652 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:57.652 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:57.652 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:57.652 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:57.652 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:57.653Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:57.653Z [INFO] agent.router.manager: shutting down +33:57.653 [INFO] TestAgent: consul server down +33:57.653 [INFO] TestAgent: shutdown complete +33:57.653 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24353 network=tcp +33:57.653 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24353 network=udp +33:57.653 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24354 network=tcp +33:58.154 [INFO] TestAgent: Waiting for endpoints to shut down +33:58.154 [INFO] TestAgent: Endpoints down === CONT TestCatalogServiceNodes -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:38.238Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:38.275Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:38.284 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:11da0db6-9cb9-4c29-d807-6fc10f193e16 Address:127.0.0.1:13328}]" -42:38.285 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-11da0db6-9cb9-4c29-d807-6fc10f193e16.dc1 127.0.0.1 -42:38.291 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-11da0db6-9cb9-4c29-d807-6fc10f193e16 127.0.0.1 -2022-09-15T21:42:38.291Z [INFO] agent.router: Initializing LAN area manager -42:38.291 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13328 [Follower]" leader= -42:38.291 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13260 network=udp -42:38.292 [INFO] TestAgent.server: Adding LAN server: server="Node-11da0db6-9cb9-4c29-d807-6fc10f193e16 (Addr: tcp/127.0.0.1:13328) (DC: dc1)" -42:38.292 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-11da0db6-9cb9-4c29-d807-6fc10f193e16.dc1 area=wan -42:38.292 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13260 network=tcp -42:38.299 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13261 network=tcp -42:38.300 [INFO] TestAgent: started state syncer -42:38.300 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13329 network=tcp -42:38.362 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:38.362 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13328 [Candidate]" term=2 -42:38.362 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:38.362 [DEBUG] TestAgent.server.raft: vote granted: from=11da0db6-9cb9-4c29-d807-6fc10f193e16 term=2 tally=1 -42:38.362 [INFO] TestAgent.server.raft: election won: tally=1 -42:38.362 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13328 [Leader]" -42:38.362 [INFO] TestAgent.server: cluster leadership acquired -42:38.363 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13328 -42:38.374 [INFO] TestAgent.server: New leader elected: payload=Node-11da0db6-9cb9-4c29-d807-6fc10f193e16 -42:38.375 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:38.383 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:38.383 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:38.383 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:38.383 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:38.383 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:38.383 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-11da0db6-9cb9-4c29-d807-6fc10f193e16 -42:38.383 [INFO] TestAgent.server: member joined, marking health alive: member=Node-11da0db6-9cb9-4c29-d807-6fc10f193e16 -42:38.395 [INFO] TestAgent.server: federation state anti-entropy synced -42:38.540 [INFO] TestAgent: Requesting shutdown -42:38.540 [INFO] TestAgent.server: shutting down server -42:38.540 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:38.540 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:38.540 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:38.540 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:38.540 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:38.541 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:38.541 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:38.541 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:38.541 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -42:38.541 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:38.541 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:38.541 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2022-09-15T21:42:38.541Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:38.541Z [INFO] agent.router.manager: shutting down -42:38.541 [INFO] TestAgent: consul server down -42:38.541 [INFO] TestAgent: shutdown complete -42:38.541 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13260 network=tcp -42:38.541 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13260 network=udp -42:38.541 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13261 network=tcp -42:39.042 [INFO] TestAgent: Waiting for endpoints to shut down -42:39.043 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogConnectServiceNodes_good (0.89s) === CONT TestCatalogRegister_checkRegistration -2022-09-15T21:42:38.238Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:38.273Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:38.292 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:73d08c69-2aad-46c5-3b8c-c2e5b57b0087 Address:127.0.0.1:13335}]" -42:38.293 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-73d08c69-2aad-46c5-3b8c-c2e5b57b0087.dc1 127.0.0.1 -42:38.294 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13335 [Follower]" leader= -42:38.294 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-73d08c69-2aad-46c5-3b8c-c2e5b57b0087 127.0.0.1 -2022-09-15T21:42:38.294Z [INFO] agent.router: Initializing LAN area manager -42:38.295 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13330 network=udp -42:38.297 [INFO] TestAgent.server: Adding LAN server: server="Node-73d08c69-2aad-46c5-3b8c-c2e5b57b0087 (Addr: tcp/127.0.0.1:13335) (DC: dc1)" -42:38.297 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-73d08c69-2aad-46c5-3b8c-c2e5b57b0087.dc1 area=wan -42:38.301 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13330 network=tcp -42:38.315 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13331 network=tcp -42:38.315 [INFO] TestAgent: started state syncer -42:38.315 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13336 network=tcp -42:38.360 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:38.360 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13335 [Candidate]" term=2 -42:38.361 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:38.361 [DEBUG] TestAgent.server.raft: vote granted: from=73d08c69-2aad-46c5-3b8c-c2e5b57b0087 term=2 tally=1 -42:38.361 [INFO] TestAgent.server.raft: election won: tally=1 -42:38.361 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13335 [Leader]" -42:38.361 [INFO] TestAgent.server: cluster leadership acquired -42:38.361 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13335 -42:38.362 [INFO] TestAgent.server: New leader elected: payload=Node-73d08c69-2aad-46c5-3b8c-c2e5b57b0087 -42:38.394 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:38.402 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:38.403 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:38.403 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:38.403 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:38.403 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:38.403 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-73d08c69-2aad-46c5-3b8c-c2e5b57b0087 -42:38.403 [INFO] TestAgent.server: member joined, marking health alive: member=Node-73d08c69-2aad-46c5-3b8c-c2e5b57b0087 -42:38.405 [INFO] TestAgent.server: federation state anti-entropy synced -42:38.454 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:38.455 [INFO] TestAgent: Synced node info -42:38.679 [INFO] TestAgent: Requesting shutdown -42:38.679 [INFO] TestAgent.server: shutting down server -42:38.679 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:38.679 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:38.679 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:38.679 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:38.679 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:38.679 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:38.679 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:38.679 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:38.679 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:38.679 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:38.679 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:38.680Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:38.680Z [INFO] agent.router.manager: shutting down -42:38.680 [INFO] TestAgent: consul server down -42:38.680 [INFO] TestAgent: shutdown complete -42:38.680 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13330 network=tcp -42:38.680 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13330 network=udp -42:38.680 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13331 network=tcp -42:39.181 [INFO] TestAgent: Waiting for endpoints to shut down -42:39.194 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogServiceNodes_ConnectProxy (1.02s) +--- PASS: TestCatalogServiceNodes_DistanceSort (0.85s) +=== CONT TestUiIndex +2021-08-13T15:33:57.326Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:57.344Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:57.349 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f91ed321-11d2-7e32-03b1-4a9d3893c0ed Address:127.0.0.1:24323}]" +33:57.350 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f91ed321-11d2-7e32-03b1-4a9d3893c0ed.dc1 127.0.0.1 +33:57.351 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24323 [Follower]" leader= +33:57.379 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f91ed321-11d2-7e32-03b1-4a9d3893c0ed 127.0.0.1 +2021-08-13T15:33:57.379Z [INFO] agent.router: Initializing LAN area manager +33:57.380 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24318 network=udp +33:57.380 [INFO] TestAgent.server: Adding LAN server: server="Node-f91ed321-11d2-7e32-03b1-4a9d3893c0ed (Addr: tcp/127.0.0.1:24323) (DC: dc1)" +33:57.380 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f91ed321-11d2-7e32-03b1-4a9d3893c0ed.dc1 area=wan +33:57.380 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24318 network=tcp +33:57.381 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24319 network=tcp +33:57.381 [INFO] TestAgent: started state syncer +33:57.381 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24324 network=tcp +33:57.423 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:57.423 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24323 [Candidate]" term=2 +33:57.423 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:57.423 [DEBUG] TestAgent.server.raft: vote granted: from=f91ed321-11d2-7e32-03b1-4a9d3893c0ed term=2 tally=1 +33:57.423 [INFO] TestAgent.server.raft: election won: tally=1 +33:57.423 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24323 [Leader]" +33:57.424 [INFO] TestAgent.server: cluster leadership acquired +33:57.424 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24323 +33:57.426 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:57.427 [INFO] TestAgent.server: New leader elected: payload=Node-f91ed321-11d2-7e32-03b1-4a9d3893c0ed +33:57.431 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:57.431 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:57.431 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:57.431 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:57.431 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:57.431 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f91ed321-11d2-7e32-03b1-4a9d3893c0ed +33:57.431 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f91ed321-11d2-7e32-03b1-4a9d3893c0ed +33:57.433 [INFO] TestAgent.server: federation state anti-entropy synced +33:57.732 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:57.732 [INFO] TestAgent: Synced node info +33:57.747 [INFO] TestAgent: Requesting shutdown +33:57.747 [INFO] TestAgent.server: shutting down server +33:57.747 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:57.747 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:57.747 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:57.747 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:57.747 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:57.747 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:57.747 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:57.747 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:57.747 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:57.747 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:57.747 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:57.748Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:57.748Z [INFO] agent.router.manager: shutting down +33:57.748 [INFO] TestAgent: consul server down +33:57.748 [INFO] TestAgent: shutdown complete +33:57.748 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24318 network=tcp +33:57.748 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24318 network=udp +33:57.748 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24319 network=tcp +33:58.248 [INFO] TestAgent: Waiting for endpoints to shut down +33:58.248 [INFO] TestAgent: Endpoints down +--- PASS: TestUiNodes (0.94s) === CONT TestCatalogServices_NodeMetaFilter -2022-09-15T21:42:38.369Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:38.465Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:38.475 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b1575f4d-bb18-66d5-cb22-5af7d0684bf6 Address:127.0.0.1:13349}]" -42:38.476 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13349 [Follower]" leader= -42:38.476 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b1575f4d-bb18-66d5-cb22-5af7d0684bf6.dc1 127.0.0.1 -42:38.478 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b1575f4d-bb18-66d5-cb22-5af7d0684bf6 127.0.0.1 -2022-09-15T21:42:38.478Z [INFO] agent.router: Initializing LAN area manager -42:38.478 [INFO] TestAgent.server: Adding LAN server: server="Node-b1575f4d-bb18-66d5-cb22-5af7d0684bf6 (Addr: tcp/127.0.0.1:13349) (DC: dc1)" -42:38.478 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b1575f4d-bb18-66d5-cb22-5af7d0684bf6.dc1 area=wan -42:38.478 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13337 network=tcp -42:38.478 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13337 network=udp -42:38.479 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13338 network=tcp -42:38.479 [INFO] TestAgent: started state syncer -42:38.479 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13350 network=tcp -42:38.541 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:38.541 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13349 [Candidate]" term=2 -42:38.541 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:38.541 [DEBUG] TestAgent.server.raft: vote granted: from=b1575f4d-bb18-66d5-cb22-5af7d0684bf6 term=2 tally=1 -42:38.541 [INFO] TestAgent.server.raft: election won: tally=1 -42:38.541 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13349 [Leader]" -42:38.542 [INFO] TestAgent.server: cluster leadership acquired -42:38.542 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13349 -42:38.544 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:38.546 [INFO] TestAgent.server: New leader elected: payload=Node-b1575f4d-bb18-66d5-cb22-5af7d0684bf6 -42:38.558 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:38.558 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:38.558 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:38.558 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:38.558 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:38.558 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b1575f4d-bb18-66d5-cb22-5af7d0684bf6 -42:38.558 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b1575f4d-bb18-66d5-cb22-5af7d0684bf6 -42:38.560 [INFO] TestAgent.server: federation state anti-entropy synced -42:38.588 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:38.589 [INFO] TestAgent: Synced node info -42:38.589 [DEBUG] TestAgent: Node info in sync -42:38.680 [INFO] TestAgent: Requesting shutdown -42:38.681 [INFO] TestAgent.server: shutting down server -42:38.681 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:38.681 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:38.681 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:38.681 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:38.681 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:38.682 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:38.682 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:38.682 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:38.682 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:38.682 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:38.682 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:38.682Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:38.682Z [INFO] agent.router.manager: shutting down -42:38.682 [INFO] TestAgent: consul server down -42:38.682 [INFO] TestAgent: shutdown complete -42:38.682 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13337 network=tcp -42:38.682 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13337 network=udp -42:38.682 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13338 network=tcp -42:39.184 [INFO] TestAgent: Waiting for endpoints to shut down -42:39.201 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogServiceNodes_DistanceSort (0.90s) +[WARN] freeport: 10 out of 45 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:33:57.386Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:57.402Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:57.405 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:206d861d-73ff-7228-b4df-67ec8f4c833a Address:127.0.0.1:24358}]" +33:57.405 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24358 [Follower]" leader= +33:57.406 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-206d861d-73ff-7228-b4df-67ec8f4c833a.dc1 127.0.0.1 +33:57.407 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-206d861d-73ff-7228-b4df-67ec8f4c833a 127.0.0.1 +2021-08-13T15:33:57.407Z [INFO] agent.router: Initializing LAN area manager +33:57.407 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24367 network=udp +33:57.407 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-206d861d-73ff-7228-b4df-67ec8f4c833a.dc1 area=wan +33:57.407 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24367 network=tcp +33:57.407 [INFO] TestAgent.server: Adding LAN server: server="Node-206d861d-73ff-7228-b4df-67ec8f4c833a (Addr: tcp/127.0.0.1:24358) (DC: dc1)" +33:57.408 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24368 network=tcp +33:57.408 [INFO] TestAgent: started state syncer +33:57.408 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24359 network=tcp +33:57.449 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:57.449 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24358 [Candidate]" term=2 +33:57.450 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:57.450 [DEBUG] TestAgent.server.raft: vote granted: from=206d861d-73ff-7228-b4df-67ec8f4c833a term=2 tally=1 +33:57.450 [INFO] TestAgent.server.raft: election won: tally=1 +33:57.450 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24358 [Leader]" +33:57.450 [INFO] TestAgent.server: cluster leadership acquired +33:57.450 [INFO] TestAgent.server: New leader elected: payload=Node-206d861d-73ff-7228-b4df-67ec8f4c833a +33:57.450 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24358 +33:57.451 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:57.453 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:57.454 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:57.454 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:57.454 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:57.454 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:57.454 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-206d861d-73ff-7228-b4df-67ec8f4c833a +33:57.454 [INFO] TestAgent.server: member joined, marking health alive: member=Node-206d861d-73ff-7228-b4df-67ec8f4c833a +33:57.454 [INFO] TestAgent.server: federation state anti-entropy synced +33:57.737 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:57.737 [INFO] TestAgent: Synced node info +33:57.828 [INFO] TestAgent: Requesting shutdown +33:57.828 [INFO] TestAgent.server: shutting down server +33:57.828 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:57.829 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:57.829 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:57.829 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:57.829 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:57.829 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:57.829 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:57.829 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:57.829 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:57.829 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:57.829 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:57.829Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:57.829Z [INFO] agent.router.manager: shutting down +33:57.829 [INFO] TestAgent: consul server down +33:57.829 [INFO] TestAgent: shutdown complete +33:57.829 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24367 network=tcp +33:57.829 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24367 network=udp +33:57.829 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24368 network=tcp +33:58.329 [INFO] TestAgent: Waiting for endpoints to shut down +33:58.329 [INFO] TestAgent: Endpoints down +--- PASS: TestShouldProcessUserEvent (1.02s) === CONT TestCatalogServices -[WARN] freeport: 4 out of 25 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:38.665Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:38.682Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:38.685 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b2e4988a-07e3-d620-d650-6eb397931b01 Address:127.0.0.1:13342}]" -42:38.686 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13342 [Follower]" leader= -42:38.687 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b2e4988a-07e3-d620-d650-6eb397931b01.dc1 127.0.0.1 -42:38.688 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b2e4988a-07e3-d620-d650-6eb397931b01 127.0.0.1 -2022-09-15T21:42:38.688Z [INFO] agent.router: Initializing LAN area manager -42:38.689 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13358 network=udp -42:38.689 [INFO] TestAgent.server: Adding LAN server: server="Node-b2e4988a-07e3-d620-d650-6eb397931b01 (Addr: tcp/127.0.0.1:13342) (DC: dc1)" -42:38.689 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b2e4988a-07e3-d620-d650-6eb397931b01.dc1 area=wan -42:38.689 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13358 network=tcp -42:38.691 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13359 network=tcp -42:38.691 [INFO] TestAgent: started state syncer -42:38.691 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13343 network=tcp -42:38.732 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:38.732 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13342 [Candidate]" term=2 -42:38.732 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:38.732 [DEBUG] TestAgent.server.raft: vote granted: from=b2e4988a-07e3-d620-d650-6eb397931b01 term=2 tally=1 -42:38.732 [INFO] TestAgent.server.raft: election won: tally=1 -42:38.732 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13342 [Leader]" -42:38.732 [INFO] TestAgent.server: cluster leadership acquired -42:38.733 [INFO] TestAgent.server: New leader elected: payload=Node-b2e4988a-07e3-d620-d650-6eb397931b01 -42:38.733 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13342 -42:38.735 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:38.739 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:38.739 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:38.739 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:38.739 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:38.739 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:38.739 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b2e4988a-07e3-d620-d650-6eb397931b01 -42:38.739 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b2e4988a-07e3-d620-d650-6eb397931b01 -42:38.740 [INFO] TestAgent.server: federation state anti-entropy synced -42:38.823 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:38.825 [INFO] TestAgent: Synced node info -42:38.898 [INFO] TestAgent: Requesting shutdown -42:38.898 [INFO] TestAgent.server: shutting down server -42:38.898 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:38.898 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:38.898 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:38.898 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:38.898 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:38.899 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:38.899 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:38.899 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:38.899Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:38.899Z [INFO] agent.router.manager: shutting down -42:38.899 [INFO] TestAgent: consul server down -42:38.899 [INFO] TestAgent: shutdown complete -42:38.899 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13358 network=tcp -42:38.899 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13358 network=udp -42:38.899 [DEBUG] TestAgent.server: shutting down leader loop -42:38.899 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:38.899 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:38.899 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:38.899 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13359 network=tcp -42:38.899 [INFO] TestAgent.server: cluster leadership lost -42:39.400 [INFO] TestAgent: Waiting for endpoints to shut down -42:39.400 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogServiceNodes_Filter (0.75s) +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:33:57.481Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:33:57.481Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:57.498Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:33:57.498Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:57.501 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2d6d4ee7-12d3-1e6d-547a-55b2cf70d0cf Address:127.0.0.1:24379}]" +33:57.501 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24379 [Follower]" leader= +33:57.501 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2d6d4ee7-12d3-1e6d-547a-55b2cf70d0cf.dc2 127.0.0.1 +33:57.502 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2d6d4ee7-12d3-1e6d-547a-55b2cf70d0cf 127.0.0.1 +2021-08-13T15:33:57.502Z [INFO] agent.router: Initializing LAN area manager +33:57.503 [INFO] TestAgent.server: Adding LAN server: server="Node-2d6d4ee7-12d3-1e6d-547a-55b2cf70d0cf (Addr: tcp/127.0.0.1:24379) (DC: dc2)" +33:57.503 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2d6d4ee7-12d3-1e6d-547a-55b2cf70d0cf.dc2 area=wan +33:57.503 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24360 network=tcp +33:57.503 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24360 network=udp +33:57.504 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24361 network=tcp +33:57.504 [INFO] TestAgent: started state syncer +33:57.504 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24380 network=tcp +33:57.541 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:57.541 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24379 [Candidate]" term=2 +33:57.542 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:57.542 [DEBUG] TestAgent.server.raft: vote granted: from=2d6d4ee7-12d3-1e6d-547a-55b2cf70d0cf term=2 tally=1 +33:57.542 [INFO] TestAgent.server.raft: election won: tally=1 +33:57.542 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24379 [Leader]" +33:57.542 [INFO] TestAgent.server: cluster leadership acquired +33:57.542 [INFO] TestAgent.server: New leader elected: payload=Node-2d6d4ee7-12d3-1e6d-547a-55b2cf70d0cf +33:57.542 [INFO] TestAgent.server: initializing acls +33:57.542 [INFO] TestAgent.server: Created ACL 'global-management' policy +33:57.543 [INFO] TestAgent.server: Created ACL anonymous token from configuration +33:57.543 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +33:57.543 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +33:57.543 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-2d6d4ee7-12d3-1e6d-547a-55b2cf70d0cf +33:57.543 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-2d6d4ee7-12d3-1e6d-547a-55b2cf70d0cf.dc2 +33:57.543 [INFO] TestAgent.server: Updating LAN server: server="Node-2d6d4ee7-12d3-1e6d-547a-55b2cf70d0cf (Addr: tcp/127.0.0.1:24379) (DC: dc2)" +33:57.544 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-2d6d4ee7-12d3-1e6d-547a-55b2cf70d0cf.dc2 area=wan +33:57.545 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:57.547 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:57.547 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:57.547 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:57.547 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:57.547 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:57.547 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2d6d4ee7-12d3-1e6d-547a-55b2cf70d0cf +33:57.547 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2d6d4ee7-12d3-1e6d-547a-55b2cf70d0cf +33:57.548 [INFO] TestAgent.server: federation state anti-entropy synced +33:57.548 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2d6d4ee7-12d3-1e6d-547a-55b2cf70d0cf +33:57.626 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:24329] +33:57.627 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:24329 +33:57.629 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e627ba65-61c2-37b5-02c9-7c19029c0bf3.dc1 127.0.0.1 +33:57.629 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 +33:57.631 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e627ba65-61c2-37b5-02c9-7c19029c0bf3.dc1 area=wan +33:57.633 [INFO] TestAgent: Requesting shutdown +33:57.633 [INFO] TestAgent.server: shutting down server +33:57.633 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +33:57.633 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +33:57.633 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:57.633 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:57.633 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:57.633 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:57.633 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:57.633 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:57.633 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:57.633 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:57.633 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +33:57.633 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +33:57.633 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:57.633 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:57.633 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:57.633 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:57.633Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:57.633Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:57.633Z [INFO] agent.router.manager: shutting down +33:57.634 [INFO] TestAgent: consul server down +33:57.634 [INFO] TestAgent: shutdown complete +33:57.634 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24360 network=tcp +33:57.634 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24360 network=udp +33:57.634 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24361 network=tcp +33:58.134 [INFO] TestAgent: Waiting for endpoints to shut down +33:58.134 [INFO] TestAgent: Endpoints down +2021-08-13T15:33:57.327Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:33:57.327Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:57.370Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:33:57.370Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:57.377 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e627ba65-61c2-37b5-02c9-7c19029c0bf3 Address:127.0.0.1:24330}]" +33:57.378 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e627ba65-61c2-37b5-02c9-7c19029c0bf3.dc1 127.0.0.1 +33:57.387 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24330 [Follower]" leader= +33:57.387 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e627ba65-61c2-37b5-02c9-7c19029c0bf3 127.0.0.1 +2021-08-13T15:33:57.387Z [INFO] agent.router: Initializing LAN area manager +33:57.389 [INFO] TestAgent.server: Adding LAN server: server="Node-e627ba65-61c2-37b5-02c9-7c19029c0bf3 (Addr: tcp/127.0.0.1:24330) (DC: dc1)" +33:57.389 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e627ba65-61c2-37b5-02c9-7c19029c0bf3.dc1 area=wan +33:57.391 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24325 network=udp +33:57.391 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24325 network=tcp +33:57.392 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24326 network=tcp +33:57.392 [INFO] TestAgent: started state syncer +33:57.392 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24331 network=tcp +33:57.431 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:57.431 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24330 [Candidate]" term=2 +33:57.431 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:57.431 [DEBUG] TestAgent.server.raft: vote granted: from=e627ba65-61c2-37b5-02c9-7c19029c0bf3 term=2 tally=1 +33:57.431 [INFO] TestAgent.server.raft: election won: tally=1 +33:57.431 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24330 [Leader]" +33:57.431 [INFO] TestAgent.server: cluster leadership acquired +33:57.432 [INFO] TestAgent.server: initializing acls +33:57.432 [INFO] TestAgent.server: Created ACL 'global-management' policy +33:57.433 [INFO] TestAgent.server: Created ACL anonymous token from configuration +33:57.433 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +33:57.433 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +33:57.433 [INFO] TestAgent.server: New leader elected: payload=Node-e627ba65-61c2-37b5-02c9-7c19029c0bf3 +33:57.433 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-e627ba65-61c2-37b5-02c9-7c19029c0bf3 +33:57.433 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-e627ba65-61c2-37b5-02c9-7c19029c0bf3.dc1 +33:57.433 [INFO] TestAgent.server: Updating LAN server: server="Node-e627ba65-61c2-37b5-02c9-7c19029c0bf3 (Addr: tcp/127.0.0.1:24330) (DC: dc1)" +33:57.434 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-e627ba65-61c2-37b5-02c9-7c19029c0bf3.dc1 area=wan +33:57.435 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:57.438 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:57.438 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:57.438 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:57.438 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:57.438 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:57.438 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e627ba65-61c2-37b5-02c9-7c19029c0bf3 +33:57.438 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e627ba65-61c2-37b5-02c9-7c19029c0bf3 +33:57.439 [INFO] TestAgent.server: federation state anti-entropy synced +33:57.439 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e627ba65-61c2-37b5-02c9-7c19029c0bf3 +33:57.606 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:57.606 [INFO] TestAgent: Synced node info +33:57.606 [DEBUG] TestAgent: Node info in sync +33:57.628 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:52588 +33:57.628 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2d6d4ee7-12d3-1e6d-547a-55b2cf70d0cf.dc2 127.0.0.1 +33:57.629 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2d6d4ee7-12d3-1e6d-547a-55b2cf70d0cf.dc2 area=wan +33:58.134 [INFO] TestAgent: Requesting shutdown +33:58.134 [INFO] TestAgent.server: shutting down server +33:58.134 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:58.134 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:58.134 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:58.134 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:58.134 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +33:58.134 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +33:58.134 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:58.134 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:58.134 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:58.134 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +33:58.134 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:58.134 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +33:58.134 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:58.134 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:58.134 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:58.134Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:58.135Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:58.135Z [INFO] agent.router.manager: shutting down +33:58.135 [INFO] TestAgent: consul server down +33:58.135 [INFO] TestAgent: shutdown complete +33:58.135 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24325 network=tcp +33:58.135 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24325 network=udp +33:58.135 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24326 network=tcp +33:58.635 [INFO] TestAgent: Waiting for endpoints to shut down +33:58.635 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogServiceNodes_WanTranslation (1.33s) === CONT TestCatalogNodes_DistanceSort -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:38.709Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:38.740Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:38.744 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e05a26c5-3b4d-4799-8302-47596d366726 Address:127.0.0.1:13370}]" -42:38.744 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13370 [Follower]" leader= -42:38.745 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e05a26c5-3b4d-4799-8302-47596d366726.dc1 127.0.0.1 -42:38.746 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e05a26c5-3b4d-4799-8302-47596d366726 127.0.0.1 -2022-09-15T21:42:38.746Z [INFO] agent.router: Initializing LAN area manager -42:38.754 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13365 network=udp -42:38.755 [INFO] TestAgent.server: Adding LAN server: server="Node-e05a26c5-3b4d-4799-8302-47596d366726 (Addr: tcp/127.0.0.1:13370) (DC: dc1)" -42:38.755 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e05a26c5-3b4d-4799-8302-47596d366726.dc1 area=wan -42:38.756 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13365 network=tcp -42:38.770 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13366 network=tcp -42:38.770 [INFO] TestAgent: started state syncer -42:38.770 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13371 network=tcp -42:38.814 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:38.814 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13370 [Candidate]" term=2 -42:38.814 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:38.814 [DEBUG] TestAgent.server.raft: vote granted: from=e05a26c5-3b4d-4799-8302-47596d366726 term=2 tally=1 -42:38.814 [INFO] TestAgent.server.raft: election won: tally=1 -42:38.814 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13370 [Leader]" -42:38.815 [INFO] TestAgent.server: cluster leadership acquired -42:38.815 [INFO] TestAgent.server: New leader elected: payload=Node-e05a26c5-3b4d-4799-8302-47596d366726 -42:38.815 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13370 -42:38.817 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:38.826 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:38.826 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:38.826 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:38.826 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:38.826 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:38.826 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e05a26c5-3b4d-4799-8302-47596d366726 -42:38.826 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e05a26c5-3b4d-4799-8302-47596d366726 -42:38.828 [INFO] TestAgent.server: federation state anti-entropy synced -42:38.861 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:38.862 [INFO] TestAgent: Synced node info -42:39.095 [INFO] TestAgent: Requesting shutdown -42:39.095 [INFO] TestAgent.server: shutting down server -42:39.095 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:39.095 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:39.095 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:39.095 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:39.095 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:39.095 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:39.095 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:39.095 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:39.095 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:39.095 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:39.095 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:39.095Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:39.095Z [INFO] agent.router.manager: shutting down -42:39.096 [INFO] TestAgent: consul server down -42:39.096 [INFO] TestAgent: shutdown complete -42:39.096 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13365 network=tcp -42:39.096 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13365 network=udp -42:39.096 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13366 network=tcp -42:39.602 [INFO] TestAgent: Waiting for endpoints to shut down -42:39.602 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogServiceNodes_NodeMetaFilter (0.91s) +2021-08-13T15:33:57.984Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:57.999Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:58.002 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8af7a66e-83b9-7423-e6c8-dcf3a636f5a4 Address:127.0.0.1:24372}]" +33:58.002 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24372 [Follower]" leader= +33:58.003 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8af7a66e-83b9-7423-e6c8-dcf3a636f5a4.dc1 127.0.0.1 +33:58.004 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8af7a66e-83b9-7423-e6c8-dcf3a636f5a4 127.0.0.1 +2021-08-13T15:33:58.004Z [INFO] agent.router: Initializing LAN area manager +33:58.004 [INFO] TestAgent.server: Adding LAN server: server="Node-8af7a66e-83b9-7423-e6c8-dcf3a636f5a4 (Addr: tcp/127.0.0.1:24372) (DC: dc1)" +33:58.004 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8af7a66e-83b9-7423-e6c8-dcf3a636f5a4.dc1 area=wan +33:58.004 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24381 network=tcp +33:58.004 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24381 network=udp +33:58.005 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24382 network=tcp +33:58.005 [INFO] TestAgent: started state syncer +33:58.005 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24373 network=tcp +33:58.071 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:58.071 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24372 [Candidate]" term=2 +33:58.072 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:58.072 [DEBUG] TestAgent.server.raft: vote granted: from=8af7a66e-83b9-7423-e6c8-dcf3a636f5a4 term=2 tally=1 +33:58.072 [INFO] TestAgent.server.raft: election won: tally=1 +33:58.072 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24372 [Leader]" +33:58.072 [INFO] TestAgent.server: cluster leadership acquired +33:58.072 [INFO] TestAgent.server: New leader elected: payload=Node-8af7a66e-83b9-7423-e6c8-dcf3a636f5a4 +33:58.072 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24372 +33:58.073 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:58.076 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:58.076 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:58.076 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:58.076 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:58.076 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:58.076 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8af7a66e-83b9-7423-e6c8-dcf3a636f5a4 +33:58.076 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8af7a66e-83b9-7423-e6c8-dcf3a636f5a4 +33:58.077 [INFO] TestAgent.server: federation state anti-entropy synced +33:58.086 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:58.087 [INFO] TestAgent: Synced node info +33:58.211 [INFO] TestAgent: Requesting shutdown +33:58.211 [INFO] TestAgent.server: shutting down server +33:58.211 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:58.211 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:58.211 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:58.211 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:58.211 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:58.211 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:58.211 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:58.211 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:58.211 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:58.212 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:58.212 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:58.212Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:58.212Z [INFO] agent.router.manager: shutting down +33:58.212 [INFO] TestAgent: consul server down +33:58.212 [INFO] TestAgent: shutdown complete +33:58.212 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24381 network=tcp +33:58.212 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24381 network=udp +33:58.212 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24382 network=tcp +33:58.712 [INFO] TestAgent: Waiting for endpoints to shut down +33:58.712 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogServiceNodes_Filter (0.75s) === CONT TestCatalogNodes_Blocking -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._tcp_(qType:=33)_compressed=false -2022-09-15T21:42:39.094Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:39.202Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:39.205 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c80c79e8-1f13-08a9-bad8-fb858967d6d2 Address:127.0.0.1:13377}]" -42:39.206 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13377 [Follower]" leader= -42:39.207 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c80c79e8-1f13-08a9-bad8-fb858967d6d2.dc1 127.0.0.1 -42:39.208 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c80c79e8-1f13-08a9-bad8-fb858967d6d2 127.0.0.1 -2022-09-15T21:42:39.208Z [INFO] agent.router: Initializing LAN area manager -42:39.208 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13400 network=udp -42:39.209 [INFO] TestAgent.server: Adding LAN server: server="Node-c80c79e8-1f13-08a9-bad8-fb858967d6d2 (Addr: tcp/127.0.0.1:13377) (DC: dc1)" -42:39.209 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c80c79e8-1f13-08a9-bad8-fb858967d6d2.dc1 area=wan -42:39.209 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13400 network=tcp -42:39.218 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13401 network=tcp -42:39.219 [INFO] TestAgent: started state syncer -42:39.219 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13378 network=tcp -42:39.262 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:39.262 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13377 [Candidate]" term=2 -42:39.262 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:39.262 [DEBUG] TestAgent.server.raft: vote granted: from=c80c79e8-1f13-08a9-bad8-fb858967d6d2 term=2 tally=1 -42:39.262 [INFO] TestAgent.server.raft: election won: tally=1 -42:39.262 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13377 [Leader]" -42:39.263 [INFO] TestAgent.server: cluster leadership acquired -42:39.263 [INFO] TestAgent.server: New leader elected: payload=Node-c80c79e8-1f13-08a9-bad8-fb858967d6d2 -42:39.263 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13377 -42:39.268 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:39.271 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:39.271 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:39.271 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:39.271 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:39.271 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:39.271 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c80c79e8-1f13-08a9-bad8-fb858967d6d2 -42:39.271 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c80c79e8-1f13-08a9-bad8-fb858967d6d2 -42:39.281 [INFO] TestAgent.server: federation state anti-entropy synced -42:39.400 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:39.401 [INFO] TestAgent: Synced node info -42:39.452 [INFO] TestAgent: Requesting shutdown -42:39.452 [INFO] TestAgent.server: shutting down server -42:39.452 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:39.452 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:39.452 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:39.452 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:39.452 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:39.452 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:39.453 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:39.453 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:39.453 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:39.453 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:39.453 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:39.453Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:39.453Z [INFO] agent.router.manager: shutting down -42:39.453 [INFO] TestAgent: consul server down -42:39.453 [INFO] TestAgent: shutdown complete -42:39.453 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13400 network=tcp -42:39.453 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13400 network=udp -42:39.453 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13401 network=tcp -42:39.962 [INFO] TestAgent: Waiting for endpoints to shut down -42:39.962 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogRegister_checkRegistration (0.90s) +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:33:58.149Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:58.178Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:58.186 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:78124153-88d3-caec-c0c1-77258f307b95 Address:127.0.0.1:24386}]" +33:58.187 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24386 [Follower]" leader= +33:58.191 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-78124153-88d3-caec-c0c1-77258f307b95.dc1 127.0.0.1 +33:58.192 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-78124153-88d3-caec-c0c1-77258f307b95 127.0.0.1 +2021-08-13T15:33:58.192Z [INFO] agent.router: Initializing LAN area manager +33:58.193 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24374 network=udp +33:58.193 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-78124153-88d3-caec-c0c1-77258f307b95.dc1 area=wan +33:58.194 [INFO] TestAgent.server: Adding LAN server: server="Node-78124153-88d3-caec-c0c1-77258f307b95 (Addr: tcp/127.0.0.1:24386) (DC: dc1)" +33:58.195 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24374 network=tcp +33:58.198 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24375 network=tcp +33:58.199 [INFO] TestAgent: started state syncer +33:58.199 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24387 network=tcp +33:58.233 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:58.233 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24386 [Candidate]" term=2 +33:58.233 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:58.233 [DEBUG] TestAgent.server.raft: vote granted: from=78124153-88d3-caec-c0c1-77258f307b95 term=2 tally=1 +33:58.233 [INFO] TestAgent.server.raft: election won: tally=1 +33:58.233 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24386 [Leader]" +33:58.234 [INFO] TestAgent.server: cluster leadership acquired +33:58.234 [INFO] TestAgent.server: New leader elected: payload=Node-78124153-88d3-caec-c0c1-77258f307b95 +33:58.234 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24386 +33:58.235 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:58.238 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:58.238 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:58.238 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:58.238 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:58.238 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:58.238 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-78124153-88d3-caec-c0c1-77258f307b95 +33:58.238 [INFO] TestAgent.server: member joined, marking health alive: member=Node-78124153-88d3-caec-c0c1-77258f307b95 +33:58.239 [INFO] TestAgent.server: federation state anti-entropy synced +33:58.286 [INFO] TestAgent: Requesting shutdown +33:58.286 [INFO] TestAgent.server: shutting down server +33:58.286 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:58.286 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:58.286 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:58.286 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:58.286 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:58.286 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:58.286 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:58.286 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:58.286 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:58.286 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:58.286 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:58.286 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:58.286Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:58.286Z [INFO] agent.router.manager: shutting down +33:58.286 [INFO] TestAgent: consul server down +33:58.286 [INFO] TestAgent: shutdown complete +33:58.286 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24374 network=tcp +33:58.286 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24374 network=udp +33:58.286 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24375 network=tcp +33:58.787 [INFO] TestAgent: Waiting for endpoints to shut down +33:58.787 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogServiceNodes_NodeMetaFilter (0.65s) === CONT TestCatalogNodes_WanTranslation -2022-09-15T21:42:38.703Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:38.703Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:38.720Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:38.720Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:38.723 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:be97a196-6152-0b30-d1d4-f2443f633bfe Address:127.0.0.1:13363}]" -42:38.724 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-be97a196-6152-0b30-d1d4-f2443f633bfe.dc2 127.0.0.1 -42:38.725 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13363 [Follower]" leader= -42:38.725 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-be97a196-6152-0b30-d1d4-f2443f633bfe 127.0.0.1 -2022-09-15T21:42:38.725Z [INFO] agent.router: Initializing LAN area manager -42:38.725 [INFO] TestAgent.server: Adding LAN server: server="Node-be97a196-6152-0b30-d1d4-f2443f633bfe (Addr: tcp/127.0.0.1:13363) (DC: dc2)" -42:38.726 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13344 network=udp -42:38.726 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-be97a196-6152-0b30-d1d4-f2443f633bfe.dc2 area=wan -42:38.726 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13344 network=tcp -42:38.727 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13345 network=tcp -42:38.727 [INFO] TestAgent: started state syncer -42:38.727 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13364 network=tcp -42:38.778 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -42:38.782 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:38.782 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13363 [Candidate]" term=2 -42:38.782 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:38.782 [DEBUG] TestAgent.server.raft: vote granted: from=be97a196-6152-0b30-d1d4-f2443f633bfe term=2 tally=1 -42:38.782 [INFO] TestAgent.server.raft: election won: tally=1 -42:38.782 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13363 [Leader]" -42:38.782 [INFO] TestAgent.server: cluster leadership acquired -42:38.783 [INFO] TestAgent.server: initializing acls -42:38.784 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:38.784 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:38.785 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:38.785 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:38.785 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-be97a196-6152-0b30-d1d4-f2443f633bfe -42:38.785 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-be97a196-6152-0b30-d1d4-f2443f633bfe.dc2 -42:38.787 [INFO] TestAgent.server: New leader elected: payload=Node-be97a196-6152-0b30-d1d4-f2443f633bfe -42:38.787 [INFO] TestAgent.server: Updating LAN server: server="Node-be97a196-6152-0b30-d1d4-f2443f633bfe (Addr: tcp/127.0.0.1:13363) (DC: dc2)" -42:38.788 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-be97a196-6152-0b30-d1d4-f2443f633bfe.dc2 area=wan -42:38.806 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:38.815 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:38.815 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:38.815 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:38.815 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:38.815 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:38.816 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-be97a196-6152-0b30-d1d4-f2443f633bfe -42:38.816 [INFO] TestAgent.server: member joined, marking health alive: member=Node-be97a196-6152-0b30-d1d4-f2443f633bfe -42:38.819 [INFO] TestAgent.server: federation state anti-entropy synced -42:38.819 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-be97a196-6152-0b30-d1d4-f2443f633bfe -42:38.949 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:13355] -42:38.954 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:13355 -42:38.956 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c6444a41-5891-d830-856e-cb5782c744d3.dc1 127.0.0.1 -42:38.956 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 -42:38.957 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c6444a41-5891-d830-856e-cb5782c744d3.dc1 area=wan -42:38.966 [INFO] TestAgent: Requesting shutdown -42:38.966 [INFO] TestAgent.server: shutting down server -42:38.966 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:38.966 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:38.966 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:38.966 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:38.966 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:38.966 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:38.966 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:38.967 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:38.967 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:38.967 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:38.967 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:38.967 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:38.967 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:38.967 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:38.967 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:38.967 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:38.967Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:38.967Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:38.967Z [INFO] agent.router.manager: shutting down -42:38.967 [INFO] TestAgent: consul server down -42:38.967 [INFO] TestAgent: shutdown complete -42:38.967 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13344 network=tcp -42:38.967 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13344 network=udp -42:38.967 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13345 network=tcp -42:39.470 [INFO] TestAgent: Waiting for endpoints to shut down -42:39.470 [INFO] TestAgent: Endpoints down -2022-09-15T21:42:38.484Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:38.484Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:38.521Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:38.521Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:38.533 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c6444a41-5891-d830-856e-cb5782c744d3 Address:127.0.0.1:13356}]" -42:38.541 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13356 [Follower]" leader= -42:38.563 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c6444a41-5891-d830-856e-cb5782c744d3.dc1 127.0.0.1 -42:38.564 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c6444a41-5891-d830-856e-cb5782c744d3 127.0.0.1 -2022-09-15T21:42:38.564Z [INFO] agent.router: Initializing LAN area manager -42:38.565 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13351 network=udp -42:38.565 [INFO] TestAgent.server: Adding LAN server: server="Node-c6444a41-5891-d830-856e-cb5782c744d3 (Addr: tcp/127.0.0.1:13356) (DC: dc1)" -42:38.565 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c6444a41-5891-d830-856e-cb5782c744d3.dc1 area=wan -42:38.566 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13351 network=tcp -42:38.567 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13352 network=tcp -42:38.567 [INFO] TestAgent: started state syncer -42:38.567 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13357 network=tcp -42:38.602 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:38.602 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13356 [Candidate]" term=2 -42:38.602 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:38.602 [DEBUG] TestAgent.server.raft: vote granted: from=c6444a41-5891-d830-856e-cb5782c744d3 term=2 tally=1 -42:38.602 [INFO] TestAgent.server.raft: election won: tally=1 -42:38.602 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13356 [Leader]" -42:38.603 [INFO] TestAgent.server: cluster leadership acquired -42:38.603 [INFO] TestAgent.server: initializing acls -42:38.604 [INFO] TestAgent.server: New leader elected: payload=Node-c6444a41-5891-d830-856e-cb5782c744d3 -42:38.604 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:38.610 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:38.610 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:38.610 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:38.610 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-c6444a41-5891-d830-856e-cb5782c744d3 -42:38.611 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-c6444a41-5891-d830-856e-cb5782c744d3.dc1 -42:38.612 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:38.614 [INFO] TestAgent.server: Updating LAN server: server="Node-c6444a41-5891-d830-856e-cb5782c744d3 (Addr: tcp/127.0.0.1:13356) (DC: dc1)" -42:38.614 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-c6444a41-5891-d830-856e-cb5782c744d3.dc1 area=wan -42:38.616 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:38.616 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:38.616 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:38.616 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:38.616 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:38.616 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c6444a41-5891-d830-856e-cb5782c744d3 -42:38.616 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c6444a41-5891-d830-856e-cb5782c744d3 -42:38.617 [INFO] TestAgent.server: federation state anti-entropy synced -42:38.618 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c6444a41-5891-d830-856e-cb5782c744d3 -42:38.633 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:38.634 [INFO] TestAgent: Synced node info -42:38.954 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:42868 -42:38.955 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-be97a196-6152-0b30-d1d4-f2443f633bfe.dc2 127.0.0.1 -42:38.955 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-be97a196-6152-0b30-d1d4-f2443f633bfe.dc2 area=wan -42:39.314 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:39.314 [DEBUG] TestAgent: Node info in sync -42:39.314 [DEBUG] TestAgent: Node info in sync -42:39.470 [INFO] TestAgent: Requesting shutdown -42:39.471 [INFO] TestAgent.server: shutting down server -42:39.471 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:39.471 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:39.471 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:39.471 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:39.471 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:39.471 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:39.471 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:39.472 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:39.472 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:39.472 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:39.472 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:39.472 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:39.472 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:39.472 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:39.472 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:39.472Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:39.472Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:39.472Z [INFO] agent.router.manager: shutting down -42:39.473 [INFO] TestAgent: consul server down -42:39.473 [INFO] TestAgent: shutdown complete -42:39.473 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13351 network=tcp -42:39.473 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13351 network=udp -42:39.473 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13352 network=tcp -42:39.983 [INFO] TestAgent: Waiting for endpoints to shut down -42:39.983 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogServiceNodes_WanTranslation (1.53s) +2021-08-13T15:33:58.178Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:58.205Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:58.208 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c2813a58-2389-6177-0177-6cb81fa434a2 Address:127.0.0.1:24393}]" +33:58.208 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24393 [Follower]" leader= +33:58.209 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c2813a58-2389-6177-0177-6cb81fa434a2.dc1 127.0.0.1 +33:58.210 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c2813a58-2389-6177-0177-6cb81fa434a2 127.0.0.1 +2021-08-13T15:33:58.210Z [INFO] agent.router: Initializing LAN area manager +33:58.210 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24388 network=udp +33:58.211 [INFO] TestAgent.server: Adding LAN server: server="Node-c2813a58-2389-6177-0177-6cb81fa434a2 (Addr: tcp/127.0.0.1:24393) (DC: dc1)" +33:58.211 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c2813a58-2389-6177-0177-6cb81fa434a2.dc1 area=wan +33:58.211 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24388 network=tcp +33:58.212 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24389 network=tcp +33:58.212 [INFO] TestAgent: started state syncer +33:58.212 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24394 network=tcp +33:58.259 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:58.259 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24393 [Candidate]" term=2 +33:58.259 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:58.259 [DEBUG] TestAgent.server.raft: vote granted: from=c2813a58-2389-6177-0177-6cb81fa434a2 term=2 tally=1 +33:58.259 [INFO] TestAgent.server.raft: election won: tally=1 +33:58.259 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24393 [Leader]" +33:58.260 [INFO] TestAgent.server: cluster leadership acquired +33:58.260 [INFO] TestAgent.server: New leader elected: payload=Node-c2813a58-2389-6177-0177-6cb81fa434a2 +33:58.260 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24393 +33:58.261 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:58.264 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:58.264 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:58.264 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:58.264 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:58.264 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:58.264 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c2813a58-2389-6177-0177-6cb81fa434a2 +33:58.264 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c2813a58-2389-6177-0177-6cb81fa434a2 +33:58.265 [INFO] TestAgent.server: federation state anti-entropy synced +33:58.298 [INFO] TestAgent: Requesting shutdown +33:58.299 [INFO] TestAgent.server: shutting down server +33:58.299 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:58.299 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:58.299 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:58.299 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:58.299 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:58.299 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:58.299 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:58.299 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:58.299 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:58.299 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:58.299 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:58.299 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:58.299Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:58.299Z [INFO] agent.router.manager: shutting down +33:58.299 [INFO] TestAgent: consul server down +33:58.299 [INFO] TestAgent: shutdown complete +33:58.299 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24388 network=tcp +33:58.299 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24388 network=udp +33:58.299 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24389 network=tcp +33:58.800 [INFO] TestAgent: Waiting for endpoints to shut down +33:58.800 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogServiceNodes (0.65s) === CONT TestCatalogNodes_Filter -2022-09-15T21:42:38.986Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:39.074Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:39.090 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cc9c4a8c-882a-ba5f-a6dd-a6ce1bda0f48 Address:127.0.0.1:13391}]" -42:39.091 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-cc9c4a8c-882a-ba5f-a6dd-a6ce1bda0f48.dc1 127.0.0.1 -42:39.092 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-cc9c4a8c-882a-ba5f-a6dd-a6ce1bda0f48 127.0.0.1 -2022-09-15T21:42:39.092Z [INFO] agent.router: Initializing LAN area manager -42:39.093 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13372 network=udp -42:39.093 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13391 [Follower]" leader= -42:39.094 [INFO] TestAgent.server: Adding LAN server: server="Node-cc9c4a8c-882a-ba5f-a6dd-a6ce1bda0f48 (Addr: tcp/127.0.0.1:13391) (DC: dc1)" -42:39.094 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-cc9c4a8c-882a-ba5f-a6dd-a6ce1bda0f48.dc1 area=wan -42:39.095 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13372 network=tcp -42:39.109 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13373 network=tcp -42:39.109 [INFO] TestAgent: started state syncer -42:39.109 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13392 network=tcp -42:39.156 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:39.156 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13391 [Candidate]" term=2 -42:39.157 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:39.157 [DEBUG] TestAgent.server.raft: vote granted: from=cc9c4a8c-882a-ba5f-a6dd-a6ce1bda0f48 term=2 tally=1 -42:39.157 [INFO] TestAgent.server.raft: election won: tally=1 -42:39.157 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13391 [Leader]" -42:39.158 [INFO] TestAgent.server: cluster leadership acquired -42:39.159 [INFO] TestAgent.server: New leader elected: payload=Node-cc9c4a8c-882a-ba5f-a6dd-a6ce1bda0f48 -42:39.159 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13391 -42:39.165 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:39.172 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:39.172 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:39.172 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:39.172 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:39.172 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:39.172 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cc9c4a8c-882a-ba5f-a6dd-a6ce1bda0f48 -42:39.172 [INFO] TestAgent.server: member joined, marking health alive: member=Node-cc9c4a8c-882a-ba5f-a6dd-a6ce1bda0f48 -42:39.202 [INFO] TestAgent: Synced node info -42:39.202 [INFO] TestAgent.server: federation state anti-entropy synced -42:39.519 [INFO] TestAgent: Requesting shutdown -42:39.519 [INFO] TestAgent.server: shutting down server -42:39.519 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:39.519 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:39.519 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:39.519 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:39.519 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:39.520 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:39.520 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:39.520 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:39.520 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:39.520 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:39.520 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:39.520Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:39.520Z [INFO] agent.router.manager: shutting down -42:39.521 [INFO] TestAgent: consul server down -42:39.521 [INFO] TestAgent: shutdown complete -42:39.521 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13372 network=tcp -42:39.521 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13372 network=udp -42:39.521 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13373 network=tcp -42:40.021 [INFO] TestAgent: Waiting for endpoints to shut down -42:40.021 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogServiceNodes (1.05s) +2021-08-13T15:33:58.194Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:58.214Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:58.216 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9298d0c1-668a-e7de-3e74-cc663964a2bc Address:127.0.0.1:24400}]" +33:58.216 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24400 [Follower]" leader= +33:58.217 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9298d0c1-668a-e7de-3e74-cc663964a2bc.dc1 127.0.0.1 +33:58.218 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9298d0c1-668a-e7de-3e74-cc663964a2bc 127.0.0.1 +2021-08-13T15:33:58.218Z [INFO] agent.router: Initializing LAN area manager +33:58.218 [INFO] TestAgent.server: Adding LAN server: server="Node-9298d0c1-668a-e7de-3e74-cc663964a2bc (Addr: tcp/127.0.0.1:24400) (DC: dc1)" +33:58.218 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9298d0c1-668a-e7de-3e74-cc663964a2bc.dc1 area=wan +33:58.218 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24395 network=udp +33:58.218 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24395 network=tcp +33:58.219 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24396 network=tcp +33:58.219 [INFO] TestAgent: started state syncer +33:58.219 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24401 network=tcp +33:58.263 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:58.263 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24400 [Candidate]" term=2 +33:58.263 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:58.263 [DEBUG] TestAgent.server.raft: vote granted: from=9298d0c1-668a-e7de-3e74-cc663964a2bc term=2 tally=1 +33:58.263 [INFO] TestAgent.server.raft: election won: tally=1 +33:58.263 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24400 [Leader]" +33:58.263 [INFO] TestAgent.server: cluster leadership acquired +33:58.263 [INFO] TestAgent.server: New leader elected: payload=Node-9298d0c1-668a-e7de-3e74-cc663964a2bc +33:58.263 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24400 +33:58.264 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:58.266 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:58.266 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:58.266 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:58.266 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:58.266 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:58.266 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9298d0c1-668a-e7de-3e74-cc663964a2bc +33:58.266 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9298d0c1-668a-e7de-3e74-cc663964a2bc +33:58.267 [INFO] TestAgent.server: federation state anti-entropy synced +33:58.430 [INFO] TestAgent: Requesting shutdown +33:58.430 [INFO] TestAgent.server: shutting down server +33:58.430 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:58.430 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:58.430 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:58.430 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:58.430 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:58.430 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:58.430 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:58.430 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:58.431 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:58.430 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:58.430 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:58.431 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:58.431Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:58.431Z [INFO] agent.router.manager: shutting down +33:58.431 [INFO] TestAgent: consul server down +33:58.431 [INFO] TestAgent: shutdown complete +33:58.431 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24395 network=tcp +33:58.431 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24395 network=udp +33:58.431 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24396 network=tcp +33:58.932 [INFO] TestAgent: Waiting for endpoints to shut down +33:58.932 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogRegister_checkRegistration (0.78s) === CONT TestCatalogNodes_MetaFilter -2022-09-15T21:42:39.271Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:39.340Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:39.346 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:615266df-dcbf-f205-1a3f-6a3cf863c910 Address:127.0.0.1:13398}]" -42:39.347 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-615266df-dcbf-f205-1a3f-6a3cf863c910.dc1 127.0.0.1 -42:39.347 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13398 [Follower]" leader= -42:39.350 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-615266df-dcbf-f205-1a3f-6a3cf863c910 127.0.0.1 -2022-09-15T21:42:39.350Z [INFO] agent.router: Initializing LAN area manager -42:39.355 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13386 network=udp -42:39.355 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-615266df-dcbf-f205-1a3f-6a3cf863c910.dc1 area=wan -42:39.356 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13386 network=tcp -42:39.358 [INFO] TestAgent.server: Adding LAN server: server="Node-615266df-dcbf-f205-1a3f-6a3cf863c910 (Addr: tcp/127.0.0.1:13398) (DC: dc1)" -42:39.358 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13387 network=tcp -42:39.358 [INFO] TestAgent: started state syncer -42:39.358 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13399 network=tcp -42:39.414 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:39.414 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13398 [Candidate]" term=2 -42:39.414 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:39.414 [DEBUG] TestAgent.server.raft: vote granted: from=615266df-dcbf-f205-1a3f-6a3cf863c910 term=2 tally=1 -42:39.414 [INFO] TestAgent.server.raft: election won: tally=1 -42:39.414 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13398 [Leader]" -42:39.415 [INFO] TestAgent.server: cluster leadership acquired -42:39.415 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13398 -42:39.416 [INFO] TestAgent.server: New leader elected: payload=Node-615266df-dcbf-f205-1a3f-6a3cf863c910 -42:39.423 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:39.426 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:39.426 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:39.426 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:39.426 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:39.426 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:39.427 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-615266df-dcbf-f205-1a3f-6a3cf863c910 -42:39.427 [INFO] TestAgent.server: member joined, marking health alive: member=Node-615266df-dcbf-f205-1a3f-6a3cf863c910 -42:39.428 [INFO] TestAgent.server: federation state anti-entropy synced -42:39.522 [INFO] TestAgent: Requesting shutdown -42:39.522 [INFO] TestAgent.server: shutting down server -42:39.522 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:39.522 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:39.522 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:39.522 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:39.522 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:39.523 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:39.523 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:39.523 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:39.523 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:39.523 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:39.523 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:39.523 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:39.523Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:39.523Z [INFO] agent.router.manager: shutting down -42:39.524 [INFO] TestAgent: consul server down -42:39.524 [INFO] TestAgent: shutdown complete -42:39.524 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13386 network=tcp -42:39.524 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13386 network=udp -42:39.524 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13387 network=tcp -42:40.024 [INFO] TestAgent: Waiting for endpoints to shut down -42:40.024 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogServices (0.82s) +2021-08-13T15:33:58.265Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:58.280Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:58.283 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:16ddad04-6cbe-2d33-5ce6-b929c5597971 Address:127.0.0.1:24421}]" +33:58.283 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24421 [Follower]" leader= +33:58.284 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-16ddad04-6cbe-2d33-5ce6-b929c5597971.dc1 127.0.0.1 +33:58.285 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-16ddad04-6cbe-2d33-5ce6-b929c5597971 127.0.0.1 +2021-08-13T15:33:58.285Z [INFO] agent.router: Initializing LAN area manager +33:58.285 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24409 network=udp +33:58.285 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-16ddad04-6cbe-2d33-5ce6-b929c5597971.dc1 area=wan +33:58.285 [INFO] TestAgent.server: Adding LAN server: server="Node-16ddad04-6cbe-2d33-5ce6-b929c5597971 (Addr: tcp/127.0.0.1:24421) (DC: dc1)" +33:58.285 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24409 network=tcp +33:58.286 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24410 network=tcp +33:58.286 [INFO] TestAgent: started state syncer +33:58.286 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24422 network=tcp +33:58.329 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:58.329 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24421 [Candidate]" term=2 +33:58.329 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:58.329 [DEBUG] TestAgent.server.raft: vote granted: from=16ddad04-6cbe-2d33-5ce6-b929c5597971 term=2 tally=1 +33:58.329 [INFO] TestAgent.server.raft: election won: tally=1 +33:58.329 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24421 [Leader]" +33:58.330 [INFO] TestAgent.server: cluster leadership acquired +33:58.330 [INFO] TestAgent.server: New leader elected: payload=Node-16ddad04-6cbe-2d33-5ce6-b929c5597971 +33:58.330 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24421 +33:58.331 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:58.333 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:58.333 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:58.333 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:58.333 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:58.333 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:58.334 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-16ddad04-6cbe-2d33-5ce6-b929c5597971 +33:58.334 [INFO] TestAgent.server: member joined, marking health alive: member=Node-16ddad04-6cbe-2d33-5ce6-b929c5597971 +33:58.335 [INFO] TestAgent.server: federation state anti-entropy synced +33:58.464 [INFO] TestAgent: Requesting shutdown +33:58.464 [INFO] TestAgent.server: shutting down server +33:58.464 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:58.464 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:58.464 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:58.464 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:58.464 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:58.464 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:58.464 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:58.464 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:58.464 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:58.464 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:58.464 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:58.464 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:58.464Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:58.464Z [INFO] agent.router.manager: shutting down +33:58.464 [INFO] TestAgent: consul server down +33:58.464 [INFO] TestAgent: shutdown complete +33:58.464 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24409 network=tcp +33:58.465 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24409 network=udp +33:58.465 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24410 network=tcp +33:58.965 [INFO] TestAgent: Waiting for endpoints to shut down +33:58.965 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogServices_NodeMetaFilter (0.72s) === CONT TestCatalogNodes -[WARN] freeport: 4 out of 39 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:39.262Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:39.312Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:39.321 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:08e5c9a6-8d10-0cca-9bfe-ab47f749a6aa Address:127.0.0.1:13384}]" -42:39.322 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13384 [Follower]" leader= -42:39.323 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-08e5c9a6-8d10-0cca-9bfe-ab47f749a6aa.dc1 127.0.0.1 -42:39.328 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-08e5c9a6-8d10-0cca-9bfe-ab47f749a6aa 127.0.0.1 -2022-09-15T21:42:39.328Z [INFO] agent.router: Initializing LAN area manager -42:39.329 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13379 network=udp -42:39.331 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13379 network=tcp -42:39.332 [INFO] TestAgent.server: Adding LAN server: server="Node-08e5c9a6-8d10-0cca-9bfe-ab47f749a6aa (Addr: tcp/127.0.0.1:13384) (DC: dc1)" -42:39.332 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-08e5c9a6-8d10-0cca-9bfe-ab47f749a6aa.dc1 area=wan -42:39.333 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13380 network=tcp -42:39.333 [INFO] TestAgent: started state syncer -42:39.335 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13385 network=tcp -42:39.359 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:39.361 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13384 [Candidate]" term=2 -42:39.361 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:39.361 [DEBUG] TestAgent.server.raft: vote granted: from=08e5c9a6-8d10-0cca-9bfe-ab47f749a6aa term=2 tally=1 -42:39.361 [INFO] TestAgent.server.raft: election won: tally=1 -42:39.361 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13384 [Leader]" -42:39.361 [INFO] TestAgent.server: cluster leadership acquired -42:39.362 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13384 -42:39.364 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:39.365 [INFO] TestAgent.server: New leader elected: payload=Node-08e5c9a6-8d10-0cca-9bfe-ab47f749a6aa -42:39.367 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:39.368 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:39.368 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:39.368 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:39.368 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:39.371 [INFO] TestAgent.server: federation state anti-entropy synced -42:39.371 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-08e5c9a6-8d10-0cca-9bfe-ab47f749a6aa -42:39.371 [INFO] TestAgent.server: member joined, marking health alive: member=Node-08e5c9a6-8d10-0cca-9bfe-ab47f749a6aa -42:39.414 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:39.416 [INFO] TestAgent: Synced node info -42:39.416 [DEBUG] TestAgent: Node info in sync -42:39.553 [INFO] TestAgent: Requesting shutdown -42:39.555 [INFO] TestAgent.server: shutting down server -42:39.555 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:39.555 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:39.555 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:39.555 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:39.555 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:39.555 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:39.555 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:39.555 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:39.555 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:39.555 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:39.555 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:39.556Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:39.556Z [INFO] agent.router.manager: shutting down -42:39.556 [INFO] TestAgent: consul server down -42:39.556 [INFO] TestAgent: shutdown complete -42:39.556 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13379 network=tcp -42:39.556 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13379 network=udp -42:39.556 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13380 network=tcp -42:40.065 [INFO] TestAgent: Waiting for endpoints to shut down -42:40.065 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogServices_NodeMetaFilter (0.87s) +[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:33:58.204Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:58.220Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:58.223 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f725ff20-b892-4c23-ca21-044b2b8eca7a Address:127.0.0.1:24407}]" +33:58.223 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24407 [Follower]" leader= +33:58.224 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f725ff20-b892-4c23-ca21-044b2b8eca7a.dc1 127.0.0.1 +33:58.224 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f725ff20-b892-4c23-ca21-044b2b8eca7a 127.0.0.1 +2021-08-13T15:33:58.225Z [INFO] agent.router: Initializing LAN area manager +33:58.225 [INFO] TestAgent.server: Adding LAN server: server="Node-f725ff20-b892-4c23-ca21-044b2b8eca7a (Addr: tcp/127.0.0.1:24407) (DC: dc1)" +33:58.225 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f725ff20-b892-4c23-ca21-044b2b8eca7a.dc1 area=wan +33:58.225 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24402 network=udp +33:58.225 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24402 network=tcp +33:58.226 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24403 network=tcp +33:58.226 [INFO] TestAgent: started state syncer +33:58.226 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24408 network=tcp +33:58.280 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:58.280 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24407 [Candidate]" term=2 +33:58.280 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:58.280 [DEBUG] TestAgent.server.raft: vote granted: from=f725ff20-b892-4c23-ca21-044b2b8eca7a term=2 tally=1 +33:58.280 [INFO] TestAgent.server.raft: election won: tally=1 +33:58.280 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24407 [Leader]" +33:58.280 [INFO] TestAgent.server: cluster leadership acquired +33:58.280 [INFO] TestAgent.server: New leader elected: payload=Node-f725ff20-b892-4c23-ca21-044b2b8eca7a +33:58.281 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24407 +33:58.282 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:58.284 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:58.284 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:58.284 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:58.284 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:58.284 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:58.284 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f725ff20-b892-4c23-ca21-044b2b8eca7a +33:58.284 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f725ff20-b892-4c23-ca21-044b2b8eca7a +33:58.285 [INFO] TestAgent.server: federation state anti-entropy synced +33:58.493 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:58.494 [INFO] TestAgent: Synced node info +33:58.607 [INFO] TestAgent: Requesting shutdown +33:58.607 [INFO] TestAgent.server: shutting down server +33:58.607 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:58.607 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:58.607 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:58.607 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:58.607 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:58.607 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:58.607 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:58.608 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:58.608 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:58.608 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:58.608 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:58.608Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:58.608Z [INFO] agent.router.manager: shutting down +33:58.608 [INFO] TestAgent: consul server down +33:58.608 [INFO] TestAgent: shutdown complete +33:58.608 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24402 network=tcp +33:58.608 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24402 network=udp +33:58.608 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24403 network=tcp +33:59.111 [INFO] TestAgent: Waiting for endpoints to shut down +33:59.111 [INFO] TestAgent: Endpoints down +--- PASS: TestUiIndex (0.96s) === CONT TestCatalogDatacenters -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:39.659Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:39.744Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:39.758 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7d2b1c4c-41c0-9f39-57ce-0bebf4c9d30a Address:127.0.0.1:13419}]" -42:39.759 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7d2b1c4c-41c0-9f39-57ce-0bebf4c9d30a.dc1 127.0.0.1 -42:39.760 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7d2b1c4c-41c0-9f39-57ce-0bebf4c9d30a 127.0.0.1 -2022-09-15T21:42:39.761Z [INFO] agent.router: Initializing LAN area manager -42:39.761 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13393 network=udp -42:39.761 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13419 [Follower]" leader= -42:39.762 [INFO] TestAgent.server: Adding LAN server: server="Node-7d2b1c4c-41c0-9f39-57ce-0bebf4c9d30a (Addr: tcp/127.0.0.1:13419) (DC: dc1)" -42:39.762 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7d2b1c4c-41c0-9f39-57ce-0bebf4c9d30a.dc1 area=wan -42:39.764 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13393 network=tcp -42:39.766 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13394 network=tcp -42:39.766 [INFO] TestAgent: started state syncer -42:39.766 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13420 network=tcp -42:39.797 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:39.798 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13419 [Candidate]" term=2 -42:39.798 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:39.798 [DEBUG] TestAgent.server.raft: vote granted: from=7d2b1c4c-41c0-9f39-57ce-0bebf4c9d30a term=2 tally=1 -42:39.798 [INFO] TestAgent.server.raft: election won: tally=1 -42:39.798 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13419 [Leader]" -42:39.798 [INFO] TestAgent.server: cluster leadership acquired -42:39.799 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13419 -42:39.799 [INFO] TestAgent.server: New leader elected: payload=Node-7d2b1c4c-41c0-9f39-57ce-0bebf4c9d30a -42:39.800 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:39.804 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:39.804 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:39.804 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:39.804 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:39.804 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:39.804 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7d2b1c4c-41c0-9f39-57ce-0bebf4c9d30a -42:39.804 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7d2b1c4c-41c0-9f39-57ce-0bebf4c9d30a -42:39.805 [INFO] TestAgent.server: federation state anti-entropy synced -42:39.810 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:39.811 [INFO] TestAgent: Synced node info -42:39.931 [INFO] TestAgent: Requesting shutdown -42:39.931 [INFO] TestAgent.server: shutting down server -42:39.931 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:39.931 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:39.931 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:39.931 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:39.931 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:39.932 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:39.932 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:39.932 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:39.932 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:39.932 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:39.932 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:39.932Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:39.932Z [INFO] agent.router.manager: shutting down -42:39.932 [INFO] TestAgent: consul server down -42:39.932 [INFO] TestAgent: shutdown complete -42:39.932 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13393 network=tcp -42:39.932 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13393 network=udp -42:39.932 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13394 network=tcp -42:40.434 [INFO] TestAgent: Waiting for endpoints to shut down -42:40.434 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogNodes_Blocking (0.83s) +2021-08-13T15:33:58.346Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:58.361Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:58.364 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:10aff637-b44d-9879-e6e9-871dae2ac16a Address:127.0.0.1:24414}]" +33:58.364 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24414 [Follower]" leader= +33:58.364 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-10aff637-b44d-9879-e6e9-871dae2ac16a.dc1 127.0.0.1 +33:58.365 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-10aff637-b44d-9879-e6e9-871dae2ac16a 127.0.0.1 +2021-08-13T15:33:58.365Z [INFO] agent.router: Initializing LAN area manager +33:58.365 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-10aff637-b44d-9879-e6e9-871dae2ac16a.dc1 area=wan +33:58.365 [INFO] TestAgent.server: Adding LAN server: server="Node-10aff637-b44d-9879-e6e9-871dae2ac16a (Addr: tcp/127.0.0.1:24414) (DC: dc1)" +33:58.366 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24423 network=udp +33:58.366 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24423 network=tcp +33:58.366 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24424 network=tcp +33:58.367 [INFO] TestAgent: started state syncer +33:58.367 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24415 network=tcp +33:58.409 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:58.409 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24414 [Candidate]" term=2 +33:58.409 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:58.409 [DEBUG] TestAgent.server.raft: vote granted: from=10aff637-b44d-9879-e6e9-871dae2ac16a term=2 tally=1 +33:58.409 [INFO] TestAgent.server.raft: election won: tally=1 +33:58.409 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24414 [Leader]" +33:58.409 [INFO] TestAgent.server: cluster leadership acquired +33:58.409 [INFO] TestAgent.server: New leader elected: payload=Node-10aff637-b44d-9879-e6e9-871dae2ac16a +33:58.410 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24414 +33:58.411 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:58.414 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:58.414 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:58.414 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:58.414 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:58.414 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:58.414 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-10aff637-b44d-9879-e6e9-871dae2ac16a +33:58.414 [INFO] TestAgent.server: member joined, marking health alive: member=Node-10aff637-b44d-9879-e6e9-871dae2ac16a +33:58.415 [INFO] TestAgent.server: federation state anti-entropy synced +33:58.520 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:58.520 [INFO] TestAgent: Synced node info +33:58.520 [DEBUG] TestAgent: Node info in sync +33:58.766 [INFO] TestAgent: Requesting shutdown +33:58.766 [INFO] TestAgent.server: shutting down server +33:58.766 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:58.766 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:58.766 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:58.766 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:58.766 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:58.766 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:58.766 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:58.766 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:58.766 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:58.766 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:58.766 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:58.766Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:58.766Z [INFO] agent.router.manager: shutting down +33:58.766 [INFO] TestAgent: consul server down +33:58.766 [INFO] TestAgent: shutdown complete +33:58.767 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24423 network=tcp +33:58.767 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24423 network=udp +33:58.767 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24424 network=tcp +33:59.267 [INFO] TestAgent: Waiting for endpoints to shut down +33:59.267 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogServices (0.94s) === CONT TestCatalogDeregister -2022-09-15T21:42:39.488Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:39.569Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:39.574 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:948fcc09-5019-a689-c393-66c590976816 Address:127.0.0.1:13405}]" -42:39.574 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13405 [Follower]" leader= -42:39.575 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-948fcc09-5019-a689-c393-66c590976816.dc1 127.0.0.1 -42:39.577 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-948fcc09-5019-a689-c393-66c590976816 127.0.0.1 -2022-09-15T21:42:39.577Z [INFO] agent.router: Initializing LAN area manager -42:39.577 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13414 network=udp -42:39.578 [INFO] TestAgent.server: Adding LAN server: server="Node-948fcc09-5019-a689-c393-66c590976816 (Addr: tcp/127.0.0.1:13405) (DC: dc1)" -42:39.578 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13414 network=tcp -42:39.578 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-948fcc09-5019-a689-c393-66c590976816.dc1 area=wan -42:39.579 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13415 network=tcp -42:39.579 [INFO] TestAgent: started state syncer -42:39.579 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13406 network=tcp -42:39.618 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:39.618 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13405 [Candidate]" term=2 -42:39.618 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:39.618 [DEBUG] TestAgent.server.raft: vote granted: from=948fcc09-5019-a689-c393-66c590976816 term=2 tally=1 -42:39.618 [INFO] TestAgent.server.raft: election won: tally=1 -42:39.618 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13405 [Leader]" -42:39.619 [INFO] TestAgent.server: cluster leadership acquired -42:39.619 [INFO] TestAgent.server: New leader elected: payload=Node-948fcc09-5019-a689-c393-66c590976816 -42:39.619 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13405 -42:39.621 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:39.626 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:39.626 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:39.626 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:39.626 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:39.626 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:39.626 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-948fcc09-5019-a689-c393-66c590976816 -42:39.626 [INFO] TestAgent.server: member joined, marking health alive: member=Node-948fcc09-5019-a689-c393-66c590976816 -42:39.628 [INFO] TestAgent.server: federation state anti-entropy synced -42:39.768 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:39.775 [INFO] TestAgent: Synced node info -42:39.775 [DEBUG] TestAgent: Node info in sync -42:39.938 [INFO] TestAgent: Requesting shutdown -42:39.938 [INFO] TestAgent.server: shutting down server -42:39.938 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:39.938 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:39.938 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:39.938 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:39.938 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:39.938 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:39.938 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:39.939 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:39.939 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:39.939 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:39.939 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:39.939Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:39.939Z [INFO] agent.router.manager: shutting down -42:39.939 [INFO] TestAgent: consul server down -42:39.939 [INFO] TestAgent: shutdown complete -42:39.939 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13414 network=tcp -42:39.939 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13414 network=udp -42:39.939 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13415 network=tcp -42:40.440 [INFO] TestAgent: Waiting for endpoints to shut down -42:40.440 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogNodes_DistanceSort (1.04s) +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:33:58.730Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:58.747Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:58.750 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b419613e-a97e-36f0-ac4b-e7ffcadd728e Address:127.0.0.1:24435}]" +33:58.750 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24435 [Follower]" leader= +33:58.751 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b419613e-a97e-36f0-ac4b-e7ffcadd728e.dc1 127.0.0.1 +33:58.752 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b419613e-a97e-36f0-ac4b-e7ffcadd728e 127.0.0.1 +2021-08-13T15:33:58.752Z [INFO] agent.router: Initializing LAN area manager +33:58.752 [INFO] TestAgent.server: Adding LAN server: server="Node-b419613e-a97e-36f0-ac4b-e7ffcadd728e (Addr: tcp/127.0.0.1:24435) (DC: dc1)" +33:58.752 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b419613e-a97e-36f0-ac4b-e7ffcadd728e.dc1 area=wan +33:58.752 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24430 network=udp +33:58.752 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24430 network=tcp +33:58.753 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24431 network=tcp +33:58.753 [INFO] TestAgent: started state syncer +33:58.753 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24436 network=tcp +33:58.819 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:58.819 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24435 [Candidate]" term=2 +33:58.820 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:58.820 [DEBUG] TestAgent.server.raft: vote granted: from=b419613e-a97e-36f0-ac4b-e7ffcadd728e term=2 tally=1 +33:58.820 [INFO] TestAgent.server.raft: election won: tally=1 +33:58.820 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24435 [Leader]" +33:58.820 [INFO] TestAgent.server: cluster leadership acquired +33:58.820 [INFO] TestAgent.server: New leader elected: payload=Node-b419613e-a97e-36f0-ac4b-e7ffcadd728e +33:58.820 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24435 +33:58.822 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:58.824 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:58.824 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:58.824 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:58.824 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:58.824 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:58.825 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b419613e-a97e-36f0-ac4b-e7ffcadd728e +33:58.825 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b419613e-a97e-36f0-ac4b-e7ffcadd728e +33:58.826 [INFO] TestAgent.server: federation state anti-entropy synced +33:58.858 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:58.859 [INFO] TestAgent: Synced node info +33:58.859 [DEBUG] TestAgent: Node info in sync +33:58.975 [INFO] TestAgent: Requesting shutdown +33:58.975 [INFO] TestAgent.server: shutting down server +33:58.975 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:58.975 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:58.975 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:58.975 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:58.975 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:58.975 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:58.975 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:58.975 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:58.975 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:58.975 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:58.975 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:58.975Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:58.975Z [INFO] agent.router.manager: shutting down +33:58.975 [INFO] TestAgent: consul server down +33:58.975 [INFO] TestAgent: shutdown complete +33:58.975 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24430 network=tcp +33:58.976 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24430 network=udp +33:58.976 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24431 network=tcp +33:59.476 [INFO] TestAgent: Waiting for endpoints to shut down +33:59.476 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogNodes_Blocking (0.76s) === CONT TestCatalogRegister_Service_InvalidAddress -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._udp_(qType:=33)_compressed=true -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._udp_(qType:=33)_compressed=false -[WARN] freeport: 4 out of 4 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=255)_compressed=true#01 +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:33:58.816Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:58.833Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:58.836 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8993bbb7-5e71-9bf4-0c14-b6ee4e4138c7 Address:127.0.0.1:24449}]" +33:58.836 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24449 [Follower]" leader= +33:58.837 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8993bbb7-5e71-9bf4-0c14-b6ee4e4138c7.dc1 127.0.0.1 +33:58.837 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8993bbb7-5e71-9bf4-0c14-b6ee4e4138c7 127.0.0.1 +2021-08-13T15:33:58.837Z [INFO] agent.router: Initializing LAN area manager +33:58.838 [INFO] TestAgent.server: Adding LAN server: server="Node-8993bbb7-5e71-9bf4-0c14-b6ee4e4138c7 (Addr: tcp/127.0.0.1:24449) (DC: dc1)" +33:58.838 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8993bbb7-5e71-9bf4-0c14-b6ee4e4138c7.dc1 area=wan +33:58.838 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24444 network=tcp +33:58.838 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24444 network=udp +33:58.839 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24445 network=tcp +33:58.839 [INFO] TestAgent: started state syncer +33:58.839 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24450 network=tcp +33:58.904 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:58.904 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24449 [Candidate]" term=2 +33:58.904 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:58.904 [DEBUG] TestAgent.server.raft: vote granted: from=8993bbb7-5e71-9bf4-0c14-b6ee4e4138c7 term=2 tally=1 +33:58.904 [INFO] TestAgent.server.raft: election won: tally=1 +33:58.904 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24449 [Leader]" +33:58.904 [INFO] TestAgent.server: cluster leadership acquired +33:58.904 [INFO] TestAgent.server: New leader elected: payload=Node-8993bbb7-5e71-9bf4-0c14-b6ee4e4138c7 +33:58.905 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24449 +33:58.906 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:58.912 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:58.912 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:58.912 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:58.912 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:58.912 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:58.913 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8993bbb7-5e71-9bf4-0c14-b6ee4e4138c7 +33:58.913 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8993bbb7-5e71-9bf4-0c14-b6ee4e4138c7 +33:58.914 [INFO] TestAgent.server: federation state anti-entropy synced +33:59.034 [INFO] TestAgent: Requesting shutdown +33:59.034 [INFO] TestAgent.server: shutting down server +33:59.034 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:59.034 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:59.034 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:59.034 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:59.034 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:59.034 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:59.034 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:59.034 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:59.034 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:59.034 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:59.034 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:59.034 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:59.035Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:59.035Z [INFO] agent.router.manager: shutting down +33:59.035 [INFO] TestAgent: consul server down +33:59.035 [INFO] TestAgent: shutdown complete +33:59.035 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24444 network=tcp +33:59.035 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24444 network=udp +33:59.035 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24445 network=tcp +33:59.535 [INFO] TestAgent: Waiting for endpoints to shut down +33:59.535 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogNodes_Filter (0.74s) +=== CONT TestAgent_JoinWAN_viaMeshGateway +2021-08-13T15:33:58.982Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:59.001Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:59.004 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1de52c51-1a4a-6f61-bb45-e0f0b25e1c86 Address:127.0.0.1:24463}]" +33:59.005 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1de52c51-1a4a-6f61-bb45-e0f0b25e1c86.dc1 127.0.0.1 +33:59.006 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1de52c51-1a4a-6f61-bb45-e0f0b25e1c86 127.0.0.1 +2021-08-13T15:33:59.006Z [INFO] agent.router: Initializing LAN area manager +33:59.006 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24458 network=udp +33:59.006 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24463 [Follower]" leader= +33:59.007 [INFO] TestAgent.server: Adding LAN server: server="Node-1de52c51-1a4a-6f61-bb45-e0f0b25e1c86 (Addr: tcp/127.0.0.1:24463) (DC: dc1)" +33:59.007 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1de52c51-1a4a-6f61-bb45-e0f0b25e1c86.dc1 area=wan +33:59.007 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24458 network=tcp +33:59.008 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24459 network=tcp +33:59.008 [INFO] TestAgent: started state syncer +33:59.008 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24464 network=tcp +33:59.062 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:59.062 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24463 [Candidate]" term=2 +33:59.062 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:59.062 [DEBUG] TestAgent.server.raft: vote granted: from=1de52c51-1a4a-6f61-bb45-e0f0b25e1c86 term=2 tally=1 +33:59.062 [INFO] TestAgent.server.raft: election won: tally=1 +33:59.062 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24463 [Leader]" +33:59.062 [INFO] TestAgent.server: cluster leadership acquired +33:59.063 [INFO] TestAgent.server: New leader elected: payload=Node-1de52c51-1a4a-6f61-bb45-e0f0b25e1c86 +33:59.063 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24463 +33:59.065 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:59.068 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:59.068 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:59.068 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:59.068 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:59.068 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:59.068 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1de52c51-1a4a-6f61-bb45-e0f0b25e1c86 +33:59.068 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1de52c51-1a4a-6f61-bb45-e0f0b25e1c86 +33:59.069 [INFO] TestAgent.server: federation state anti-entropy synced +33:59.069 [INFO] TestAgent: Synced node info +33:59.069 [DEBUG] TestAgent: Node info in sync +33:59.189 [INFO] TestAgent: Requesting shutdown +33:59.189 [INFO] TestAgent.server: shutting down server +33:59.189 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:59.189 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:59.189 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:59.189 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:59.189 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:59.189 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:59.189 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:59.189 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:59.189 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:59.189 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:59.189 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:59.189Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:59.189Z [INFO] agent.router.manager: shutting down +33:59.189 [INFO] TestAgent: consul server down +33:59.189 [INFO] TestAgent: shutdown complete +33:59.189 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24458 network=tcp +33:59.189 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24458 network=udp +33:59.189 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24459 network=tcp +33:59.690 [INFO] TestAgent: Waiting for endpoints to shut down +33:59.690 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogNodes (0.72s) +=== CONT TestAgent_RerouteNewHTTPChecks +2021-08-13T15:33:58.949Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:58.965Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:58.968 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ef4777d5-73a8-6c5e-b44d-725aa4bf4efc Address:127.0.0.1:24456}]" +33:58.969 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ef4777d5-73a8-6c5e-b44d-725aa4bf4efc.dc1 127.0.0.1 +33:58.970 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ef4777d5-73a8-6c5e-b44d-725aa4bf4efc 127.0.0.1 +2021-08-13T15:33:58.970Z [INFO] agent.router: Initializing LAN area manager +33:58.971 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24451 network=udp +33:58.971 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24456 [Follower]" leader= +33:58.971 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24451 network=tcp +33:58.971 [INFO] TestAgent.server: Adding LAN server: server="Node-ef4777d5-73a8-6c5e-b44d-725aa4bf4efc (Addr: tcp/127.0.0.1:24456) (DC: dc1)" +33:58.971 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ef4777d5-73a8-6c5e-b44d-725aa4bf4efc.dc1 area=wan +33:58.972 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24452 network=tcp +33:58.972 [INFO] TestAgent: started state syncer +33:58.972 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24457 network=tcp +33:59.040 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:59.040 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24456 [Candidate]" term=2 +33:59.040 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:59.040 [DEBUG] TestAgent.server.raft: vote granted: from=ef4777d5-73a8-6c5e-b44d-725aa4bf4efc term=2 tally=1 +33:59.040 [INFO] TestAgent.server.raft: election won: tally=1 +33:59.040 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24456 [Leader]" +33:59.040 [INFO] TestAgent.server: cluster leadership acquired +33:59.041 [INFO] TestAgent.server: New leader elected: payload=Node-ef4777d5-73a8-6c5e-b44d-725aa4bf4efc +33:59.041 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24456 +33:59.042 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:59.044 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:59.044 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:59.044 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:59.044 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:59.044 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:59.045 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ef4777d5-73a8-6c5e-b44d-725aa4bf4efc +33:59.045 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ef4777d5-73a8-6c5e-b44d-725aa4bf4efc +33:59.045 [INFO] TestAgent.server: federation state anti-entropy synced +33:59.252 [INFO] TestAgent: Requesting shutdown +33:59.252 [INFO] TestAgent.server: shutting down server +33:59.252 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:59.252 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:59.252 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:59.252 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:59.252 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:59.252 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:59.253 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:59.252 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:59.252 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:59.253 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:59.253 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:59.253 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:59.253Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:59.253Z [INFO] agent.router.manager: shutting down +33:59.253 [INFO] TestAgent: consul server down +33:59.253 [INFO] TestAgent: shutdown complete +33:59.253 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24451 network=tcp +33:59.253 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24451 network=udp +33:59.253 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24452 network=tcp +33:59.753 [INFO] TestAgent: Waiting for endpoints to shut down +33:59.753 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogNodes_MetaFilter (0.82s) +=== CONT TestAgent_RerouteExistingHTTPChecks === RUN TestCatalogRegister_Service_InvalidAddress/addr_0.0.0.0 === RUN TestCatalogRegister_Service_InvalidAddress/addr_:: === RUN TestCatalogRegister_Service_InvalidAddress/addr_[::] -2022-09-15T21:42:40.128Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:40.170Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:40.174 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1717caa0-f6b8-a407-0c58-97fe0e79a2bf Address:127.0.0.1:13461}]" -42:40.174 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13461 [Follower]" leader= -42:40.175 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1717caa0-f6b8-a407-0c58-97fe0e79a2bf.dc1 127.0.0.1 -42:40.177 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1717caa0-f6b8-a407-0c58-97fe0e79a2bf 127.0.0.1 -2022-09-15T21:42:40.177Z [INFO] agent.router: Initializing LAN area manager -42:40.177 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13442 network=udp -42:40.178 [INFO] TestAgent.server: Adding LAN server: server="Node-1717caa0-f6b8-a407-0c58-97fe0e79a2bf (Addr: tcp/127.0.0.1:13461) (DC: dc1)" -42:40.178 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1717caa0-f6b8-a407-0c58-97fe0e79a2bf.dc1 area=wan -42:40.178 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13442 network=tcp -42:40.179 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13443 network=tcp -42:40.179 [INFO] TestAgent: started state syncer -42:40.179 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13462 network=tcp -42:40.216 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:40.216 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13461 [Candidate]" term=2 -42:40.216 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:40.216 [DEBUG] TestAgent.server.raft: vote granted: from=1717caa0-f6b8-a407-0c58-97fe0e79a2bf term=2 tally=1 -42:40.216 [INFO] TestAgent.server.raft: election won: tally=1 -42:40.216 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13461 [Leader]" -42:40.216 [INFO] TestAgent.server: cluster leadership acquired -42:40.218 [INFO] TestAgent.server: New leader elected: payload=Node-1717caa0-f6b8-a407-0c58-97fe0e79a2bf -42:40.229 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13461 -42:40.232 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:40.249 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:40.249 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:40.249 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:40.249 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:40.249 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:40.250 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1717caa0-f6b8-a407-0c58-97fe0e79a2bf -42:40.250 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1717caa0-f6b8-a407-0c58-97fe0e79a2bf -42:40.331 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:40.332 [INFO] TestAgent: Synced node info -42:40.332 [DEBUG] TestAgent: Node info in sync -42:40.367 [INFO] TestAgent: Requesting shutdown -42:40.367 [INFO] TestAgent.server: shutting down server -42:40.367 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:40.367 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:40.367 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:40.367 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:40.367 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:40.367 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="shutdown waiting for leader" -42:40.367 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:40.367 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:40.367 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:40.367 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:40.368 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:40.368Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:40.368Z [INFO] agent.router.manager: shutting down -42:40.368 [INFO] TestAgent: consul server down -42:40.368 [INFO] TestAgent: shutdown complete -42:40.368 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13442 network=tcp -42:40.368 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13442 network=udp -42:40.368 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13443 network=tcp -42:40.872 [INFO] TestAgent: Waiting for endpoints to shut down -42:40.872 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogDatacenters (0.81s) +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! === CONT TestAgent_JoinWAN_viaMeshGateway -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:40.063Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:40.129Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:40.166 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f4bc308e-d511-6448-d690-d1616341964a Address:127.0.0.1:13489}]" -42:40.166 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13489 [Follower]" leader= -42:40.168 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f4bc308e-d511-6448-d690-d1616341964a.dc1 127.0.0.1 -42:40.170 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f4bc308e-d511-6448-d690-d1616341964a 127.0.0.1 -2022-09-15T21:42:40.170Z [INFO] agent.router: Initializing LAN area manager -42:40.170 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13428 network=udp -42:40.170 [INFO] TestAgent.server: Adding LAN server: server="Node-f4bc308e-d511-6448-d690-d1616341964a (Addr: tcp/127.0.0.1:13489) (DC: dc1)" -42:40.170 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f4bc308e-d511-6448-d690-d1616341964a.dc1 area=wan -42:40.171 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13428 network=tcp -42:40.171 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13429 network=tcp -42:40.172 [INFO] TestAgent: started state syncer -42:40.172 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13490 network=tcp -42:40.216 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:40.217 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13489 [Candidate]" term=2 -42:40.217 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:40.217 [DEBUG] TestAgent.server.raft: vote granted: from=f4bc308e-d511-6448-d690-d1616341964a term=2 tally=1 -42:40.217 [INFO] TestAgent.server.raft: election won: tally=1 -42:40.217 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13489 [Leader]" -42:40.217 [INFO] TestAgent.server: cluster leadership acquired -42:40.217 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13489 -42:40.218 [INFO] TestAgent.server: New leader elected: payload=Node-f4bc308e-d511-6448-d690-d1616341964a -42:40.226 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:40.238 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:40.238 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:40.238 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:40.238 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:40.238 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:40.239 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f4bc308e-d511-6448-d690-d1616341964a -42:40.239 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f4bc308e-d511-6448-d690-d1616341964a -42:40.245 [INFO] TestAgent.server: federation state anti-entropy synced -42:40.519 [INFO] TestAgent: Requesting shutdown -42:40.520 [INFO] TestAgent.server: shutting down server -42:40.520 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:40.520 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:40.520 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:40.520 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:40.520 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:40.520 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:40.520 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:40.520 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:40.520 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:40.520 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:40.521 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:40.525 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:40.525Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:40.526Z [INFO] agent.router.manager: shutting down -42:40.526 [INFO] TestAgent: consul server down -42:40.526 [INFO] TestAgent: shutdown complete -42:40.526 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13428 network=tcp -42:40.526 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13428 network=udp -42:40.526 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13429 network=tcp -42:41.029 [INFO] TestAgent: Waiting for endpoints to shut down -42:41.029 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogNodes (1.00s) -=== CONT TestAgent_RerouteNewHTTPChecks -2022-09-15T21:42:40.077Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:40.144Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:40.161 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:46c52f86-5dd9-74d9-7051-59aba5a4a7af Address:127.0.0.1:13426}]" -42:40.161 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13426 [Follower]" leader= -42:40.162 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-46c52f86-5dd9-74d9-7051-59aba5a4a7af.dc1 127.0.0.1 -42:40.163 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-46c52f86-5dd9-74d9-7051-59aba5a4a7af 127.0.0.1 -2022-09-15T21:42:40.163Z [INFO] agent.router: Initializing LAN area manager -42:40.163 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-46c52f86-5dd9-74d9-7051-59aba5a4a7af.dc1 area=wan -42:40.163 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13421 network=udp -42:40.163 [INFO] TestAgent.server: Adding LAN server: server="Node-46c52f86-5dd9-74d9-7051-59aba5a4a7af (Addr: tcp/127.0.0.1:13426) (DC: dc1)" -42:40.163 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13421 network=tcp -42:40.164 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13422 network=tcp -42:40.164 [INFO] TestAgent: started state syncer -42:40.165 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13427 network=tcp -42:40.218 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:40.218 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13426 [Candidate]" term=2 -42:40.218 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:40.218 [DEBUG] TestAgent.server.raft: vote granted: from=46c52f86-5dd9-74d9-7051-59aba5a4a7af term=2 tally=1 -42:40.218 [INFO] TestAgent.server.raft: election won: tally=1 -42:40.218 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13426 [Leader]" -42:40.219 [INFO] TestAgent.server: cluster leadership acquired -42:40.219 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13426 -42:40.220 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:40.222 [INFO] TestAgent.server: New leader elected: payload=Node-46c52f86-5dd9-74d9-7051-59aba5a4a7af -42:40.233 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:40.233 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:40.233 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:40.233 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:40.233 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:40.233 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-46c52f86-5dd9-74d9-7051-59aba5a4a7af -42:40.233 [INFO] TestAgent.server: member joined, marking health alive: member=Node-46c52f86-5dd9-74d9-7051-59aba5a4a7af -42:40.239 [INFO] TestAgent.server: federation state anti-entropy synced -42:40.374 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:40.375 [INFO] TestAgent: Synced node info -42:40.375 [DEBUG] TestAgent: Node info in sync -42:40.551 [INFO] TestAgent: Requesting shutdown -42:40.551 [INFO] TestAgent.server: shutting down server -42:40.551 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:40.551 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:40.551 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:40.551 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:40.551 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:40.552 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:40.552 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:40.552 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:40.552 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:40.552 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:40.552 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:40.552Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:40.552Z [INFO] agent.router.manager: shutting down -42:40.552 [INFO] TestAgent: consul server down -42:40.552 [INFO] TestAgent: shutdown complete -42:40.552 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13421 network=tcp -42:40.552 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13421 network=udp -42:40.553 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13422 network=tcp -42:41.054 [INFO] TestAgent: Waiting for endpoints to shut down -42:41.054 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogNodes_MetaFilter (1.04s) -=== CONT TestAgent_RerouteExistingHTTPChecks -2022-09-15T21:42:40.073Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:40.142Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:40.147 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:708aba04-1483-c409-9336-a3cbaad6a103 Address:127.0.0.1:13412}]" -42:40.148 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13412 [Follower]" leader= -42:40.149 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-708aba04-1483-c409-9336-a3cbaad6a103.dc1 127.0.0.1 -42:40.159 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-708aba04-1483-c409-9336-a3cbaad6a103 127.0.0.1 -2022-09-15T21:42:40.159Z [INFO] agent.router: Initializing LAN area manager -42:40.159 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13456 network=udp -42:40.159 [INFO] TestAgent.server: Adding LAN server: server="Node-708aba04-1483-c409-9336-a3cbaad6a103 (Addr: tcp/127.0.0.1:13412) (DC: dc1)" -42:40.160 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-708aba04-1483-c409-9336-a3cbaad6a103.dc1 area=wan -42:40.160 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13456 network=tcp -42:40.161 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13457 network=tcp -42:40.161 [INFO] TestAgent: started state syncer -42:40.161 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13413 network=tcp -42:40.206 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:40.206 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13412 [Candidate]" term=2 -42:40.207 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:40.207 [DEBUG] TestAgent.server.raft: vote granted: from=708aba04-1483-c409-9336-a3cbaad6a103 term=2 tally=1 -42:40.207 [INFO] TestAgent.server.raft: election won: tally=1 -42:40.207 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13412 [Leader]" -42:40.207 [INFO] TestAgent.server: cluster leadership acquired -42:40.208 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13412 -42:40.209 [INFO] TestAgent.server: New leader elected: payload=Node-708aba04-1483-c409-9336-a3cbaad6a103 -42:40.223 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:40.235 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:40.235 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:40.235 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:40.235 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:40.235 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:40.235 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-708aba04-1483-c409-9336-a3cbaad6a103 -42:40.235 [INFO] TestAgent.server: member joined, marking health alive: member=Node-708aba04-1483-c409-9336-a3cbaad6a103 -42:40.243 [INFO] TestAgent.server: federation state anti-entropy synced -42:40.298 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:40.299 [INFO] TestAgent: Synced node info -42:40.584 [INFO] TestAgent: Requesting shutdown -42:40.584 [INFO] TestAgent.server: shutting down server -42:40.584 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:40.584 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:40.584 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:40.584 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:40.584 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:40.584 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:40.584 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:40.584 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:40.584 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:40.584 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:40.584 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:40.585Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:40.585Z [INFO] agent.router.manager: shutting down -42:40.585 [INFO] TestAgent: consul server down -42:40.585 [INFO] TestAgent: shutdown complete -42:40.585 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13456 network=tcp -42:40.585 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13456 network=udp -42:40.585 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13457 network=tcp -42:41.090 [INFO] TestAgent: Waiting for endpoints to shut down -42:41.091 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogNodes_Filter (1.11s) + agent_test.go:4395: routing {bob.,}server.dc1.consul => 127.0.0.1:24508 + agent_test.go:4396: routing {betty.,}server.dc2.consul => 127.0.0.1:24487 + agent_test.go:4397: routing {bonnie.,}server.dc3.consul => 127.0.0.1:24488 +2021-08-13T15:33:58.678Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:58.695Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:58.698 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3d02259b-5c24-7191-e6f5-eea7d92631f3 Address:127.0.0.1:24428}]" +33:58.698 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24428 [Follower]" leader= +33:58.699 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3d02259b-5c24-7191-e6f5-eea7d92631f3.dc1 127.0.0.1 +33:58.700 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3d02259b-5c24-7191-e6f5-eea7d92631f3 127.0.0.1 +2021-08-13T15:33:58.700Z [INFO] agent.router: Initializing LAN area manager +33:58.701 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24416 network=udp +33:58.701 [INFO] TestAgent.server: Adding LAN server: server="Node-3d02259b-5c24-7191-e6f5-eea7d92631f3 (Addr: tcp/127.0.0.1:24428) (DC: dc1)" +33:58.701 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3d02259b-5c24-7191-e6f5-eea7d92631f3.dc1 area=wan +33:58.701 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24416 network=tcp +33:58.702 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24417 network=tcp +33:58.702 [INFO] TestAgent: started state syncer +33:58.702 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24429 network=tcp +33:58.757 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:58.757 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24428 [Candidate]" term=2 +33:58.757 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:58.758 [DEBUG] TestAgent.server.raft: vote granted: from=3d02259b-5c24-7191-e6f5-eea7d92631f3 term=2 tally=1 +33:58.758 [INFO] TestAgent.server.raft: election won: tally=1 +33:58.758 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24428 [Leader]" +33:58.758 [INFO] TestAgent.server: cluster leadership acquired +33:58.758 [INFO] TestAgent.server: New leader elected: payload=Node-3d02259b-5c24-7191-e6f5-eea7d92631f3 +33:58.758 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24428 +33:58.759 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:58.762 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:58.762 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:58.762 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:58.762 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:58.762 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:58.762 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3d02259b-5c24-7191-e6f5-eea7d92631f3 +33:58.762 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3d02259b-5c24-7191-e6f5-eea7d92631f3 +33:58.763 [INFO] TestAgent.server: federation state anti-entropy synced +33:58.893 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:58.893 [INFO] TestAgent: Synced node info +33:59.138 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:59.138 [DEBUG] TestAgent: Node info in sync +33:59.138 [DEBUG] TestAgent: Node info in sync +33:59.426 [INFO] TestAgent: Requesting shutdown +33:59.426 [INFO] TestAgent.server: shutting down server +33:59.426 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:59.426 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:59.426 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:59.426 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:59.426 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:59.426 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:59.426 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:59.426 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:59.426 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:59.426 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:59.426 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:59.426Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:59.426Z [INFO] agent.router.manager: shutting down +33:59.426 [INFO] TestAgent: consul server down +33:59.426 [INFO] TestAgent: shutdown complete +33:59.426 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24416 network=tcp +33:59.426 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24416 network=udp +33:59.427 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24417 network=tcp +33:59.927 [INFO] TestAgent: Waiting for endpoints to shut down +33:59.927 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogNodes_DistanceSort (1.29s) === CONT TestAgent_consulConfig_RaftTrailingLogs -[WARN] freeport: 4 out of 25 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:40.513Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:40.538Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:40.549 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6c4cd269-8ef2-276a-aa3e-8ba73a4761e6 Address:127.0.0.1:13468}]" -42:40.550 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13468 [Follower]" leader= -42:40.563 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6c4cd269-8ef2-276a-aa3e-8ba73a4761e6.dc1 127.0.0.1 -42:40.564 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6c4cd269-8ef2-276a-aa3e-8ba73a4761e6 127.0.0.1 -2022-09-15T21:42:40.564Z [INFO] agent.router: Initializing LAN area manager -42:40.565 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13463 network=udp -42:40.565 [INFO] TestAgent.server: Adding LAN server: server="Node-6c4cd269-8ef2-276a-aa3e-8ba73a4761e6 (Addr: tcp/127.0.0.1:13468) (DC: dc1)" -42:40.565 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6c4cd269-8ef2-276a-aa3e-8ba73a4761e6.dc1 area=wan -42:40.570 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13463 network=tcp -42:40.571 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13464 network=tcp -42:40.572 [INFO] TestAgent: started state syncer -42:40.572 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13469 network=tcp -42:40.599 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:40.600 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13468 [Candidate]" term=2 -42:40.600 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:40.600 [DEBUG] TestAgent.server.raft: vote granted: from=6c4cd269-8ef2-276a-aa3e-8ba73a4761e6 term=2 tally=1 -42:40.600 [INFO] TestAgent.server.raft: election won: tally=1 -42:40.600 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13468 [Leader]" -42:40.600 [INFO] TestAgent.server: cluster leadership acquired -42:40.600 [INFO] TestAgent.server: New leader elected: payload=Node-6c4cd269-8ef2-276a-aa3e-8ba73a4761e6 -42:40.600 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13468 -42:40.602 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:40.604 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:40.604 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:40.604 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:40.604 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:40.604 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:40.604 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6c4cd269-8ef2-276a-aa3e-8ba73a4761e6 -42:40.605 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6c4cd269-8ef2-276a-aa3e-8ba73a4761e6 -42:40.606 [INFO] TestAgent.server: federation state anti-entropy synced -42:40.851 [INFO] TestAgent: Requesting shutdown -42:40.851 [INFO] TestAgent.server: shutting down server -42:40.851 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:40.851 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:40.851 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:40.851 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:40.851 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:40.852 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:40.852 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:40.852 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:40.852 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:40.852 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:40.852 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:40.852 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:40.852Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:40.852Z [INFO] agent.router.manager: shutting down -42:40.852 [INFO] TestAgent: consul server down -42:40.852 [INFO] TestAgent: shutdown complete -42:40.852 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13463 network=tcp -42:40.852 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13463 network=udp -42:40.852 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13464 network=tcp -42:41.353 [INFO] TestAgent: Waiting for endpoints to shut down -42:41.353 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogRegister_Service_InvalidAddress (0.91s) +2021-08-13T15:33:59.310Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:59.327Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:59.330 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6a87a68e-0dd7-a044-201d-40188d48b1a4 Address:127.0.0.1:24491}]" +33:59.330 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24491 [Follower]" leader= +33:59.330 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6a87a68e-0dd7-a044-201d-40188d48b1a4.dc1 127.0.0.1 +33:59.331 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6a87a68e-0dd7-a044-201d-40188d48b1a4 127.0.0.1 +2021-08-13T15:33:59.331Z [INFO] agent.router: Initializing LAN area manager +33:59.332 [INFO] TestAgent.server: Adding LAN server: server="Node-6a87a68e-0dd7-a044-201d-40188d48b1a4 (Addr: tcp/127.0.0.1:24491) (DC: dc1)" +33:59.332 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6a87a68e-0dd7-a044-201d-40188d48b1a4.dc1 area=wan +33:59.332 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24479 network=tcp +33:59.332 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24479 network=udp +33:59.333 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24480 network=tcp +33:59.333 [INFO] TestAgent: started state syncer +33:59.333 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24492 network=tcp +33:59.398 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:59.398 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24491 [Candidate]" term=2 +33:59.398 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:59.398 [DEBUG] TestAgent.server.raft: vote granted: from=6a87a68e-0dd7-a044-201d-40188d48b1a4 term=2 tally=1 +33:59.398 [INFO] TestAgent.server.raft: election won: tally=1 +33:59.398 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24491 [Leader]" +33:59.398 [INFO] TestAgent.server: cluster leadership acquired +33:59.398 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24491 +33:59.399 [INFO] TestAgent.server: New leader elected: payload=Node-6a87a68e-0dd7-a044-201d-40188d48b1a4 +33:59.400 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:59.402 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:59.402 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:59.402 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:59.402 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:59.402 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:59.402 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6a87a68e-0dd7-a044-201d-40188d48b1a4 +33:59.402 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6a87a68e-0dd7-a044-201d-40188d48b1a4 +33:59.404 [INFO] TestAgent.server: federation state anti-entropy synced +33:59.493 [INFO] TestAgent: Requesting shutdown +33:59.493 [INFO] TestAgent.server: shutting down server +33:59.493 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:59.493 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:59.493 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:59.493 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:59.493 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:59.493 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:59.493 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:59.493 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:59.493 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:59.493 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:59.493 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:59.494 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:59.494Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:59.494Z [INFO] agent.router.manager: shutting down +33:59.494 [INFO] TestAgent: consul server down +33:59.494 [INFO] TestAgent: shutdown complete +33:59.494 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24479 network=tcp +33:59.494 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24479 network=udp +33:59.494 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24480 network=tcp +33:59.994 [INFO] TestAgent: Waiting for endpoints to shut down +33:59.994 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogDeregister (0.73s) +=== CONT TestAgent_consulConfig_AutoEncryptAllowTLS +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:33:59.131Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:59.148Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:59.151 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:261040c0-8056-d00d-0a48-04fe0e8678dc Address:127.0.0.1:24477}]" +33:59.151 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24477 [Follower]" leader= +33:59.152 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-261040c0-8056-d00d-0a48-04fe0e8678dc.dc1 127.0.0.1 +33:59.153 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-261040c0-8056-d00d-0a48-04fe0e8678dc 127.0.0.1 +2021-08-13T15:33:59.153Z [INFO] agent.router: Initializing LAN area manager +33:59.153 [INFO] TestAgent.server: Adding LAN server: server="Node-261040c0-8056-d00d-0a48-04fe0e8678dc (Addr: tcp/127.0.0.1:24477) (DC: dc1)" +33:59.153 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-261040c0-8056-d00d-0a48-04fe0e8678dc.dc1 area=wan +33:59.153 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24472 network=udp +33:59.153 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24472 network=tcp +33:59.154 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24473 network=tcp +33:59.154 [INFO] TestAgent: started state syncer +33:59.154 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24478 network=tcp +33:59.188 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:59.188 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24477 [Candidate]" term=2 +33:59.188 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:59.188 [DEBUG] TestAgent.server.raft: vote granted: from=261040c0-8056-d00d-0a48-04fe0e8678dc term=2 tally=1 +33:59.188 [INFO] TestAgent.server.raft: election won: tally=1 +33:59.188 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24477 [Leader]" +33:59.189 [INFO] TestAgent.server: cluster leadership acquired +33:59.189 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24477 +33:59.189 [INFO] TestAgent.server: New leader elected: payload=Node-261040c0-8056-d00d-0a48-04fe0e8678dc +33:59.190 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:59.192 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:59.192 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:59.192 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:59.192 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:59.192 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:59.192 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-261040c0-8056-d00d-0a48-04fe0e8678dc +33:59.192 [INFO] TestAgent.server: member joined, marking health alive: member=Node-261040c0-8056-d00d-0a48-04fe0e8678dc +33:59.193 [INFO] TestAgent.server: federation state anti-entropy synced +33:59.239 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:59.239 [INFO] TestAgent: Synced node info +33:59.582 [INFO] TestAgent: Requesting shutdown +33:59.582 [INFO] TestAgent.server: shutting down server +33:59.582 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:59.582 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:59.582 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:59.582 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:59.582 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:59.582 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:59.582 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:59.582 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:59.582 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:59.582 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:59.582 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:59.582Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:59.582Z [INFO] agent.router.manager: shutting down +33:59.582 [INFO] TestAgent: consul server down +33:59.582 [INFO] TestAgent: shutdown complete +33:59.582 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24472 network=tcp +33:59.582 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24472 network=udp +33:59.582 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24473 network=tcp +34:00.083 [INFO] TestAgent: Waiting for endpoints to shut down +34:00.083 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogDatacenters (0.97s) +=== CONT TestAgent_ReloadConfigTLSConfigFailure +2021-08-13T15:33:59.493Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:59.509Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:59.511 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:714965df-acf0-4184-1e70-cf6e44058356 Address:127.0.0.1:24484}]" +33:59.512 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24484 [Follower]" leader= +33:59.512 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-714965df-acf0-4184-1e70-cf6e44058356.dc1 127.0.0.1 +33:59.513 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-714965df-acf0-4184-1e70-cf6e44058356 127.0.0.1 +2021-08-13T15:33:59.513Z [INFO] agent.router: Initializing LAN area manager +33:59.513 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24493 network=udp +33:59.513 [INFO] TestAgent.server: Adding LAN server: server="Node-714965df-acf0-4184-1e70-cf6e44058356 (Addr: tcp/127.0.0.1:24484) (DC: dc1)" +33:59.513 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-714965df-acf0-4184-1e70-cf6e44058356.dc1 area=wan +33:59.514 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24493 network=tcp +33:59.514 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24494 network=tcp +33:59.515 [INFO] TestAgent: started state syncer +33:59.515 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24485 network=tcp +33:59.568 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:59.568 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24484 [Candidate]" term=2 +33:59.568 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:59.568 [DEBUG] TestAgent.server.raft: vote granted: from=714965df-acf0-4184-1e70-cf6e44058356 term=2 tally=1 +33:59.569 [INFO] TestAgent.server.raft: election won: tally=1 +33:59.569 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24484 [Leader]" +33:59.569 [INFO] TestAgent.server: cluster leadership acquired +33:59.569 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24484 +33:59.570 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:59.571 [INFO] TestAgent.server: New leader elected: payload=Node-714965df-acf0-4184-1e70-cf6e44058356 +33:59.573 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:59.573 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:59.573 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:59.573 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:59.573 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:59.573 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-714965df-acf0-4184-1e70-cf6e44058356 +33:59.573 [INFO] TestAgent.server: member joined, marking health alive: member=Node-714965df-acf0-4184-1e70-cf6e44058356 +33:59.574 [INFO] TestAgent.server: federation state anti-entropy synced +33:59.759 [INFO] TestAgent: Requesting shutdown +33:59.759 [INFO] TestAgent.server: shutting down server +33:59.759 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:59.759 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:59.759 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:59.759 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:59.759 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:59.759 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:59.759 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:59.759 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:59.759 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:59.759 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:59.759 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:59.759 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:59.759Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:59.759Z [INFO] agent.router.manager: shutting down +33:59.759 [INFO] TestAgent: consul server down +33:59.759 [INFO] TestAgent: shutdown complete +33:59.759 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24493 network=tcp +33:59.759 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24493 network=udp +33:59.759 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24494 network=tcp +34:00.260 [INFO] TestAgent: Waiting for endpoints to shut down +34:00.260 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogRegister_Service_InvalidAddress (0.78s) --- PASS: TestCatalogRegister_Service_InvalidAddress/addr_0.0.0.0 (0.00s) --- PASS: TestCatalogRegister_Service_InvalidAddress/addr_:: (0.00s) --- PASS: TestCatalogRegister_Service_InvalidAddress/addr_[::] (0.00s) -=== CONT TestAgent_consulConfig_AutoEncryptAllowTLS -2022-09-15T21:42:40.504Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:40.538Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:40.574 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bb8fd888-4b4f-8ea1-e465-e8e230341a20 Address:127.0.0.1:13447}]" -42:40.574 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13447 [Follower]" leader= -42:40.575 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bb8fd888-4b4f-8ea1-e465-e8e230341a20.dc1 127.0.0.1 -42:40.576 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bb8fd888-4b4f-8ea1-e465-e8e230341a20 127.0.0.1 -2022-09-15T21:42:40.576Z [INFO] agent.router: Initializing LAN area manager -42:40.577 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13484 network=udp -42:40.577 [INFO] TestAgent.server: Adding LAN server: server="Node-bb8fd888-4b4f-8ea1-e465-e8e230341a20 (Addr: tcp/127.0.0.1:13447) (DC: dc1)" -42:40.577 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bb8fd888-4b4f-8ea1-e465-e8e230341a20.dc1 area=wan -42:40.578 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13484 network=tcp -42:40.579 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13485 network=tcp -42:40.580 [INFO] TestAgent: started state syncer -42:40.580 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13448 network=tcp -42:40.617 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:40.617 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13447 [Candidate]" term=2 -42:40.618 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:40.618 [DEBUG] TestAgent.server.raft: vote granted: from=bb8fd888-4b4f-8ea1-e465-e8e230341a20 term=2 tally=1 -42:40.618 [INFO] TestAgent.server.raft: election won: tally=1 -42:40.618 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13447 [Leader]" -42:40.618 [INFO] TestAgent.server: cluster leadership acquired -42:40.618 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13447 -42:40.620 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:40.622 [INFO] TestAgent.server: New leader elected: payload=Node-bb8fd888-4b4f-8ea1-e465-e8e230341a20 -42:40.623 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:40.623 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:40.623 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:40.623 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:40.623 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:40.623 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bb8fd888-4b4f-8ea1-e465-e8e230341a20 -42:40.623 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bb8fd888-4b4f-8ea1-e465-e8e230341a20 -42:40.624 [INFO] TestAgent.server: federation state anti-entropy synced -42:40.872 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:40.873 [INFO] TestAgent: Synced node info -42:40.873 [DEBUG] TestAgent: Node info in sync -42:40.879 [INFO] TestAgent: Requesting shutdown -42:40.879 [INFO] TestAgent.server: shutting down server -42:40.879 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:40.879 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:40.879 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:40.879 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:40.879 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:40.879 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:40.879 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:40.879 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:40.879 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:40.879 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:40.879 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:40.880Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:40.880Z [INFO] agent.router.manager: shutting down -42:40.880 [INFO] TestAgent: consul server down -42:40.880 [INFO] TestAgent: shutdown complete -42:40.880 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13484 network=tcp -42:40.880 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13484 network=udp -42:40.880 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13485 network=tcp -42:41.380 [INFO] TestAgent: Waiting for endpoints to shut down -42:41.380 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogDeregister (0.95s) -=== CONT TestAgent_ReloadConfigTLSConfigFailure -=== CONT TestAgent_JoinWAN_viaMeshGateway - agent_test.go:4395: routing {bob.,}server.dc1.consul => 127.0.0.1:13451 - agent_test.go:4396: routing {betty.,}server.dc2.consul => 127.0.0.1:13450 - agent_test.go:4397: routing {bonnie.,}server.dc3.consul => 127.0.0.1:13554 -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=255)_compressed=false#01 -2022-09-15T21:42:40.356Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:40.356Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:40.416Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:40.416Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:40.426 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:45efda03-6614-dbaf-ce17-5d05a61675fc Address:127.0.0.1:13440}]" -42:40.428 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13440 [Follower]" leader= -42:40.430 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-45efda03-6614-dbaf-ce17-5d05a61675fc.dc2 127.0.0.1 -42:40.440 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-45efda03-6614-dbaf-ce17-5d05a61675fc 127.0.0.1 -2022-09-15T21:42:40.440Z [INFO] agent.router: Initializing LAN area manager -42:40.440 [INFO] TestAgent.server: Adding LAN server: server="Node-45efda03-6614-dbaf-ce17-5d05a61675fc (Addr: tcp/127.0.0.1:13440) (DC: dc2)" -42:40.440 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-45efda03-6614-dbaf-ce17-5d05a61675fc.dc2 area=wan -42:40.440 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13435 network=tcp -42:40.440 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13435 network=udp -42:40.441 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13436 network=tcp -42:40.441 [INFO] TestAgent: started state syncer -42:40.442 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13441 network=tcp -42:40.494 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -42:40.494 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:40.494 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13440 [Candidate]" term=2 -42:40.494 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:40.494 [DEBUG] TestAgent.server.raft: vote granted: from=45efda03-6614-dbaf-ce17-5d05a61675fc term=2 tally=1 -42:40.494 [INFO] TestAgent.server.raft: election won: tally=1 -42:40.494 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13440 [Leader]" -42:40.495 [INFO] TestAgent.server: cluster leadership acquired -42:40.495 [INFO] TestAgent.server: initializing acls -42:40.496 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:40.496 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:40.497 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:40.497 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:40.497 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-45efda03-6614-dbaf-ce17-5d05a61675fc -42:40.497 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-45efda03-6614-dbaf-ce17-5d05a61675fc.dc2 -42:40.499 [INFO] TestAgent.server: New leader elected: payload=Node-45efda03-6614-dbaf-ce17-5d05a61675fc -42:40.499 [INFO] TestAgent.server: Updating LAN server: server="Node-45efda03-6614-dbaf-ce17-5d05a61675fc (Addr: tcp/127.0.0.1:13440) (DC: dc2)" -42:40.499 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-45efda03-6614-dbaf-ce17-5d05a61675fc.dc2 area=wan -42:40.507 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:40.515 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:40.515 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:40.515 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:40.515 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:40.515 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:40.516 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-45efda03-6614-dbaf-ce17-5d05a61675fc -42:40.516 [INFO] TestAgent.server: member joined, marking health alive: member=Node-45efda03-6614-dbaf-ce17-5d05a61675fc -42:40.517 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-45efda03-6614-dbaf-ce17-5d05a61675fc -42:40.518 [INFO] TestAgent.server: federation state anti-entropy synced -42:40.572 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:40.573 [INFO] TestAgent: Synced node info -42:40.573 [DEBUG] TestAgent: Node info in sync -42:40.724 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:13432] -42:40.725 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:13432 -42:40.726 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-910867d4-467c-5e90-9a25-7fdc1b5995a0.dc1 127.0.0.1 -42:40.726 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 -42:40.726 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-910867d4-467c-5e90-9a25-7fdc1b5995a0.dc1 area=wan -42:40.730 [INFO] TestAgent: Requesting shutdown -42:40.730 [INFO] TestAgent.server: shutting down server -42:40.730 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:40.730 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:40.730 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:40.730 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:40.730 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:40.730 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:40.730 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:40.730 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:40.730 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:40.730 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:40.730 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:40.730 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:40.730 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:40.730 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:40.730 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:40.731Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:40.731Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:40.731Z [INFO] agent.router.manager: shutting down -42:40.731 [INFO] TestAgent: consul server down -42:40.731 [INFO] TestAgent: shutdown complete -42:40.731 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13435 network=tcp -42:40.731 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13435 network=udp -42:40.731 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13436 network=tcp -42:41.236 [INFO] TestAgent: Waiting for endpoints to shut down -42:41.236 [INFO] TestAgent: Endpoints down -2022-09-15T21:42:40.039Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:40.039Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:40.132Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:40.132Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:40.143 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:910867d4-467c-5e90-9a25-7fdc1b5995a0 Address:127.0.0.1:13433}]" -42:40.144 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-910867d4-467c-5e90-9a25-7fdc1b5995a0.dc1 127.0.0.1 -42:40.146 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13433 [Follower]" leader= -42:40.149 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-910867d4-467c-5e90-9a25-7fdc1b5995a0 127.0.0.1 -2022-09-15T21:42:40.149Z [INFO] agent.router: Initializing LAN area manager -42:40.159 [INFO] TestAgent.server: Adding LAN server: server="Node-910867d4-467c-5e90-9a25-7fdc1b5995a0 (Addr: tcp/127.0.0.1:13433) (DC: dc1)" -42:40.159 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-910867d4-467c-5e90-9a25-7fdc1b5995a0.dc1 area=wan -42:40.160 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13407 network=tcp -42:40.161 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13407 network=udp -42:40.163 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13408 network=tcp -42:40.163 [INFO] TestAgent: started state syncer -42:40.163 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13434 network=tcp -42:40.202 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:40.202 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13433 [Candidate]" term=2 -42:40.203 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:40.203 [DEBUG] TestAgent.server.raft: vote granted: from=910867d4-467c-5e90-9a25-7fdc1b5995a0 term=2 tally=1 -42:40.203 [INFO] TestAgent.server.raft: election won: tally=1 -42:40.203 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13433 [Leader]" -42:40.203 [INFO] TestAgent.server: cluster leadership acquired -42:40.203 [INFO] TestAgent.server: initializing acls -42:40.204 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:40.205 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:40.205 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:40.205 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:40.205 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-910867d4-467c-5e90-9a25-7fdc1b5995a0 -42:40.206 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-910867d4-467c-5e90-9a25-7fdc1b5995a0.dc1 -42:40.206 [INFO] TestAgent.server: New leader elected: payload=Node-910867d4-467c-5e90-9a25-7fdc1b5995a0 -42:40.206 [INFO] TestAgent.server: Updating LAN server: server="Node-910867d4-467c-5e90-9a25-7fdc1b5995a0 (Addr: tcp/127.0.0.1:13433) (DC: dc1)" -42:40.206 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-910867d4-467c-5e90-9a25-7fdc1b5995a0.dc1 area=wan -42:40.209 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:40.212 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:40.212 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:40.212 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:40.212 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:40.212 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:40.213 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-910867d4-467c-5e90-9a25-7fdc1b5995a0 -42:40.213 [INFO] TestAgent.server: member joined, marking health alive: member=Node-910867d4-467c-5e90-9a25-7fdc1b5995a0 -42:40.215 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-910867d4-467c-5e90-9a25-7fdc1b5995a0 -42:40.216 [INFO] TestAgent.server: federation state anti-entropy synced -42:40.394 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:40.395 [INFO] TestAgent: Synced node info -42:40.725 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:51492 -42:40.726 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-45efda03-6614-dbaf-ce17-5d05a61675fc.dc2 127.0.0.1 -42:40.726 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-45efda03-6614-dbaf-ce17-5d05a61675fc.dc2 area=wan -42:41.236 [INFO] TestAgent: Requesting shutdown -42:41.236 [INFO] TestAgent.server: shutting down server -42:41.236 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:41.236 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:41.236 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:41.236 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:41.236 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:41.236 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:41.236 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:41.236 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:41.236 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:41.236 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:41.236 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:41.236 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:41.236 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:41.236 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:41.236 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:41.237Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:41.237Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:41.237Z [INFO] agent.router.manager: shutting down -42:41.237 [INFO] TestAgent: consul server down -42:41.237 [INFO] TestAgent: shutdown complete -42:41.237 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13407 network=tcp -42:41.237 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13407 network=udp -42:41.237 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13408 network=tcp -42:41.742 [INFO] TestAgent: Waiting for endpoints to shut down -42:41.742 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogNodes_WanTranslation (1.78s) === CONT TestAgent_ReloadConfigIncomingRPCConfig -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:41.127Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:41.184Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:41.195 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ddeaf0cf-bd9c-2ece-a2ff-98aef80ef6ff Address:127.0.0.1:13493}]" -42:41.197 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ddeaf0cf-bd9c-2ece-a2ff-98aef80ef6ff.dc1 127.0.0.1 -42:41.197 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13493 [Follower]" leader= -42:41.197 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ddeaf0cf-bd9c-2ece-a2ff-98aef80ef6ff 127.0.0.1 -2022-09-15T21:42:41.197Z [INFO] agent.router: Initializing LAN area manager -42:41.198 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ddeaf0cf-bd9c-2ece-a2ff-98aef80ef6ff.dc1 area=wan -42:41.198 [INFO] TestAgent.server: Adding LAN server: server="Node-ddeaf0cf-bd9c-2ece-a2ff-98aef80ef6ff (Addr: tcp/127.0.0.1:13493) (DC: dc1)" -42:41.198 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13481 network=udp -42:41.198 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13481 network=tcp -42:41.199 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13482 network=tcp -42:41.200 [INFO] TestAgent: started state syncer -42:41.200 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13494 network=tcp -42:41.250 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:41.250 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13493 [Candidate]" term=2 -42:41.250 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:41.251 [DEBUG] TestAgent.server.raft: vote granted: from=ddeaf0cf-bd9c-2ece-a2ff-98aef80ef6ff term=2 tally=1 -42:41.251 [INFO] TestAgent.server.raft: election won: tally=1 -42:41.251 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13493 [Leader]" -42:41.251 [INFO] TestAgent.server: cluster leadership acquired -42:41.252 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13493 -42:41.254 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:41.258 [INFO] TestAgent.server: New leader elected: payload=Node-ddeaf0cf-bd9c-2ece-a2ff-98aef80ef6ff -42:41.262 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:41.262 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:41.262 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:41.262 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:41.262 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:41.262 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ddeaf0cf-bd9c-2ece-a2ff-98aef80ef6ff -42:41.262 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ddeaf0cf-bd9c-2ece-a2ff-98aef80ef6ff -42:41.265 [INFO] TestAgent.server: federation state anti-entropy synced -42:41.311 [INFO] TestAgent: Requesting shutdown -42:41.311 [INFO] TestAgent.server: shutting down server -42:41.311 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:41.311 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:41.311 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:41.311 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:41.311 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:41.312 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:41.312 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:41.312 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:41.312 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:41.312 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:41.312 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:41.312 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:41.312Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:41.312Z [INFO] agent.router.manager: shutting down -42:41.312 [INFO] TestAgent: consul server down -42:41.312 [INFO] TestAgent: shutdown complete -42:41.312 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13481 network=tcp -42:41.312 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13481 network=udp -42:41.313 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13482 network=tcp -42:41.814 [INFO] TestAgent: Waiting for endpoints to shut down -42:41.814 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_consulConfig_RaftTrailingLogs (0.73s) +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:33:59.089Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:33:59.089Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:59.107Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:33:59.107Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:59.110 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c570a1dc-29d1-a25a-b3a1-a4662da1bba2 Address:127.0.0.1:24470}]" +33:59.111 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24470 [Follower]" leader= +33:59.111 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c570a1dc-29d1-a25a-b3a1-a4662da1bba2.dc2 127.0.0.1 +33:59.112 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c570a1dc-29d1-a25a-b3a1-a4662da1bba2 127.0.0.1 +2021-08-13T15:33:59.112Z [INFO] agent.router: Initializing LAN area manager +33:59.112 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24465 network=udp +33:59.113 [INFO] TestAgent.server: Adding LAN server: server="Node-c570a1dc-29d1-a25a-b3a1-a4662da1bba2 (Addr: tcp/127.0.0.1:24470) (DC: dc2)" +33:59.113 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c570a1dc-29d1-a25a-b3a1-a4662da1bba2.dc2 area=wan +33:59.113 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24465 network=tcp +33:59.114 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24466 network=tcp +33:59.114 [INFO] TestAgent: started state syncer +33:59.114 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24471 network=tcp +33:59.155 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:59.156 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24470 [Candidate]" term=2 +33:59.156 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:59.156 [DEBUG] TestAgent.server.raft: vote granted: from=c570a1dc-29d1-a25a-b3a1-a4662da1bba2 term=2 tally=1 +33:59.156 [INFO] TestAgent.server.raft: election won: tally=1 +33:59.156 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24470 [Leader]" +33:59.156 [INFO] TestAgent.server: cluster leadership acquired +33:59.156 [INFO] TestAgent.server: New leader elected: payload=Node-c570a1dc-29d1-a25a-b3a1-a4662da1bba2 +33:59.156 [INFO] TestAgent.server: initializing acls +33:59.157 [INFO] TestAgent.server: Created ACL 'global-management' policy +33:59.157 [INFO] TestAgent.server: Created ACL anonymous token from configuration +33:59.157 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +33:59.157 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +33:59.157 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-c570a1dc-29d1-a25a-b3a1-a4662da1bba2 +33:59.158 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-c570a1dc-29d1-a25a-b3a1-a4662da1bba2.dc2 +33:59.159 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:59.161 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:59.161 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:59.161 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:59.161 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:59.161 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:59.161 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c570a1dc-29d1-a25a-b3a1-a4662da1bba2 +33:59.161 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c570a1dc-29d1-a25a-b3a1-a4662da1bba2 +33:59.162 [INFO] TestAgent.server: Updating LAN server: server="Node-c570a1dc-29d1-a25a-b3a1-a4662da1bba2 (Addr: tcp/127.0.0.1:24470) (DC: dc2)" +33:59.162 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c570a1dc-29d1-a25a-b3a1-a4662da1bba2 +33:59.162 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-c570a1dc-29d1-a25a-b3a1-a4662da1bba2.dc2 area=wan +33:59.163 [INFO] TestAgent.server: federation state anti-entropy synced +33:59.360 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:24441] +33:59.360 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:24441 +33:59.362 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ba99dc40-f0af-0bc6-4d94-1949abeca13e.dc1 127.0.0.1 +33:59.362 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 +33:59.363 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ba99dc40-f0af-0bc6-4d94-1949abeca13e.dc1 area=wan +33:59.372 [INFO] TestAgent: Requesting shutdown +33:59.372 [INFO] TestAgent.server: shutting down server +33:59.372 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +33:59.372 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:59.372 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:59.372 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:59.372 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:59.372 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +33:59.372 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:59.372 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:59.372 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:59.372 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:59.372 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +33:59.372 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:59.372 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:59.372 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:59.372 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +33:59.372 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:59.372Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:59.372Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:59.372Z [INFO] agent.router.manager: shutting down +33:59.373 [INFO] TestAgent: consul server down +33:59.373 [INFO] TestAgent: shutdown complete +33:59.373 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24465 network=tcp +33:59.373 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24465 network=udp +33:59.373 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24466 network=tcp +33:59.873 [INFO] TestAgent: Waiting for endpoints to shut down +33:59.873 [INFO] TestAgent: Endpoints down +2021-08-13T15:33:58.805Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:33:58.805Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:58.821Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:33:58.821Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:58.824 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ba99dc40-f0af-0bc6-4d94-1949abeca13e Address:127.0.0.1:24442}]" +33:58.824 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24442 [Follower]" leader= +33:58.825 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ba99dc40-f0af-0bc6-4d94-1949abeca13e.dc1 127.0.0.1 +33:58.826 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ba99dc40-f0af-0bc6-4d94-1949abeca13e 127.0.0.1 +2021-08-13T15:33:58.826Z [INFO] agent.router: Initializing LAN area manager +33:58.826 [INFO] TestAgent.server: Adding LAN server: server="Node-ba99dc40-f0af-0bc6-4d94-1949abeca13e (Addr: tcp/127.0.0.1:24442) (DC: dc1)" +33:58.826 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ba99dc40-f0af-0bc6-4d94-1949abeca13e.dc1 area=wan +33:58.826 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24437 network=udp +33:58.826 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24437 network=tcp +33:58.827 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24438 network=tcp +33:58.827 [INFO] TestAgent: started state syncer +33:58.827 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24443 network=tcp +33:58.876 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +33:58.891 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:58.891 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24442 [Candidate]" term=2 +33:58.891 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:58.891 [DEBUG] TestAgent.server.raft: vote granted: from=ba99dc40-f0af-0bc6-4d94-1949abeca13e term=2 tally=1 +33:58.891 [INFO] TestAgent.server.raft: election won: tally=1 +33:58.891 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24442 [Leader]" +33:58.891 [INFO] TestAgent.server: cluster leadership acquired +33:58.892 [INFO] TestAgent.server: New leader elected: payload=Node-ba99dc40-f0af-0bc6-4d94-1949abeca13e +33:58.892 [INFO] TestAgent.server: initializing acls +33:58.892 [INFO] TestAgent.server: Created ACL 'global-management' policy +33:58.893 [INFO] TestAgent.server: Created ACL anonymous token from configuration +33:58.893 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +33:58.893 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +33:58.893 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-ba99dc40-f0af-0bc6-4d94-1949abeca13e +33:58.893 [INFO] TestAgent.server: Updating LAN server: server="Node-ba99dc40-f0af-0bc6-4d94-1949abeca13e (Addr: tcp/127.0.0.1:24442) (DC: dc1)" +33:58.893 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-ba99dc40-f0af-0bc6-4d94-1949abeca13e.dc1 +33:58.893 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-ba99dc40-f0af-0bc6-4d94-1949abeca13e.dc1 area=wan +33:58.895 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:58.897 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:58.897 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:58.897 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:58.897 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:58.897 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:58.897 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ba99dc40-f0af-0bc6-4d94-1949abeca13e +33:58.897 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ba99dc40-f0af-0bc6-4d94-1949abeca13e +33:58.898 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ba99dc40-f0af-0bc6-4d94-1949abeca13e +33:58.899 [INFO] TestAgent.server: federation state anti-entropy synced +33:59.288 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:59.289 [INFO] TestAgent: Synced node info +33:59.289 [DEBUG] TestAgent: Node info in sync +33:59.361 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:60322 +33:59.361 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c570a1dc-29d1-a25a-b3a1-a4662da1bba2.dc2 127.0.0.1 +33:59.362 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c570a1dc-29d1-a25a-b3a1-a4662da1bba2.dc2 area=wan +33:59.873 [INFO] TestAgent: Requesting shutdown +33:59.873 [INFO] TestAgent.server: shutting down server +33:59.873 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:59.873 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +33:59.873 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +33:59.873 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:59.873 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:59.873 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:59.873 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:59.873 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:59.873 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:59.873 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:59.873 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +33:59.873 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +33:59.873 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:59.873 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:59.873 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:59.873Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:59.873Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:59.874Z [INFO] agent.router.manager: shutting down +33:59.874 [INFO] TestAgent: consul server down +33:59.874 [INFO] TestAgent: shutdown complete +33:59.874 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24437 network=tcp +33:59.874 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24437 network=udp +33:59.874 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24438 network=tcp +34:00.374 [INFO] TestAgent: Waiting for endpoints to shut down +34:00.374 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogNodes_WanTranslation (1.59s) === CONT TestAgent_ReloadConfigOutgoingRPCConfig -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:41.120Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:41.145Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:41.154 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ba2a1e7b-b226-2f2d-0fd5-45c5b8413271 Address:127.0.0.1:13479}]" -42:41.156 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ba2a1e7b-b226-2f2d-0fd5-45c5b8413271.dc1 127.0.0.1 -42:41.157 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ba2a1e7b-b226-2f2d-0fd5-45c5b8413271 127.0.0.1 -2022-09-15T21:42:41.157Z [INFO] agent.router: Initializing LAN area manager -42:41.157 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13479 [Follower]" leader= -42:41.158 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13474 network=udp -42:41.162 [INFO] TestAgent.server: Adding LAN server: server="Node-ba2a1e7b-b226-2f2d-0fd5-45c5b8413271 (Addr: tcp/127.0.0.1:13479) (DC: dc1)" -42:41.162 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ba2a1e7b-b226-2f2d-0fd5-45c5b8413271.dc1 area=wan -42:41.165 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13474 network=tcp -42:41.171 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13475 network=tcp -42:41.171 [INFO] TestAgent: started state syncer -42:41.171 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13480 network=tcp -42:41.208 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:41.208 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13479 [Candidate]" term=2 -42:41.208 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:41.208 [DEBUG] TestAgent.server.raft: vote granted: from=ba2a1e7b-b226-2f2d-0fd5-45c5b8413271 term=2 tally=1 -42:41.208 [INFO] TestAgent.server.raft: election won: tally=1 -42:41.208 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13479 [Leader]" -42:41.209 [INFO] TestAgent.server: cluster leadership acquired -42:41.211 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13479 -42:41.213 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:41.214 [INFO] TestAgent.server: New leader elected: payload=Node-ba2a1e7b-b226-2f2d-0fd5-45c5b8413271 -42:41.217 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:41.217 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:41.217 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:41.217 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:41.217 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:41.217 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ba2a1e7b-b226-2f2d-0fd5-45c5b8413271 -42:41.217 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ba2a1e7b-b226-2f2d-0fd5-45c5b8413271 -42:41.218 [INFO] TestAgent.server: federation state anti-entropy synced -42:41.532 [WARN] TestAgent: check has interval below minimum: check=http minimum_interval=1s -42:41.532 [WARN] TestAgent: check has interval below minimum: check=grpc minimum_interval=1s -42:41.532 [INFO] TestAgent: Requesting shutdown -42:41.532 [INFO] TestAgent.server: shutting down server -42:41.532 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:41.532 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:41.532 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:41.532 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:41.532 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:41.533 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:41.533 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:41.533 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:41.533 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:41.533 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:41.533 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:41.533 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:41.546Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:41.546Z [INFO] agent.router.manager: shutting down -42:41.546 [INFO] TestAgent: consul server down -42:41.546 [INFO] TestAgent: shutdown complete -42:41.546 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13474 network=tcp -42:41.546 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13474 network=udp -42:41.546 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13475 network=tcp -42:42.046 [INFO] TestAgent: Waiting for endpoints to shut down -42:42.046 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_RerouteExistingHTTPChecks (0.99s) +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:33:59.706Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:59.721Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:59.723 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c9368ad1-4a27-f677-1adf-2b057736eb49 Address:127.0.0.1:24501}]" +33:59.723 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24501 [Follower]" leader= +33:59.724 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c9368ad1-4a27-f677-1adf-2b057736eb49.dc1 127.0.0.1 +33:59.725 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c9368ad1-4a27-f677-1adf-2b057736eb49 127.0.0.1 +2021-08-13T15:33:59.725Z [INFO] agent.router: Initializing LAN area manager +33:59.725 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24510 network=udp +33:59.726 [INFO] TestAgent.server: Adding LAN server: server="Node-c9368ad1-4a27-f677-1adf-2b057736eb49 (Addr: tcp/127.0.0.1:24501) (DC: dc1)" +33:59.726 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c9368ad1-4a27-f677-1adf-2b057736eb49.dc1 area=wan +33:59.726 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24510 network=tcp +33:59.727 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24511 network=tcp +33:59.727 [INFO] TestAgent: started state syncer +33:59.727 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24502 network=tcp +33:59.795 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:59.795 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24501 [Candidate]" term=2 +33:59.795 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:59.795 [DEBUG] TestAgent.server.raft: vote granted: from=c9368ad1-4a27-f677-1adf-2b057736eb49 term=2 tally=1 +33:59.795 [INFO] TestAgent.server.raft: election won: tally=1 +33:59.795 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24501 [Leader]" +33:59.796 [INFO] TestAgent.server: cluster leadership acquired +33:59.796 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24501 +33:59.797 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:59.804 [INFO] TestAgent.server: New leader elected: payload=Node-c9368ad1-4a27-f677-1adf-2b057736eb49 +33:59.806 [INFO] TestAgent: Synced node info +33:59.810 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:59.811 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:59.811 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:59.811 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:59.811 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:59.811 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c9368ad1-4a27-f677-1adf-2b057736eb49 +33:59.811 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c9368ad1-4a27-f677-1adf-2b057736eb49 +33:59.820 [INFO] TestAgent.server: federation state anti-entropy synced +34:00.028 [WARN] TestAgent: check has interval below minimum: check=http minimum_interval=1s +34:00.029 [WARN] TestAgent: check has interval below minimum: check=grpc minimum_interval=1s +34:00.029 [INFO] TestAgent: Requesting shutdown +34:00.029 [INFO] TestAgent.server: shutting down server +34:00.029 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:00.029 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:00.029 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:00.029 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:00.029 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:00.029 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:00.029 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:00.029 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:00.029 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:00.029 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:00.029 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:00.030Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:00.030Z [INFO] agent.router.manager: shutting down +34:00.030 [INFO] TestAgent: consul server down +34:00.030 [INFO] TestAgent: shutdown complete +34:00.030 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24510 network=tcp +34:00.030 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24510 network=udp +34:00.030 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24511 network=tcp +34:00.531 [INFO] TestAgent: Waiting for endpoints to shut down +34:00.531 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_RerouteNewHTTPChecks (0.84s) === CONT TestAgent_SecurityChecks -2022-09-15T21:42:41.075Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:41.132Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:41.154 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3159b46e-466d-6768-c9c7-9fd55676d056 Address:127.0.0.1:13472}]" -42:41.156 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3159b46e-466d-6768-c9c7-9fd55676d056.dc1 127.0.0.1 -42:41.157 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3159b46e-466d-6768-c9c7-9fd55676d056 127.0.0.1 -2022-09-15T21:42:41.157Z [INFO] agent.router: Initializing LAN area manager -42:41.162 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13472 [Follower]" leader= -42:41.164 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3159b46e-466d-6768-c9c7-9fd55676d056.dc1 area=wan -42:41.164 [INFO] TestAgent.server: Adding LAN server: server="Node-3159b46e-466d-6768-c9c7-9fd55676d056 (Addr: tcp/127.0.0.1:13472) (DC: dc1)" -42:41.165 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13453 network=tcp -42:41.165 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13453 network=udp -42:41.167 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13454 network=tcp -42:41.167 [INFO] TestAgent: started state syncer -42:41.168 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13473 network=tcp -42:41.205 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:41.205 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13472 [Candidate]" term=2 -42:41.205 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:41.205 [DEBUG] TestAgent.server.raft: vote granted: from=3159b46e-466d-6768-c9c7-9fd55676d056 term=2 tally=1 -42:41.205 [INFO] TestAgent.server.raft: election won: tally=1 -42:41.205 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13472 [Leader]" -42:41.206 [INFO] TestAgent.server: cluster leadership acquired -42:41.206 [INFO] TestAgent.server: New leader elected: payload=Node-3159b46e-466d-6768-c9c7-9fd55676d056 -42:41.207 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13472 -42:41.210 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:41.213 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:41.213 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:41.213 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:41.213 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:41.213 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:41.216 [INFO] TestAgent.server: federation state anti-entropy synced -42:41.234 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3159b46e-466d-6768-c9c7-9fd55676d056 -42:41.234 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3159b46e-466d-6768-c9c7-9fd55676d056 -42:41.488 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:41.494 [INFO] TestAgent: Synced node info -42:41.494 [DEBUG] TestAgent: Node info in sync -42:41.566 [WARN] TestAgent: check has interval below minimum: check=http minimum_interval=1s -42:41.567 [WARN] TestAgent: check has interval below minimum: check=grpc minimum_interval=1s -42:41.567 [INFO] TestAgent: Requesting shutdown -42:41.567 [INFO] TestAgent.server: shutting down server -42:41.567 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:41.567 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:41.567 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:41.567 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:41.567 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:41.567 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:41.567 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:41.567 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:41.567 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:41.567 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:41.567 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:41.568Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:41.568Z [INFO] agent.router.manager: shutting down -42:41.568 [INFO] TestAgent: consul server down -42:41.568 [INFO] TestAgent: shutdown complete -42:41.568 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13453 network=tcp -42:41.568 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13453 network=udp -42:41.568 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13454 network=tcp -42:42.069 [INFO] TestAgent: Waiting for endpoints to shut down -42:42.069 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_RerouteNewHTTPChecks (1.04s) +2021-08-13T15:33:59.945Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:59.961Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:59.964 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1c4c72bb-ded2-5f48-aa12-d599d7875a97 Address:127.0.0.1:24522}]" +33:59.965 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1c4c72bb-ded2-5f48-aa12-d599d7875a97.dc1 127.0.0.1 +33:59.965 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1c4c72bb-ded2-5f48-aa12-d599d7875a97 127.0.0.1 +2021-08-13T15:33:59.965Z [INFO] agent.router: Initializing LAN area manager +33:59.966 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24517 network=udp +33:59.966 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24522 [Follower]" leader= +33:59.966 [INFO] TestAgent.server: Adding LAN server: server="Node-1c4c72bb-ded2-5f48-aa12-d599d7875a97 (Addr: tcp/127.0.0.1:24522) (DC: dc1)" +33:59.966 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1c4c72bb-ded2-5f48-aa12-d599d7875a97.dc1 area=wan +33:59.966 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24517 network=tcp +33:59.967 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24518 network=tcp +33:59.967 [INFO] TestAgent: started state syncer +33:59.968 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24523 network=tcp +34:00.010 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:00.010 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24522 [Candidate]" term=2 +34:00.010 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:00.010 [DEBUG] TestAgent.server.raft: vote granted: from=1c4c72bb-ded2-5f48-aa12-d599d7875a97 term=2 tally=1 +34:00.010 [INFO] TestAgent.server.raft: election won: tally=1 +34:00.010 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24522 [Leader]" +34:00.011 [INFO] TestAgent.server: cluster leadership acquired +34:00.011 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24522 +34:00.011 [INFO] TestAgent.server: New leader elected: payload=Node-1c4c72bb-ded2-5f48-aa12-d599d7875a97 +34:00.012 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:00.014 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:00.014 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:00.014 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:00.014 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:00.014 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:00.014 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1c4c72bb-ded2-5f48-aa12-d599d7875a97 +34:00.014 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1c4c72bb-ded2-5f48-aa12-d599d7875a97 +34:00.016 [INFO] TestAgent.server: federation state anti-entropy synced +34:00.047 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:00.048 [INFO] TestAgent: Synced node info +34:00.050 [INFO] TestAgent: Requesting shutdown +34:00.050 [INFO] TestAgent.server: shutting down server +34:00.050 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:00.050 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:00.050 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:00.050 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:00.050 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:00.050 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:00.050 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:00.050 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:00.050 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:00.050 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:00.050 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:00.051Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:00.051Z [INFO] agent.router.manager: shutting down +34:00.051 [INFO] TestAgent: consul server down +34:00.051 [INFO] TestAgent: shutdown complete +34:00.051 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24517 network=tcp +34:00.051 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24517 network=udp +34:00.051 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24518 network=tcp +34:00.551 [INFO] TestAgent: Waiting for endpoints to shut down +34:00.551 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_consulConfig_RaftTrailingLogs (0.62s) === CONT TestAgent_loadTokens -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=255)_compressed=true#01 -2022-09-15T21:42:41.423Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:41.465Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:41.471 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d9c81d94-9030-53f2-4209-2a9a29de105f Address:127.0.0.1:13500}]" -42:41.472 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d9c81d94-9030-53f2-4209-2a9a29de105f.dc1 127.0.0.1 -42:41.473 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d9c81d94-9030-53f2-4209-2a9a29de105f 127.0.0.1 -2022-09-15T21:42:41.473Z [INFO] agent.router: Initializing LAN area manager -42:41.473 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13500 [Follower]" leader= -42:41.474 [INFO] TestAgent.server: Adding LAN server: server="Node-d9c81d94-9030-53f2-4209-2a9a29de105f (Addr: tcp/127.0.0.1:13500) (DC: dc1)" -42:41.474 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d9c81d94-9030-53f2-4209-2a9a29de105f.dc1 area=wan -42:41.474 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13495 network=tcp -42:41.474 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13495 network=udp -42:41.475 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13496 network=tcp -42:41.475 [INFO] TestAgent: started state syncer -42:41.475 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13501 network=tcp -42:41.518 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:41.518 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13500 [Candidate]" term=2 -42:41.518 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:41.518 [DEBUG] TestAgent.server.raft: vote granted: from=d9c81d94-9030-53f2-4209-2a9a29de105f term=2 tally=1 -42:41.518 [INFO] TestAgent.server.raft: election won: tally=1 -42:41.518 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13500 [Leader]" -42:41.519 [INFO] TestAgent.server: cluster leadership acquired -42:41.519 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13500 -42:41.521 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:41.522 [INFO] TestAgent.server: New leader elected: payload=Node-d9c81d94-9030-53f2-4209-2a9a29de105f -42:41.537 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:41.538 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:41.538 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:41.538 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:41.538 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:41.538 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d9c81d94-9030-53f2-4209-2a9a29de105f -42:41.538 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d9c81d94-9030-53f2-4209-2a9a29de105f -42:41.541 [INFO] TestAgent.server: federation state anti-entropy synced -42:41.604 [INFO] TestAgent: Requesting shutdown -42:41.604 [INFO] TestAgent.server: shutting down server -42:41.604 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:41.604 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:41.604 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:41.604 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:41.605 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:41.605 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:41.605 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:41.605 [INFO] TestAgent.server: shutting down Connect CA roots monitor -42:41.605 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:41.605 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:41.605 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:41.605 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:41.605 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:41.605Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:41.605Z [INFO] agent.router.manager: shutting down -42:41.605 [INFO] TestAgent: consul server down -42:41.605 [INFO] TestAgent: shutdown complete -42:41.605 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13495 network=tcp -42:41.606 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13495 network=udp -42:41.606 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13496 network=tcp -42:42.110 [INFO] TestAgent: Waiting for endpoints to shut down -42:42.110 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_consulConfig_AutoEncryptAllowTLS (0.76s) +2021-08-13T15:33:59.770Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:59.829Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:59.831 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a312e0e7-7fa3-e4b5-fb71-eb1cbc1cfcdd Address:127.0.0.1:24515}]" +33:59.831 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24515 [Follower]" leader= +33:59.832 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a312e0e7-7fa3-e4b5-fb71-eb1cbc1cfcdd.dc1 127.0.0.1 +33:59.833 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a312e0e7-7fa3-e4b5-fb71-eb1cbc1cfcdd 127.0.0.1 +2021-08-13T15:33:59.833Z [INFO] agent.router: Initializing LAN area manager +33:59.833 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24503 network=udp +33:59.834 [INFO] TestAgent.server: Adding LAN server: server="Node-a312e0e7-7fa3-e4b5-fb71-eb1cbc1cfcdd (Addr: tcp/127.0.0.1:24515) (DC: dc1)" +33:59.834 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a312e0e7-7fa3-e4b5-fb71-eb1cbc1cfcdd.dc1 area=wan +33:59.834 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24503 network=tcp +33:59.835 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24504 network=tcp +33:59.835 [INFO] TestAgent: started state syncer +33:59.835 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24516 network=tcp +33:59.886 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:59.887 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24515 [Candidate]" term=2 +33:59.887 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:59.887 [DEBUG] TestAgent.server.raft: vote granted: from=a312e0e7-7fa3-e4b5-fb71-eb1cbc1cfcdd term=2 tally=1 +33:59.887 [INFO] TestAgent.server.raft: election won: tally=1 +33:59.887 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24515 [Leader]" +33:59.887 [INFO] TestAgent.server: cluster leadership acquired +33:59.887 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24515 +33:59.887 [INFO] TestAgent.server: New leader elected: payload=Node-a312e0e7-7fa3-e4b5-fb71-eb1cbc1cfcdd +33:59.888 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:59.891 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:59.891 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:59.891 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:59.891 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:59.891 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:59.891 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a312e0e7-7fa3-e4b5-fb71-eb1cbc1cfcdd +33:59.891 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a312e0e7-7fa3-e4b5-fb71-eb1cbc1cfcdd +33:59.891 [INFO] TestAgent.server: federation state anti-entropy synced +33:59.995 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:59.995 [INFO] TestAgent: Synced node info +33:59.995 [DEBUG] TestAgent: Node info in sync +34:00.122 [WARN] TestAgent: check has interval below minimum: check=http minimum_interval=1s +34:00.122 [WARN] TestAgent: check has interval below minimum: check=grpc minimum_interval=1s +34:00.122 [INFO] TestAgent: Requesting shutdown +34:00.123 [INFO] TestAgent.server: shutting down server +34:00.123 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:00.123 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:00.123 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:00.123 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:00.123 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:00.123 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:00.123 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:00.123 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:00.123 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:00.123 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:00.123 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:00.123Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:00.123Z [INFO] agent.router.manager: shutting down +34:00.123 [INFO] TestAgent: consul server down +34:00.123 [INFO] TestAgent: shutdown complete +34:00.123 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24503 network=tcp +34:00.123 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24503 network=udp +34:00.123 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24504 network=tcp +34:00.624 [INFO] TestAgent: Waiting for endpoints to shut down +34:00.624 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_RerouteExistingHTTPChecks (0.87s) === CONT TestAgent_reloadWatchesHTTPS -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=255)_compressed=false#01 -[WARN] freeport: 4 out of 25 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._tcp_(qType:=255)_compressed=true#01 +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! === RUN TestAgent_loadTokens/original-configuration === RUN TestAgent_loadTokens/updated-configuration === RUN TestAgent_loadTokens/persisted-tokens @@ -34860,245 +23818,299 @@ === RUN TestAgent_loadTokens/partial-persisted === RUN TestAgent_loadTokens/persistence-error-not-json === RUN TestAgent_loadTokens/persistence-error-wrong-top-level -2022-09-15T21:42:41.973Z [WARN] TestAgent_ReloadConfigTLSConfigFailure: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:41.459Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:41.519Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:41.531 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2119d10c-e796-b526-a431-cdd9ad497b7d Address:127.0.0.1:13507}]" -42:41.533 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2119d10c-e796-b526-a431-cdd9ad497b7d.dc1 127.0.0.1 -42:41.534 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13507 [Follower]" leader= -42:41.558 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2119d10c-e796-b526-a431-cdd9ad497b7d 127.0.0.1 -2022-09-15T21:42:41.558Z [INFO] agent.router: Initializing LAN area manager -42:41.559 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13502 network=udp -42:41.559 [INFO] TestAgent.server: Adding LAN server: server="Node-2119d10c-e796-b526-a431-cdd9ad497b7d (Addr: tcp/127.0.0.1:13507) (DC: dc1)" -42:41.559 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2119d10c-e796-b526-a431-cdd9ad497b7d.dc1 area=wan -42:41.559 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13502 network=tcp -42:41.561 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13503 network=tcp -42:41.561 [INFO] TestAgent: started state syncer -42:41.561 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13508 network=tcp -42:41.586 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:41.586 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13507 [Candidate]" term=2 -42:41.587 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:41.587 [DEBUG] TestAgent.server.raft: vote granted: from=2119d10c-e796-b526-a431-cdd9ad497b7d term=2 tally=1 -42:41.587 [INFO] TestAgent.server.raft: election won: tally=1 -42:41.587 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13507 [Leader]" -42:41.587 [INFO] TestAgent.server: cluster leadership acquired -42:41.587 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13507 -42:41.588 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:41.590 [INFO] TestAgent.server: New leader elected: payload=Node-2119d10c-e796-b526-a431-cdd9ad497b7d -42:41.591 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:41.591 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:41.591 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:41.591 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:41.591 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:41.591 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2119d10c-e796-b526-a431-cdd9ad497b7d -42:41.591 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2119d10c-e796-b526-a431-cdd9ad497b7d -42:41.592 [INFO] TestAgent.server: federation state anti-entropy synced -42:41.739 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:41.743 [INFO] TestAgent: Synced node info -42:41.743 [DEBUG] TestAgent: Node info in sync -42:41.843 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:41.843 [DEBUG] TestAgent: Node info in sync -42:41.973 [INFO] TestAgent: Requesting shutdown -42:41.973 [INFO] TestAgent.server: shutting down server -42:41.973 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:41.974 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:41.974 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:41.974Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:41.974Z [INFO] agent.router.manager: shutting down -42:41.974 [INFO] TestAgent: consul server down -42:41.974 [INFO] TestAgent: shutdown complete -42:41.974 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13502 network=tcp -42:41.974 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13502 network=udp -42:41.974 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13503 network=tcp -42:42.475 [INFO] TestAgent: Waiting for endpoints to shut down -42:42.475 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_ReloadConfigTLSConfigFailure (1.09s) +2021-08-13T15:34:00.012Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:00.031Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:00.034 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0a43a3bf-954b-00d0-ba17-dc67c6fc3edc Address:127.0.0.1:24529}]" +34:00.034 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24529 [Follower]" leader= +34:00.035 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0a43a3bf-954b-00d0-ba17-dc67c6fc3edc.dc1 127.0.0.1 +34:00.036 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0a43a3bf-954b-00d0-ba17-dc67c6fc3edc 127.0.0.1 +2021-08-13T15:34:00.036Z [INFO] agent.router: Initializing LAN area manager +34:00.036 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24524 network=udp +34:00.037 [INFO] TestAgent.server: Adding LAN server: server="Node-0a43a3bf-954b-00d0-ba17-dc67c6fc3edc (Addr: tcp/127.0.0.1:24529) (DC: dc1)" +34:00.037 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0a43a3bf-954b-00d0-ba17-dc67c6fc3edc.dc1 area=wan +34:00.037 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24524 network=tcp +34:00.038 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24525 network=tcp +34:00.038 [INFO] TestAgent: started state syncer +34:00.038 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24530 network=tcp +34:00.074 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:00.074 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24529 [Candidate]" term=2 +34:00.075 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:00.075 [DEBUG] TestAgent.server.raft: vote granted: from=0a43a3bf-954b-00d0-ba17-dc67c6fc3edc term=2 tally=1 +34:00.075 [INFO] TestAgent.server.raft: election won: tally=1 +34:00.075 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24529 [Leader]" +34:00.075 [INFO] TestAgent.server: cluster leadership acquired +34:00.075 [INFO] TestAgent.server: New leader elected: payload=Node-0a43a3bf-954b-00d0-ba17-dc67c6fc3edc +34:00.075 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24529 +34:00.076 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:00.079 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:00.079 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:00.079 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:00.079 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:00.079 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:00.079 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0a43a3bf-954b-00d0-ba17-dc67c6fc3edc +34:00.079 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0a43a3bf-954b-00d0-ba17-dc67c6fc3edc +34:00.079 [INFO] TestAgent.server: federation state anti-entropy synced +34:00.134 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:00.135 [INFO] TestAgent: Synced node info +34:00.368 [INFO] TestAgent: Requesting shutdown +34:00.368 [INFO] TestAgent.server: shutting down server +34:00.368 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:00.368 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:00.368 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:00.368 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:00.368 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:00.368 [INFO] TestAgent.server: shutting down Connect CA roots monitor +34:00.368 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:00.368 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:00.368 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:00.368 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:00.368 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:00.368 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:00.368Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:00.368Z [INFO] agent.router.manager: shutting down +34:00.368 [INFO] TestAgent: consul server down +34:00.368 [INFO] TestAgent: shutdown complete +34:00.368 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24524 network=tcp +34:00.368 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24524 network=udp +34:00.369 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24525 network=tcp +34:00.869 [INFO] TestAgent: Waiting for endpoints to shut down +34:00.869 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_consulConfig_AutoEncryptAllowTLS (0.87s) === CONT TestAgent_reloadWatches -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:42.087Z [WARN] TestAgent_ReloadConfigIncomingRPCConfig: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:41.805Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:41.859Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:41.874 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:35256248-04e9-bd6a-c7f0-e653d759d17e Address:127.0.0.1:13535}]" -42:41.876 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13535 [Follower]" leader= -42:41.886 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-35256248-04e9-bd6a-c7f0-e653d759d17e.dc1 127.0.0.1 -42:41.887 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-35256248-04e9-bd6a-c7f0-e653d759d17e 127.0.0.1 -2022-09-15T21:42:41.887Z [INFO] agent.router: Initializing LAN area manager -42:41.888 [INFO] TestAgent.server: Adding LAN server: server="Node-35256248-04e9-bd6a-c7f0-e653d759d17e (Addr: tcp/127.0.0.1:13535) (DC: dc1)" -42:41.888 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-35256248-04e9-bd6a-c7f0-e653d759d17e.dc1 area=wan -42:41.888 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13509 network=udp -42:41.888 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13509 network=tcp -42:41.889 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13510 network=tcp -42:41.889 [INFO] TestAgent: started state syncer -42:41.889 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13536 network=tcp -42:41.946 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:41.946 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13535 [Candidate]" term=2 -42:41.946 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:41.946 [DEBUG] TestAgent.server.raft: vote granted: from=35256248-04e9-bd6a-c7f0-e653d759d17e term=2 tally=1 -42:41.946 [INFO] TestAgent.server.raft: election won: tally=1 -42:41.946 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13535 [Leader]" -42:41.946 [INFO] TestAgent.server: cluster leadership acquired -42:41.947 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13535 -42:41.948 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:41.956 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:41.956 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:41.956 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:41.956 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:41.956 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:41.957 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-35256248-04e9-bd6a-c7f0-e653d759d17e -42:41.957 [INFO] TestAgent.server: member joined, marking health alive: member=Node-35256248-04e9-bd6a-c7f0-e653d759d17e -42:41.960 [INFO] TestAgent.server: New leader elected: payload=Node-35256248-04e9-bd6a-c7f0-e653d759d17e -42:41.968 [INFO] TestAgent.server: federation state anti-entropy synced -42:42.017 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:42.018 [INFO] TestAgent: Synced node info -42:42.089 [INFO] TestAgent: Requesting shutdown -42:42.097 [INFO] TestAgent.server: shutting down server -42:42.098 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:42.098 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:42.098 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:42.098Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:42.098Z [INFO] agent.router.manager: shutting down -42:42.098 [INFO] TestAgent: consul server down -42:42.098 [INFO] TestAgent: shutdown complete -42:42.098 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13509 network=tcp -42:42.098 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13509 network=udp -42:42.098 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13510 network=tcp -42:42.602 [INFO] TestAgent: Waiting for endpoints to shut down -42:42.602 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_ReloadConfigIncomingRPCConfig (0.86s) +2021-08-13T15:34:00.458Z [WARN] TestAgent_ReloadConfigTLSConfigFailure: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:00.100Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:00.117Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:00.121 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:60954432-3ff7-6f2a-4b72-18c8de4599e9 Address:127.0.0.1:24536}]" +34:00.121 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24536 [Follower]" leader= +34:00.121 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-60954432-3ff7-6f2a-4b72-18c8de4599e9.dc1 127.0.0.1 +34:00.122 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-60954432-3ff7-6f2a-4b72-18c8de4599e9 127.0.0.1 +2021-08-13T15:34:00.122Z [INFO] agent.router: Initializing LAN area manager +34:00.123 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24531 network=udp +34:00.123 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-60954432-3ff7-6f2a-4b72-18c8de4599e9.dc1 area=wan +34:00.123 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24531 network=tcp +34:00.124 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24532 network=tcp +34:00.124 [INFO] TestAgent: started state syncer +34:00.124 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24537 network=tcp +34:00.125 [INFO] TestAgent.server: Adding LAN server: server="Node-60954432-3ff7-6f2a-4b72-18c8de4599e9 (Addr: tcp/127.0.0.1:24536) (DC: dc1)" +34:00.182 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:00.182 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24536 [Candidate]" term=2 +34:00.182 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:00.182 [DEBUG] TestAgent.server.raft: vote granted: from=60954432-3ff7-6f2a-4b72-18c8de4599e9 term=2 tally=1 +34:00.182 [INFO] TestAgent.server.raft: election won: tally=1 +34:00.182 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24536 [Leader]" +34:00.182 [INFO] TestAgent.server: cluster leadership acquired +34:00.182 [INFO] TestAgent.server: New leader elected: payload=Node-60954432-3ff7-6f2a-4b72-18c8de4599e9 +34:00.182 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24536 +34:00.184 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:00.186 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:00.186 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:00.186 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:00.186 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:00.186 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:00.186 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-60954432-3ff7-6f2a-4b72-18c8de4599e9 +34:00.186 [INFO] TestAgent.server: member joined, marking health alive: member=Node-60954432-3ff7-6f2a-4b72-18c8de4599e9 +34:00.187 [INFO] TestAgent.server: federation state anti-entropy synced +34:00.269 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:00.270 [INFO] TestAgent: Synced node info +34:00.458 [INFO] TestAgent: Requesting shutdown +34:00.458 [INFO] TestAgent.server: shutting down server +34:00.458 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:00.458 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:00.458 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:00.458Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:00.458Z [INFO] agent.router.manager: shutting down +34:00.459 [INFO] TestAgent: consul server down +34:00.459 [INFO] TestAgent: shutdown complete +34:00.459 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24531 network=tcp +34:00.459 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24531 network=udp +34:00.459 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24532 network=tcp +34:00.959 [INFO] TestAgent: Waiting for endpoints to shut down +34:00.959 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_ReloadConfigTLSConfigFailure (0.88s) === CONT TestAgent_GetCoordinate -2022-09-15T21:42:42.202Z [WARN] TestAgent_ReloadConfigOutgoingRPCConfig: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:41.888Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:41.930Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:41.940 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:475c01c4-337e-14d5-b10f-b7c8b893ba74 Address:127.0.0.1:13521}]" -42:41.941 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-475c01c4-337e-14d5-b10f-b7c8b893ba74.dc1 127.0.0.1 -42:41.942 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13521 [Follower]" leader= -42:41.944 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-475c01c4-337e-14d5-b10f-b7c8b893ba74 127.0.0.1 -2022-09-15T21:42:41.944Z [INFO] agent.router: Initializing LAN area manager -42:41.944 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13537 network=udp -42:41.944 [INFO] TestAgent.server: Adding LAN server: server="Node-475c01c4-337e-14d5-b10f-b7c8b893ba74 (Addr: tcp/127.0.0.1:13521) (DC: dc1)" -42:41.944 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-475c01c4-337e-14d5-b10f-b7c8b893ba74.dc1 area=wan -42:41.945 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13537 network=tcp -42:41.958 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13538 network=tcp -42:41.958 [INFO] TestAgent: started state syncer -42:41.958 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13522 network=tcp -42:42.015 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:42.015 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13521 [Candidate]" term=2 -42:42.028 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:42.028 [DEBUG] TestAgent.server.raft: vote granted: from=475c01c4-337e-14d5-b10f-b7c8b893ba74 term=2 tally=1 -42:42.028 [INFO] TestAgent.server.raft: election won: tally=1 -42:42.028 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13521 [Leader]" -42:42.029 [INFO] TestAgent.server: cluster leadership acquired -42:42.029 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13521 -42:42.034 [INFO] TestAgent.server: New leader elected: payload=Node-475c01c4-337e-14d5-b10f-b7c8b893ba74 -42:42.039 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:42.045 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:42.045 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:42.045 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:42.045 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:42.045 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:42.046 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-475c01c4-337e-14d5-b10f-b7c8b893ba74 -42:42.046 [INFO] TestAgent.server: member joined, marking health alive: member=Node-475c01c4-337e-14d5-b10f-b7c8b893ba74 -42:42.047 [INFO] TestAgent.server: federation state anti-entropy synced -42:42.204 [INFO] TestAgent: Requesting shutdown -42:42.204 [INFO] TestAgent.server: shutting down server -42:42.204 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:42.205 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:42.205 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:42.205 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:42.205Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:42.205Z [INFO] agent.router.manager: shutting down -42:42.205 [INFO] TestAgent: consul server down -42:42.205 [INFO] TestAgent: shutdown complete -42:42.205 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13537 network=tcp -42:42.205 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13537 network=udp -42:42.206 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13538 network=tcp -42:42.707 [INFO] TestAgent: Waiting for endpoints to shut down -42:42.707 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_ReloadConfigOutgoingRPCConfig (0.89s) +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:00.524Z [WARN] TestAgent_ReloadConfigOutgoingRPCConfig: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:00.392Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:00.409Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:00.413 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:04522398-e42d-e2c7-945d-cb5a462e7695 Address:127.0.0.1:24550}]" +34:00.413 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24550 [Follower]" leader= +34:00.414 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-04522398-e42d-e2c7-945d-cb5a462e7695.dc1 127.0.0.1 +34:00.414 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-04522398-e42d-e2c7-945d-cb5a462e7695 127.0.0.1 +2021-08-13T15:34:00.414Z [INFO] agent.router: Initializing LAN area manager +34:00.415 [INFO] TestAgent.server: Adding LAN server: server="Node-04522398-e42d-e2c7-945d-cb5a462e7695 (Addr: tcp/127.0.0.1:24550) (DC: dc1)" +34:00.415 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-04522398-e42d-e2c7-945d-cb5a462e7695.dc1 area=wan +34:00.415 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24545 network=udp +34:00.415 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24545 network=tcp +34:00.416 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24546 network=tcp +34:00.416 [INFO] TestAgent: started state syncer +34:00.416 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24551 network=tcp +34:00.471 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:00.471 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24550 [Candidate]" term=2 +34:00.471 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:00.471 [DEBUG] TestAgent.server.raft: vote granted: from=04522398-e42d-e2c7-945d-cb5a462e7695 term=2 tally=1 +34:00.471 [INFO] TestAgent.server.raft: election won: tally=1 +34:00.471 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24550 [Leader]" +34:00.471 [INFO] TestAgent.server: cluster leadership acquired +34:00.472 [INFO] TestAgent.server: New leader elected: payload=Node-04522398-e42d-e2c7-945d-cb5a462e7695 +34:00.472 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24550 +34:00.473 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:00.475 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:00.475 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:00.475 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:00.475 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:00.475 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:00.476 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-04522398-e42d-e2c7-945d-cb5a462e7695 +34:00.476 [INFO] TestAgent.server: member joined, marking health alive: member=Node-04522398-e42d-e2c7-945d-cb5a462e7695 +34:00.477 [INFO] TestAgent.server: federation state anti-entropy synced +34:00.526 [INFO] TestAgent: Requesting shutdown +34:00.526 [INFO] TestAgent.server: shutting down server +34:00.526 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:00.526 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:00.526 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:00.526 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:00.526Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:00.526Z [INFO] agent.router.manager: shutting down +34:00.526 [INFO] TestAgent: consul server down +34:00.526 [INFO] TestAgent: shutdown complete +34:00.526 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24545 network=tcp +34:00.526 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24545 network=udp +34:00.526 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24546 network=tcp +34:01.027 [INFO] TestAgent: Waiting for endpoints to shut down +34:01.027 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_ReloadConfigOutgoingRPCConfig (0.65s) === CONT TestAgent_purgeCheckState -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:42.121Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:42.202Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:42.207 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f4d3965a-4676-d004-0eea-7c277491f898 Address:127.0.0.1:13549}]" -42:42.207 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13549 [Follower]" leader= -42:42.208 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f4d3965a-4676-d004-0eea-7c277491f898.dc1 127.0.0.1 -42:42.214 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f4d3965a-4676-d004-0eea-7c277491f898 127.0.0.1 -2022-09-15T21:42:42.214Z [INFO] agent.router: Initializing LAN area manager -42:42.215 [INFO] TestAgent.server: Adding LAN server: server="Node-f4d3965a-4676-d004-0eea-7c277491f898 (Addr: tcp/127.0.0.1:13549) (DC: dc1)" -42:42.215 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f4d3965a-4676-d004-0eea-7c277491f898.dc1 area=wan -42:42.215 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13530 network=udp -42:42.215 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13530 network=tcp -42:42.216 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13531 network=tcp -42:42.216 [INFO] TestAgent: started state syncer -42:42.216 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13550 network=tcp -42:42.266 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -42:42.274 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:42.274 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13549 [Candidate]" term=2 -42:42.274 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:42.274 [DEBUG] TestAgent.server.raft: vote granted: from=f4d3965a-4676-d004-0eea-7c277491f898 term=2 tally=1 -42:42.274 [INFO] TestAgent.server.raft: election won: tally=1 -42:42.274 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13549 [Leader]" -42:42.274 [INFO] TestAgent.server: cluster leadership acquired -42:42.275 [INFO] TestAgent.server: initializing acls -42:42.275 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:42.276 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:42.276 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:42.276 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:42.276 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-f4d3965a-4676-d004-0eea-7c277491f898 -42:42.276 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-f4d3965a-4676-d004-0eea-7c277491f898.dc1 -42:42.276 [INFO] TestAgent.server: New leader elected: payload=Node-f4d3965a-4676-d004-0eea-7c277491f898 -42:42.277 [INFO] TestAgent.server: Updating LAN server: server="Node-f4d3965a-4676-d004-0eea-7c277491f898 (Addr: tcp/127.0.0.1:13549) (DC: dc1)" -42:42.277 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-f4d3965a-4676-d004-0eea-7c277491f898.dc1 area=wan -42:42.277 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:42.283 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:42.283 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:42.283 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:42.283 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:42.283 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:42.283 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f4d3965a-4676-d004-0eea-7c277491f898 -42:42.283 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f4d3965a-4676-d004-0eea-7c277491f898 -42:42.284 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f4d3965a-4676-d004-0eea-7c277491f898 -42:42.284 [INFO] TestAgent.server: federation state anti-entropy synced -42:42.332 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="ACL not found" -42:42.422 [WARN] TestAgent: "default" token present in both the configuration and persisted token store, using the persisted token -42:42.422 [WARN] TestAgent: "agent" token present in both the configuration and persisted token store, using the persisted token -42:42.422 [WARN] TestAgent: "agent_master" token present in both the configuration and persisted token store, using the persisted token -42:42.422 [WARN] TestAgent: "replication" token present in both the configuration and persisted token store, using the persisted token -42:42.422 [WARN] TestAgent: "default" token present in both the configuration and persisted token store, using the persisted token -42:42.422 [WARN] TestAgent: "agent" token present in both the configuration and persisted token store, using the persisted token -42:42.422 [WARN] TestAgent: "agent_master" token present in both the configuration and persisted token store, using the persisted token -42:42.422 [WARN] TestAgent: "replication" token present in both the configuration and persisted token store, using the persisted token -42:42.422 [WARN] TestAgent: "agent" token present in both the configuration and persisted token store, using the persisted token -42:42.422 [WARN] TestAgent: "agent_master" token present in both the configuration and persisted token store, using the persisted token -42:42.423 [WARN] TestAgent: unable to load persisted tokens: error="failed to decode tokens file "/tmp/consul-test/TestAgent_loadTokens-TestAgent-agent031003756/acl-tokens.json": invalid character '\x01' looking for beginning of value" -42:42.423 [WARN] TestAgent: unable to load persisted tokens: error="failed to decode tokens file "/tmp/consul-test/TestAgent_loadTokens-TestAgent-agent031003756/acl-tokens.json": json: cannot unmarshal array into Go value of type agent.persistedTokens" -42:42.423 [INFO] TestAgent: Requesting shutdown -42:42.423 [INFO] TestAgent.server: shutting down server -42:42.423 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:42.423 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:42.423 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:42.423 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:42.423 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:42.423 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:42.423 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:42.423 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:42.423 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:42.423 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:42.423 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:42.423 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:42.423 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:42.423 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:42.424 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:42.424Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:42.424Z [INFO] agent.router.manager: shutting down -42:42.424 [INFO] TestAgent: consul server down -42:42.424 [INFO] TestAgent: shutdown complete -42:42.424 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13530 network=tcp -42:42.424 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13530 network=udp -42:42.424 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13531 network=tcp -42:42.926 [INFO] TestAgent: Waiting for endpoints to shut down -42:42.926 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_loadTokens (0.86s) +2021-08-13T15:34:00.727Z [WARN] TestAgent_ReloadConfigIncomingRPCConfig: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:00.277Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:00.323Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:00.335 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cbd5bc49-ddf4-fc51-9c35-f3e132d2339b Address:127.0.0.1:24543}]" +34:00.337 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-cbd5bc49-ddf4-fc51-9c35-f3e132d2339b.dc1 127.0.0.1 +34:00.338 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-cbd5bc49-ddf4-fc51-9c35-f3e132d2339b 127.0.0.1 +2021-08-13T15:34:00.338Z [INFO] agent.router: Initializing LAN area manager +34:00.338 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24538 network=udp +34:00.338 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24543 [Follower]" leader= +34:00.339 [INFO] TestAgent.server: Adding LAN server: server="Node-cbd5bc49-ddf4-fc51-9c35-f3e132d2339b (Addr: tcp/127.0.0.1:24543) (DC: dc1)" +34:00.339 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-cbd5bc49-ddf4-fc51-9c35-f3e132d2339b.dc1 area=wan +34:00.339 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24538 network=tcp +34:00.341 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24539 network=tcp +34:00.341 [INFO] TestAgent: started state syncer +34:00.341 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24544 network=tcp +34:00.397 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:00.397 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24543 [Candidate]" term=2 +34:00.398 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:00.398 [DEBUG] TestAgent.server.raft: vote granted: from=cbd5bc49-ddf4-fc51-9c35-f3e132d2339b term=2 tally=1 +34:00.398 [INFO] TestAgent.server.raft: election won: tally=1 +34:00.398 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24543 [Leader]" +34:00.398 [INFO] TestAgent.server: cluster leadership acquired +34:00.398 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24543 +34:00.399 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:00.400 [INFO] TestAgent.server: New leader elected: payload=Node-cbd5bc49-ddf4-fc51-9c35-f3e132d2339b +34:00.402 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:00.402 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:00.402 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:00.402 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:00.402 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:00.402 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cbd5bc49-ddf4-fc51-9c35-f3e132d2339b +34:00.402 [INFO] TestAgent.server: member joined, marking health alive: member=Node-cbd5bc49-ddf4-fc51-9c35-f3e132d2339b +34:00.403 [INFO] TestAgent.server: federation state anti-entropy synced +34:00.724 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:00.725 [INFO] TestAgent: Synced node info +34:00.729 [INFO] TestAgent: Requesting shutdown +34:00.729 [INFO] TestAgent.server: shutting down server +34:00.729 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:00.729 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:00.729 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:00.730Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:00.730Z [INFO] agent.router.manager: shutting down +34:00.730 [INFO] TestAgent: consul server down +34:00.730 [INFO] TestAgent: shutdown complete +34:00.730 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24538 network=tcp +34:00.730 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24538 network=udp +34:00.730 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24539 network=tcp +34:01.230 [INFO] TestAgent: Waiting for endpoints to shut down +34:01.230 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_ReloadConfigIncomingRPCConfig (0.97s) +=== CONT TestAgent_loadCheckState +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:00.568Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:00.584Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:00.587 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d99015fb-5ba2-642a-f8bf-b3576a558db9 Address:127.0.0.1:24564}]" +34:00.587 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24564 [Follower]" leader= +34:00.588 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d99015fb-5ba2-642a-f8bf-b3576a558db9.dc1 127.0.0.1 +34:00.589 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d99015fb-5ba2-642a-f8bf-b3576a558db9 127.0.0.1 +2021-08-13T15:34:00.589Z [INFO] agent.router: Initializing LAN area manager +34:00.589 [INFO] TestAgent.server: Adding LAN server: server="Node-d99015fb-5ba2-642a-f8bf-b3576a558db9 (Addr: tcp/127.0.0.1:24564) (DC: dc1)" +34:00.589 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d99015fb-5ba2-642a-f8bf-b3576a558db9.dc1 area=wan +34:00.589 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24559 network=tcp +34:00.589 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24559 network=udp +34:00.590 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24560 network=tcp +34:00.590 [INFO] TestAgent: started state syncer +34:00.590 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24565 network=tcp +34:00.633 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:00.633 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24564 [Candidate]" term=2 +34:00.634 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:00.634 [DEBUG] TestAgent.server.raft: vote granted: from=d99015fb-5ba2-642a-f8bf-b3576a558db9 term=2 tally=1 +34:00.634 [INFO] TestAgent.server.raft: election won: tally=1 +34:00.634 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24564 [Leader]" +34:00.634 [INFO] TestAgent.server: cluster leadership acquired +34:00.634 [INFO] TestAgent.server: initializing acls +34:00.634 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:00.635 [INFO] TestAgent.server: New leader elected: payload=Node-d99015fb-5ba2-642a-f8bf-b3576a558db9 +34:00.635 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:00.635 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:00.635 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:00.636 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-d99015fb-5ba2-642a-f8bf-b3576a558db9 +34:00.636 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-d99015fb-5ba2-642a-f8bf-b3576a558db9.dc1 +34:00.637 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:00.638 [INFO] TestAgent.server: Updating LAN server: server="Node-d99015fb-5ba2-642a-f8bf-b3576a558db9 (Addr: tcp/127.0.0.1:24564) (DC: dc1)" +34:00.638 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-d99015fb-5ba2-642a-f8bf-b3576a558db9.dc1 area=wan +34:00.644 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:00.644 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:00.644 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:00.644 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:00.644 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:00.644 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d99015fb-5ba2-642a-f8bf-b3576a558db9 +34:00.644 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d99015fb-5ba2-642a-f8bf-b3576a558db9 +34:00.645 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d99015fb-5ba2-642a-f8bf-b3576a558db9 +34:00.646 [INFO] TestAgent.server: federation state anti-entropy synced +34:00.814 [WARN] TestAgent: "default" token present in both the configuration and persisted token store, using the persisted token +34:00.814 [WARN] TestAgent: "agent" token present in both the configuration and persisted token store, using the persisted token +34:00.814 [WARN] TestAgent: "agent_master" token present in both the configuration and persisted token store, using the persisted token +34:00.814 [WARN] TestAgent: "replication" token present in both the configuration and persisted token store, using the persisted token +34:00.815 [WARN] TestAgent: "default" token present in both the configuration and persisted token store, using the persisted token +34:00.815 [WARN] TestAgent: "agent" token present in both the configuration and persisted token store, using the persisted token +34:00.815 [WARN] TestAgent: "agent_master" token present in both the configuration and persisted token store, using the persisted token +34:00.815 [WARN] TestAgent: "replication" token present in both the configuration and persisted token store, using the persisted token +34:00.815 [WARN] TestAgent: "agent" token present in both the configuration and persisted token store, using the persisted token +34:00.815 [WARN] TestAgent: "agent_master" token present in both the configuration and persisted token store, using the persisted token +34:00.815 [WARN] TestAgent: unable to load persisted tokens: error="failed to decode tokens file "/tmp/consul-test/TestAgent_loadTokens-TestAgent-agent180615254/acl-tokens.json": invalid character '\x01' looking for beginning of value" +34:00.815 [WARN] TestAgent: unable to load persisted tokens: error="failed to decode tokens file "/tmp/consul-test/TestAgent_loadTokens-TestAgent-agent180615254/acl-tokens.json": json: cannot unmarshal array into Go value of type agent.persistedTokens" +34:00.815 [INFO] TestAgent: Requesting shutdown +34:00.816 [INFO] TestAgent.server: shutting down server +34:00.816 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:00.816 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:00.816 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:00.816 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:00.816 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:00.816 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:00.816 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:00.816 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:00.816 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:00.816 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:00.816 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:00.816 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:00.816 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +34:00.816 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:00.816 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +2021-08-13T15:34:00.816Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:00.816Z [INFO] agent.router.manager: shutting down +34:00.816 [INFO] TestAgent: consul server down +34:00.816 [INFO] TestAgent: shutdown complete +34:00.816 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24559 network=tcp +34:00.816 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24559 network=udp +34:00.816 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24560 network=tcp +34:01.317 [INFO] TestAgent: Waiting for endpoints to shut down +34:01.317 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_loadTokens (0.77s) --- PASS: TestAgent_loadTokens/original-configuration (0.00s) --- PASS: TestAgent_loadTokens/updated-configuration (0.00s) --- PASS: TestAgent_loadTokens/persisted-tokens (0.00s) @@ -35106,5275 +24118,7910 @@ --- PASS: TestAgent_loadTokens/partial-persisted (0.00s) --- PASS: TestAgent_loadTokens/persistence-error-not-json (0.00s) --- PASS: TestAgent_loadTokens/persistence-error-wrong-top-level (0.00s) -=== CONT TestAgent_loadCheckState -2022-09-15T21:42:42.149Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:42.198Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:42.204 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ef9ee48d-7a29-7cfd-0d2b-e399bb2b31b1 Address:127.0.0.1:13569}]" -42:42.205 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13569 [Follower]" leader= -42:42.206 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ef9ee48d-7a29-7cfd-0d2b-e399bb2b31b1.dc1 127.0.0.1 -42:42.209 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ef9ee48d-7a29-7cfd-0d2b-e399bb2b31b1 127.0.0.1 -2022-09-15T21:42:42.209Z [INFO] agent.router: Initializing LAN area manager -42:42.210 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ef9ee48d-7a29-7cfd-0d2b-e399bb2b31b1.dc1 area=wan -42:42.210 [INFO] TestAgent.server: Adding LAN server: server="Node-ef9ee48d-7a29-7cfd-0d2b-e399bb2b31b1 (Addr: tcp/127.0.0.1:13569) (DC: dc1)" -42:42.211 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13551 network=udp -42:42.211 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13551 network=tcp -42:42.212 [INFO] TestAgent: Started HTTPS server: address=127.0.0.1:13553 network=tcp -42:42.212 [INFO] TestAgent: started state syncer -42:42.212 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13570 network=tcp -42:42.256 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:42.256 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13569 [Candidate]" term=2 -42:42.256 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:42.256 [DEBUG] TestAgent.server.raft: vote granted: from=ef9ee48d-7a29-7cfd-0d2b-e399bb2b31b1 term=2 tally=1 -42:42.256 [INFO] TestAgent.server.raft: election won: tally=1 -42:42.256 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13569 [Leader]" -42:42.257 [INFO] TestAgent.server: cluster leadership acquired -42:42.257 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13569 -42:42.257 [INFO] TestAgent.server: New leader elected: payload=Node-ef9ee48d-7a29-7cfd-0d2b-e399bb2b31b1 -42:42.258 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:42.261 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:42.261 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:42.261 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:42.261 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:42.261 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:42.261 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ef9ee48d-7a29-7cfd-0d2b-e399bb2b31b1 -42:42.261 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ef9ee48d-7a29-7cfd-0d2b-e399bb2b31b1 -42:42.262 [INFO] TestAgent.server: federation state anti-entropy synced -42:42.453 [INFO] TestAgent: Requesting shutdown -42:42.453 [INFO] TestAgent.server: shutting down server -42:42.453 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:42.453 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:42.453 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:42.453 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:42.453 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:42.453 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:42.453 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:42.453 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:42.453 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:42.453 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:42.453 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:42.453 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:42.454Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:42.454Z [INFO] agent.router.manager: shutting down -42:42.454 [INFO] TestAgent: consul server down -42:42.454 [INFO] TestAgent: shutdown complete -42:42.454 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13551 network=tcp -42:42.454 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13551 network=udp -42:42.454 [INFO] TestAgent: Stopping server: protocol=HTTPS address=127.0.0.1:13553 network=tcp -42:42.958 [INFO] TestAgent: Waiting for endpoints to shut down -42:42.958 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_reloadWatchesHTTPS (0.85s) === CONT TestAgent_persistCheckState -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._tcp_(qType:=255)_compressed=false#01 ---- PASS: TestAgent_SecurityChecks (1.07s) +--- PASS: TestAgent_SecurityChecks (0.84s) === CONT TestAgent_loadChecks_checkFails -2022-09-15T21:42:42.493Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:42.545Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:42.549 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a240b714-d93b-84b9-7d6f-238d2e899542 Address:127.0.0.1:13542}]" -42:42.549 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13542 [Follower]" leader= -42:42.553 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a240b714-d93b-84b9-7d6f-238d2e899542.dc1 127.0.0.1 -42:42.554 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a240b714-d93b-84b9-7d6f-238d2e899542 127.0.0.1 -2022-09-15T21:42:42.554Z [INFO] agent.router: Initializing LAN area manager -42:42.555 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13571 network=udp -42:42.555 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a240b714-d93b-84b9-7d6f-238d2e899542.dc1 area=wan -42:42.555 [INFO] TestAgent.server: Adding LAN server: server="Node-a240b714-d93b-84b9-7d6f-238d2e899542 (Addr: tcp/127.0.0.1:13542) (DC: dc1)" -42:42.555 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13571 network=tcp -42:42.556 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13572 network=tcp -42:42.556 [INFO] TestAgent: started state syncer -42:42.556 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13543 network=tcp -42:42.613 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:42.613 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13542 [Candidate]" term=2 -42:42.614 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:42.615 [DEBUG] TestAgent.server.raft: vote granted: from=a240b714-d93b-84b9-7d6f-238d2e899542 term=2 tally=1 -42:42.615 [INFO] TestAgent.server.raft: election won: tally=1 -42:42.615 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13542 [Leader]" -42:42.615 [INFO] TestAgent.server: cluster leadership acquired -42:42.615 [INFO] TestAgent.server: New leader elected: payload=Node-a240b714-d93b-84b9-7d6f-238d2e899542 -42:42.616 [INFO] TestAgent: Requesting shutdown -42:42.624 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13542 -42:42.625 [INFO] TestAgent.server: shutting down server -42:42.625 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:42.625 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:42.625 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:42.625Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:42.625Z [INFO] agent.router.manager: shutting down -42:42.626 [ERROR] TestAgent.server: failed to establish leadership: error="error configuring provider: raft is already shutdown" -42:42.626 [INFO] TestAgent: consul server down -42:42.626 [INFO] TestAgent: shutdown complete -42:42.626 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13571 network=tcp -42:42.626 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13571 network=udp -42:42.626 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13572 network=tcp -42:43.126 [INFO] TestAgent: Waiting for endpoints to shut down -42:43.126 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_reloadWatches (0.65s) +2021-08-13T15:34:00.643Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:00.718Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:00.721 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f75ff51a-3973-97ea-1a53-0580961793a0 Address:127.0.0.1:24578}]" +34:00.722 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f75ff51a-3973-97ea-1a53-0580961793a0.dc1 127.0.0.1 +34:00.723 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f75ff51a-3973-97ea-1a53-0580961793a0 127.0.0.1 +2021-08-13T15:34:00.723Z [INFO] agent.router: Initializing LAN area manager +34:00.723 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24578 [Follower]" leader= +34:00.723 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24566 network=udp +34:00.723 [INFO] TestAgent.server: Adding LAN server: server="Node-f75ff51a-3973-97ea-1a53-0580961793a0 (Addr: tcp/127.0.0.1:24578) (DC: dc1)" +34:00.723 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f75ff51a-3973-97ea-1a53-0580961793a0.dc1 area=wan +34:00.723 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24566 network=tcp +34:00.724 [INFO] TestAgent: Started HTTPS server: address=127.0.0.1:24575 network=tcp +34:00.724 [INFO] TestAgent: started state syncer +34:00.725 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24579 network=tcp +34:00.761 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:00.761 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24578 [Candidate]" term=2 +34:00.762 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:00.762 [DEBUG] TestAgent.server.raft: vote granted: from=f75ff51a-3973-97ea-1a53-0580961793a0 term=2 tally=1 +34:00.762 [INFO] TestAgent.server.raft: election won: tally=1 +34:00.762 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24578 [Leader]" +34:00.762 [INFO] TestAgent.server: cluster leadership acquired +34:00.762 [INFO] TestAgent.server: New leader elected: payload=Node-f75ff51a-3973-97ea-1a53-0580961793a0 +34:00.762 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24578 +34:00.763 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:00.766 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:00.766 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:00.766 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:00.766 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:00.766 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:00.766 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f75ff51a-3973-97ea-1a53-0580961793a0 +34:00.766 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f75ff51a-3973-97ea-1a53-0580961793a0 +34:00.767 [INFO] TestAgent.server: federation state anti-entropy synced +34:00.883 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:00.883 [INFO] TestAgent: Synced node info +34:00.942 [INFO] TestAgent: Requesting shutdown +34:00.942 [INFO] TestAgent.server: shutting down server +34:00.942 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:00.942 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:00.942 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:00.942 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:00.942 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:00.942 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:00.942 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:00.942 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:00.942 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:00.942 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:00.942 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:00.943Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:00.943Z [INFO] agent.router.manager: shutting down +34:00.943 [INFO] TestAgent: consul server down +34:00.943 [INFO] TestAgent: shutdown complete +34:00.943 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24566 network=tcp +34:00.943 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24566 network=udp +34:00.943 [INFO] TestAgent: Stopping server: protocol=HTTPS address=127.0.0.1:24575 network=tcp +34:01.443 [INFO] TestAgent: Waiting for endpoints to shut down +34:01.443 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_reloadWatchesHTTPS (0.82s) === CONT TestAgent_checkStateSnapshot -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:42.747Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:42.807Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:42.811 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:68c5bc06-6639-54a1-ab4f-06765bb69f2d Address:127.0.0.1:13576}]" -42:42.812 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-68c5bc06-6639-54a1-ab4f-06765bb69f2d.dc1 127.0.0.1 -42:42.813 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-68c5bc06-6639-54a1-ab4f-06765bb69f2d 127.0.0.1 -2022-09-15T21:42:42.813Z [INFO] agent.router: Initializing LAN area manager -42:42.814 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13576 [Follower]" leader= -42:42.814 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13516 network=udp -42:42.814 [INFO] TestAgent.server: Adding LAN server: server="Node-68c5bc06-6639-54a1-ab4f-06765bb69f2d (Addr: tcp/127.0.0.1:13576) (DC: dc1)" -42:42.814 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-68c5bc06-6639-54a1-ab4f-06765bb69f2d.dc1 area=wan -42:42.815 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13516 network=tcp -42:42.816 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13517 network=tcp -42:42.816 [INFO] TestAgent: started state syncer -42:42.816 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13577 network=tcp -42:42.874 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:42.875 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13576 [Candidate]" term=2 -42:42.877 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:42.877 [DEBUG] TestAgent.server.raft: vote granted: from=68c5bc06-6639-54a1-ab4f-06765bb69f2d term=2 tally=1 -42:42.877 [INFO] TestAgent.server.raft: election won: tally=1 -42:42.877 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13576 [Leader]" -42:42.878 [INFO] TestAgent.server: cluster leadership acquired -42:42.878 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13576 -42:42.881 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:42.883 [INFO] TestAgent.server: New leader elected: payload=Node-68c5bc06-6639-54a1-ab4f-06765bb69f2d -42:42.884 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:42.885 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:42.885 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:42.885 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:42.885 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:42.885 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-68c5bc06-6639-54a1-ab4f-06765bb69f2d -42:42.885 [INFO] TestAgent.server: member joined, marking health alive: member=Node-68c5bc06-6639-54a1-ab4f-06765bb69f2d -42:42.887 [INFO] TestAgent.server: federation state anti-entropy synced -42:42.926 [INFO] TestAgent: Requesting shutdown -42:42.927 [INFO] TestAgent.server: shutting down server -42:42.927 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:42.927 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:42.927 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:42.927 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:42.927 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:42.927 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:42.927 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:42.927 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:42.927 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:42.927 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:42.927 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:42.927 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:42.928Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:42.928Z [INFO] agent.router.manager: shutting down -42:42.928 [INFO] TestAgent: consul server down -42:42.928 [INFO] TestAgent: shutdown complete -42:42.928 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13516 network=tcp -42:42.928 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13516 network=udp -42:42.928 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13517 network=tcp -42:43.431 [INFO] TestAgent: Waiting for endpoints to shut down -42:43.431 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_purgeCheckState (0.72s) +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:00.887Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:00.904Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:00.907 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5e285323-79ce-82d7-b9f9-af4c348cba01 Address:127.0.0.1:24571}]" +34:00.907 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24571 [Follower]" leader= +34:00.908 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5e285323-79ce-82d7-b9f9-af4c348cba01.dc1 127.0.0.1 +34:00.909 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5e285323-79ce-82d7-b9f9-af4c348cba01 127.0.0.1 +2021-08-13T15:34:00.909Z [INFO] agent.router: Initializing LAN area manager +34:00.909 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5e285323-79ce-82d7-b9f9-af4c348cba01.dc1 area=wan +34:00.910 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24580 network=tcp +34:00.910 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24580 network=udp +34:00.910 [INFO] TestAgent.server: Adding LAN server: server="Node-5e285323-79ce-82d7-b9f9-af4c348cba01 (Addr: tcp/127.0.0.1:24571) (DC: dc1)" +34:00.910 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24581 network=tcp +34:00.911 [INFO] TestAgent: started state syncer +34:00.911 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24572 network=tcp +34:00.950 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:00.950 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24571 [Candidate]" term=2 +34:00.950 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:00.950 [DEBUG] TestAgent.server.raft: vote granted: from=5e285323-79ce-82d7-b9f9-af4c348cba01 term=2 tally=1 +34:00.950 [INFO] TestAgent.server.raft: election won: tally=1 +34:00.950 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24571 [Leader]" +34:00.951 [INFO] TestAgent.server: cluster leadership acquired +34:00.951 [INFO] TestAgent.server: New leader elected: payload=Node-5e285323-79ce-82d7-b9f9-af4c348cba01 +34:00.951 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24571 +34:00.952 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:00.955 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:00.955 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:00.955 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:00.955 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:00.955 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:00.955 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5e285323-79ce-82d7-b9f9-af4c348cba01 +34:00.955 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5e285323-79ce-82d7-b9f9-af4c348cba01 +34:00.956 [INFO] TestAgent.server: federation state anti-entropy synced +34:01.050 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:01.051 [INFO] TestAgent: Synced node info +34:01.051 [DEBUG] TestAgent: Node info in sync +34:01.111 [INFO] TestAgent: Requesting shutdown +34:01.111 [INFO] TestAgent.server: shutting down server +34:01.111 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:01.111 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:01.111 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:01.111 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:01.111 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:01.112 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:01.112 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:01.112 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:01.112 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:01.112 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:01.112 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:01.112Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:01.112Z [INFO] agent.router.manager: shutting down +34:01.112 [INFO] TestAgent: consul server down +34:01.112 [INFO] TestAgent: shutdown complete +34:01.112 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24580 network=tcp +34:01.112 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24580 network=udp +34:01.112 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24581 network=tcp +34:01.619 [INFO] TestAgent: Waiting for endpoints to shut down +34:01.619 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_reloadWatches (0.76s) === CONT TestAgent_NodeMaintenanceMode -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._udp_(qType:=255)_compressed=true#01 -[WARN] freeport: 4 out of 4 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:43.016Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:43.101Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:43.104 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:48178c80-4a46-76ee-4f04-0d3400e5255d Address:127.0.0.1:13562}]" -42:43.105 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13562 [Follower]" leader= -42:43.105 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-48178c80-4a46-76ee-4f04-0d3400e5255d.dc1 127.0.0.1 -42:43.106 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-48178c80-4a46-76ee-4f04-0d3400e5255d 127.0.0.1 -2022-09-15T21:42:43.106Z [INFO] agent.router: Initializing LAN area manager -42:43.107 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13578 network=udp -42:43.107 [INFO] TestAgent.server: Adding LAN server: server="Node-48178c80-4a46-76ee-4f04-0d3400e5255d (Addr: tcp/127.0.0.1:13562) (DC: dc1)" -42:43.107 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-48178c80-4a46-76ee-4f04-0d3400e5255d.dc1 area=wan -42:43.107 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13578 network=tcp -42:43.108 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13579 network=tcp -42:43.108 [INFO] TestAgent: started state syncer -42:43.108 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13563 network=tcp -42:43.166 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:43.166 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13562 [Candidate]" term=2 -42:43.167 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:43.167 [DEBUG] TestAgent.server.raft: vote granted: from=48178c80-4a46-76ee-4f04-0d3400e5255d term=2 tally=1 -42:43.167 [INFO] TestAgent.server.raft: election won: tally=1 -42:43.167 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13562 [Leader]" -42:43.167 [INFO] TestAgent.server: cluster leadership acquired -42:43.167 [INFO] TestAgent.server: New leader elected: payload=Node-48178c80-4a46-76ee-4f04-0d3400e5255d -42:43.167 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13562 -42:43.168 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:43.176 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:43.176 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:43.176 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:43.176 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:43.176 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:43.176 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-48178c80-4a46-76ee-4f04-0d3400e5255d -42:43.176 [INFO] TestAgent.server: member joined, marking health alive: member=Node-48178c80-4a46-76ee-4f04-0d3400e5255d -42:43.178 [INFO] TestAgent.server: federation state anti-entropy synced -42:43.335 [DEBUG] TestAgent: check state expired, not restoring: check=check1 -42:43.335 [INFO] TestAgent: Requesting shutdown -42:43.339 [INFO] TestAgent.server: shutting down server -42:43.339 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:43.339 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:43.339 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:43.339 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:43.339 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:43.339 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:43.339 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:43.339 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:43.339 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:43.339 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:43.339 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:43.339 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:43.339Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:43.339Z [INFO] agent.router.manager: shutting down -42:43.340 [INFO] TestAgent: consul server down -42:43.340 [INFO] TestAgent: shutdown complete -42:43.340 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13578 network=tcp -42:43.340 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13578 network=udp -42:43.340 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13579 network=tcp -42:43.840 [INFO] TestAgent: Waiting for endpoints to shut down -42:43.840 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_loadCheckState (0.91s) +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:01.250Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:01.268Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:01.271 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:27bd84d2-9700-3419-3b45-b997d8142017 Address:127.0.0.1:24599}]" +34:01.271 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24599 [Follower]" leader= +34:01.272 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-27bd84d2-9700-3419-3b45-b997d8142017.dc1 127.0.0.1 +34:01.275 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-27bd84d2-9700-3419-3b45-b997d8142017 127.0.0.1 +2021-08-13T15:34:01.275Z [INFO] agent.router: Initializing LAN area manager +34:01.277 [INFO] TestAgent.server: Adding LAN server: server="Node-27bd84d2-9700-3419-3b45-b997d8142017 (Addr: tcp/127.0.0.1:24599) (DC: dc1)" +34:01.277 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-27bd84d2-9700-3419-3b45-b997d8142017.dc1 area=wan +34:01.277 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24594 network=tcp +34:01.279 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24594 network=udp +34:01.280 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24595 network=tcp +34:01.280 [INFO] TestAgent: started state syncer +34:01.281 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24600 network=tcp +34:01.330 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:01.330 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24599 [Candidate]" term=2 +34:01.330 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:01.330 [DEBUG] TestAgent.server.raft: vote granted: from=27bd84d2-9700-3419-3b45-b997d8142017 term=2 tally=1 +34:01.330 [INFO] TestAgent.server.raft: election won: tally=1 +34:01.330 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24599 [Leader]" +34:01.330 [INFO] TestAgent.server: cluster leadership acquired +34:01.331 [INFO] TestAgent.server: New leader elected: payload=Node-27bd84d2-9700-3419-3b45-b997d8142017 +34:01.331 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24599 +34:01.332 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:01.335 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:01.335 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:01.335 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:01.335 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:01.335 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:01.335 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-27bd84d2-9700-3419-3b45-b997d8142017 +34:01.335 [INFO] TestAgent.server: member joined, marking health alive: member=Node-27bd84d2-9700-3419-3b45-b997d8142017 +34:01.336 [INFO] TestAgent.server: federation state anti-entropy synced +34:01.341 [DEBUG] TestAgent: check state expired, not restoring: check=check1 +34:01.341 [INFO] TestAgent: Requesting shutdown +34:01.341 [INFO] TestAgent.server: shutting down server +34:01.341 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:01.341 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:01.341 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:01.341 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:01.341 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:01.341 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:01.341 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:01.341 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:01.341 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:01.341 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:01.341 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:01.341 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:01.342Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:01.342Z [INFO] agent.router.manager: shutting down +34:01.342 [INFO] TestAgent: consul server down +34:01.342 [INFO] TestAgent: shutdown complete +34:01.342 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24594 network=tcp +34:01.342 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24594 network=udp +34:01.342 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24595 network=tcp +34:01.842 [INFO] TestAgent: Waiting for endpoints to shut down +34:01.842 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_loadCheckState (0.61s) === CONT TestAgent_AddCheck_restoresSnapshot -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._udp_(qType:=255)_compressed=false#01 -2022-09-15T21:42:43.023Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:43.052Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:43.063 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:983b981e-121a-7361-a86c-ccdb7783ae9e Address:127.0.0.1:13590}]" -42:43.064 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-983b981e-121a-7361-a86c-ccdb7783ae9e.dc1 127.0.0.1 -42:43.065 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13590 [Follower]" leader= -42:43.065 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-983b981e-121a-7361-a86c-ccdb7783ae9e 127.0.0.1 -2022-09-15T21:42:43.065Z [INFO] agent.router: Initializing LAN area manager -42:43.066 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-983b981e-121a-7361-a86c-ccdb7783ae9e.dc1 area=wan -42:43.066 [INFO] TestAgent.server: Adding LAN server: server="Node-983b981e-121a-7361-a86c-ccdb7783ae9e (Addr: tcp/127.0.0.1:13590) (DC: dc1)" -42:43.072 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13564 network=udp -42:43.073 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13564 network=tcp -42:43.075 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13565 network=tcp -42:43.075 [INFO] TestAgent: started state syncer -42:43.076 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13591 network=tcp -42:43.123 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:43.123 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13590 [Candidate]" term=2 -42:43.123 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:43.123 [DEBUG] TestAgent.server.raft: vote granted: from=983b981e-121a-7361-a86c-ccdb7783ae9e term=2 tally=1 -42:43.123 [INFO] TestAgent.server.raft: election won: tally=1 -42:43.123 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13590 [Leader]" -42:43.123 [INFO] TestAgent.server: cluster leadership acquired -42:43.124 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13590 -42:43.124 [INFO] TestAgent.server: New leader elected: payload=Node-983b981e-121a-7361-a86c-ccdb7783ae9e -42:43.125 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:43.127 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:43.127 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:43.127 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:43.127 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:43.127 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:43.127 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-983b981e-121a-7361-a86c-ccdb7783ae9e -42:43.127 [INFO] TestAgent.server: member joined, marking health alive: member=Node-983b981e-121a-7361-a86c-ccdb7783ae9e -42:43.128 [INFO] TestAgent.server: federation state anti-entropy synced -42:43.344 [INFO] TestAgent: Requesting shutdown -42:43.345 [INFO] TestAgent.server: shutting down server -42:43.345 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:43.345 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:43.345 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:43.345 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:43.345 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:43.345 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:43.345 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:43.345 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:43.345 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:43.345 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:43.345 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:43.345 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:43.345Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:43.345Z [INFO] agent.router.manager: shutting down -42:43.346 [INFO] TestAgent: consul server down -42:43.346 [INFO] TestAgent: shutdown complete -42:43.346 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13564 network=tcp -42:43.346 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13564 network=udp -42:43.346 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13565 network=tcp -42:43.850 [INFO] TestAgent: Waiting for endpoints to shut down -42:43.850 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_persistCheckState (0.89s) +2021-08-13T15:34:01.044Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:01.060Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:01.063 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:36b90b4b-7ff8-2acf-d73f-25630f0019ef Address:127.0.0.1:24592}]" +34:01.063 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24592 [Follower]" leader= +34:01.064 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-36b90b4b-7ff8-2acf-d73f-25630f0019ef.dc1 127.0.0.1 +34:01.065 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-36b90b4b-7ff8-2acf-d73f-25630f0019ef 127.0.0.1 +2021-08-13T15:34:01.065Z [INFO] agent.router: Initializing LAN area manager +34:01.065 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24587 network=udp +34:01.065 [INFO] TestAgent.server: Adding LAN server: server="Node-36b90b4b-7ff8-2acf-d73f-25630f0019ef (Addr: tcp/127.0.0.1:24592) (DC: dc1)" +34:01.065 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-36b90b4b-7ff8-2acf-d73f-25630f0019ef.dc1 area=wan +34:01.066 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24587 network=tcp +34:01.067 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24588 network=tcp +34:01.067 [INFO] TestAgent: started state syncer +34:01.067 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24593 network=tcp +34:01.131 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:01.131 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24592 [Candidate]" term=2 +34:01.131 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:01.131 [DEBUG] TestAgent.server.raft: vote granted: from=36b90b4b-7ff8-2acf-d73f-25630f0019ef term=2 tally=1 +34:01.131 [INFO] TestAgent.server.raft: election won: tally=1 +34:01.131 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24592 [Leader]" +34:01.131 [INFO] TestAgent.server: cluster leadership acquired +34:01.132 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24592 +34:01.132 [INFO] TestAgent.server: New leader elected: payload=Node-36b90b4b-7ff8-2acf-d73f-25630f0019ef +34:01.133 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:01.136 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:01.136 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:01.136 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:01.136 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:01.136 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:01.136 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-36b90b4b-7ff8-2acf-d73f-25630f0019ef +34:01.136 [INFO] TestAgent.server: member joined, marking health alive: member=Node-36b90b4b-7ff8-2acf-d73f-25630f0019ef +34:01.138 [INFO] TestAgent.server: federation state anti-entropy synced +34:01.350 [INFO] TestAgent: Requesting shutdown +34:01.350 [INFO] TestAgent.server: shutting down server +34:01.350 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:01.350 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:01.350 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:01.350 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:01.350 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:01.350 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:01.350 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:01.350 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:01.350 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:01.350 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:01.350 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:01.350 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:01.350Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:01.350Z [INFO] agent.router.manager: shutting down +34:01.350 [INFO] TestAgent: consul server down +34:01.350 [INFO] TestAgent: shutdown complete +34:01.350 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24587 network=tcp +34:01.350 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24587 network=udp +34:01.350 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24588 network=tcp +34:01.851 [INFO] TestAgent: Waiting for endpoints to shut down +34:01.851 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_purgeCheckState (0.82s) === CONT TestAgent_Service_MaintenanceMode -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:43.143Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:43.159Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:43.162 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6a268b4f-62e8-4e9c-bbe4-9f93c0f15e11 Address:127.0.0.1:13618}]" -42:43.163 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6a268b4f-62e8-4e9c-bbe4-9f93c0f15e11.dc1 127.0.0.1 -42:43.164 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6a268b4f-62e8-4e9c-bbe4-9f93c0f15e11 127.0.0.1 -2022-09-15T21:42:43.164Z [INFO] agent.router: Initializing LAN area manager -42:43.165 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13613 network=udp -42:43.165 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13618 [Follower]" leader= -42:43.165 [INFO] TestAgent.server: Adding LAN server: server="Node-6a268b4f-62e8-4e9c-bbe4-9f93c0f15e11 (Addr: tcp/127.0.0.1:13618) (DC: dc1)" -42:43.165 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6a268b4f-62e8-4e9c-bbe4-9f93c0f15e11.dc1 area=wan -42:43.165 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13613 network=tcp -42:43.166 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13614 network=tcp -42:43.167 [INFO] TestAgent: started state syncer -42:43.167 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13619 network=tcp -42:43.212 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:43.212 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13618 [Candidate]" term=2 -42:43.213 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:43.213 [DEBUG] TestAgent.server.raft: vote granted: from=6a268b4f-62e8-4e9c-bbe4-9f93c0f15e11 term=2 tally=1 -42:43.213 [INFO] TestAgent.server.raft: election won: tally=1 -42:43.213 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13618 [Leader]" -42:43.213 [INFO] TestAgent.server: cluster leadership acquired -42:43.213 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13618 -42:43.213 [INFO] TestAgent.server: New leader elected: payload=Node-6a268b4f-62e8-4e9c-bbe4-9f93c0f15e11 -42:43.215 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:43.217 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:43.217 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:43.217 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:43.217 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:43.217 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:43.217 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6a268b4f-62e8-4e9c-bbe4-9f93c0f15e11 -42:43.217 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6a268b4f-62e8-4e9c-bbe4-9f93c0f15e11 -42:43.218 [INFO] TestAgent.server: federation state anti-entropy synced -42:43.446 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:43.447 [INFO] TestAgent: Synced node info -42:43.447 [DEBUG] TestAgent: Node info in sync -42:43.530 [DEBUG] TestAgent: removed check: check=service:redis -42:43.530 [DEBUG] TestAgent: restored health check from file: check=service:redis file=/tmp/consul-test/TestAgent_checkStateSnapshot-TestAgent-agent171116515/checks/60a2ef12de014a05ecdc850d9aab46da -42:43.530 [INFO] TestAgent: Requesting shutdown -42:43.531 [INFO] TestAgent.server: shutting down server -42:43.531 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:43.531 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:43.531 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:43.531 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:43.531 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:43.531 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:43.531 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:43.531 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:43.531 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:43.531 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:43.531 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:43.531Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:43.531Z [INFO] agent.router.manager: shutting down -42:43.532 [INFO] TestAgent: consul server down -42:43.532 [INFO] TestAgent: shutdown complete -42:43.532 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13613 network=tcp -42:43.532 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13613 network=udp -42:43.532 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13614 network=tcp -42:44.034 [INFO] TestAgent: Waiting for endpoints to shut down -42:44.034 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_checkStateSnapshot (0.91s) +2021-08-13T15:34:01.333Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:01.349Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:01.352 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c5b3a761-4d8a-88d8-84ca-88c251c237ed Address:127.0.0.1:24606}]" +34:01.352 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24606 [Follower]" leader= +34:01.353 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c5b3a761-4d8a-88d8-84ca-88c251c237ed.dc1 127.0.0.1 +34:01.354 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c5b3a761-4d8a-88d8-84ca-88c251c237ed 127.0.0.1 +2021-08-13T15:34:01.354Z [INFO] agent.router: Initializing LAN area manager +34:01.354 [INFO] TestAgent.server: Adding LAN server: server="Node-c5b3a761-4d8a-88d8-84ca-88c251c237ed (Addr: tcp/127.0.0.1:24606) (DC: dc1)" +34:01.354 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c5b3a761-4d8a-88d8-84ca-88c251c237ed.dc1 area=wan +34:01.354 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24601 network=tcp +34:01.354 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24601 network=udp +34:01.355 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24602 network=tcp +34:01.355 [INFO] TestAgent: started state syncer +34:01.356 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24607 network=tcp +34:01.406 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:01.406 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24606 [Candidate]" term=2 +34:01.406 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:01.406 [DEBUG] TestAgent.server.raft: vote granted: from=c5b3a761-4d8a-88d8-84ca-88c251c237ed term=2 tally=1 +34:01.406 [INFO] TestAgent.server.raft: election won: tally=1 +34:01.406 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24606 [Leader]" +34:01.406 [INFO] TestAgent.server: cluster leadership acquired +34:01.407 [INFO] TestAgent.server: New leader elected: payload=Node-c5b3a761-4d8a-88d8-84ca-88c251c237ed +34:01.407 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24606 +34:01.408 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:01.410 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:01.410 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:01.410 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:01.410 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:01.410 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:01.410 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c5b3a761-4d8a-88d8-84ca-88c251c237ed +34:01.410 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c5b3a761-4d8a-88d8-84ca-88c251c237ed +34:01.412 [INFO] TestAgent.server: federation state anti-entropy synced +34:01.488 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:01.489 [INFO] TestAgent: Synced node info +34:01.489 [DEBUG] TestAgent: Node info in sync +34:01.521 [INFO] TestAgent: Requesting shutdown +34:01.521 [INFO] TestAgent.server: shutting down server +34:01.521 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:01.521 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:01.521 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:01.521 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:01.521 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:01.521 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:01.521 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:01.521 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:01.521 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:01.521 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:01.521 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:01.521Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:01.521Z [INFO] agent.router.manager: shutting down +34:01.521 [INFO] TestAgent: consul server down +34:01.521 [INFO] TestAgent: shutdown complete +34:01.522 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24601 network=tcp +34:01.522 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24601 network=udp +34:01.522 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24602 network=tcp +34:02.022 [INFO] TestAgent: Waiting for endpoints to shut down +34:02.022 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_persistCheckState (0.71s) === CONT TestAgent_unloadChecks -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=1)_compressed=true#01 -2022-09-15T21:42:43.133Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:43.150Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:43.153 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7b6d3e88-3357-53ab-a4fd-b939d33ac506 Address:127.0.0.1:13611}]" -42:43.153 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13611 [Follower]" leader= -42:43.154 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7b6d3e88-3357-53ab-a4fd-b939d33ac506.dc1 127.0.0.1 -42:43.155 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7b6d3e88-3357-53ab-a4fd-b939d33ac506 127.0.0.1 -2022-09-15T21:42:43.155Z [INFO] agent.router: Initializing LAN area manager -42:43.155 [INFO] TestAgent.server: Adding LAN server: server="Node-7b6d3e88-3357-53ab-a4fd-b939d33ac506 (Addr: tcp/127.0.0.1:13611) (DC: dc1)" -42:43.155 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7b6d3e88-3357-53ab-a4fd-b939d33ac506.dc1 area=wan -42:43.155 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13599 network=udp -42:43.155 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13599 network=tcp -42:43.156 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13600 network=tcp -42:43.156 [INFO] TestAgent: started state syncer -42:43.156 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13612 network=tcp -42:43.203 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:43.203 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13611 [Candidate]" term=2 -42:43.203 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:43.203 [DEBUG] TestAgent.server.raft: vote granted: from=7b6d3e88-3357-53ab-a4fd-b939d33ac506 term=2 tally=1 -42:43.203 [INFO] TestAgent.server.raft: election won: tally=1 -42:43.203 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13611 [Leader]" -42:43.203 [INFO] TestAgent.server: cluster leadership acquired -42:43.204 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13611 -42:43.204 [INFO] TestAgent.server: New leader elected: payload=Node-7b6d3e88-3357-53ab-a4fd-b939d33ac506 -42:43.205 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:43.207 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:43.207 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:43.207 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:43.207 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:43.207 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:43.208 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7b6d3e88-3357-53ab-a4fd-b939d33ac506 -42:43.208 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7b6d3e88-3357-53ab-a4fd-b939d33ac506 -42:43.208 [INFO] TestAgent.server: federation state anti-entropy synced -42:43.358 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:43.360 [INFO] TestAgent: Synced node info -42:43.360 [DEBUG] TestAgent: Node info in sync -42:43.578 [WARN] TestAgent: Failed to restore check: check=service:redis error="ServiceID "nope" does not exist" -42:43.579 [DEBUG] TestAgent: restored health check from file: check=service:redis file=/tmp/consul-test/TestAgent_loadChecks_checkFails-TestAgent-agent859542932/checks/60a2ef12de014a05ecdc850d9aab46da -42:43.579 [INFO] TestAgent: Requesting shutdown -42:43.579 [INFO] TestAgent.server: shutting down server -42:43.579 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:43.579 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:43.579 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:43.579 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:43.579 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:43.579 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:43.579 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:43.580 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:43.580 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -42:43.580 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:43.580 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2022-09-15T21:42:43.580Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:43.581Z [INFO] agent.router.manager: shutting down -42:43.581 [INFO] TestAgent: consul server down -42:43.581 [INFO] TestAgent: shutdown complete -42:43.581 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13599 network=tcp -42:43.581 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13599 network=udp -42:43.581 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13600 network=tcp -42:44.086 [INFO] TestAgent: Waiting for endpoints to shut down -42:44.086 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_loadChecks_checkFails (0.97s) +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:01.387Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:01.403Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:01.406 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f0bff5e1-7161-4cbd-28d5-afd74995776d Address:127.0.0.1:24613}]" +34:01.406 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24613 [Follower]" leader= +34:01.407 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f0bff5e1-7161-4cbd-28d5-afd74995776d.dc1 127.0.0.1 +34:01.408 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f0bff5e1-7161-4cbd-28d5-afd74995776d 127.0.0.1 +2021-08-13T15:34:01.408Z [INFO] agent.router: Initializing LAN area manager +34:01.408 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24608 network=udp +34:01.408 [INFO] TestAgent.server: Adding LAN server: server="Node-f0bff5e1-7161-4cbd-28d5-afd74995776d (Addr: tcp/127.0.0.1:24613) (DC: dc1)" +34:01.408 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f0bff5e1-7161-4cbd-28d5-afd74995776d.dc1 area=wan +34:01.409 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24608 network=tcp +34:01.409 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24609 network=tcp +34:01.409 [INFO] TestAgent: started state syncer +34:01.410 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24614 network=tcp +34:01.463 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:01.463 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24613 [Candidate]" term=2 +34:01.463 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:01.463 [DEBUG] TestAgent.server.raft: vote granted: from=f0bff5e1-7161-4cbd-28d5-afd74995776d term=2 tally=1 +34:01.463 [INFO] TestAgent.server.raft: election won: tally=1 +34:01.464 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24613 [Leader]" +34:01.464 [INFO] TestAgent.server: cluster leadership acquired +34:01.464 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24613 +34:01.465 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:01.466 [INFO] TestAgent.server: New leader elected: payload=Node-f0bff5e1-7161-4cbd-28d5-afd74995776d +34:01.468 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:01.468 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:01.468 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:01.468 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:01.468 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:01.468 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f0bff5e1-7161-4cbd-28d5-afd74995776d +34:01.468 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f0bff5e1-7161-4cbd-28d5-afd74995776d +34:01.469 [INFO] TestAgent.server: federation state anti-entropy synced +34:01.470 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:01.470 [INFO] TestAgent: Synced node info +34:01.470 [DEBUG] TestAgent: Node info in sync +34:01.732 [WARN] TestAgent: Failed to restore check: check=service:redis error="ServiceID "nope" does not exist" +34:01.732 [DEBUG] TestAgent: restored health check from file: check=service:redis file=/tmp/consul-test/TestAgent_loadChecks_checkFails-TestAgent-agent769823259/checks/60a2ef12de014a05ecdc850d9aab46da +34:01.732 [INFO] TestAgent: Requesting shutdown +34:01.732 [INFO] TestAgent.server: shutting down server +34:01.732 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:01.732 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:01.732 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:01.732 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:01.733 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:01.733 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:01.733 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:01.733 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:01.733 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:01.733 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:01.733 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:01.733Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:01.733Z [INFO] agent.router.manager: shutting down +34:01.733 [INFO] TestAgent: consul server down +34:01.733 [INFO] TestAgent: shutdown complete +34:01.733 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24608 network=tcp +34:01.733 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24608 network=udp +34:01.733 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24609 network=tcp +34:02.233 [INFO] TestAgent: Waiting for endpoints to shut down +34:02.233 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_loadChecks_checkFails (0.86s) === CONT TestAgent_loadChecks_token -2022-09-15T21:42:43.428Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:43.523Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:43.527 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:023d1ba7-873b-6759-69b2-d8d624aed6e4 Address:127.0.0.1:13639}]" -42:43.528 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-023d1ba7-873b-6759-69b2-d8d624aed6e4.dc1 127.0.0.1 -42:43.529 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13639 [Follower]" leader= -42:43.529 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-023d1ba7-873b-6759-69b2-d8d624aed6e4 127.0.0.1 -2022-09-15T21:42:43.529Z [INFO] agent.router: Initializing LAN area manager -42:43.534 [INFO] TestAgent.server: Adding LAN server: server="Node-023d1ba7-873b-6759-69b2-d8d624aed6e4 (Addr: tcp/127.0.0.1:13639) (DC: dc1)" -42:43.534 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-023d1ba7-873b-6759-69b2-d8d624aed6e4.dc1 area=wan -42:43.534 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13627 network=tcp -42:43.534 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13627 network=udp -42:43.536 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13628 network=tcp -42:43.536 [INFO] TestAgent: started state syncer -42:43.536 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13640 network=tcp -42:43.599 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:43.600 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13639 [Candidate]" term=2 -42:43.600 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:43.600 [DEBUG] TestAgent.server.raft: vote granted: from=023d1ba7-873b-6759-69b2-d8d624aed6e4 term=2 tally=1 -42:43.600 [INFO] TestAgent.server.raft: election won: tally=1 -42:43.600 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13639 [Leader]" -42:43.600 [INFO] TestAgent.server: cluster leadership acquired -42:43.601 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13639 -42:43.601 [INFO] TestAgent.server: New leader elected: payload=Node-023d1ba7-873b-6759-69b2-d8d624aed6e4 -42:43.602 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:43.605 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:43.605 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:43.605 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:43.605 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:43.605 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:43.605 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-023d1ba7-873b-6759-69b2-d8d624aed6e4 -42:43.605 [INFO] TestAgent.server: member joined, marking health alive: member=Node-023d1ba7-873b-6759-69b2-d8d624aed6e4 -42:43.607 [INFO] TestAgent.server: federation state anti-entropy synced -42:43.648 [INFO] TestAgent: Requesting shutdown -42:43.648 [INFO] TestAgent.server: shutting down server -42:43.648 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:43.648 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:43.648 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:43.648 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:43.648 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:43.648 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:43.648 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:43.648 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:43.648 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:43.648 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:43.648 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:43.648 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:43.648Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:43.649Z [INFO] agent.router.manager: shutting down -42:43.649 [INFO] TestAgent: consul server down -42:43.649 [INFO] TestAgent: shutdown complete -42:43.649 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13627 network=tcp -42:43.649 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13627 network=udp -42:43.649 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13628 network=tcp -42:44.151 [INFO] TestAgent: Waiting for endpoints to shut down -42:44.151 [INFO] TestAgent: Endpoints down -2022-09-15T21:42:42.647Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:42.719Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:42.723 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:88d0b454-3fd1-e69d-507b-d98158dc1187 Address:127.0.0.1:13514}]" -42:42.725 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-88d0b454-3fd1-e69d-507b-d98158dc1187.dc1 127.0.0.1 -42:42.726 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13514 [Follower]" leader= -42:42.726 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-88d0b454-3fd1-e69d-507b-d98158dc1187 127.0.0.1 -2022-09-15T21:42:42.726Z [INFO] agent.router: Initializing LAN area manager -42:42.727 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13544 network=udp -42:42.727 [INFO] TestAgent.server: Adding LAN server: server="Node-88d0b454-3fd1-e69d-507b-d98158dc1187 (Addr: tcp/127.0.0.1:13514) (DC: dc1)" -42:42.727 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-88d0b454-3fd1-e69d-507b-d98158dc1187.dc1 area=wan -42:42.727 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13544 network=tcp -42:42.728 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13545 network=tcp -42:42.728 [INFO] TestAgent: started state syncer -42:42.728 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13515 network=tcp -42:42.774 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:42.774 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13514 [Candidate]" term=2 -42:42.782 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:42.782 [DEBUG] TestAgent.server.raft: vote granted: from=88d0b454-3fd1-e69d-507b-d98158dc1187 term=2 tally=1 -42:42.782 [INFO] TestAgent.server.raft: election won: tally=1 -42:42.782 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13514 [Leader]" -42:42.783 [INFO] TestAgent.server: cluster leadership acquired -42:42.784 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13514 -42:42.786 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:42.787 [INFO] TestAgent.server: New leader elected: payload=Node-88d0b454-3fd1-e69d-507b-d98158dc1187 -42:42.797 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:42.797 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:42.797 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:42.797 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:42.797 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:42.797 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-88d0b454-3fd1-e69d-507b-d98158dc1187 -42:42.797 [INFO] TestAgent.server: member joined, marking health alive: member=Node-88d0b454-3fd1-e69d-507b-d98158dc1187 -42:42.803 [INFO] TestAgent.server: federation state anti-entropy synced -42:42.814 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:42.815 [INFO] TestAgent: Synced node info -42:42.886 [INFO] TestAgent: Requesting shutdown -42:42.886 [INFO] TestAgent.server: shutting down server -42:42.886 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:42.886 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:42.886 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:42.886 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:42.886 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:42.886 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:42.887 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:42.887 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:42.887 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:42.887 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:42.887 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:42.887Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:42.887Z [INFO] agent.router.manager: shutting down -42:42.887 [INFO] TestAgent: consul server down -42:42.887 [INFO] TestAgent: shutdown complete -42:42.887 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13544 network=tcp -42:42.887 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13544 network=udp -42:42.887 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13545 network=tcp -42:43.395 [INFO] TestAgent: Waiting for endpoints to shut down -42:43.395 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_GetCoordinate (1.55s) +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:01.674Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:01.693Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:01.696 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e2a28686-1b46-9027-801c-31133e869900 Address:127.0.0.1:24648}]" +34:01.696 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24648 [Follower]" leader= +34:01.697 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e2a28686-1b46-9027-801c-31133e869900.dc1 127.0.0.1 +34:01.698 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e2a28686-1b46-9027-801c-31133e869900 127.0.0.1 +2021-08-13T15:34:01.698Z [INFO] agent.router: Initializing LAN area manager +34:01.698 [INFO] TestAgent.server: Adding LAN server: server="Node-e2a28686-1b46-9027-801c-31133e869900 (Addr: tcp/127.0.0.1:24648) (DC: dc1)" +34:01.698 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e2a28686-1b46-9027-801c-31133e869900.dc1 area=wan +34:01.698 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24636 network=udp +34:01.698 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24636 network=tcp +34:01.699 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24637 network=tcp +34:01.699 [INFO] TestAgent: started state syncer +34:01.699 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24649 network=tcp +34:01.742 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:01.742 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24648 [Candidate]" term=2 +34:01.743 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:01.743 [DEBUG] TestAgent.server.raft: vote granted: from=e2a28686-1b46-9027-801c-31133e869900 term=2 tally=1 +34:01.743 [INFO] TestAgent.server.raft: election won: tally=1 +34:01.743 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24648 [Leader]" +34:01.743 [INFO] TestAgent.server: cluster leadership acquired +34:01.743 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24648 +34:01.743 [INFO] TestAgent.server: New leader elected: payload=Node-e2a28686-1b46-9027-801c-31133e869900 +34:01.744 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:01.747 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:01.747 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:01.747 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:01.747 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:01.747 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:01.747 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e2a28686-1b46-9027-801c-31133e869900 +34:01.747 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e2a28686-1b46-9027-801c-31133e869900 +34:01.747 [INFO] TestAgent.server: federation state anti-entropy synced +34:01.883 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:01.883 [INFO] TestAgent: Node entered maintenance mode +34:01.884 [INFO] TestAgent: Synced node info +34:01.884 [INFO] TestAgent: Synced check: check=_node_maintenance +34:01.884 [DEBUG] TestAgent: removed check: check=_node_maintenance +34:01.884 [INFO] TestAgent: Node left maintenance mode +34:01.885 [INFO] TestAgent: Node entered maintenance mode +34:01.885 [INFO] TestAgent: Requesting shutdown +34:01.885 [INFO] TestAgent.server: shutting down server +34:01.885 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:01.885 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:01.885 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:01.885 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:01.885 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:01.885 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:01.885 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:01.885 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:01.885 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:01.885 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:01.885 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:01.885Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:01.885Z [INFO] agent.router.manager: shutting down +34:01.886 [INFO] TestAgent: consul server down +34:01.886 [INFO] TestAgent: shutdown complete +34:01.886 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24636 network=tcp +34:01.886 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24636 network=udp +34:01.886 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24637 network=tcp +34:02.386 [INFO] TestAgent: Waiting for endpoints to shut down +34:02.386 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_NodeMaintenanceMode (0.76s) === CONT TestAgent_DeregisterPersistedSidecarAfterRestart -[WARN] freeport: 4 out of 32 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:43.523Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:43.574Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:43.587 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4d9a2fd1-e505-73f6-45c5-26344535431c Address:127.0.0.1:13604}]" -42:43.588 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4d9a2fd1-e505-73f6-45c5-26344535431c.dc1 127.0.0.1 -42:43.590 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13604 [Follower]" leader= -42:43.590 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4d9a2fd1-e505-73f6-45c5-26344535431c 127.0.0.1 -2022-09-15T21:42:43.590Z [INFO] agent.router: Initializing LAN area manager -42:43.591 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13641 network=udp -42:43.594 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4d9a2fd1-e505-73f6-45c5-26344535431c.dc1 area=wan -42:43.594 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13641 network=tcp -42:43.598 [INFO] TestAgent.server: Adding LAN server: server="Node-4d9a2fd1-e505-73f6-45c5-26344535431c (Addr: tcp/127.0.0.1:13604) (DC: dc1)" -42:43.599 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13642 network=tcp -42:43.599 [INFO] TestAgent: started state syncer -42:43.599 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13605 network=tcp -42:43.662 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:43.662 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13604 [Candidate]" term=2 -42:43.662 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:43.662 [DEBUG] TestAgent.server.raft: vote granted: from=4d9a2fd1-e505-73f6-45c5-26344535431c term=2 tally=1 -42:43.662 [INFO] TestAgent.server.raft: election won: tally=1 -42:43.662 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13604 [Leader]" -42:43.665 [INFO] TestAgent.server: cluster leadership acquired -42:43.666 [INFO] TestAgent.server: New leader elected: payload=Node-4d9a2fd1-e505-73f6-45c5-26344535431c -42:43.666 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13604 -42:43.679 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:43.682 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:43.682 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:43.682 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:43.682 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:43.682 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:43.682 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4d9a2fd1-e505-73f6-45c5-26344535431c -42:43.682 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4d9a2fd1-e505-73f6-45c5-26344535431c -42:43.683 [INFO] TestAgent.server: federation state anti-entropy synced -42:43.769 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:43.770 [INFO] TestAgent: Synced node info -42:43.829 [INFO] TestAgent: Node entered maintenance mode -42:43.829 [DEBUG] TestAgent: removed check: check=_node_maintenance -42:43.829 [INFO] TestAgent: Node left maintenance mode -42:43.829 [INFO] TestAgent: Node entered maintenance mode -42:43.829 [INFO] TestAgent: Requesting shutdown -42:43.829 [INFO] TestAgent.server: shutting down server -42:43.829 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:43.829 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:43.829 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:43.829 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:43.829 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:43.830 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:43.830 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:43.830 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:43.830 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:43.830Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:43.830Z [INFO] agent.router.manager: shutting down -42:43.830 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:43.830 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:43.830 [INFO] TestAgent: consul server down -42:43.830 [INFO] TestAgent: shutdown complete -42:43.830 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13641 network=tcp -42:43.830 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13641 network=udp -42:43.830 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13642 network=tcp -42:44.337 [INFO] TestAgent: Waiting for endpoints to shut down -42:44.337 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_NodeMaintenanceMode (0.92s) +2021-08-13T15:34:01.460Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:01.476Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:01.478 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7c96cd3f-77af-8af1-e4bb-8b154cac0a2c Address:127.0.0.1:24620}]" +34:01.479 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24620 [Follower]" leader= +34:01.479 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7c96cd3f-77af-8af1-e4bb-8b154cac0a2c.dc1 127.0.0.1 +34:01.480 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7c96cd3f-77af-8af1-e4bb-8b154cac0a2c 127.0.0.1 +2021-08-13T15:34:01.480Z [INFO] agent.router: Initializing LAN area manager +34:01.480 [INFO] TestAgent.server: Adding LAN server: server="Node-7c96cd3f-77af-8af1-e4bb-8b154cac0a2c (Addr: tcp/127.0.0.1:24620) (DC: dc1)" +34:01.481 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7c96cd3f-77af-8af1-e4bb-8b154cac0a2c.dc1 area=wan +34:01.481 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24615 network=tcp +34:01.481 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24615 network=udp +34:01.482 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24616 network=tcp +34:01.482 [INFO] TestAgent: started state syncer +34:01.482 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24621 network=tcp +34:01.517 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:01.517 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24620 [Candidate]" term=2 +34:01.517 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:01.517 [DEBUG] TestAgent.server.raft: vote granted: from=7c96cd3f-77af-8af1-e4bb-8b154cac0a2c term=2 tally=1 +34:01.517 [INFO] TestAgent.server.raft: election won: tally=1 +34:01.517 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24620 [Leader]" +34:01.517 [INFO] TestAgent.server: cluster leadership acquired +34:01.517 [INFO] TestAgent.server: New leader elected: payload=Node-7c96cd3f-77af-8af1-e4bb-8b154cac0a2c +34:01.518 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24620 +34:01.519 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:01.522 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:01.522 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:01.522 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:01.522 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:01.522 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:01.522 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7c96cd3f-77af-8af1-e4bb-8b154cac0a2c +34:01.522 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7c96cd3f-77af-8af1-e4bb-8b154cac0a2c +34:01.523 [INFO] TestAgent.server: federation state anti-entropy synced +34:01.891 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:01.891 [INFO] TestAgent: Synced node info +34:01.891 [DEBUG] TestAgent: Node info in sync +34:01.909 [DEBUG] TestAgent: removed check: check=service:redis +34:01.909 [DEBUG] TestAgent: restored health check from file: check=service:redis file=/tmp/consul-test/TestAgent_checkStateSnapshot-TestAgent-agent300732094/checks/60a2ef12de014a05ecdc850d9aab46da +34:01.909 [INFO] TestAgent: Requesting shutdown +34:01.910 [INFO] TestAgent.server: shutting down server +34:01.910 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:01.910 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:01.910 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:01.910 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:01.910 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:01.910 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:01.910 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:01.914 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:01.914 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:01.914 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:01.914 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:01.915Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:01.915Z [INFO] agent.router.manager: shutting down +34:01.915 [INFO] TestAgent: consul server down +34:01.915 [INFO] TestAgent: shutdown complete +34:01.915 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24615 network=tcp +34:01.915 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24615 network=udp +34:01.915 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24616 network=tcp +34:02.419 [INFO] TestAgent: Waiting for endpoints to shut down +34:02.419 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_checkStateSnapshot (0.98s) === CONT TestAgent_PurgeCheckOnDuplicate -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=1)_compressed=false#01 -[WARN] freeport: 4 out of 4 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:43.880Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:43.915Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:43.925 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9f5725ac-a516-da81-09fb-f52990d5acb7 Address:127.0.0.1:13660}]" -42:43.925 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13660 [Follower]" leader= -42:43.926 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9f5725ac-a516-da81-09fb-f52990d5acb7.dc1 127.0.0.1 -42:43.927 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9f5725ac-a516-da81-09fb-f52990d5acb7 127.0.0.1 -2022-09-15T21:42:43.927Z [INFO] agent.router: Initializing LAN area manager -42:43.928 [INFO] TestAgent.server: Adding LAN server: server="Node-9f5725ac-a516-da81-09fb-f52990d5acb7 (Addr: tcp/127.0.0.1:13660) (DC: dc1)" -42:43.928 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9f5725ac-a516-da81-09fb-f52990d5acb7.dc1 area=wan -42:43.928 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13585 network=udp -42:43.928 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13585 network=tcp -42:43.929 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13586 network=tcp -42:43.929 [INFO] TestAgent: started state syncer -42:43.929 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13661 network=tcp -42:43.964 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:43.964 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13660 [Candidate]" term=2 -42:43.965 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:43.965 [DEBUG] TestAgent.server.raft: vote granted: from=9f5725ac-a516-da81-09fb-f52990d5acb7 term=2 tally=1 -42:43.965 [INFO] TestAgent.server.raft: election won: tally=1 -42:43.965 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13660 [Leader]" -42:43.965 [INFO] TestAgent.server: cluster leadership acquired -42:43.965 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13660 -42:43.966 [INFO] TestAgent.server: New leader elected: payload=Node-9f5725ac-a516-da81-09fb-f52990d5acb7 -42:43.971 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:43.973 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:43.973 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:43.973 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:43.973 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:43.973 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:43.974 [INFO] TestAgent.server: federation state anti-entropy synced -42:43.982 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9f5725ac-a516-da81-09fb-f52990d5acb7 -42:43.982 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9f5725ac-a516-da81-09fb-f52990d5acb7 -42:44.219 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:44.219 [INFO] TestAgent: Synced node info -42:44.267 [INFO] TestAgent: Service entered maintenance mode: service=redis -42:44.267 [DEBUG] TestAgent: removed check: check=_service_maintenance:redis -42:44.267 [INFO] TestAgent: Service left maintenance mode: service=redis -42:44.267 [INFO] TestAgent: Service entered maintenance mode: service=redis -42:44.267 [INFO] TestAgent: Requesting shutdown -42:44.267 [INFO] TestAgent.server: shutting down server -42:44.267 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:44.267 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:44.267 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:44.267 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:44.267 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:44.267 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:44.267 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:44.268 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:44.268 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:44.268 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:44.268 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:44.268Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:44.268Z [INFO] agent.router.manager: shutting down -42:44.268 [INFO] TestAgent: consul server down -42:44.268 [INFO] TestAgent: shutdown complete -42:44.268 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13585 network=tcp -42:44.268 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13585 network=udp -42:44.269 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13586 network=tcp -42:44.774 [INFO] TestAgent: Waiting for endpoints to shut down -42:44.775 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Service_MaintenanceMode (0.93s) +2021-08-13T15:34:01.586Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:01.604Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:01.607 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c4f986d9-0330-2e29-690d-855828dc3b3b Address:127.0.0.1:24627}]" +34:01.607 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24627 [Follower]" leader= +34:01.608 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c4f986d9-0330-2e29-690d-855828dc3b3b.dc1 127.0.0.1 +34:01.622 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c4f986d9-0330-2e29-690d-855828dc3b3b 127.0.0.1 +2021-08-13T15:34:01.622Z [INFO] agent.router: Initializing LAN area manager +34:01.628 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24622 network=tcp +34:01.628 [INFO] TestAgent.server: Adding LAN server: server="Node-c4f986d9-0330-2e29-690d-855828dc3b3b (Addr: tcp/127.0.0.1:24627) (DC: dc1)" +34:01.629 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c4f986d9-0330-2e29-690d-855828dc3b3b.dc1 area=wan +34:01.629 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24622 network=udp +34:01.634 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24623 network=tcp +34:01.634 [INFO] TestAgent: started state syncer +34:01.635 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24628 network=tcp +34:01.659 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:01.659 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24627 [Candidate]" term=2 +34:01.659 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:01.659 [DEBUG] TestAgent.server.raft: vote granted: from=c4f986d9-0330-2e29-690d-855828dc3b3b term=2 tally=1 +34:01.659 [INFO] TestAgent.server.raft: election won: tally=1 +34:01.659 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24627 [Leader]" +34:01.660 [INFO] TestAgent.server: cluster leadership acquired +34:01.660 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24627 +34:01.661 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:01.667 [INFO] TestAgent.server: New leader elected: payload=Node-c4f986d9-0330-2e29-690d-855828dc3b3b +34:01.668 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:01.668 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:01.668 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:01.668 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:01.668 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:01.668 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c4f986d9-0330-2e29-690d-855828dc3b3b +34:01.668 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c4f986d9-0330-2e29-690d-855828dc3b3b +34:01.669 [INFO] TestAgent.server: federation state anti-entropy synced +34:01.911 [INFO] TestAgent: Requesting shutdown +34:01.911 [INFO] TestAgent.server: shutting down server +34:01.911 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:01.911 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:01.911 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:01.911 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:01.911 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:01.912 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:01.912 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:01.912 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:01.912 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:01.912 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:01.912 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:01.912 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:01.913Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:01.913Z [INFO] agent.router.manager: shutting down +34:01.923 [INFO] TestAgent: consul server down +34:01.923 [INFO] TestAgent: shutdown complete +34:01.923 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24622 network=tcp +34:01.923 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24622 network=udp +34:01.923 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24623 network=tcp +34:02.424 [INFO] TestAgent: Waiting for endpoints to shut down +34:02.424 [INFO] TestAgent: Endpoints down +2021-08-13T15:34:00.977Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:00.994Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:00.997 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:36109506-1204-2ad7-63b1-84ff6a21ca9b Address:127.0.0.1:24585}]" +34:00.997 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24585 [Follower]" leader= +34:00.998 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-36109506-1204-2ad7-63b1-84ff6a21ca9b.dc1 127.0.0.1 +34:00.999 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-36109506-1204-2ad7-63b1-84ff6a21ca9b 127.0.0.1 +2021-08-13T15:34:00.999Z [INFO] agent.router: Initializing LAN area manager +34:00.999 [INFO] TestAgent.server: Adding LAN server: server="Node-36109506-1204-2ad7-63b1-84ff6a21ca9b (Addr: tcp/127.0.0.1:24585) (DC: dc1)" +34:00.999 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-36109506-1204-2ad7-63b1-84ff6a21ca9b.dc1 area=wan +34:00.999 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24573 network=tcp +34:00.999 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24573 network=udp +34:01.000 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24574 network=tcp +34:01.000 [INFO] TestAgent: started state syncer +34:01.000 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24586 network=tcp +34:01.061 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:01.062 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24585 [Candidate]" term=2 +34:01.062 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:01.062 [DEBUG] TestAgent.server.raft: vote granted: from=36109506-1204-2ad7-63b1-84ff6a21ca9b term=2 tally=1 +34:01.062 [INFO] TestAgent.server.raft: election won: tally=1 +34:01.062 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24585 [Leader]" +34:01.062 [INFO] TestAgent.server: cluster leadership acquired +34:01.062 [INFO] TestAgent.server: New leader elected: payload=Node-36109506-1204-2ad7-63b1-84ff6a21ca9b +34:01.062 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24585 +34:01.064 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:01.066 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:01.066 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:01.066 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:01.066 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:01.066 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:01.066 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-36109506-1204-2ad7-63b1-84ff6a21ca9b +34:01.066 [INFO] TestAgent.server: member joined, marking health alive: member=Node-36109506-1204-2ad7-63b1-84ff6a21ca9b +34:01.066 [INFO] TestAgent: Requesting shutdown +34:01.066 [INFO] TestAgent.server: shutting down server +34:01.067 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:01.067 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:01.067 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:01.067 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:01.067 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:01.067 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:01.067 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:01.067 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:01.067 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +34:01.067 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +2021-08-13T15:34:01.067Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:01.067Z [INFO] agent.router.manager: shutting down +34:01.067 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:01.067 [INFO] TestAgent: consul server down +34:01.067 [INFO] TestAgent: shutdown complete +34:01.067 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24573 network=tcp +34:01.067 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24573 network=udp +34:01.067 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24574 network=tcp +34:01.567 [INFO] TestAgent: Waiting for endpoints to shut down +34:01.568 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_GetCoordinate (1.47s) === CONT TestAgent_PurgeCheck -2022-09-15T21:42:43.863Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:43.879Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:43.883 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1f72f8f3-58be-a43a-5e60-28dcd4088260 Address:127.0.0.1:13583}]" -42:43.884 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1f72f8f3-58be-a43a-5e60-28dcd4088260.dc1 127.0.0.1 -42:43.885 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1f72f8f3-58be-a43a-5e60-28dcd4088260 127.0.0.1 -2022-09-15T21:42:43.885Z [INFO] agent.router: Initializing LAN area manager -42:43.885 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13583 [Follower]" leader= -42:43.885 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13606 network=udp -42:43.885 [INFO] TestAgent.server: Adding LAN server: server="Node-1f72f8f3-58be-a43a-5e60-28dcd4088260 (Addr: tcp/127.0.0.1:13583) (DC: dc1)" -42:43.885 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1f72f8f3-58be-a43a-5e60-28dcd4088260.dc1 area=wan -42:43.886 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13606 network=tcp -42:43.887 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13607 network=tcp -42:43.887 [INFO] TestAgent: started state syncer -42:43.887 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13584 network=tcp -42:43.945 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:43.945 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13583 [Candidate]" term=2 -42:43.946 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:43.946 [DEBUG] TestAgent.server.raft: vote granted: from=1f72f8f3-58be-a43a-5e60-28dcd4088260 term=2 tally=1 -42:43.946 [INFO] TestAgent.server.raft: election won: tally=1 -42:43.946 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13583 [Leader]" -42:43.946 [INFO] TestAgent.server: cluster leadership acquired -42:43.946 [INFO] TestAgent.server: New leader elected: payload=Node-1f72f8f3-58be-a43a-5e60-28dcd4088260 -42:43.947 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13583 -42:43.948 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:43.951 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:43.951 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:43.951 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:43.951 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:43.951 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:43.951 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1f72f8f3-58be-a43a-5e60-28dcd4088260 -42:43.951 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1f72f8f3-58be-a43a-5e60-28dcd4088260 -42:43.952 [INFO] TestAgent.server: federation state anti-entropy synced -42:43.983 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:43.984 [INFO] TestAgent: Synced node info -42:43.984 [DEBUG] TestAgent: Node info in sync -42:44.285 [INFO] TestAgent: Requesting shutdown -42:44.285 [INFO] TestAgent.server: shutting down server -42:44.285 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:44.285 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:44.285 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:44.285 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:44.285 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:44.285 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:44.285 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:44.285 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:44.285 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:44.285 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:44.285 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:44.286Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:44.286Z [INFO] agent.router.manager: shutting down -42:44.286 [INFO] TestAgent: consul server down -42:44.286 [INFO] TestAgent: shutdown complete -42:44.286 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13606 network=tcp -42:44.286 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13606 network=udp -42:44.286 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13607 network=tcp -42:44.790 [INFO] TestAgent: Waiting for endpoints to shut down -42:44.791 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_AddCheck_restoresSnapshot (0.95s) +[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:01.859Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:01.876Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:01.879 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6ee1bfa5-66c8-ca3a-dfc5-9b6725f534d5 Address:127.0.0.1:24655}]" +34:01.880 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6ee1bfa5-66c8-ca3a-dfc5-9b6725f534d5.dc1 127.0.0.1 +34:01.881 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6ee1bfa5-66c8-ca3a-dfc5-9b6725f534d5 127.0.0.1 +2021-08-13T15:34:01.881Z [INFO] agent.router: Initializing LAN area manager +34:01.882 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24655 [Follower]" leader= +34:01.882 [INFO] TestAgent.server: Adding LAN server: server="Node-6ee1bfa5-66c8-ca3a-dfc5-9b6725f534d5 (Addr: tcp/127.0.0.1:24655) (DC: dc1)" +34:01.882 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6ee1bfa5-66c8-ca3a-dfc5-9b6725f534d5.dc1 area=wan +34:01.882 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24643 network=tcp +34:01.883 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24643 network=udp +34:01.884 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24644 network=tcp +34:01.884 [INFO] TestAgent: started state syncer +34:01.884 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24656 network=tcp +34:01.935 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:01.935 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24655 [Candidate]" term=2 +34:01.935 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:01.935 [DEBUG] TestAgent.server.raft: vote granted: from=6ee1bfa5-66c8-ca3a-dfc5-9b6725f534d5 term=2 tally=1 +34:01.935 [INFO] TestAgent.server.raft: election won: tally=1 +34:01.935 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24655 [Leader]" +34:01.936 [INFO] TestAgent.server: cluster leadership acquired +34:01.936 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24655 +34:01.937 [INFO] TestAgent.server: New leader elected: payload=Node-6ee1bfa5-66c8-ca3a-dfc5-9b6725f534d5 +34:01.938 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:01.942 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:01.942 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:01.942 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:01.942 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:01.942 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:01.942 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6ee1bfa5-66c8-ca3a-dfc5-9b6725f534d5 +34:01.942 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6ee1bfa5-66c8-ca3a-dfc5-9b6725f534d5 +34:01.952 [INFO] TestAgent.server: federation state anti-entropy synced +34:02.143 [INFO] TestAgent: Requesting shutdown +34:02.144 [INFO] TestAgent.server: shutting down server +34:02.144 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:02.144 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:02.144 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:02.144 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:02.144 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:02.144 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:02.144 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:02.144 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:02.144 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:02.144 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:02.144 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:02.144 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:02.144Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:02.144Z [INFO] agent.router.manager: shutting down +34:02.144 [INFO] TestAgent: consul server down +34:02.144 [INFO] TestAgent: shutdown complete +34:02.144 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24643 network=tcp +34:02.144 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24643 network=udp +34:02.144 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24644 network=tcp +34:02.644 [INFO] TestAgent: Waiting for endpoints to shut down +34:02.644 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_AddCheck_restoresSnapshot (0.80s) +=== CONT TestIngestUserEvent +2021-08-13T15:34:02.040Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:02.056Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:02.059 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0e7c553e-bf61-c301-635f-497d8db286d6 Address:127.0.0.1:24690}]" +34:02.059 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24690 [Follower]" leader= +34:02.060 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0e7c553e-bf61-c301-635f-497d8db286d6.dc1 127.0.0.1 +34:02.060 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0e7c553e-bf61-c301-635f-497d8db286d6 127.0.0.1 +2021-08-13T15:34:02.060Z [INFO] agent.router: Initializing LAN area manager +34:02.061 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24664 network=udp +34:02.061 [INFO] TestAgent.server: Adding LAN server: server="Node-0e7c553e-bf61-c301-635f-497d8db286d6 (Addr: tcp/127.0.0.1:24690) (DC: dc1)" +34:02.061 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0e7c553e-bf61-c301-635f-497d8db286d6.dc1 area=wan +34:02.061 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24664 network=tcp +34:02.062 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24665 network=tcp +34:02.062 [INFO] TestAgent: started state syncer +34:02.062 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24691 network=tcp +34:02.102 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:02.102 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24690 [Candidate]" term=2 +34:02.102 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:02.102 [DEBUG] TestAgent.server.raft: vote granted: from=0e7c553e-bf61-c301-635f-497d8db286d6 term=2 tally=1 +34:02.102 [INFO] TestAgent.server.raft: election won: tally=1 +34:02.102 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24690 [Leader]" +34:02.102 [INFO] TestAgent.server: cluster leadership acquired +34:02.102 [INFO] TestAgent.server: New leader elected: payload=Node-0e7c553e-bf61-c301-635f-497d8db286d6 +34:02.102 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24690 +34:02.103 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:02.106 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:02.106 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:02.106 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:02.106 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:02.106 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:02.106 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0e7c553e-bf61-c301-635f-497d8db286d6 +34:02.106 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0e7c553e-bf61-c301-635f-497d8db286d6 +34:02.107 [INFO] TestAgent.server: federation state anti-entropy synced +34:02.166 [DEBUG] TestAgent: removed check: check=service:redis +34:02.166 [INFO] TestAgent: Requesting shutdown +34:02.166 [INFO] TestAgent.server: shutting down server +34:02.166 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:02.166 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:02.166 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:02.166 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:02.166 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:02.166 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:02.166 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:02.166 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:02.166 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:02.166 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:02.166 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:02.166 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:02.167Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:02.167Z [INFO] agent.router.manager: shutting down +34:02.167 [INFO] TestAgent: consul server down +34:02.167 [INFO] TestAgent: shutdown complete +34:02.167 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24664 network=tcp +34:02.167 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24664 network=udp +34:02.167 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24665 network=tcp +34:02.667 [INFO] TestAgent: Waiting for endpoints to shut down +34:02.667 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_unloadChecks (0.64s) === CONT TestAgent_PersistCheck -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:44.084Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:44.101Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:44.104 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c27cb976-a269-cd65-a4d9-9a9b50cf9eb6 Address:127.0.0.1:13646}]" -42:44.105 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c27cb976-a269-cd65-a4d9-9a9b50cf9eb6.dc1 127.0.0.1 -42:44.106 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13646 [Follower]" leader= -42:44.107 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c27cb976-a269-cd65-a4d9-9a9b50cf9eb6 127.0.0.1 -2022-09-15T21:42:44.107Z [INFO] agent.router: Initializing LAN area manager -42:44.107 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13662 network=udp -42:44.107 [INFO] TestAgent.server: Adding LAN server: server="Node-c27cb976-a269-cd65-a4d9-9a9b50cf9eb6 (Addr: tcp/127.0.0.1:13646) (DC: dc1)" -42:44.108 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c27cb976-a269-cd65-a4d9-9a9b50cf9eb6.dc1 area=wan -42:44.108 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13662 network=tcp -42:44.109 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13663 network=tcp -42:44.109 [INFO] TestAgent: started state syncer -42:44.109 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13647 network=tcp -42:44.163 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:44.163 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13646 [Candidate]" term=2 -42:44.164 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:44.166 [DEBUG] TestAgent.server.raft: vote granted: from=c27cb976-a269-cd65-a4d9-9a9b50cf9eb6 term=2 tally=1 -42:44.167 [INFO] TestAgent.server.raft: election won: tally=1 -42:44.167 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13646 [Leader]" -42:44.167 [INFO] TestAgent.server: cluster leadership acquired -42:44.167 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13646 -42:44.167 [INFO] TestAgent.server: New leader elected: payload=Node-c27cb976-a269-cd65-a4d9-9a9b50cf9eb6 -42:44.169 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:44.173 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:44.173 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:44.173 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:44.173 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:44.173 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:44.173 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c27cb976-a269-cd65-a4d9-9a9b50cf9eb6 -42:44.173 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c27cb976-a269-cd65-a4d9-9a9b50cf9eb6 -42:44.174 [INFO] TestAgent.server: federation state anti-entropy synced -42:44.228 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:44.229 [INFO] TestAgent: Synced node info -42:44.229 [DEBUG] TestAgent: Node info in sync -42:44.529 [DEBUG] TestAgent: removed check: check=service:redis -42:44.529 [INFO] TestAgent: Requesting shutdown -42:44.530 [INFO] TestAgent.server: shutting down server -42:44.530 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:44.530 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:44.530 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:44.530 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:44.530 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:44.530 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:44.530 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:44.530 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:44.530 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:44.530 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:44.530 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:44.530Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:44.531Z [INFO] agent.router.manager: shutting down -42:44.531 [INFO] TestAgent: consul server down -42:44.531 [INFO] TestAgent: shutdown complete -42:44.531 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13662 network=tcp -42:44.531 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13662 network=udp -42:44.531 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13663 network=tcp -42:45.032 [INFO] TestAgent: Waiting for endpoints to shut down -42:45.038 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_unloadChecks (1.00s) +2021-08-13T15:34:01.888Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:01.966Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:01.969 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1ccbc66e-5c57-de90-c69e-1a85cc148a0f Address:127.0.0.1:24662}]" +34:01.969 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24662 [Follower]" leader= +34:01.977 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1ccbc66e-5c57-de90-c69e-1a85cc148a0f.dc1 127.0.0.1 +34:01.978 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1ccbc66e-5c57-de90-c69e-1a85cc148a0f 127.0.0.1 +2021-08-13T15:34:01.978Z [INFO] agent.router: Initializing LAN area manager +34:01.979 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24657 network=udp +34:01.979 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1ccbc66e-5c57-de90-c69e-1a85cc148a0f.dc1 area=wan +34:01.979 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24657 network=tcp +34:01.980 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24658 network=tcp +34:01.979 [INFO] TestAgent.server: Adding LAN server: server="Node-1ccbc66e-5c57-de90-c69e-1a85cc148a0f (Addr: tcp/127.0.0.1:24662) (DC: dc1)" +34:01.980 [INFO] TestAgent: started state syncer +34:01.980 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24663 network=tcp +34:02.028 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:02.028 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24662 [Candidate]" term=2 +34:02.028 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:02.028 [DEBUG] TestAgent.server.raft: vote granted: from=1ccbc66e-5c57-de90-c69e-1a85cc148a0f term=2 tally=1 +34:02.028 [INFO] TestAgent.server.raft: election won: tally=1 +34:02.028 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24662 [Leader]" +34:02.028 [INFO] TestAgent.server: cluster leadership acquired +34:02.028 [INFO] TestAgent.server: New leader elected: payload=Node-1ccbc66e-5c57-de90-c69e-1a85cc148a0f +34:02.028 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24662 +34:02.030 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:02.032 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:02.032 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:02.032 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:02.032 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:02.032 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:02.032 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1ccbc66e-5c57-de90-c69e-1a85cc148a0f +34:02.032 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1ccbc66e-5c57-de90-c69e-1a85cc148a0f +34:02.033 [INFO] TestAgent.server: federation state anti-entropy synced +34:02.220 [INFO] TestAgent: Service entered maintenance mode: service=redis +34:02.220 [DEBUG] TestAgent: removed check: check=_service_maintenance:redis +34:02.220 [INFO] TestAgent: Service left maintenance mode: service=redis +34:02.220 [INFO] TestAgent: Service entered maintenance mode: service=redis +34:02.220 [INFO] TestAgent: Requesting shutdown +34:02.220 [INFO] TestAgent.server: shutting down server +34:02.220 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:02.220 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:02.220 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:02.220 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:02.220 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:02.221 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:02.221 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:02.221 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:02.221 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:02.221 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:02.221 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:02.221 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:02.221Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:02.221Z [INFO] agent.router.manager: shutting down +34:02.221 [INFO] TestAgent: consul server down +34:02.221 [INFO] TestAgent: shutdown complete +34:02.221 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24657 network=tcp +34:02.221 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24657 network=udp +34:02.221 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24658 network=tcp +34:02.722 [INFO] TestAgent: Waiting for endpoints to shut down +34:02.722 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Service_MaintenanceMode (0.87s) +=== CONT TestTxnEndpoint_UpdateCheck +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:02.252Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:02.270Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:02.273 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6ad289ec-f9df-0f91-b933-c6ea32ea7fd6 Address:127.0.0.1:24669}]" +34:02.273 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24669 [Follower]" leader= +34:02.274 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6ad289ec-f9df-0f91-b933-c6ea32ea7fd6.dc1 127.0.0.1 +34:02.275 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6ad289ec-f9df-0f91-b933-c6ea32ea7fd6 127.0.0.1 +2021-08-13T15:34:02.275Z [INFO] agent.router: Initializing LAN area manager +34:02.275 [INFO] TestAgent.server: Adding LAN server: server="Node-6ad289ec-f9df-0f91-b933-c6ea32ea7fd6 (Addr: tcp/127.0.0.1:24669) (DC: dc1)" +34:02.275 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6ad289ec-f9df-0f91-b933-c6ea32ea7fd6.dc1 area=wan +34:02.275 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24692 network=tcp +34:02.275 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24692 network=udp +34:02.276 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24693 network=tcp +34:02.276 [INFO] TestAgent: started state syncer +34:02.276 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24670 network=tcp +34:02.316 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:02.316 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24669 [Candidate]" term=2 +34:02.316 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:02.316 [DEBUG] TestAgent.server.raft: vote granted: from=6ad289ec-f9df-0f91-b933-c6ea32ea7fd6 term=2 tally=1 +34:02.316 [INFO] TestAgent.server.raft: election won: tally=1 +34:02.316 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24669 [Leader]" +34:02.316 [INFO] TestAgent.server: cluster leadership acquired +34:02.316 [INFO] TestAgent.server: New leader elected: payload=Node-6ad289ec-f9df-0f91-b933-c6ea32ea7fd6 +34:02.316 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24669 +34:02.318 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:02.321 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:02.321 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:02.321 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:02.321 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:02.321 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:02.321 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6ad289ec-f9df-0f91-b933-c6ea32ea7fd6 +34:02.321 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6ad289ec-f9df-0f91-b933-c6ea32ea7fd6 +34:02.321 [INFO] TestAgent.server: federation state anti-entropy synced +34:02.411 [INFO] TestAgent: Requesting shutdown +34:02.411 [INFO] TestAgent.server: shutting down server +34:02.411 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:02.411 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:02.411 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:02.411 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:02.411 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:02.411 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:02.411 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:02.412 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:02.412 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:02.412 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:02.412 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:02.412 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:02.412Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:02.412Z [INFO] agent.router.manager: shutting down +34:02.412 [INFO] TestAgent: consul server down +34:02.412 [INFO] TestAgent: shutdown complete +34:02.412 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24692 network=tcp +34:02.412 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24692 network=udp +34:02.412 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24693 network=tcp +34:02.912 [INFO] TestAgent: Waiting for endpoints to shut down +34:02.912 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_loadChecks_token (0.68s) === CONT TestAgent_updateTTLCheck -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=1)_compressed=true#01 -2022-09-15T21:42:44.119Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:44.147Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:44.151 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4c392f40-bb07-d75a-6a01-28cbd8c1d343 Address:127.0.0.1:13653}]" -42:44.152 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13653 [Follower]" leader= -42:44.153 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4c392f40-bb07-d75a-6a01-28cbd8c1d343.dc1 127.0.0.1 -42:44.154 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4c392f40-bb07-d75a-6a01-28cbd8c1d343 127.0.0.1 -2022-09-15T21:42:44.154Z [INFO] agent.router: Initializing LAN area manager -42:44.155 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13648 network=udp -42:44.155 [INFO] TestAgent.server: Adding LAN server: server="Node-4c392f40-bb07-d75a-6a01-28cbd8c1d343 (Addr: tcp/127.0.0.1:13653) (DC: dc1)" -42:44.155 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4c392f40-bb07-d75a-6a01-28cbd8c1d343.dc1 area=wan -42:44.155 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13648 network=tcp -42:44.156 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13649 network=tcp -42:44.156 [INFO] TestAgent: started state syncer -42:44.156 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13654 network=tcp -42:44.210 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:44.210 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13653 [Candidate]" term=2 -42:44.211 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:44.211 [DEBUG] TestAgent.server.raft: vote granted: from=4c392f40-bb07-d75a-6a01-28cbd8c1d343 term=2 tally=1 -42:44.211 [INFO] TestAgent.server.raft: election won: tally=1 -42:44.211 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13653 [Leader]" -42:44.212 [INFO] TestAgent.server: cluster leadership acquired -42:44.212 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13653 -42:44.215 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:44.215 [INFO] TestAgent.server: New leader elected: payload=Node-4c392f40-bb07-d75a-6a01-28cbd8c1d343 -42:44.217 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:44.217 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:44.217 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:44.217 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:44.217 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:44.217 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4c392f40-bb07-d75a-6a01-28cbd8c1d343 -42:44.217 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4c392f40-bb07-d75a-6a01-28cbd8c1d343 -42:44.222 [INFO] TestAgent.server: federation state anti-entropy synced -42:44.475 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:44.476 [INFO] TestAgent: Synced node info -42:44.477 [INFO] TestAgent: Synced check: check=rabbitmq -42:44.477 [DEBUG] TestAgent: Node info in sync -42:44.477 [DEBUG] TestAgent: Check in sync: check=rabbitmq -42:44.562 [INFO] TestAgent: Requesting shutdown -42:44.562 [INFO] TestAgent.server: shutting down server -42:44.562 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:44.562 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:44.562 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:44.562 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:44.562 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:44.562 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:44.562 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:44.562 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:44.562 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:44.562 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:44.563 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:44.563Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:44.563Z [INFO] agent.router.manager: shutting down -42:44.563 [INFO] TestAgent: consul server down -42:44.563 [INFO] TestAgent: shutdown complete -42:44.563 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13648 network=tcp -42:44.563 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13648 network=udp -42:44.563 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13649 network=tcp -42:45.064 [INFO] TestAgent: Waiting for endpoints to shut down -42:45.064 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_loadChecks_token (0.98s) +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:02.662Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:02.679Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:02.682 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d6df0a4f-4384-f033-8f1e-a1596b889c3a Address:127.0.0.1:24683}]" +34:02.682 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24683 [Follower]" leader= +34:02.683 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d6df0a4f-4384-f033-8f1e-a1596b889c3a.dc1 127.0.0.1 +34:02.684 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d6df0a4f-4384-f033-8f1e-a1596b889c3a 127.0.0.1 +2021-08-13T15:34:02.684Z [INFO] agent.router: Initializing LAN area manager +34:02.684 [INFO] TestAgent.server: Adding LAN server: server="Node-d6df0a4f-4384-f033-8f1e-a1596b889c3a (Addr: tcp/127.0.0.1:24683) (DC: dc1)" +34:02.684 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d6df0a4f-4384-f033-8f1e-a1596b889c3a.dc1 area=wan +34:02.684 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24699 network=udp +34:02.685 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24699 network=tcp +34:02.685 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24700 network=tcp +34:02.685 [INFO] TestAgent: started state syncer +34:02.686 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24684 network=tcp +34:02.719 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:02.719 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24683 [Candidate]" term=2 +34:02.719 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:02.719 [DEBUG] TestAgent.server.raft: vote granted: from=d6df0a4f-4384-f033-8f1e-a1596b889c3a term=2 tally=1 +34:02.719 [INFO] TestAgent.server.raft: election won: tally=1 +34:02.719 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24683 [Leader]" +34:02.720 [INFO] TestAgent.server: cluster leadership acquired +34:02.720 [INFO] TestAgent.server: New leader elected: payload=Node-d6df0a4f-4384-f033-8f1e-a1596b889c3a +34:02.720 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24683 +34:02.721 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:02.724 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:02.724 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:02.724 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:02.724 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:02.724 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:02.724 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d6df0a4f-4384-f033-8f1e-a1596b889c3a +34:02.724 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d6df0a4f-4384-f033-8f1e-a1596b889c3a +34:02.726 [INFO] TestAgent.server: federation state anti-entropy synced +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.790 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.791 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.792 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [DEBUG] TestAgent: new event: event_name=test event_id= +34:02.793 [INFO] TestAgent: Requesting shutdown +34:02.793 [INFO] TestAgent.server: shutting down server +34:02.793 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:02.793 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:02.793 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:02.793 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:02.793 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:02.793 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:02.793 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:02.793 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:02.793 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:02.793 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:02.793 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:02.794 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:02.794Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:02.794Z [INFO] agent.router.manager: shutting down +34:02.794 [INFO] TestAgent: consul server down +34:02.794 [INFO] TestAgent: shutdown complete +34:02.794 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24699 network=tcp +34:02.794 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24699 network=udp +34:02.794 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24700 network=tcp +34:03.294 [INFO] TestAgent: Waiting for endpoints to shut down +34:03.294 [INFO] TestAgent: Endpoints down +--- PASS: TestIngestUserEvent (0.65s) +=== CONT TestTxnEndpoint_KV_Actions +=== RUN TestTxnEndpoint_KV_Actions/#00 +2021-08-13T15:34:02.495Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:02.512Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:02.515 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9a1de784-d047-94c8-5001-9c2f04fffef8 Address:127.0.0.1:24697}]" +34:02.515 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24697 [Follower]" leader= +34:02.516 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9a1de784-d047-94c8-5001-9c2f04fffef8.dc1 127.0.0.1 +34:02.517 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9a1de784-d047-94c8-5001-9c2f04fffef8 127.0.0.1 +2021-08-13T15:34:02.517Z [INFO] agent.router: Initializing LAN area manager +34:02.517 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24678 network=udp +34:02.518 [INFO] TestAgent.server: Adding LAN server: server="Node-9a1de784-d047-94c8-5001-9c2f04fffef8 (Addr: tcp/127.0.0.1:24697) (DC: dc1)" +34:02.518 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9a1de784-d047-94c8-5001-9c2f04fffef8.dc1 area=wan +34:02.518 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24678 network=tcp +34:02.519 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24679 network=tcp +34:02.519 [INFO] TestAgent: started state syncer +34:02.519 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24698 network=tcp +34:02.568 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:02.568 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24697 [Candidate]" term=2 +34:02.569 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:02.569 [DEBUG] TestAgent.server.raft: vote granted: from=9a1de784-d047-94c8-5001-9c2f04fffef8 term=2 tally=1 +34:02.569 [INFO] TestAgent.server.raft: election won: tally=1 +34:02.569 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24697 [Leader]" +34:02.569 [INFO] TestAgent.server: cluster leadership acquired +34:02.569 [INFO] TestAgent.server: New leader elected: payload=Node-9a1de784-d047-94c8-5001-9c2f04fffef8 +34:02.569 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24697 +34:02.571 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:02.573 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:02.573 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:02.573 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:02.573 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:02.573 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:02.574 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9a1de784-d047-94c8-5001-9c2f04fffef8 +34:02.574 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9a1de784-d047-94c8-5001-9c2f04fffef8 +34:02.574 [INFO] TestAgent.server: federation state anti-entropy synced +34:02.873 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:02.874 [INFO] TestAgent: Synced node info +34:02.874 [DEBUG] TestAgent: Node info in sync +34:02.952 [DEBUG] TestAgent: removed check: check=mem +34:02.952 [DEBUG] TestAgent: removed check: check=mem +34:02.952 [INFO] TestAgent: Requesting shutdown +34:02.952 [INFO] TestAgent.server: shutting down server +34:02.952 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:02.952 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:02.952 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:02.952 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:02.952 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:02.953 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:02.953 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:02.953 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:02.953 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:02.953 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:02.953 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:02.954Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:02.954Z [INFO] agent.router.manager: shutting down +34:02.958 [INFO] TestAgent: consul server down +34:02.958 [INFO] TestAgent: shutdown complete +34:02.958 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24678 network=tcp +34:02.958 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24678 network=udp +34:02.958 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24679 network=tcp +34:03.458 [INFO] TestAgent: Waiting for endpoints to shut down +34:03.458 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_PurgeCheck (1.03s) === CONT TestAgent_HTTPCheck_TLSSkipVerify -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=1)_compressed=false#01 -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._tcp_(qType:=1)_compressed=true#01 -2022-09-15T21:42:44.736Z [WARN] agent: Node name "Node efea7d8d-8767-cb27-e43d-f0029d17e147" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. -2022-09-15T21:42:44.794Z [WARN] agent.auto_config: Node name "Node efea7d8d-8767-cb27-e43d-f0029d17e147" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. -42:44.795 [INFO] Agent2.client.serf.lan: serf: EventMemberJoin: Node efea7d8d-8767-cb27-e43d-f0029d17e147 127.0.0.1 -2022-09-15T21:42:44.795Z [INFO] agent.router: Initializing LAN area manager -42:44.796 [DEBUG] Agent2: restored service definition from file: service=svc-sidecar-proxy file=/tmp/consul-test/TestAgent_DeregisterPersistedSidecarAfterRestart-TestAgent-agent867356651/services/18ed17ea501b78b6980ecfa920fcf57a -42:44.796 [DEBUG] Agent2: restored service definition from file: service=svc file=/tmp/consul-test/TestAgent_DeregisterPersistedSidecarAfterRestart-TestAgent-agent867356651/services/961e38b5146c0b07d078f53dec9b699a -42:44.796 [INFO] Agent2: Started DNS server: address=127.0.0.1:13683 network=udp -42:44.796 [WARN] Agent2.client.serf.lan: serf: Failed to re-join any previously known node -2022-09-15T21:42:44.797Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.797Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.797Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.797Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.797Z [WARN] agent.router.manager: No servers available -42:44.797 [INFO] Agent2: Started DNS server: address=127.0.0.1:13683 network=tcp -2022-09-15T21:42:44.797Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.797Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.797Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.797Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.797Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.797Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.797Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.797Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.797Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.797Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.797Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.798Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.798Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.798Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.798Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.798Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.798Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.798Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.798Z [WARN] agent.router.manager: No servers available -42:44.798 [ERROR] Agent2.proxycfg: watch error: service_id=svc-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" -42:44.798 [ERROR] Agent2.proxycfg: watch error: service_id=svc-sidecar-proxy id=leaf error="error filling agent cache: No known Consul servers" -42:44.798 [ERROR] Agent2.proxycfg: watch error: service_id=svc-sidecar-proxy id=intentions error="error filling agent cache: No known Consul servers" -2022-09-15T21:42:44.798Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.798Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.798Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.798Z [WARN] agent.router.manager: No servers available -42:44.798 [INFO] Agent2: Started HTTP server: address=127.0.0.1:13684 network=tcp -2022-09-15T21:42:44.798Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.798Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.798Z [WARN] agent.router.manager: No servers available -42:44.798 [INFO] Agent2: started state syncer -2022-09-15T21:42:44.798Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.798Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.798Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.798Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.798Z [WARN] agent.router.manager: No servers available -42:44.798 [ERROR] Agent2.proxycfg: watch error: service_id=svc-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" -42:44.798 [ERROR] Agent2.proxycfg: watch error: service_id=svc-sidecar-proxy id=leaf error="error filling agent cache: No known Consul servers" -42:44.798 [ERROR] Agent2.proxycfg: watch error: service_id=svc-sidecar-proxy id=intentions error="error filling agent cache: No known Consul servers" -2022-09-15T21:42:44.798Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.798Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.798Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.798Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.799Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.799Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.799Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.799Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.799Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.799Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.799Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:44.799Z [WARN] agent.router.manager: No servers available -42:44.799 [ERROR] Agent2.proxycfg: watch error: service_id=svc-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" -42:44.799 [ERROR] Agent2.proxycfg: watch error: service_id=svc-sidecar-proxy id=leaf error="error filling agent cache: No known Consul servers" -42:44.799 [ERROR] Agent2.proxycfg: watch error: service_id=svc-sidecar-proxy id=intentions error="error filling agent cache: No known Consul servers" -42:44.799 [ERROR] Agent2.proxycfg: watch error: service_id=svc-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" -42:44.799 [ERROR] Agent2.proxycfg: watch error: service_id=svc-sidecar-proxy id=leaf error="error filling agent cache: No known Consul servers" -42:44.799 [ERROR] Agent2.proxycfg: watch error: service_id=svc-sidecar-proxy id=intentions error="error filling agent cache: No known Consul servers" -42:44.799 [INFO] Agent2: Started gRPC server: address=127.0.0.1:13696 network=tcp -2022-09-15T21:42:44.799Z [WARN] agent.router.manager: No servers available -42:44.799 [ERROR] Agent2.anti_entropy: failed to sync remote state: error="No known Consul servers" -42:44.799 [DEBUG] Agent2: removed service: service=svc -42:44.799 [DEBUG] Agent2: removed service: service=svc-sidecar-proxy -42:44.799 [INFO] Agent2: Requesting shutdown -42:44.800 [INFO] Agent2.client: shutting down client -42:44.800 [ERROR] Agent2.proxycfg: watch error: service_id=svc-sidecar-proxy id=service-http-checks:svc error="error filling agent cache: Internal cache failure: service 'svc' not in agent state" -42:44.800 [WARN] Agent2.client.serf.lan: serf: Shutdown without a Leave -42:44.800 [INFO] Agent2: consul client down -42:44.800 [INFO] Agent2: shutdown complete -42:44.800 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:13683 network=tcp -42:44.800 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:13683 network=udp -42:44.800 [INFO] Agent2: Stopping server: protocol=HTTP address=127.0.0.1:13684 network=tcp -42:45.302 [INFO] Agent2: Waiting for endpoints to shut down -42:45.302 [INFO] Agent2: Endpoints down -2022-09-15T21:42:44.177Z [WARN] agent: Node name "Node efea7d8d-8767-cb27-e43d-f0029d17e147" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. -2022-09-15T21:42:44.209Z [WARN] agent.auto_config: Node name "Node efea7d8d-8767-cb27-e43d-f0029d17e147" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. -42:44.211 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node efea7d8d-8767-cb27-e43d-f0029d17e147 127.0.0.1 -2022-09-15T21:42:44.211Z [INFO] agent.router: Initializing LAN area manager -42:44.211 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13655 network=udp -42:44.211 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13655 network=tcp -42:44.212 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13656 network=tcp -42:44.212 [INFO] TestAgent: started state syncer -42:44.214 [INFO] TestAgent: Requesting shutdown -42:44.214 [INFO] TestAgent.client: shutting down client -42:44.214 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave -42:44.214 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13668 network=tcp -42:44.214 [ERROR] TestAgent: gRPC server failed: error="grpc: the server has been stopped" -2022-09-15T21:42:44.214Z [WARN] agent.router.manager: No servers available -42:44.214 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" -42:44.215 [INFO] TestAgent: consul client down -42:44.215 [INFO] TestAgent: shutdown complete -42:44.215 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13655 network=tcp -42:44.215 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13655 network=udp -42:44.215 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13656 network=tcp -42:44.717 [INFO] TestAgent: Waiting for endpoints to shut down -42:44.717 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_DeregisterPersistedSidecarAfterRestart (1.15s) +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:03.031Z [WARN] agent: Node name "Node e44220eb-7196-8311-d4b6-fea71f7edc51" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. +2021-08-13T15:34:03.052Z [WARN] agent.auto_config: Node name "Node e44220eb-7196-8311-d4b6-fea71f7edc51" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. +34:03.053 [INFO] Agent2.client.serf.lan: serf: EventMemberJoin: Node e44220eb-7196-8311-d4b6-fea71f7edc51 127.0.0.1 +2021-08-13T15:34:03.053Z [INFO] agent.router: Initializing LAN area manager +34:03.053 [WARN] Agent2.client.serf.lan: serf: Failed to re-join any previously known node +34:03.053 [DEBUG] Agent2: restored service definition from file: service=svc-sidecar-proxy file=/tmp/consul-test/TestAgent_DeregisterPersistedSidecarAfterRestart-TestAgent-agent836360013/services/18ed17ea501b78b6980ecfa920fcf57a +34:03.054 [DEBUG] Agent2: restored service definition from file: service=svc file=/tmp/consul-test/TestAgent_DeregisterPersistedSidecarAfterRestart-TestAgent-agent836360013/services/961e38b5146c0b07d078f53dec9b699a +34:03.054 [INFO] Agent2: Started DNS server: address=127.0.0.1:24727 network=udp +34:03.054 [INFO] Agent2: Started DNS server: address=127.0.0.1:24727 network=tcp +2021-08-13T15:34:03.054Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:03.054Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:03.054Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:03.054Z [WARN] agent.router.manager: No servers available +34:03.054 [ERROR] Agent2.proxycfg: watch error: service_id=svc-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" +2021-08-13T15:34:03.054Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:03.054Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:03.054Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:03.054Z [WARN] agent.router.manager: No servers available +34:03.054 [ERROR] Agent2.proxycfg: watch error: service_id=svc-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" +34:03.055 [ERROR] Agent2.proxycfg: watch error: service_id=svc-sidecar-proxy id=leaf error="error filling agent cache: No known Consul servers" +2021-08-13T15:34:03.055Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:03.055Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:03.055Z [WARN] agent.router.manager: No servers available +34:03.055 [ERROR] Agent2.proxycfg: watch error: service_id=svc-sidecar-proxy id=intentions error="error filling agent cache: No known Consul servers" +34:03.055 [ERROR] Agent2.proxycfg: watch error: service_id=svc-sidecar-proxy id=intentions error="error filling agent cache: No known Consul servers" +2021-08-13T15:34:03.055Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:03.055Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:03.055Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:03.055Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:03.055Z [WARN] agent.router.manager: No servers available +34:03.055 [ERROR] Agent2.proxycfg: watch error: service_id=svc-sidecar-proxy id=leaf error="error filling agent cache: No known Consul servers" +34:03.055 [ERROR] Agent2.proxycfg: watch error: service_id=svc-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" +34:03.055 [ERROR] Agent2.proxycfg: watch error: service_id=svc-sidecar-proxy id=intentions error="error filling agent cache: No known Consul servers" +2021-08-13T15:34:03.055Z [WARN] agent.router.manager: No servers available +34:03.055 [ERROR] Agent2.proxycfg: watch error: service_id=svc-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" +2021-08-13T15:34:03.055Z [WARN] agent.router.manager: No servers available +34:03.055 [ERROR] Agent2.proxycfg: watch error: service_id=svc-sidecar-proxy id=leaf error="error filling agent cache: No known Consul servers" +2021-08-13T15:34:03.055Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:03.055Z [WARN] agent.router.manager: No servers available +34:03.055 [ERROR] Agent2.proxycfg: watch error: service_id=svc-sidecar-proxy id=intentions error="error filling agent cache: No known Consul servers" +2021-08-13T15:34:03.055Z [WARN] agent.router.manager: No servers available +34:03.055 [INFO] Agent2: Started HTTP server: address=127.0.0.1:24728 network=tcp +2021-08-13T15:34:03.055Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:03.055Z [WARN] agent.router.manager: No servers available +34:03.055 [ERROR] Agent2.proxycfg: watch error: service_id=svc-sidecar-proxy id=leaf error="error filling agent cache: No known Consul servers" +2021-08-13T15:34:03.055Z [WARN] agent.router.manager: No servers available +34:03.055 [INFO] Agent2: started state syncer +34:03.055 [INFO] Agent2: Started gRPC server: address=127.0.0.1:24733 network=tcp +2021-08-13T15:34:03.055Z [WARN] agent.router.manager: No servers available +34:03.055 [ERROR] Agent2.anti_entropy: failed to sync remote state: error="No known Consul servers" +34:03.056 [DEBUG] Agent2: removed service: service=svc +34:03.056 [DEBUG] Agent2: removed service: service=svc-sidecar-proxy +34:03.056 [INFO] Agent2: Requesting shutdown +34:03.056 [INFO] Agent2.client: shutting down client +34:03.056 [WARN] Agent2.client.serf.lan: serf: Shutdown without a Leave +34:03.056 [INFO] Agent2: consul client down +34:03.056 [INFO] Agent2: shutdown complete +34:03.056 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:24727 network=tcp +34:03.056 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:24727 network=udp +34:03.056 [INFO] Agent2: Stopping server: protocol=HTTP address=127.0.0.1:24728 network=tcp +34:03.557 [INFO] Agent2: Waiting for endpoints to shut down +34:03.557 [INFO] Agent2: Endpoints down +2021-08-13T15:34:02.405Z [WARN] agent: Node name "Node e44220eb-7196-8311-d4b6-fea71f7edc51" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. +2021-08-13T15:34:02.423Z [WARN] agent.auto_config: Node name "Node e44220eb-7196-8311-d4b6-fea71f7edc51" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. +34:02.424 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node e44220eb-7196-8311-d4b6-fea71f7edc51 127.0.0.1 +2021-08-13T15:34:02.424Z [INFO] agent.router: Initializing LAN area manager +34:02.427 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24671 network=tcp +34:02.427 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24671 network=udp +34:02.428 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24672 network=tcp +34:02.428 [INFO] TestAgent: started state syncer +34:02.430 [INFO] TestAgent: Requesting shutdown +34:02.430 [INFO] TestAgent.client: shutting down client +34:02.430 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave +34:02.430 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24712 network=tcp +34:02.430 [ERROR] TestAgent: gRPC server failed: error="grpc: the server has been stopped" +2021-08-13T15:34:02.430Z [WARN] agent.router.manager: No servers available +34:02.430 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" +34:02.431 [INFO] TestAgent: consul client down +34:02.431 [INFO] TestAgent: shutdown complete +34:02.431 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24671 network=tcp +34:02.431 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24671 network=udp +34:02.431 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24672 network=tcp +34:02.935 [INFO] TestAgent: Waiting for endpoints to shut down +34:02.935 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_DeregisterPersistedSidecarAfterRestart (1.17s) +=== CONT TestTxnEndpoint_Bad_Size_Ops +2021-08-13T15:34:03.044Z [WARN] agent: Node name "Node 56b99bd9-cc07-e6a6-b304-2741cf6214de" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. +2021-08-13T15:34:03.044Z [WARN] agent: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ +2021-08-13T15:34:03.061Z [WARN] agent.auto_config: Node name "Node 56b99bd9-cc07-e6a6-b304-2741cf6214de" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. +2021-08-13T15:34:03.061Z [WARN] agent.auto_config: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ +34:03.062 [INFO] Agent2.client.serf.lan: serf: EventMemberJoin: Node 56b99bd9-cc07-e6a6-b304-2741cf6214de 127.0.0.1 +2021-08-13T15:34:03.062Z [INFO] agent.router: Initializing LAN area manager +34:03.062 [WARN] Agent2.client.serf.lan: serf: Failed to re-join any previously known node +34:03.062 [DEBUG] Agent2: check exists, not restoring from file: check=mem file=/tmp/consul-test/TestAgent_PurgeCheckOnDuplicate-TestAgent-agent904022600/checks/afc4fc7e48a0710a1dc94ef3e8bc5764 +34:03.062 [INFO] Agent2: Started DNS server: address=127.0.0.1:24734 network=tcp +34:03.062 [INFO] Agent2: Started DNS server: address=127.0.0.1:24734 network=udp +34:03.063 [INFO] Agent2: Started HTTP server: address=127.0.0.1:24735 network=tcp +34:03.063 [INFO] Agent2: started state syncer +34:03.063 [INFO] Agent2: Started gRPC server: address=127.0.0.1:24740 network=tcp +2021-08-13T15:34:03.064Z [WARN] agent.router.manager: No servers available +34:03.064 [ERROR] Agent2.anti_entropy: failed to sync remote state: error="No known Consul servers" +34:03.064 [INFO] Agent2: Requesting shutdown +34:03.064 [INFO] Agent2.client: shutting down client +34:03.064 [WARN] Agent2.client.serf.lan: serf: Shutdown without a Leave +34:03.064 [INFO] Agent2: consul client down +34:03.064 [INFO] Agent2: shutdown complete +34:03.064 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:24734 network=tcp +34:03.065 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:24734 network=udp +34:03.065 [INFO] Agent2: Stopping server: protocol=HTTP address=127.0.0.1:24735 network=tcp +34:03.565 [INFO] Agent2: Waiting for endpoints to shut down +34:03.565 [INFO] Agent2: Endpoints down +2021-08-13T15:34:02.465Z [WARN] agent: Node name "Node 56b99bd9-cc07-e6a6-b304-2741cf6214de" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. +2021-08-13T15:34:02.465Z [WARN] agent: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ +2021-08-13T15:34:02.508Z [WARN] agent.auto_config: Node name "Node 56b99bd9-cc07-e6a6-b304-2741cf6214de" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. +2021-08-13T15:34:02.508Z [WARN] agent.auto_config: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ +34:02.510 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node 56b99bd9-cc07-e6a6-b304-2741cf6214de 127.0.0.1 +2021-08-13T15:34:02.510Z [INFO] agent.router: Initializing LAN area manager +34:02.510 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24713 network=udp +34:02.510 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24713 network=tcp +34:02.511 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24714 network=tcp +34:02.512 [INFO] TestAgent: started state syncer +34:02.513 [INFO] TestAgent: Requesting shutdown +34:02.513 [INFO] TestAgent.client: shutting down client +34:02.513 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave +34:02.513 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24677 network=tcp +34:02.513 [ERROR] TestAgent: gRPC server failed: error="grpc: the server has been stopped" +2021-08-13T15:34:02.513Z [WARN] agent.router.manager: No servers available +34:02.513 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" +34:02.513 [INFO] TestAgent: consul client down +34:02.513 [INFO] TestAgent: shutdown complete +34:02.513 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24713 network=tcp +34:02.513 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24713 network=udp +34:02.513 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24714 network=tcp +34:03.013 [INFO] TestAgent: Waiting for endpoints to shut down +34:03.013 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_PurgeCheckOnDuplicate (1.15s) +=== CONT TestTxnEndpoint_Bad_Size_Net +=== RUN TestTxnEndpoint_Bad_Size_Net/exceeds_default_limits +2021-08-13T15:34:02.741Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:02.759Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:02.762 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b3d386e3-db1e-6773-31d5-942761e33bbf Address:127.0.0.1:24718}]" +34:02.763 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24718 [Follower]" leader= +34:02.765 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b3d386e3-db1e-6773-31d5-942761e33bbf.dc1 127.0.0.1 +34:02.766 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b3d386e3-db1e-6773-31d5-942761e33bbf 127.0.0.1 +2021-08-13T15:34:02.766Z [INFO] agent.router: Initializing LAN area manager +34:02.766 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24706 network=udp +34:02.767 [INFO] TestAgent.server: Adding LAN server: server="Node-b3d386e3-db1e-6773-31d5-942761e33bbf (Addr: tcp/127.0.0.1:24718) (DC: dc1)" +34:02.767 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b3d386e3-db1e-6773-31d5-942761e33bbf.dc1 area=wan +34:02.767 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24706 network=tcp +34:02.768 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24707 network=tcp +34:02.768 [INFO] TestAgent: started state syncer +34:02.768 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24719 network=tcp +34:02.800 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:02.800 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24718 [Candidate]" term=2 +34:02.801 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:02.801 [DEBUG] TestAgent.server.raft: vote granted: from=b3d386e3-db1e-6773-31d5-942761e33bbf term=2 tally=1 +34:02.801 [INFO] TestAgent.server.raft: election won: tally=1 +34:02.801 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24718 [Leader]" +34:02.801 [INFO] TestAgent.server: cluster leadership acquired +34:02.801 [INFO] TestAgent.server: New leader elected: payload=Node-b3d386e3-db1e-6773-31d5-942761e33bbf +34:02.801 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24718 +34:02.802 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:02.805 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:02.805 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:02.805 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:02.805 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:02.805 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:02.805 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b3d386e3-db1e-6773-31d5-942761e33bbf +34:02.805 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b3d386e3-db1e-6773-31d5-942761e33bbf +34:02.806 [INFO] TestAgent.server: federation state anti-entropy synced +34:02.839 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:02.839 [INFO] TestAgent: Synced node info +34:02.839 [DEBUG] TestAgent: Node info in sync +34:03.074 [INFO] TestAgent: Requesting shutdown +34:03.074 [INFO] TestAgent.server: shutting down server +34:03.074 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:03.074 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:03.074 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:03.074 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:03.074 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:03.074 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:03.074 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:03.074 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:03.074 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:03.074 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:03.074 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:03.074Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:03.074Z [INFO] agent.router.manager: shutting down +34:03.074 [INFO] TestAgent: consul server down +34:03.074 [INFO] TestAgent: shutdown complete +34:03.074 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24706 network=tcp +34:03.075 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24706 network=udp +34:03.075 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24707 network=tcp +34:03.575 [INFO] TestAgent: Waiting for endpoints to shut down +34:03.575 [INFO] TestAgent: Endpoints down +--- PASS: TestTxnEndpoint_UpdateCheck (0.85s) === CONT TestAgent_RemoveCheck -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:44.999Z [WARN] agent: Node name "Node 55c2ab6e-5c91-d90d-b197-13df377f46f9" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. -2022-09-15T21:42:44.999Z [WARN] agent: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ -2022-09-15T21:42:45.044Z [WARN] agent.auto_config: Node name "Node 55c2ab6e-5c91-d90d-b197-13df377f46f9" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. -2022-09-15T21:42:45.044Z [WARN] agent.auto_config: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ -42:45.046 [INFO] Agent2.client.serf.lan: serf: EventMemberJoin: Node 55c2ab6e-5c91-d90d-b197-13df377f46f9 127.0.0.1 -2022-09-15T21:42:45.046Z [INFO] agent.router: Initializing LAN area manager -42:45.046 [WARN] Agent2.client.serf.lan: serf: Failed to re-join any previously known node -42:45.046 [DEBUG] Agent2: check exists, not restoring from file: check=mem file=/tmp/consul-test/TestAgent_PurgeCheckOnDuplicate-TestAgent-agent140762702/checks/afc4fc7e48a0710a1dc94ef3e8bc5764 -42:45.046 [INFO] Agent2: Started DNS server: address=127.0.0.1:13704 network=udp -42:45.047 [INFO] Agent2: Started DNS server: address=127.0.0.1:13704 network=tcp -42:45.047 [INFO] Agent2: Started HTTP server: address=127.0.0.1:13705 network=tcp -42:45.048 [INFO] Agent2: started state syncer -42:45.048 [INFO] Agent2: Requesting shutdown -42:45.048 [INFO] Agent2.client: shutting down client -42:45.048 [WARN] Agent2.client.serf.lan: serf: Shutdown without a Leave -42:45.049 [INFO] Agent2: Started gRPC server: address=127.0.0.1:13717 network=tcp -42:45.049 [ERROR] Agent2: gRPC server failed: error="grpc: the server has been stopped" -2022-09-15T21:42:45.049Z [WARN] agent.router.manager: No servers available -42:45.049 [ERROR] Agent2.anti_entropy: failed to sync remote state: error="No known Consul servers" -42:45.049 [INFO] Agent2: consul client down -42:45.049 [INFO] Agent2: shutdown complete -42:45.049 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:13704 network=tcp -42:45.049 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:13704 network=udp -42:45.049 [INFO] Agent2: Stopping server: protocol=HTTP address=127.0.0.1:13705 network=tcp -42:45.550 [INFO] Agent2: Waiting for endpoints to shut down -42:45.564 [INFO] Agent2: Endpoints down -2022-09-15T21:42:44.414Z [WARN] agent: Node name "Node 55c2ab6e-5c91-d90d-b197-13df377f46f9" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. -2022-09-15T21:42:44.414Z [WARN] agent: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ -2022-09-15T21:42:44.461Z [WARN] agent.auto_config: Node name "Node 55c2ab6e-5c91-d90d-b197-13df377f46f9" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. -2022-09-15T21:42:44.461Z [WARN] agent.auto_config: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ -42:44.463 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node 55c2ab6e-5c91-d90d-b197-13df377f46f9 127.0.0.1 -2022-09-15T21:42:44.463Z [INFO] agent.router: Initializing LAN area manager -42:44.463 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13669 network=udp -42:44.464 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13669 network=tcp -42:44.465 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13670 network=tcp -42:44.465 [INFO] TestAgent: started state syncer -42:44.465 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13682 network=tcp -2022-09-15T21:42:44.465Z [WARN] agent.router.manager: No servers available -42:44.465 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" -42:44.466 [INFO] TestAgent: Requesting shutdown -42:44.467 [INFO] TestAgent.client: shutting down client -42:44.467 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave -42:44.467 [INFO] TestAgent: consul client down -42:44.467 [INFO] TestAgent: shutdown complete -42:44.467 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13669 network=tcp -42:44.467 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13669 network=udp -42:44.467 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13670 network=tcp -42:44.967 [INFO] TestAgent: Waiting for endpoints to shut down -42:44.968 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_PurgeCheckOnDuplicate (1.21s) +2021-08-13T15:34:02.930Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:03.013Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:03.024 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0c772f37-c4da-2d62-0740-282493ce9fd8 Address:127.0.0.1:24725}]" +34:03.025 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0c772f37-c4da-2d62-0740-282493ce9fd8.dc1 127.0.0.1 +34:03.026 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0c772f37-c4da-2d62-0740-282493ce9fd8 127.0.0.1 +2021-08-13T15:34:03.026Z [INFO] agent.router: Initializing LAN area manager +34:03.026 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24720 network=udp +34:03.026 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24725 [Follower]" leader= +34:03.027 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0c772f37-c4da-2d62-0740-282493ce9fd8.dc1 area=wan +34:03.027 [INFO] TestAgent.server: Adding LAN server: server="Node-0c772f37-c4da-2d62-0740-282493ce9fd8 (Addr: tcp/127.0.0.1:24725) (DC: dc1)" +34:03.027 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24720 network=tcp +34:03.028 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24721 network=tcp +34:03.028 [INFO] TestAgent: started state syncer +34:03.028 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24726 network=tcp +34:03.072 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:03.072 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24725 [Candidate]" term=2 +34:03.072 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:03.072 [DEBUG] TestAgent.server.raft: vote granted: from=0c772f37-c4da-2d62-0740-282493ce9fd8 term=2 tally=1 +34:03.072 [INFO] TestAgent.server.raft: election won: tally=1 +34:03.072 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24725 [Leader]" +34:03.072 [INFO] TestAgent.server: cluster leadership acquired +34:03.072 [INFO] TestAgent.server: New leader elected: payload=Node-0c772f37-c4da-2d62-0740-282493ce9fd8 +34:03.072 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24725 +34:03.074 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:03.076 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:03.076 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:03.076 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:03.076 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:03.076 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:03.077 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0c772f37-c4da-2d62-0740-282493ce9fd8 +34:03.077 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0c772f37-c4da-2d62-0740-282493ce9fd8 +34:03.077 [INFO] TestAgent.server: federation state anti-entropy synced +34:03.207 [DEBUG] TestAgent: Check status updated: check=mem status=passing +34:03.207 [DEBUG] TestAgent: Check status updated: check=mem status=critical +34:03.207 [INFO] TestAgent: Requesting shutdown +34:03.207 [INFO] TestAgent.server: shutting down server +34:03.207 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:03.207 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:03.207 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:03.207 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:03.207 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:03.207 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:03.207 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:03.207 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:03.207 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:03.207 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:03.207 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:03.208 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:03.208Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:03.208Z [INFO] agent.router.manager: shutting down +34:03.208 [INFO] TestAgent: consul server down +34:03.208 [INFO] TestAgent: shutdown complete +34:03.208 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24720 network=tcp +34:03.208 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24720 network=udp +34:03.208 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24721 network=tcp +34:03.709 [INFO] TestAgent: Waiting for endpoints to shut down +34:03.709 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_updateTTLCheck (0.80s) +=== CONT TestTxnEndpoint_Bad_Size_Item +=== RUN TestTxnEndpoint_Bad_Size_Item/exceeds_default_limits +2021-08-13T15:34:03.225Z [WARN] agent: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ +2021-08-13T15:34:03.242Z [WARN] agent.auto_config: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ +34:03.243 [INFO] Agent2.client.serf.lan: serf: EventMemberJoin: Node-07c6ef9c-5420-63c4-731a-2098df8e1019 127.0.0.1 +2021-08-13T15:34:03.243Z [INFO] agent.router: Initializing LAN area manager +34:03.243 [INFO] Agent2.client.serf.lan: serf: Attempting re-join to previously known node: Node-2bc77c89-1bad-8f92-d2c3-2fda50173eff: 127.0.0.1:24702 +34:03.243 [DEBUG] Agent2: restored health check from file: check=mem file=/tmp/consul-test/TestAgent_PersistCheck-TestAgent-agent270284497/checks/afc4fc7e48a0710a1dc94ef3e8bc5764 +34:03.243 [DEBUG] Agent2.client.memberlist.lan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:24702: connect: connection refused +34:03.243 [WARN] Agent2.client.serf.lan: serf: Failed to re-join any previously known node +34:03.243 [INFO] Agent2: Started DNS server: address=127.0.0.1:24741 network=tcp +34:03.243 [INFO] Agent2: Started DNS server: address=127.0.0.1:24741 network=udp +34:03.244 [INFO] Agent2: Started HTTP server: address=127.0.0.1:24742 network=tcp +34:03.244 [INFO] Agent2: started state syncer +34:03.244 [INFO] Agent2: Started gRPC server: address=127.0.0.1:24747 network=tcp +2021-08-13T15:34:03.245Z [WARN] agent.router.manager: No servers available +34:03.245 [ERROR] Agent2.anti_entropy: failed to sync remote state: error="No known Consul servers" +34:03.245 [INFO] Agent2: Requesting shutdown +34:03.245 [INFO] Agent2.client: shutting down client +34:03.245 [WARN] Agent2.client.serf.lan: serf: Shutdown without a Leave +34:03.245 [INFO] Agent2: consul client down +34:03.245 [INFO] Agent2: shutdown complete +34:03.245 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:24741 network=tcp +34:03.246 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:24741 network=udp +34:03.246 [INFO] Agent2: Stopping server: protocol=HTTP address=127.0.0.1:24742 network=tcp +34:03.746 [INFO] Agent2: Waiting for endpoints to shut down +34:03.746 [INFO] Agent2: Endpoints down +2021-08-13T15:34:02.685Z [WARN] agent: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ +2021-08-13T15:34:02.703Z [WARN] agent.auto_config: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ +34:02.704 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-2bc77c89-1bad-8f92-d2c3-2fda50173eff 127.0.0.1 +2021-08-13T15:34:02.704Z [INFO] agent.router: Initializing LAN area manager +34:02.704 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24685 network=tcp +34:02.704 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24685 network=udp +34:02.705 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24686 network=tcp +34:02.705 [INFO] TestAgent: started state syncer +34:02.707 [INFO] TestAgent: Requesting shutdown +34:02.707 [INFO] TestAgent.client: shutting down client +34:02.707 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave +34:02.707 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24705 network=tcp +34:02.707 [ERROR] TestAgent: gRPC server failed: error="grpc: the server has been stopped" +2021-08-13T15:34:02.708Z [WARN] agent.router.manager: No servers available +34:02.708 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" +34:02.708 [INFO] TestAgent: consul client down +34:02.708 [INFO] TestAgent: shutdown complete +34:02.708 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24685 network=tcp +34:02.708 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24685 network=udp +34:02.708 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24686 network=tcp +34:03.208 [INFO] TestAgent: Waiting for endpoints to shut down +34:03.208 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_PersistCheck (1.08s) === CONT TestAgent_AddCheck_Alias_userAndSetToken -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:44.815Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:44.846Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:44.850 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a91cc447-f50e-cdae-9c90-1e22ee849161 Address:127.0.0.1:13688}]" -42:44.851 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a91cc447-f50e-cdae-9c90-1e22ee849161.dc1 127.0.0.1 -42:44.852 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a91cc447-f50e-cdae-9c90-1e22ee849161 127.0.0.1 -2022-09-15T21:42:44.852Z [INFO] agent.router: Initializing LAN area manager -42:44.853 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13697 network=udp -42:44.853 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13688 [Follower]" leader= -42:44.854 [INFO] TestAgent.server: Adding LAN server: server="Node-a91cc447-f50e-cdae-9c90-1e22ee849161 (Addr: tcp/127.0.0.1:13688) (DC: dc1)" -42:44.854 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a91cc447-f50e-cdae-9c90-1e22ee849161.dc1 area=wan -42:44.854 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13697 network=tcp -42:44.856 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13698 network=tcp -42:44.856 [INFO] TestAgent: started state syncer -42:44.856 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13689 network=tcp -42:44.906 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:44.906 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13688 [Candidate]" term=2 -42:44.906 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:44.906 [DEBUG] TestAgent.server.raft: vote granted: from=a91cc447-f50e-cdae-9c90-1e22ee849161 term=2 tally=1 -42:44.906 [INFO] TestAgent.server.raft: election won: tally=1 -42:44.906 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13688 [Leader]" -42:44.907 [INFO] TestAgent.server: cluster leadership acquired -42:44.907 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13688 -42:44.909 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:44.909 [INFO] TestAgent.server: New leader elected: payload=Node-a91cc447-f50e-cdae-9c90-1e22ee849161 -42:44.913 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:44.913 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:44.918 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:44.918 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:44.918 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:44.918 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a91cc447-f50e-cdae-9c90-1e22ee849161 -42:44.918 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a91cc447-f50e-cdae-9c90-1e22ee849161 -42:44.928 [INFO] TestAgent.server: federation state anti-entropy synced -42:45.170 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:45.170 [INFO] TestAgent: Synced node info -42:45.170 [DEBUG] TestAgent: Node info in sync -42:45.276 [DEBUG] TestAgent: removed check: check=mem -42:45.276 [DEBUG] TestAgent: removed check: check=mem -42:45.276 [INFO] TestAgent: Requesting shutdown -42:45.276 [INFO] TestAgent.server: shutting down server -42:45.276 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:45.276 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:45.276 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:45.276 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:45.276 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:45.276 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:45.276 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:45.276 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:45.276 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:45.276 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:45.277 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:45.277Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:45.277Z [INFO] agent.router.manager: shutting down -42:45.277 [INFO] TestAgent: consul server down -42:45.277 [INFO] TestAgent: shutdown complete -42:45.277 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13697 network=tcp -42:45.277 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13697 network=udp -42:45.277 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13698 network=tcp -42:45.778 [INFO] TestAgent: Waiting for endpoints to shut down -42:45.778 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_PurgeCheck (1.00s) +[WARN] freeport: 10 out of 66 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:03.312Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:03.328Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:03.331 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:48989602-53a9-8d6f-a2b9-1f4cce94860d Address:127.0.0.1:24753}]" +34:03.331 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24753 [Follower]" leader= +34:03.332 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-48989602-53a9-8d6f-a2b9-1f4cce94860d.dc1 127.0.0.1 +34:03.333 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-48989602-53a9-8d6f-a2b9-1f4cce94860d 127.0.0.1 +2021-08-13T15:34:03.333Z [INFO] agent.router: Initializing LAN area manager +34:03.333 [INFO] TestAgent.server: Adding LAN server: server="Node-48989602-53a9-8d6f-a2b9-1f4cce94860d (Addr: tcp/127.0.0.1:24753) (DC: dc1)" +34:03.333 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-48989602-53a9-8d6f-a2b9-1f4cce94860d.dc1 area=wan +34:03.333 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24748 network=tcp +34:03.333 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24748 network=udp +34:03.334 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24749 network=tcp +34:03.334 [INFO] TestAgent: started state syncer +34:03.334 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24754 network=tcp +34:03.374 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:03.374 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24753 [Candidate]" term=2 +34:03.374 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:03.374 [DEBUG] TestAgent.server.raft: vote granted: from=48989602-53a9-8d6f-a2b9-1f4cce94860d term=2 tally=1 +34:03.374 [INFO] TestAgent.server.raft: election won: tally=1 +34:03.374 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24753 [Leader]" +34:03.374 [INFO] TestAgent.server: cluster leadership acquired +34:03.374 [INFO] TestAgent.server: New leader elected: payload=Node-48989602-53a9-8d6f-a2b9-1f4cce94860d +34:03.374 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24753 +34:03.376 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:03.378 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:03.378 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:03.378 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:03.378 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:03.378 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:03.378 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-48989602-53a9-8d6f-a2b9-1f4cce94860d +34:03.378 [INFO] TestAgent.server: member joined, marking health alive: member=Node-48989602-53a9-8d6f-a2b9-1f4cce94860d +34:03.379 [INFO] TestAgent.server: federation state anti-entropy synced +34:03.460 [INFO] TestAgent: Requesting shutdown +34:03.460 [INFO] TestAgent.server: shutting down server +34:03.460 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:03.460 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:03.460 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:03.460 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:03.460 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:03.461 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:03.461 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:03.461 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:03.461 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:03.461 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:03.461 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:03.461 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:03.461Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:03.461Z [INFO] agent.router.manager: shutting down +34:03.461 [INFO] TestAgent: consul server down +34:03.461 [INFO] TestAgent: shutdown complete +34:03.461 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24748 network=tcp +34:03.461 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24748 network=udp +34:03.461 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24749 network=tcp +34:03.963 [INFO] TestAgent: Waiting for endpoints to shut down +34:03.963 [INFO] TestAgent: Endpoints down +=== RUN TestTxnEndpoint_KV_Actions/#01 +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:04.010Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:04.026Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:04.029 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e6a64872-6d96-ff75-0dbc-63637c3a91bc Address:127.0.0.1:24815}]" +34:04.029 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24815 [Follower]" leader= +34:04.030 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e6a64872-6d96-ff75-0dbc-63637c3a91bc.dc1 127.0.0.1 +34:04.031 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e6a64872-6d96-ff75-0dbc-63637c3a91bc 127.0.0.1 +2021-08-13T15:34:04.031Z [INFO] agent.router: Initializing LAN area manager +34:04.031 [INFO] TestAgent.server: Adding LAN server: server="Node-e6a64872-6d96-ff75-0dbc-63637c3a91bc (Addr: tcp/127.0.0.1:24815) (DC: dc1)" +34:04.031 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e6a64872-6d96-ff75-0dbc-63637c3a91bc.dc1 area=wan +34:04.031 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24803 network=udp +34:04.031 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24803 network=tcp +34:04.032 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24804 network=tcp +34:04.032 [INFO] TestAgent: started state syncer +34:04.032 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24816 network=tcp +34:04.072 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:04.072 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24815 [Candidate]" term=2 +34:04.072 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:04.072 [DEBUG] TestAgent.server.raft: vote granted: from=e6a64872-6d96-ff75-0dbc-63637c3a91bc term=2 tally=1 +34:04.072 [INFO] TestAgent.server.raft: election won: tally=1 +34:04.072 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24815 [Leader]" +34:04.072 [INFO] TestAgent.server: cluster leadership acquired +34:04.072 [INFO] TestAgent.server: New leader elected: payload=Node-e6a64872-6d96-ff75-0dbc-63637c3a91bc +34:04.072 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24815 +34:04.074 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:04.076 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:04.077 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:04.077 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:04.077 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:04.077 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:04.077 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e6a64872-6d96-ff75-0dbc-63637c3a91bc +34:04.077 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e6a64872-6d96-ff75-0dbc-63637c3a91bc +34:04.077 [INFO] TestAgent.server: federation state anti-entropy synced +34:04.214 [INFO] TestAgent: Requesting shutdown +34:04.214 [INFO] TestAgent.server: shutting down server +34:04.214 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:04.214 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:04.214 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:04.214 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:04.214 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:04.214 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:04.214 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:04.215 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:04.215 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:04.215 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:04.215 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:04.215 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:04.217Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:04.217Z [INFO] agent.router.manager: shutting down +34:04.218 [INFO] TestAgent: consul server down +34:04.218 [INFO] TestAgent: shutdown complete +34:04.218 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24803 network=tcp +34:04.218 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24803 network=udp +34:04.218 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24804 network=tcp +34:04.218 [INFO] TestAgent: Waiting for endpoints to shut down +34:04.218 [INFO] TestAgent: Endpoints down +--- PASS: TestTxnEndpoint_KV_Actions (0.92s) + --- PASS: TestTxnEndpoint_KV_Actions/#00 (0.69s) + --- PASS: TestTxnEndpoint_KV_Actions/#01 (0.23s) +=== CONT TestTxnEndpoint_Bad_JSON +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:03.574Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:03.590Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:03.594 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:95b6169d-7bc4-ba9b-4383-865018837ab9 Address:127.0.0.1:24760}]" +34:03.594 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-95b6169d-7bc4-ba9b-4383-865018837ab9.dc1 127.0.0.1 +34:03.595 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24760 [Follower]" leader= +34:03.596 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-95b6169d-7bc4-ba9b-4383-865018837ab9 127.0.0.1 +2021-08-13T15:34:03.596Z [INFO] agent.router: Initializing LAN area manager +34:03.596 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24769 network=udp +34:03.597 [INFO] TestAgent.server: Adding LAN server: server="Node-95b6169d-7bc4-ba9b-4383-865018837ab9 (Addr: tcp/127.0.0.1:24760) (DC: dc1)" +34:03.597 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-95b6169d-7bc4-ba9b-4383-865018837ab9.dc1 area=wan +34:03.597 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24769 network=tcp +34:03.598 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24770 network=tcp +34:03.598 [INFO] TestAgent: started state syncer +34:03.598 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24761 network=tcp +34:03.637 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:03.637 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24760 [Candidate]" term=2 +34:03.637 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:03.637 [DEBUG] TestAgent.server.raft: vote granted: from=95b6169d-7bc4-ba9b-4383-865018837ab9 term=2 tally=1 +34:03.637 [INFO] TestAgent.server.raft: election won: tally=1 +34:03.637 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24760 [Leader]" +34:03.637 [INFO] TestAgent.server: cluster leadership acquired +34:03.637 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24760 +34:03.639 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:03.639 [INFO] TestAgent.server: New leader elected: payload=Node-95b6169d-7bc4-ba9b-4383-865018837ab9 +34:03.641 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:03.641 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:03.641 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:03.641 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:03.641 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:03.641 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-95b6169d-7bc4-ba9b-4383-865018837ab9 +34:03.641 [INFO] TestAgent.server: member joined, marking health alive: member=Node-95b6169d-7bc4-ba9b-4383-865018837ab9 +34:03.642 [INFO] TestAgent.server: federation state anti-entropy synced +34:03.795 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:03.796 [INFO] TestAgent: Synced node info +34:03.894 [INFO] TestAgent: Requesting shutdown +34:03.894 [INFO] TestAgent.server: shutting down server +34:03.894 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:03.894 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:03.894 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:03.894 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:03.894 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:03.895 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:03.895 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:03.895 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:03.895 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:03.895 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:03.895 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:03.895Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:03.895Z [INFO] agent.router.manager: shutting down +34:03.895 [INFO] TestAgent: consul server down +34:03.895 [INFO] TestAgent: shutdown complete +34:03.895 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24769 network=tcp +34:03.895 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24769 network=udp +34:03.895 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24770 network=tcp +34:04.395 [INFO] TestAgent: Waiting for endpoints to shut down +34:04.395 [INFO] TestAgent: Endpoints down +--- PASS: TestTxnEndpoint_Bad_Size_Ops (0.84s) === CONT TestAgent_AddCheck_Alias_userToken -2022-09-15T21:42:45.428Z [WARN] agent: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ -2022-09-15T21:42:45.466Z [WARN] agent.auto_config: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ -42:45.467 [INFO] Agent2.client.serf.lan: serf: EventMemberJoin: Node-d291fd84-c5f0-e7af-4d07-e0568ac66689 127.0.0.1 -2022-09-15T21:42:45.467Z [INFO] agent.router: Initializing LAN area manager -42:45.468 [DEBUG] Agent2: restored health check from file: check=mem file=/tmp/consul-test/TestAgent_PersistCheck-TestAgent-agent180598960/checks/afc4fc7e48a0710a1dc94ef3e8bc5764 -42:45.468 [INFO] Agent2.client.serf.lan: serf: Attempting re-join to previously known node: Node-cf5d7a91-985e-1b32-e251-3f90be700c53: 127.0.0.1:13693 -42:45.469 [INFO] Agent2: Started DNS server: address=127.0.0.1:13739 network=tcp -42:45.469 [DEBUG] Agent2.client.memberlist.lan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:13693: connect: connection refused -42:45.469 [WARN] Agent2.client.serf.lan: serf: Failed to re-join any previously known node -42:45.469 [INFO] Agent2: Started DNS server: address=127.0.0.1:13739 network=udp -42:45.475 [INFO] Agent2: Started HTTP server: address=127.0.0.1:13740 network=tcp -42:45.475 [INFO] Agent2: started state syncer -42:45.477 [INFO] Agent2: Started gRPC server: address=127.0.0.1:13731 network=tcp -2022-09-15T21:42:45.477Z [WARN] agent.router.manager: No servers available -42:45.477 [ERROR] Agent2.anti_entropy: failed to sync remote state: error="No known Consul servers" -42:45.477 [INFO] Agent2: Requesting shutdown -42:45.477 [INFO] Agent2.client: shutting down client -42:45.477 [WARN] Agent2.client.serf.lan: serf: Shutdown without a Leave -42:45.477 [INFO] Agent2: consul client down -42:45.477 [INFO] Agent2: shutdown complete -42:45.478 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:13739 network=tcp -42:45.482 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:13739 network=udp -42:45.482 [INFO] Agent2: Stopping server: protocol=HTTP address=127.0.0.1:13740 network=tcp -42:45.982 [INFO] Agent2: Waiting for endpoints to shut down -42:45.983 [INFO] Agent2: Endpoints down -2022-09-15T21:42:44.817Z [WARN] agent: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ -2022-09-15T21:42:44.903Z [WARN] agent.auto_config: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ -42:44.904 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-cf5d7a91-985e-1b32-e251-3f90be700c53 127.0.0.1 -2022-09-15T21:42:44.904Z [INFO] agent.router: Initializing LAN area manager -42:44.905 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13690 network=udp -42:44.906 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13690 network=tcp -42:44.907 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13691 network=tcp -42:44.907 [INFO] TestAgent: started state syncer -42:44.909 [INFO] TestAgent: Requesting shutdown -42:44.909 [INFO] TestAgent.client: shutting down client -42:44.909 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave -42:44.909 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13703 network=tcp -42:44.909 [ERROR] TestAgent: gRPC server failed: error="grpc: the server has been stopped" -2022-09-15T21:42:44.910Z [WARN] agent.router.manager: No servers available -42:44.910 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" -42:44.910 [INFO] TestAgent: consul client down -42:44.910 [INFO] TestAgent: shutdown complete -42:44.910 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13690 network=tcp -42:44.910 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13690 network=udp -42:44.910 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13691 network=tcp -42:45.411 [INFO] TestAgent: Waiting for endpoints to shut down -42:45.411 [INFO] TestAgent: Endpoints down +2021-08-13T15:34:03.726Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:03.742Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:03.745 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:611e448b-5803-beda-72b6-53f4899cc915 Address:127.0.0.1:24794}]" +34:03.745 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24794 [Follower]" leader= +34:03.746 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-611e448b-5803-beda-72b6-53f4899cc915.dc1 127.0.0.1 +34:03.747 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-611e448b-5803-beda-72b6-53f4899cc915 127.0.0.1 +2021-08-13T15:34:03.747Z [INFO] agent.router: Initializing LAN area manager +34:03.748 [INFO] TestAgent.server: Adding LAN server: server="Node-611e448b-5803-beda-72b6-53f4899cc915 (Addr: tcp/127.0.0.1:24794) (DC: dc1)" +34:03.748 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-611e448b-5803-beda-72b6-53f4899cc915.dc1 area=wan +34:03.748 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24776 network=udp +34:03.748 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24776 network=tcp +34:03.749 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24777 network=tcp +34:03.749 [INFO] TestAgent: started state syncer +34:03.749 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24795 network=tcp +34:03.802 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:03.802 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24794 [Candidate]" term=2 +34:03.802 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:03.802 [DEBUG] TestAgent.server.raft: vote granted: from=611e448b-5803-beda-72b6-53f4899cc915 term=2 tally=1 +34:03.802 [INFO] TestAgent.server.raft: election won: tally=1 +34:03.802 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24794 [Leader]" +34:03.803 [INFO] TestAgent.server: cluster leadership acquired +34:03.803 [INFO] TestAgent.server: New leader elected: payload=Node-611e448b-5803-beda-72b6-53f4899cc915 +34:03.803 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24794 +34:03.804 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:03.807 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:03.807 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:03.807 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:03.807 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:03.807 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:03.807 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-611e448b-5803-beda-72b6-53f4899cc915 +34:03.807 [INFO] TestAgent.server: member joined, marking health alive: member=Node-611e448b-5803-beda-72b6-53f4899cc915 +34:03.808 [INFO] TestAgent.server: federation state anti-entropy synced +34:03.989 [INFO] TestAgent: Requesting shutdown +34:03.989 [INFO] TestAgent.server: shutting down server +34:03.989 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:03.989 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:03.989 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:03.989 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:03.989 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:03.989 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:03.989 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:03.989 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:03.989 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +34:03.989 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:03.989 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:03.989 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +2021-08-13T15:34:03.990Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:03.990Z [INFO] agent.router.manager: shutting down +34:03.990 [INFO] TestAgent: consul server down +34:03.990 [INFO] TestAgent: shutdown complete +34:03.990 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24776 network=tcp +34:03.990 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24776 network=udp +34:03.990 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24777 network=tcp +34:04.490 [INFO] TestAgent: Waiting for endpoints to shut down +34:04.490 [INFO] TestAgent: Endpoints down +=== RUN TestTxnEndpoint_Bad_Size_Item/exceeds_configured_max_txn_len +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:03.609Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:03.609Z [WARN] agent: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ +2021-08-13T15:34:03.643Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:03.643Z [WARN] agent.auto_config: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ +34:03.646 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bdd9f41d-92d1-cb41-7de9-467694dcd657 Address:127.0.0.1:24774}]" +34:03.646 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24774 [Follower]" leader= +34:03.647 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bdd9f41d-92d1-cb41-7de9-467694dcd657.dc1 127.0.0.1 +34:03.648 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bdd9f41d-92d1-cb41-7de9-467694dcd657 127.0.0.1 +2021-08-13T15:34:03.648Z [INFO] agent.router: Initializing LAN area manager +34:03.648 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bdd9f41d-92d1-cb41-7de9-467694dcd657.dc1 area=wan +34:03.648 [INFO] TestAgent.server: Adding LAN server: server="Node-bdd9f41d-92d1-cb41-7de9-467694dcd657 (Addr: tcp/127.0.0.1:24774) (DC: dc1)" +34:03.648 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24783 network=tcp +34:03.648 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24783 network=udp +34:03.649 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24784 network=tcp +34:03.649 [INFO] TestAgent: started state syncer +34:03.649 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24775 network=tcp +34:03.694 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:03.694 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24774 [Candidate]" term=2 +34:03.694 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:03.694 [DEBUG] TestAgent.server.raft: vote granted: from=bdd9f41d-92d1-cb41-7de9-467694dcd657 term=2 tally=1 +34:03.694 [INFO] TestAgent.server.raft: election won: tally=1 +34:03.694 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24774 [Leader]" +34:03.694 [INFO] TestAgent.server: cluster leadership acquired +34:03.694 [INFO] TestAgent.server: New leader elected: payload=Node-bdd9f41d-92d1-cb41-7de9-467694dcd657 +34:03.695 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24774 +34:03.696 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:03.699 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:03.699 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:03.699 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:03.699 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:03.699 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:03.699 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bdd9f41d-92d1-cb41-7de9-467694dcd657 +34:03.699 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bdd9f41d-92d1-cb41-7de9-467694dcd657 +34:03.699 [INFO] TestAgent.server: federation state anti-entropy synced +34:03.778 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:03.782 [INFO] TestAgent: Synced node info +34:04.049 [DEBUG] TestAgent: removed check: check=mem +34:04.049 [DEBUG] TestAgent: removed check: check=mem +34:04.049 [INFO] TestAgent: Requesting shutdown +34:04.049 [INFO] TestAgent.server: shutting down server +34:04.049 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:04.049 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:04.049 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:04.049 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:04.050 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:04.050 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:04.050 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:04.050 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:04.050 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:04.050 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:04.050 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:04.050Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:04.050Z [INFO] agent.router.manager: shutting down +34:04.050 [INFO] TestAgent: consul server down +34:04.050 [INFO] TestAgent: shutdown complete +34:04.050 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24783 network=tcp +34:04.050 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24783 network=udp +34:04.050 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24784 network=tcp +34:04.550 [INFO] TestAgent: Waiting for endpoints to shut down +34:04.550 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_RemoveCheck (0.98s) +=== CONT TestStatusPeersSecondary +2021-08-13T15:34:03.777Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:03.800Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:03.803 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f7b52377-40f8-2851-5081-332e46c03f3f Address:127.0.0.1:24801}]" +34:03.803 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24801 [Follower]" leader= +34:03.804 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f7b52377-40f8-2851-5081-332e46c03f3f.dc1 127.0.0.1 +34:03.805 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f7b52377-40f8-2851-5081-332e46c03f3f 127.0.0.1 +2021-08-13T15:34:03.805Z [INFO] agent.router: Initializing LAN area manager +34:03.806 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24796 network=udp +34:03.806 [INFO] TestAgent.server: Adding LAN server: server="Node-f7b52377-40f8-2851-5081-332e46c03f3f (Addr: tcp/127.0.0.1:24801) (DC: dc1)" +34:03.806 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f7b52377-40f8-2851-5081-332e46c03f3f.dc1 area=wan +34:03.806 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24796 network=tcp +34:03.807 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24797 network=tcp +34:03.807 [INFO] TestAgent: started state syncer +34:03.807 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24802 network=tcp +34:03.869 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:03.869 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24801 [Candidate]" term=2 +34:03.869 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:03.869 [DEBUG] TestAgent.server.raft: vote granted: from=f7b52377-40f8-2851-5081-332e46c03f3f term=2 tally=1 +34:03.869 [INFO] TestAgent.server.raft: election won: tally=1 +34:03.869 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24801 [Leader]" +34:03.869 [INFO] TestAgent.server: cluster leadership acquired +34:03.870 [INFO] TestAgent.server: New leader elected: payload=Node-f7b52377-40f8-2851-5081-332e46c03f3f +34:03.870 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24801 +34:03.871 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:03.874 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:03.874 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:03.874 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:03.874 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:03.874 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:03.874 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f7b52377-40f8-2851-5081-332e46c03f3f +34:03.874 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f7b52377-40f8-2851-5081-332e46c03f3f +34:03.875 [INFO] TestAgent.server: federation state anti-entropy synced +34:04.103 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:04.103 [INFO] TestAgent: Synced node info +34:04.140 [INFO] TestAgent: Requesting shutdown +34:04.141 [INFO] TestAgent.server: shutting down server +34:04.141 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:04.141 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:04.141 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:04.141 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:04.141 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:04.141 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:04.141 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:04.141 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:04.141 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:04.141 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:04.141 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:04.141Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:04.141Z [INFO] agent.router.manager: shutting down +34:04.141 [INFO] TestAgent: consul server down +34:04.141 [INFO] TestAgent: shutdown complete +34:04.141 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24796 network=tcp +34:04.141 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24796 network=udp +34:04.141 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24797 network=tcp +34:04.642 [INFO] TestAgent: Waiting for endpoints to shut down +34:04.642 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_AddCheck_Alias_userAndSetToken (0.90s) === CONT TestAgent_AddCheck_Alias_setToken ---- PASS: TestAgent_PersistCheck (1.19s) -2022-09-15T21:42:45.061Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:45.078Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:45.081 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5e3b7f3b-d622-d62f-7049-a04568c0e228 Address:127.0.0.1:13709}]" -42:45.081 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13709 [Follower]" leader= -42:45.082 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5e3b7f3b-d622-d62f-7049-a04568c0e228.dc1 127.0.0.1 -42:45.083 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5e3b7f3b-d622-d62f-7049-a04568c0e228 127.0.0.1 -2022-09-15T21:42:45.083Z [INFO] agent.router: Initializing LAN area manager -42:45.084 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13718 network=udp -42:45.084 [INFO] TestAgent.server: Adding LAN server: server="Node-5e3b7f3b-d622-d62f-7049-a04568c0e228 (Addr: tcp/127.0.0.1:13709) (DC: dc1)" -42:45.084 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5e3b7f3b-d622-d62f-7049-a04568c0e228.dc1 area=wan -42:45.084 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13718 network=tcp -42:45.085 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13719 network=tcp -42:45.085 [INFO] TestAgent: started state syncer -42:45.085 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13710 network=tcp -42:45.129 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:45.129 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13709 [Candidate]" term=2 -42:45.130 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:45.130 [DEBUG] TestAgent.server.raft: vote granted: from=5e3b7f3b-d622-d62f-7049-a04568c0e228 term=2 tally=1 -42:45.130 [INFO] TestAgent.server.raft: election won: tally=1 -42:45.130 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13709 [Leader]" -42:45.130 [INFO] TestAgent.server: cluster leadership acquired -42:45.130 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13709 -42:45.131 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:45.132 [INFO] TestAgent.server: New leader elected: payload=Node-5e3b7f3b-d622-d62f-7049-a04568c0e228 -42:45.134 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:45.134 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:45.134 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:45.134 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:45.134 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:45.134 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5e3b7f3b-d622-d62f-7049-a04568c0e228 -42:45.134 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5e3b7f3b-d622-d62f-7049-a04568c0e228 -42:45.135 [INFO] TestAgent.server: federation state anti-entropy synced -42:45.242 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:45.243 [INFO] TestAgent: Synced node info -42:45.243 [DEBUG] TestAgent: Node info in sync -42:45.482 [DEBUG] TestAgent: Check status updated: check=mem status=passing -42:45.483 [DEBUG] TestAgent: Check status updated: check=mem status=critical -42:45.483 [INFO] TestAgent: Requesting shutdown -42:45.483 [INFO] TestAgent.server: shutting down server -42:45.483 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:45.483 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:45.483 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:45.483 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:45.483 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:45.483 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:45.483 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:45.483 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:45.483 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:45.483 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:45.484 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:45.484Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:45.484Z [INFO] agent.router.manager: shutting down -42:45.484 [INFO] TestAgent: consul server down -42:45.484 [INFO] TestAgent: shutdown complete -42:45.484 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13718 network=tcp -42:45.484 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13718 network=udp -42:45.484 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13719 network=tcp -42:45.986 [INFO] TestAgent: Waiting for endpoints to shut down -42:45.986 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_updateTTLCheck (0.95s) +2021-08-13T15:34:03.595Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:03.632Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:03.635 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a4b6e1f9-ed7e-2489-4fe5-ce6adcb2937d Address:127.0.0.1:24781}]" +34:03.635 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24781 [Follower]" leader= +34:03.636 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a4b6e1f9-ed7e-2489-4fe5-ce6adcb2937d.dc1 127.0.0.1 +34:03.637 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a4b6e1f9-ed7e-2489-4fe5-ce6adcb2937d 127.0.0.1 +2021-08-13T15:34:03.637Z [INFO] agent.router: Initializing LAN area manager +34:03.638 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24762 network=udp +34:03.638 [INFO] TestAgent.server: Adding LAN server: server="Node-a4b6e1f9-ed7e-2489-4fe5-ce6adcb2937d (Addr: tcp/127.0.0.1:24781) (DC: dc1)" +34:03.638 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a4b6e1f9-ed7e-2489-4fe5-ce6adcb2937d.dc1 area=wan +34:03.638 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24762 network=tcp +34:03.643 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24763 network=tcp +34:03.643 [INFO] TestAgent: started state syncer +34:03.643 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24782 network=tcp +34:03.705 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:03.705 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24781 [Candidate]" term=2 +34:03.706 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:03.706 [DEBUG] TestAgent.server.raft: vote granted: from=a4b6e1f9-ed7e-2489-4fe5-ce6adcb2937d term=2 tally=1 +34:03.706 [INFO] TestAgent.server.raft: election won: tally=1 +34:03.706 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24781 [Leader]" +34:03.706 [INFO] TestAgent.server: cluster leadership acquired +34:03.706 [INFO] TestAgent.server: New leader elected: payload=Node-a4b6e1f9-ed7e-2489-4fe5-ce6adcb2937d +34:03.706 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24781 +34:03.708 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:03.710 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:03.710 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:03.710 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:03.710 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:03.710 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:03.710 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a4b6e1f9-ed7e-2489-4fe5-ce6adcb2937d +34:03.710 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a4b6e1f9-ed7e-2489-4fe5-ce6adcb2937d +34:03.711 [INFO] TestAgent.server: federation state anti-entropy synced +34:03.839 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:03.840 [INFO] TestAgent: Synced node info +34:03.840 [DEBUG] TestAgent: Node info in sync +34:04.195 [INFO] TestAgent: Requesting shutdown +34:04.195 [INFO] TestAgent.server: shutting down server +34:04.195 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:04.195 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:04.195 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:04.195 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:04.195 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:04.196 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:04.196 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:04.196 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:04.196 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:04.196 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:04.196 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:04.196Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:04.196Z [INFO] agent.router.manager: shutting down +34:04.196 [INFO] TestAgent: consul server down +34:04.196 [INFO] TestAgent: shutdown complete +34:04.196 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24762 network=tcp +34:04.196 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24762 network=udp +34:04.207 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24763 network=tcp +34:04.708 [INFO] TestAgent: Waiting for endpoints to shut down +34:04.708 [INFO] TestAgent: Endpoints down +=== RUN TestTxnEndpoint_Bad_Size_Net/exceeds_configured_max_txn_len +2021-08-13T15:34:03.476Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:03.505Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:03.508 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b77b8e29-5c29-7f9c-c5fd-b2e503693d6c Address:127.0.0.1:24767}]" +34:03.508 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24767 [Follower]" leader= +34:03.509 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b77b8e29-5c29-7f9c-c5fd-b2e503693d6c.dc1 127.0.0.1 +34:03.510 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b77b8e29-5c29-7f9c-c5fd-b2e503693d6c 127.0.0.1 +2021-08-13T15:34:03.510Z [INFO] agent.router: Initializing LAN area manager +34:03.510 [INFO] TestAgent.server: Adding LAN server: server="Node-b77b8e29-5c29-7f9c-c5fd-b2e503693d6c (Addr: tcp/127.0.0.1:24767) (DC: dc1)" +34:03.510 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b77b8e29-5c29-7f9c-c5fd-b2e503693d6c.dc1 area=wan +34:03.511 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24755 network=udp +34:03.511 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24755 network=tcp +34:03.512 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24756 network=tcp +34:03.512 [INFO] TestAgent: started state syncer +34:03.512 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24768 network=tcp +34:03.557 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:03.557 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24767 [Candidate]" term=2 +34:03.557 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:03.557 [DEBUG] TestAgent.server.raft: vote granted: from=b77b8e29-5c29-7f9c-c5fd-b2e503693d6c term=2 tally=1 +34:03.557 [INFO] TestAgent.server.raft: election won: tally=1 +34:03.557 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24767 [Leader]" +34:03.557 [INFO] TestAgent.server: cluster leadership acquired +34:03.557 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24767 +34:03.559 [INFO] TestAgent.server: New leader elected: payload=Node-b77b8e29-5c29-7f9c-c5fd-b2e503693d6c +34:03.559 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:03.562 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:03.562 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:03.562 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:03.562 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:03.562 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:03.562 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b77b8e29-5c29-7f9c-c5fd-b2e503693d6c +34:03.562 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b77b8e29-5c29-7f9c-c5fd-b2e503693d6c +34:03.563 [INFO] TestAgent.server: federation state anti-entropy synced +34:03.712 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:03.712 [INFO] TestAgent: Synced node info +34:03.712 [DEBUG] TestAgent: Node info in sync +34:03.969 [WARN] TestAgent: check has interval below minimum: check=tls minimum_interval=1s +34:04.218 [DEBUG] TestAgent: Check status updated: check=tls status=passing +34:04.229 [INFO] TestAgent: Requesting shutdown +34:04.229 [INFO] TestAgent.server: shutting down server +34:04.229 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:04.229 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:04.229 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:04.229 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:04.229 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:04.229 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:04.229 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:04.229 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:04.229 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:04.229 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:04.229 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:04.229Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:04.229Z [INFO] agent.router.manager: shutting down +34:04.229 [INFO] TestAgent: consul server down +34:04.229 [INFO] TestAgent: shutdown complete +34:04.229 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24755 network=tcp +34:04.230 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24755 network=udp +34:04.230 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24756 network=tcp +34:04.730 [INFO] TestAgent: Waiting for endpoints to shut down +34:04.730 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_HTTPCheck_TLSSkipVerify (1.27s) +=== CONT TestStatusPeers +[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:04.413Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:04.430Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:04.433 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:68a92d78-041f-8e8f-042c-0dc4a51250ae Address:127.0.0.1:24822}]" +34:04.433 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24822 [Follower]" leader= +34:04.434 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-68a92d78-041f-8e8f-042c-0dc4a51250ae.dc1 127.0.0.1 +34:04.435 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-68a92d78-041f-8e8f-042c-0dc4a51250ae 127.0.0.1 +2021-08-13T15:34:04.435Z [INFO] agent.router: Initializing LAN area manager +34:04.435 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-68a92d78-041f-8e8f-042c-0dc4a51250ae.dc1 area=wan +34:04.435 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24810 network=tcp +34:04.435 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24810 network=udp +34:04.436 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24811 network=tcp +34:04.436 [INFO] TestAgent: started state syncer +34:04.436 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24823 network=tcp +34:04.436 [INFO] TestAgent.server: Adding LAN server: server="Node-68a92d78-041f-8e8f-042c-0dc4a51250ae (Addr: tcp/127.0.0.1:24822) (DC: dc1)" +34:04.491 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:04.491 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24822 [Candidate]" term=2 +34:04.491 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:04.491 [DEBUG] TestAgent.server.raft: vote granted: from=68a92d78-041f-8e8f-042c-0dc4a51250ae term=2 tally=1 +34:04.491 [INFO] TestAgent.server.raft: election won: tally=1 +34:04.491 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24822 [Leader]" +34:04.492 [INFO] TestAgent.server: cluster leadership acquired +34:04.492 [INFO] TestAgent.server: New leader elected: payload=Node-68a92d78-041f-8e8f-042c-0dc4a51250ae +34:04.492 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24822 +34:04.493 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:04.496 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:04.496 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:04.496 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:04.496 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:04.496 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:04.496 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-68a92d78-041f-8e8f-042c-0dc4a51250ae +34:04.496 [INFO] TestAgent.server: member joined, marking health alive: member=Node-68a92d78-041f-8e8f-042c-0dc4a51250ae +34:04.497 [INFO] TestAgent.server: federation state anti-entropy synced +34:04.561 [INFO] TestAgent: Requesting shutdown +34:04.561 [INFO] TestAgent.server: shutting down server +34:04.561 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:04.561 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:04.561 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:04.561 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:04.561 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:04.561 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:04.561 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:04.561 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:04.561 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:04.561 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:04.561 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:04.561 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:04.561Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:04.561Z [INFO] agent.router.manager: shutting down +34:04.561 [INFO] TestAgent: consul server down +34:04.561 [INFO] TestAgent: shutdown complete +34:04.561 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24810 network=tcp +34:04.562 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24810 network=udp +34:04.562 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24811 network=tcp +34:05.062 [INFO] TestAgent: Waiting for endpoints to shut down +34:05.062 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_AddCheck_Alias_userToken (0.67s) === CONT TestAgent_AddCheck_Alias -[WARN] freeport: 4 out of 32 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._tcp_(qType:=1)_compressed=false#01 -2022-09-15T21:42:45.327Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:45.327Z [WARN] agent: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ -2022-09-15T21:42:45.345Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:45.345Z [WARN] agent.auto_config: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ -42:45.348 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:dd403b39-7e63-96cd-e908-d72ba9e87908 Address:127.0.0.1:13737}]" -42:45.349 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-dd403b39-7e63-96cd-e908-d72ba9e87908.dc1 127.0.0.1 -42:45.350 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13737 [Follower]" leader= -42:45.358 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-dd403b39-7e63-96cd-e908-d72ba9e87908 127.0.0.1 -2022-09-15T21:42:45.358Z [INFO] agent.router: Initializing LAN area manager -42:45.359 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13725 network=udp -42:45.359 [INFO] TestAgent.server: Adding LAN server: server="Node-dd403b39-7e63-96cd-e908-d72ba9e87908 (Addr: tcp/127.0.0.1:13737) (DC: dc1)" -42:45.359 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-dd403b39-7e63-96cd-e908-d72ba9e87908.dc1 area=wan -42:45.359 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13725 network=tcp -42:45.360 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13726 network=tcp -42:45.360 [INFO] TestAgent: started state syncer -42:45.360 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13738 network=tcp -42:45.387 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:45.387 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13737 [Candidate]" term=2 -42:45.387 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:45.387 [DEBUG] TestAgent.server.raft: vote granted: from=dd403b39-7e63-96cd-e908-d72ba9e87908 term=2 tally=1 -42:45.387 [INFO] TestAgent.server.raft: election won: tally=1 -42:45.387 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13737 [Leader]" -42:45.387 [INFO] TestAgent.server: cluster leadership acquired -42:45.387 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13737 -42:45.389 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:45.393 [INFO] TestAgent.server: New leader elected: payload=Node-dd403b39-7e63-96cd-e908-d72ba9e87908 -42:45.396 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:45.396 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:45.396 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:45.396 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:45.396 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:45.396 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-dd403b39-7e63-96cd-e908-d72ba9e87908 -42:45.396 [INFO] TestAgent.server: member joined, marking health alive: member=Node-dd403b39-7e63-96cd-e908-d72ba9e87908 -42:45.397 [INFO] TestAgent.server: federation state anti-entropy synced -42:45.502 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:45.503 [INFO] TestAgent: Synced node info -42:45.503 [DEBUG] TestAgent: Node info in sync -42:45.665 [DEBUG] TestAgent: removed check: check=mem -42:45.665 [DEBUG] TestAgent: removed check: check=mem -42:45.665 [INFO] TestAgent: Requesting shutdown -42:45.665 [INFO] TestAgent.server: shutting down server -42:45.665 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:45.665 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:45.665 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:45.665 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:45.665 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:45.666 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:45.666 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:45.666 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:45.666 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:45.666 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:45.666 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:45.666Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:45.666Z [INFO] agent.router.manager: shutting down -42:45.666 [INFO] TestAgent: consul server down -42:45.666 [INFO] TestAgent: shutdown complete -42:45.666 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13725 network=tcp -42:45.666 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13725 network=udp -42:45.666 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13726 network=tcp -42:46.167 [INFO] TestAgent: Waiting for endpoints to shut down -42:46.167 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_RemoveCheck (0.87s) +2021-08-13T15:34:04.235Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:04.252Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:04.255 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:39bba131-4ec5-93cf-625e-651921b542d9 Address:127.0.0.1:24808}]" +34:04.255 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24808 [Follower]" leader= +34:04.256 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-39bba131-4ec5-93cf-625e-651921b542d9.dc1 127.0.0.1 +34:04.257 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-39bba131-4ec5-93cf-625e-651921b542d9 127.0.0.1 +2021-08-13T15:34:04.257Z [INFO] agent.router: Initializing LAN area manager +34:04.258 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24817 network=udp +34:04.258 [INFO] TestAgent.server: Adding LAN server: server="Node-39bba131-4ec5-93cf-625e-651921b542d9 (Addr: tcp/127.0.0.1:24808) (DC: dc1)" +34:04.258 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-39bba131-4ec5-93cf-625e-651921b542d9.dc1 area=wan +34:04.258 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24817 network=tcp +34:04.259 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24818 network=tcp +34:04.259 [INFO] TestAgent: started state syncer +34:04.259 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24809 network=tcp +34:04.305 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:04.305 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24808 [Candidate]" term=2 +34:04.306 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:04.306 [DEBUG] TestAgent.server.raft: vote granted: from=39bba131-4ec5-93cf-625e-651921b542d9 term=2 tally=1 +34:04.306 [INFO] TestAgent.server.raft: election won: tally=1 +34:04.306 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24808 [Leader]" +34:04.306 [INFO] TestAgent.server: cluster leadership acquired +34:04.306 [INFO] TestAgent.server: New leader elected: payload=Node-39bba131-4ec5-93cf-625e-651921b542d9 +34:04.306 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24808 +34:04.307 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:04.310 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:04.310 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:04.310 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:04.310 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:04.310 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:04.310 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-39bba131-4ec5-93cf-625e-651921b542d9 +34:04.310 [INFO] TestAgent.server: member joined, marking health alive: member=Node-39bba131-4ec5-93cf-625e-651921b542d9 +34:04.311 [INFO] TestAgent.server: federation state anti-entropy synced +34:04.579 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:04.580 [INFO] TestAgent: Synced node info +34:04.580 [DEBUG] TestAgent: Node info in sync +34:04.604 [INFO] TestAgent: Requesting shutdown +34:04.604 [INFO] TestAgent.server: shutting down server +34:04.604 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:04.604 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:04.604 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:04.604 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:04.604 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:04.604 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:04.604 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:04.604 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:04.604 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:04.604 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:04.604 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:04.604Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:04.604Z [INFO] agent.router.manager: shutting down +34:04.604 [INFO] TestAgent: consul server down +34:04.604 [INFO] TestAgent: shutdown complete +34:04.605 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24817 network=tcp +34:04.605 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24817 network=udp +34:04.605 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24818 network=tcp +34:05.105 [INFO] TestAgent: Waiting for endpoints to shut down +34:05.105 [INFO] TestAgent: Endpoints down +--- PASS: TestTxnEndpoint_Bad_JSON (0.89s) +=== CONT TestStatusLeaderSecondary +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:04.659Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:04.674Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:04.676 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ef31f369-81f7-dd66-8443-79fc4349d164 Address:127.0.0.1:24843}]" +34:04.677 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24843 [Follower]" leader= +34:04.677 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ef31f369-81f7-dd66-8443-79fc4349d164.dc1 127.0.0.1 +34:04.680 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ef31f369-81f7-dd66-8443-79fc4349d164 127.0.0.1 +2021-08-13T15:34:04.680Z [INFO] agent.router: Initializing LAN area manager +34:04.682 [INFO] TestAgent.server: Adding LAN server: server="Node-ef31f369-81f7-dd66-8443-79fc4349d164 (Addr: tcp/127.0.0.1:24843) (DC: dc1)" +34:04.682 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ef31f369-81f7-dd66-8443-79fc4349d164.dc1 area=wan +34:04.686 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24838 network=udp +34:04.686 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24838 network=tcp +34:04.692 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24839 network=tcp +34:04.693 [INFO] TestAgent: started state syncer +34:04.693 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24844 network=tcp +34:04.728 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:04.728 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24843 [Candidate]" term=2 +34:04.728 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:04.728 [DEBUG] TestAgent.server.raft: vote granted: from=ef31f369-81f7-dd66-8443-79fc4349d164 term=2 tally=1 +34:04.728 [INFO] TestAgent.server.raft: election won: tally=1 +34:04.728 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24843 [Leader]" +34:04.728 [INFO] TestAgent.server: cluster leadership acquired +34:04.729 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24843 +34:04.729 [INFO] TestAgent.server: New leader elected: payload=Node-ef31f369-81f7-dd66-8443-79fc4349d164 +34:04.730 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:04.733 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:04.733 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:04.733 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:04.733 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:04.733 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:04.733 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ef31f369-81f7-dd66-8443-79fc4349d164 +34:04.733 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ef31f369-81f7-dd66-8443-79fc4349d164 +34:04.734 [INFO] TestAgent.server: federation state anti-entropy synced +34:04.793 [INFO] TestAgent: Requesting shutdown +34:04.793 [INFO] TestAgent.server: shutting down server +34:04.793 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:04.793 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:04.793 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:04.793 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:04.793 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:04.793 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:04.793 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:04.793 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:04.793 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:04.793 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:04.793 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:04.793 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:04.793Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:04.794Z [INFO] agent.router.manager: shutting down +34:04.794 [INFO] TestAgent: consul server down +34:04.794 [INFO] TestAgent: shutdown complete +34:04.794 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24838 network=tcp +34:04.794 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24838 network=udp +34:04.794 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24839 network=tcp +34:05.295 [INFO] TestAgent: Waiting for endpoints to shut down +34:05.295 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_AddCheck_Alias_setToken (0.65s) === CONT TestAgent_AddCheck_GRPC -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:45.597Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:45.641Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:45.650 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:803fb849-6afe-6ef6-97ca-d8eb45e4c7e4 Address:127.0.0.1:13751}]" -42:45.650 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13751 [Follower]" leader= -42:45.651 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-803fb849-6afe-6ef6-97ca-d8eb45e4c7e4.dc1 127.0.0.1 -42:45.652 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-803fb849-6afe-6ef6-97ca-d8eb45e4c7e4 127.0.0.1 -2022-09-15T21:42:45.652Z [INFO] agent.router: Initializing LAN area manager -42:45.653 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13732 network=udp -42:45.653 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-803fb849-6afe-6ef6-97ca-d8eb45e4c7e4.dc1 area=wan -42:45.653 [INFO] TestAgent.server: Adding LAN server: server="Node-803fb849-6afe-6ef6-97ca-d8eb45e4c7e4 (Addr: tcp/127.0.0.1:13751) (DC: dc1)" -42:45.653 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13732 network=tcp -42:45.654 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13733 network=tcp -42:45.654 [INFO] TestAgent: started state syncer -42:45.654 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13752 network=tcp -42:45.722 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:45.722 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13751 [Candidate]" term=2 -42:45.722 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:45.722 [DEBUG] TestAgent.server.raft: vote granted: from=803fb849-6afe-6ef6-97ca-d8eb45e4c7e4 term=2 tally=1 -42:45.722 [INFO] TestAgent.server.raft: election won: tally=1 -42:45.722 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13751 [Leader]" -42:45.722 [INFO] TestAgent.server: cluster leadership acquired -42:45.723 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13751 -42:45.723 [INFO] TestAgent.server: New leader elected: payload=Node-803fb849-6afe-6ef6-97ca-d8eb45e4c7e4 -42:45.725 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:45.728 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:45.728 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:45.728 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:45.728 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:45.728 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:45.729 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-803fb849-6afe-6ef6-97ca-d8eb45e4c7e4 -42:45.729 [INFO] TestAgent.server: member joined, marking health alive: member=Node-803fb849-6afe-6ef6-97ca-d8eb45e4c7e4 -42:45.730 [INFO] TestAgent.server: federation state anti-entropy synced -42:45.794 [INFO] TestAgent: Requesting shutdown -42:45.794 [INFO] TestAgent.server: shutting down server -42:45.794 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:45.794 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:45.794 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:45.794 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:45.794 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:45.794 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:45.794 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:45.794 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:45.794 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -42:45.794 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:45.794 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:45.794 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2022-09-15T21:42:45.795Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:45.795Z [INFO] agent.router.manager: shutting down -42:45.795 [INFO] TestAgent: consul server down -42:45.795 [INFO] TestAgent: shutdown complete -42:45.795 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13732 network=tcp -42:45.795 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13732 network=udp -42:45.795 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13733 network=tcp -42:46.296 [INFO] TestAgent: Waiting for endpoints to shut down -42:46.296 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_AddCheck_Alias_userAndSetToken (0.73s) +2021-08-13T15:34:04.508Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:04.525Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:04.528 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a631e67f-6035-5ab1-6015-86ff79b12e33 Address:127.0.0.1:24829}]" +34:04.528 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24829 [Follower]" leader= +34:04.529 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a631e67f-6035-5ab1-6015-86ff79b12e33.dc1 127.0.0.1 +34:04.530 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a631e67f-6035-5ab1-6015-86ff79b12e33 127.0.0.1 +2021-08-13T15:34:04.530Z [INFO] agent.router: Initializing LAN area manager +34:04.530 [INFO] TestAgent.server: Adding LAN server: server="Node-a631e67f-6035-5ab1-6015-86ff79b12e33 (Addr: tcp/127.0.0.1:24829) (DC: dc1)" +34:04.530 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a631e67f-6035-5ab1-6015-86ff79b12e33.dc1 area=wan +34:04.530 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24824 network=tcp +34:04.530 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24824 network=udp +34:04.531 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24825 network=tcp +34:04.531 [INFO] TestAgent: started state syncer +34:04.531 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24830 network=tcp +34:04.571 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:04.571 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24829 [Candidate]" term=2 +34:04.571 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:04.571 [DEBUG] TestAgent.server.raft: vote granted: from=a631e67f-6035-5ab1-6015-86ff79b12e33 term=2 tally=1 +34:04.572 [INFO] TestAgent.server.raft: election won: tally=1 +34:04.572 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24829 [Leader]" +34:04.572 [INFO] TestAgent.server: cluster leadership acquired +34:04.572 [INFO] TestAgent.server: New leader elected: payload=Node-a631e67f-6035-5ab1-6015-86ff79b12e33 +34:04.572 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24829 +34:04.573 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:04.576 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:04.576 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:04.576 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:04.576 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:04.576 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:04.576 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a631e67f-6035-5ab1-6015-86ff79b12e33 +34:04.576 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a631e67f-6035-5ab1-6015-86ff79b12e33 +34:04.577 [INFO] TestAgent.server: federation state anti-entropy synced +34:04.633 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:04.634 [INFO] TestAgent: Synced node info +34:04.634 [DEBUG] TestAgent: Node info in sync +34:05.015 [INFO] TestAgent: Requesting shutdown +34:05.015 [INFO] TestAgent.server: shutting down server +34:05.015 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:05.015 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:05.015 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:05.015 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:05.016 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:05.016 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:05.016 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:05.016 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:05.016 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:05.016 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:05.016 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:05.016Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:05.016Z [INFO] agent.router.manager: shutting down +34:05.017 [INFO] TestAgent: consul server down +34:05.017 [INFO] TestAgent: shutdown complete +34:05.017 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24824 network=tcp +34:05.017 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24824 network=udp +34:05.017 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24825 network=tcp +34:05.517 [INFO] TestAgent: Waiting for endpoints to shut down +34:05.517 [INFO] TestAgent: Endpoints down +=== RUN TestTxnEndpoint_Bad_Size_Item/exceeds_default_max_kv_value_size +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:04.749Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:04.765Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:04.768 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5e1dc728-44a2-5f01-8eaa-1444006fe227 Address:127.0.0.1:24864}]" +34:04.768 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24864 [Follower]" leader= +34:04.769 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5e1dc728-44a2-5f01-8eaa-1444006fe227.dc1 127.0.0.1 +34:04.770 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5e1dc728-44a2-5f01-8eaa-1444006fe227 127.0.0.1 +2021-08-13T15:34:04.770Z [INFO] agent.router: Initializing LAN area manager +34:04.770 [INFO] TestAgent.server: Adding LAN server: server="Node-5e1dc728-44a2-5f01-8eaa-1444006fe227 (Addr: tcp/127.0.0.1:24864) (DC: dc1)" +34:04.770 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24859 network=udp +34:04.770 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5e1dc728-44a2-5f01-8eaa-1444006fe227.dc1 area=wan +34:04.771 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24859 network=tcp +34:04.772 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24860 network=tcp +34:04.772 [INFO] TestAgent: started state syncer +34:04.772 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24865 network=tcp +34:04.803 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:04.803 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24864 [Candidate]" term=2 +34:04.804 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:04.804 [DEBUG] TestAgent.server.raft: vote granted: from=5e1dc728-44a2-5f01-8eaa-1444006fe227 term=2 tally=1 +34:04.804 [INFO] TestAgent.server.raft: election won: tally=1 +34:04.804 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24864 [Leader]" +34:04.804 [INFO] TestAgent.server: cluster leadership acquired +34:04.804 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24864 +34:04.805 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:04.806 [INFO] TestAgent.server: New leader elected: payload=Node-5e1dc728-44a2-5f01-8eaa-1444006fe227 +34:04.808 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:04.808 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:04.808 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:04.808 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:04.808 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:04.808 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5e1dc728-44a2-5f01-8eaa-1444006fe227 +34:04.808 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5e1dc728-44a2-5f01-8eaa-1444006fe227 +34:04.809 [INFO] TestAgent.server: federation state anti-entropy synced +34:04.935 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:04.936 [INFO] TestAgent: Synced node info +34:05.183 [INFO] TestAgent: Requesting shutdown +34:05.183 [INFO] TestAgent.server: shutting down server +34:05.183 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:05.183 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:05.183 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:05.183 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:05.183 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:05.183 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:05.183 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:05.184 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:05.184 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:05.184 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:05.184 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:05.184Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:05.184Z [INFO] agent.router.manager: shutting down +34:05.184 [INFO] TestAgent: consul server down +34:05.184 [INFO] TestAgent: shutdown complete +34:05.184 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24859 network=tcp +34:05.184 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24859 network=udp +34:05.184 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24860 network=tcp +34:05.684 [INFO] TestAgent: Waiting for endpoints to shut down +34:05.684 [INFO] TestAgent: Endpoints down +--- PASS: TestStatusPeers (0.95s) +=== CONT TestStatusLeader +2021-08-13T15:34:05.080Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:05.104Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:05.107 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f2b9fb13-39e7-22f8-9160-f55417e95f2d Address:127.0.0.1:24871}]" +34:05.107 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24871 [Follower]" leader= +34:05.108 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f2b9fb13-39e7-22f8-9160-f55417e95f2d.dc1 127.0.0.1 +34:05.109 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f2b9fb13-39e7-22f8-9160-f55417e95f2d 127.0.0.1 +2021-08-13T15:34:05.109Z [INFO] agent.router: Initializing LAN area manager +34:05.109 [INFO] TestAgent.server: Adding LAN server: server="Node-f2b9fb13-39e7-22f8-9160-f55417e95f2d (Addr: tcp/127.0.0.1:24871) (DC: dc1)" +34:05.109 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f2b9fb13-39e7-22f8-9160-f55417e95f2d.dc1 area=wan +34:05.109 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24866 network=udp +34:05.109 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24866 network=tcp +34:05.110 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24867 network=tcp +34:05.110 [INFO] TestAgent: started state syncer +34:05.110 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24872 network=tcp +34:05.177 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:05.177 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24871 [Candidate]" term=2 +34:05.177 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:05.177 [DEBUG] TestAgent.server.raft: vote granted: from=f2b9fb13-39e7-22f8-9160-f55417e95f2d term=2 tally=1 +34:05.177 [INFO] TestAgent.server.raft: election won: tally=1 +34:05.177 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24871 [Leader]" +34:05.177 [INFO] TestAgent.server: cluster leadership acquired +34:05.177 [INFO] TestAgent.server: New leader elected: payload=Node-f2b9fb13-39e7-22f8-9160-f55417e95f2d +34:05.178 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24871 +34:05.179 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:05.182 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:05.182 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:05.182 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:05.182 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:05.182 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:05.182 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f2b9fb13-39e7-22f8-9160-f55417e95f2d +34:05.182 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f2b9fb13-39e7-22f8-9160-f55417e95f2d +34:05.183 [INFO] TestAgent.server: federation state anti-entropy synced +34:05.251 [INFO] TestAgent: Requesting shutdown +34:05.251 [INFO] TestAgent.server: shutting down server +34:05.251 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:05.251 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:05.251 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:05.251 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:05.251 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:05.251 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:05.251 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:05.251 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:05.251 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:05.251 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:05.251 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:05.252 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:05.252Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:05.252Z [INFO] agent.router.manager: shutting down +34:05.252 [INFO] TestAgent: consul server down +34:05.252 [INFO] TestAgent: shutdown complete +34:05.252 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24866 network=tcp +34:05.252 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24866 network=udp +34:05.252 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24867 network=tcp +34:05.752 [INFO] TestAgent: Waiting for endpoints to shut down +34:05.752 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_AddCheck_Alias (0.69s) === CONT TestAgent_AddCheck_ExecRemoteDisable -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:45.810Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:45.891Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:45.895 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3c408a02-d290-f298-c951-c9c2752b7a2c Address:127.0.0.1:13765}]" -42:45.895 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13765 [Follower]" leader= -42:45.896 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3c408a02-d290-f298-c951-c9c2752b7a2c.dc1 127.0.0.1 -42:45.898 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3c408a02-d290-f298-c951-c9c2752b7a2c 127.0.0.1 -2022-09-15T21:42:45.898Z [INFO] agent.router: Initializing LAN area manager -42:45.898 [INFO] TestAgent.server: Adding LAN server: server="Node-3c408a02-d290-f298-c951-c9c2752b7a2c (Addr: tcp/127.0.0.1:13765) (DC: dc1)" -42:45.898 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3c408a02-d290-f298-c951-c9c2752b7a2c.dc1 area=wan -42:45.898 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13753 network=udp -42:45.898 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13753 network=tcp -42:45.900 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13754 network=tcp -42:45.900 [INFO] TestAgent: started state syncer -42:45.900 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13766 network=tcp -42:45.934 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:45.934 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13765 [Candidate]" term=2 -42:45.934 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:45.934 [DEBUG] TestAgent.server.raft: vote granted: from=3c408a02-d290-f298-c951-c9c2752b7a2c term=2 tally=1 -42:45.934 [INFO] TestAgent.server.raft: election won: tally=1 -42:45.934 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13765 [Leader]" -42:45.935 [INFO] TestAgent.server: cluster leadership acquired -42:45.935 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13765 -42:45.937 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:45.941 [INFO] TestAgent.server: New leader elected: payload=Node-3c408a02-d290-f298-c951-c9c2752b7a2c -42:45.945 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:45.945 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:45.945 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:45.945 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:45.945 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:45.945 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3c408a02-d290-f298-c951-c9c2752b7a2c -42:45.945 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3c408a02-d290-f298-c951-c9c2752b7a2c -42:45.952 [INFO] TestAgent.server: federation state anti-entropy synced -42:46.172 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:46.173 [INFO] TestAgent: Synced node info -42:46.191 [INFO] TestAgent: Requesting shutdown -42:46.191 [INFO] TestAgent.server: shutting down server -42:46.191 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:46.191 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:46.191 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:46.191 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:46.191 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:46.191 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:46.191 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:46.191 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:46.192 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:46.192 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:46.192 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:46.192Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:46.192Z [INFO] agent.router.manager: shutting down -42:46.192 [INFO] TestAgent: consul server down -42:46.192 [INFO] TestAgent: shutdown complete -42:46.192 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13753 network=tcp -42:46.192 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13753 network=udp -42:46.192 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13754 network=tcp -42:46.701 [INFO] TestAgent: Waiting for endpoints to shut down -42:46.701 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_AddCheck_Alias_userToken (0.92s) +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:04.673Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:04.705Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:04.709 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6ab6dc63-c0ec-b7a2-1250-bac0620bd419 Address:127.0.0.1:24850}]" +34:04.709 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24850 [Follower]" leader= +34:04.710 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6ab6dc63-c0ec-b7a2-1250-bac0620bd419.secondary 127.0.0.1 +34:04.711 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6ab6dc63-c0ec-b7a2-1250-bac0620bd419 127.0.0.1 +2021-08-13T15:34:04.711Z [INFO] agent.router: Initializing LAN area manager +34:04.712 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24845 network=udp +34:04.712 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6ab6dc63-c0ec-b7a2-1250-bac0620bd419.secondary area=wan +34:04.712 [INFO] TestAgent.server: Adding LAN server: server="Node-6ab6dc63-c0ec-b7a2-1250-bac0620bd419 (Addr: tcp/127.0.0.1:24850) (DC: secondary)" +34:04.712 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24845 network=tcp +34:04.714 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24846 network=tcp +34:04.714 [INFO] TestAgent: started state syncer +34:04.714 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24851 network=tcp +34:04.766 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:04.766 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24850 [Candidate]" term=2 +34:04.767 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:04.767 [DEBUG] TestAgent.server.raft: vote granted: from=6ab6dc63-c0ec-b7a2-1250-bac0620bd419 term=2 tally=1 +34:04.767 [INFO] TestAgent.server.raft: election won: tally=1 +34:04.767 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24850 [Leader]" +34:04.767 [INFO] TestAgent.server: cluster leadership acquired +34:04.767 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24850 +34:04.767 [INFO] TestAgent.server: New leader elected: payload=Node-6ab6dc63-c0ec-b7a2-1250-bac0620bd419 +34:04.768 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:04.771 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:04.771 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:04.771 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:04.771 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:04.771 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:04.771 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6ab6dc63-c0ec-b7a2-1250-bac0620bd419 +34:04.771 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6ab6dc63-c0ec-b7a2-1250-bac0620bd419 +34:04.772 [INFO] TestAgent.server: federation state anti-entropy synced +34:04.922 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:24835] +34:04.922 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:24835 +34:04.924 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-748c0e2a-7a41-d426-3053-4fa4abeee441.primary 127.0.0.1 +34:04.924 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 +34:04.924 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-748c0e2a-7a41-d426-3053-4fa4abeee441.primary area=wan +34:04.927 [INFO] TestAgent: Requesting shutdown +34:04.927 [INFO] TestAgent.server: shutting down server +34:04.927 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:04.927 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:04.927 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:04.927 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:04.927 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:04.927 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:04.927 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:04.927 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:04.927 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:04.927 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:04.927 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:04.927 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:04.928Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:04.928Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:04.928Z [INFO] agent.router.manager: shutting down +34:04.928 [INFO] TestAgent: consul server down +34:04.928 [INFO] TestAgent: shutdown complete +34:04.928 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24845 network=tcp +34:04.928 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24845 network=udp +34:04.928 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24846 network=tcp +34:05.431 [INFO] TestAgent: Waiting for endpoints to shut down +34:05.431 [INFO] TestAgent: Endpoints down +2021-08-13T15:34:04.568Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:04.585Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:04.588 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:748c0e2a-7a41-d426-3053-4fa4abeee441 Address:127.0.0.1:24836}]" +34:04.588 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24836 [Follower]" leader= +34:04.589 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-748c0e2a-7a41-d426-3053-4fa4abeee441.primary 127.0.0.1 +34:04.590 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-748c0e2a-7a41-d426-3053-4fa4abeee441 127.0.0.1 +2021-08-13T15:34:04.590Z [INFO] agent.router: Initializing LAN area manager +34:04.590 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-748c0e2a-7a41-d426-3053-4fa4abeee441.primary area=wan +34:04.590 [INFO] TestAgent.server: Adding LAN server: server="Node-748c0e2a-7a41-d426-3053-4fa4abeee441 (Addr: tcp/127.0.0.1:24836) (DC: primary)" +34:04.590 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24831 network=tcp +34:04.590 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24831 network=udp +34:04.591 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24832 network=tcp +34:04.591 [INFO] TestAgent: started state syncer +34:04.591 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24837 network=tcp +34:04.633 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:04.633 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24836 [Candidate]" term=2 +34:04.633 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:04.633 [DEBUG] TestAgent.server.raft: vote granted: from=748c0e2a-7a41-d426-3053-4fa4abeee441 term=2 tally=1 +34:04.633 [INFO] TestAgent.server.raft: election won: tally=1 +34:04.633 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24836 [Leader]" +34:04.633 [INFO] TestAgent.server: cluster leadership acquired +34:04.633 [INFO] TestAgent.server: New leader elected: payload=Node-748c0e2a-7a41-d426-3053-4fa4abeee441 +34:04.633 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24836 +34:04.635 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:04.637 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:04.637 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:04.637 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:04.637 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:04.637 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:04.637 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-748c0e2a-7a41-d426-3053-4fa4abeee441 +34:04.637 [INFO] TestAgent.server: member joined, marking health alive: member=Node-748c0e2a-7a41-d426-3053-4fa4abeee441 +34:04.638 [INFO] TestAgent.server: federation state anti-entropy synced +34:04.694 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:04.695 [INFO] TestAgent: Synced node info +34:04.695 [DEBUG] TestAgent: Node info in sync +34:04.922 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:42284 +34:04.923 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6ab6dc63-c0ec-b7a2-1250-bac0620bd419.secondary 127.0.0.1 +34:04.924 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6ab6dc63-c0ec-b7a2-1250-bac0620bd419.secondary area=wan +34:05.431 [INFO] TestAgent: Requesting shutdown +34:05.431 [INFO] TestAgent.server: shutting down server +34:05.431 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:05.431 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:05.431 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:05.431 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:05.431 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:05.431 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:05.431 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:05.431 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:05.431 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:05.431 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +34:05.431 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +2021-08-13T15:34:05.432Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:05.432Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:05.432Z [INFO] agent.router.manager: shutting down +34:05.432 [INFO] TestAgent: consul server down +34:05.432 [INFO] TestAgent: shutdown complete +34:05.432 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24831 network=tcp +34:05.432 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24831 network=udp +34:05.432 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24832 network=tcp +34:05.933 [INFO] TestAgent: Waiting for endpoints to shut down +34:05.933 [INFO] TestAgent: Endpoints down +--- PASS: TestStatusPeersSecondary (1.38s) +=== CONT TestSnapshot_Options +=== RUN TestSnapshot_Options/GET +2021-08-13T15:34:04.730Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:04.747Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:04.750 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:25db512e-9889-d24b-a635-3449871510a3 Address:127.0.0.1:24857}]" +34:04.751 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24857 [Follower]" leader= +34:04.751 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-25db512e-9889-d24b-a635-3449871510a3.dc1 127.0.0.1 +34:04.752 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-25db512e-9889-d24b-a635-3449871510a3 127.0.0.1 +2021-08-13T15:34:04.752Z [INFO] agent.router: Initializing LAN area manager +34:04.753 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24852 network=udp +34:04.753 [INFO] TestAgent.server: Adding LAN server: server="Node-25db512e-9889-d24b-a635-3449871510a3 (Addr: tcp/127.0.0.1:24857) (DC: dc1)" +34:04.753 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-25db512e-9889-d24b-a635-3449871510a3.dc1 area=wan +34:04.753 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24852 network=tcp +34:04.754 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24853 network=tcp +34:04.754 [INFO] TestAgent: started state syncer +34:04.754 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24858 network=tcp +34:04.800 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:04.800 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24857 [Candidate]" term=2 +34:04.801 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:04.801 [DEBUG] TestAgent.server.raft: vote granted: from=25db512e-9889-d24b-a635-3449871510a3 term=2 tally=1 +34:04.801 [INFO] TestAgent.server.raft: election won: tally=1 +34:04.801 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24857 [Leader]" +34:04.801 [INFO] TestAgent.server: cluster leadership acquired +34:04.801 [INFO] TestAgent.server: New leader elected: payload=Node-25db512e-9889-d24b-a635-3449871510a3 +34:04.801 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24857 +34:04.803 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:04.806 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:04.806 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:04.806 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:04.806 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:04.806 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:04.806 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-25db512e-9889-d24b-a635-3449871510a3 +34:04.806 [INFO] TestAgent.server: member joined, marking health alive: member=Node-25db512e-9889-d24b-a635-3449871510a3 +34:04.807 [INFO] TestAgent.server: federation state anti-entropy synced +34:04.844 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:04.845 [INFO] TestAgent: Synced node info +34:05.434 [INFO] TestAgent: Requesting shutdown +34:05.434 [INFO] TestAgent.server: shutting down server +34:05.434 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:05.434 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:05.434 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:05.434 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:05.434 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:05.434 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:05.434 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:05.434 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:05.434 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:05.434 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:05.434 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:05.434Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:05.434Z [INFO] agent.router.manager: shutting down +34:05.435 [INFO] TestAgent: consul server down +34:05.435 [INFO] TestAgent: shutdown complete +34:05.435 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24852 network=tcp +34:05.435 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24852 network=udp +34:05.435 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24853 network=tcp +34:05.935 [INFO] TestAgent: Waiting for endpoints to shut down +34:05.935 [INFO] TestAgent: Endpoints down +=== RUN TestTxnEndpoint_Bad_Size_Net/exceeds_default_max_kv_value_size +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:05.313Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:05.333Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:05.337 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1407050b-cdc3-0426-2512-d4a204f491ab Address:127.0.0.1:24885}]" +34:05.338 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1407050b-cdc3-0426-2512-d4a204f491ab.dc1 127.0.0.1 +34:05.339 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24885 [Follower]" leader= +34:05.339 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1407050b-cdc3-0426-2512-d4a204f491ab 127.0.0.1 +2021-08-13T15:34:05.339Z [INFO] agent.router: Initializing LAN area manager +34:05.340 [INFO] TestAgent.server: Adding LAN server: server="Node-1407050b-cdc3-0426-2512-d4a204f491ab (Addr: tcp/127.0.0.1:24885) (DC: dc1)" +34:05.340 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1407050b-cdc3-0426-2512-d4a204f491ab.dc1 area=wan +34:05.340 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24894 network=udp +34:05.341 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24894 network=tcp +34:05.350 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24895 network=tcp +34:05.350 [INFO] TestAgent: started state syncer +34:05.351 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24886 network=tcp +34:05.409 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:05.409 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24885 [Candidate]" term=2 +34:05.410 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:05.410 [DEBUG] TestAgent.server.raft: vote granted: from=1407050b-cdc3-0426-2512-d4a204f491ab term=2 tally=1 +34:05.410 [INFO] TestAgent.server.raft: election won: tally=1 +34:05.410 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24885 [Leader]" +34:05.410 [INFO] TestAgent.server: cluster leadership acquired +34:05.410 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24885 +34:05.412 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:05.413 [INFO] TestAgent.server: New leader elected: payload=Node-1407050b-cdc3-0426-2512-d4a204f491ab +34:05.414 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:05.414 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:05.414 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:05.414 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:05.414 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:05.414 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1407050b-cdc3-0426-2512-d4a204f491ab +34:05.414 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1407050b-cdc3-0426-2512-d4a204f491ab +34:05.415 [INFO] TestAgent.server: federation state anti-entropy synced +34:05.554 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:05.555 [INFO] TestAgent: Synced node info +34:05.676 [INFO] TestAgent: Requesting shutdown +34:05.676 [INFO] TestAgent.server: shutting down server +34:05.676 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:05.676 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:05.676 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:05.676 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:05.676 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:05.676 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:05.676 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:05.676 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:05.676 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:05.676 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:05.677 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:05.677Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:05.677Z [INFO] agent.router.manager: shutting down +34:05.677 [INFO] TestAgent: consul server down +34:05.677 [INFO] TestAgent: shutdown complete +34:05.677 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24894 network=tcp +34:05.677 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24894 network=udp +34:05.677 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24895 network=tcp +34:06.177 [INFO] TestAgent: Waiting for endpoints to shut down +34:06.177 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_AddCheck_GRPC (0.88s) === CONT TestAgent_AddCheck_ExecDisable -2022-09-15T21:42:45.083Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:45.099Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:45.102 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:97ba3c5a-0b6b-f807-6c30-d2322d7ced1e Address:127.0.0.1:13723}]" -42:45.103 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13723 [Follower]" leader= -42:45.104 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-97ba3c5a-0b6b-f807-6c30-d2322d7ced1e.dc1 127.0.0.1 -42:45.105 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-97ba3c5a-0b6b-f807-6c30-d2322d7ced1e 127.0.0.1 -2022-09-15T21:42:45.105Z [INFO] agent.router: Initializing LAN area manager -42:45.105 [INFO] TestAgent.server: Adding LAN server: server="Node-97ba3c5a-0b6b-f807-6c30-d2322d7ced1e (Addr: tcp/127.0.0.1:13723) (DC: dc1)" -42:45.105 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-97ba3c5a-0b6b-f807-6c30-d2322d7ced1e.dc1 area=wan -42:45.105 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13711 network=udp -42:45.106 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13711 network=tcp -42:45.106 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13712 network=tcp -42:45.107 [INFO] TestAgent: started state syncer -42:45.107 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13724 network=tcp -42:45.167 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:45.167 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13723 [Candidate]" term=2 -42:45.167 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:45.167 [DEBUG] TestAgent.server.raft: vote granted: from=97ba3c5a-0b6b-f807-6c30-d2322d7ced1e term=2 tally=1 -42:45.167 [INFO] TestAgent.server.raft: election won: tally=1 -42:45.167 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13723 [Leader]" -42:45.167 [INFO] TestAgent.server: cluster leadership acquired -42:45.168 [INFO] TestAgent.server: New leader elected: payload=Node-97ba3c5a-0b6b-f807-6c30-d2322d7ced1e -42:45.168 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13723 -42:45.169 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:45.171 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:45.171 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:45.171 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:45.171 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:45.171 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:45.171 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-97ba3c5a-0b6b-f807-6c30-d2322d7ced1e -42:45.171 [INFO] TestAgent.server: member joined, marking health alive: member=Node-97ba3c5a-0b6b-f807-6c30-d2322d7ced1e -42:45.172 [INFO] TestAgent.server: federation state anti-entropy synced -42:45.207 [WARN] TestAgent: check has interval below minimum: check=tls minimum_interval=1s -42:45.375 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:45.376 [INFO] TestAgent: Synced node info -42:45.376 [INFO] TestAgent: Synced check: check=tls -42:46.202 [DEBUG] TestAgent: Check status updated: check=tls status=passing -42:46.234 [INFO] TestAgent: Requesting shutdown -42:46.234 [INFO] TestAgent.server: shutting down server -42:46.234 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:46.234 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:46.234 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:46.234 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:46.234 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:46.234 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:46.234 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:46.234 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:46.234 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:46.234 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:46.234 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:46.235Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:46.235Z [INFO] agent.router.manager: shutting down -42:46.235 [INFO] TestAgent: consul server down -42:46.235 [INFO] TestAgent: shutdown complete -42:46.235 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13711 network=tcp -42:46.235 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13711 network=udp -42:46.235 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13712 network=tcp -42:46.735 [INFO] TestAgent: Waiting for endpoints to shut down -42:46.739 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_HTTPCheck_TLSSkipVerify (1.68s) +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:05.701Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:05.717Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:05.720 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1cd5f26b-7e58-073d-da4d-d62eb831414f Address:127.0.0.1:24906}]" +34:05.720 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24906 [Follower]" leader= +34:05.721 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1cd5f26b-7e58-073d-da4d-d62eb831414f.dc1 127.0.0.1 +34:05.722 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1cd5f26b-7e58-073d-da4d-d62eb831414f 127.0.0.1 +2021-08-13T15:34:05.722Z [INFO] agent.router: Initializing LAN area manager +34:05.722 [INFO] TestAgent.server: Adding LAN server: server="Node-1cd5f26b-7e58-073d-da4d-d62eb831414f (Addr: tcp/127.0.0.1:24906) (DC: dc1)" +34:05.722 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1cd5f26b-7e58-073d-da4d-d62eb831414f.dc1 area=wan +34:05.722 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24880 network=tcp +34:05.722 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24880 network=udp +34:05.723 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24881 network=tcp +34:05.723 [INFO] TestAgent: started state syncer +34:05.723 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24907 network=tcp +34:05.764 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:05.764 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24906 [Candidate]" term=2 +34:05.764 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:05.764 [DEBUG] TestAgent.server.raft: vote granted: from=1cd5f26b-7e58-073d-da4d-d62eb831414f term=2 tally=1 +34:05.764 [INFO] TestAgent.server.raft: election won: tally=1 +34:05.764 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24906 [Leader]" +34:05.764 [INFO] TestAgent.server: cluster leadership acquired +34:05.765 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24906 +34:05.765 [INFO] TestAgent.server: New leader elected: payload=Node-1cd5f26b-7e58-073d-da4d-d62eb831414f +34:05.766 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:05.768 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:05.768 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:05.768 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:05.768 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:05.768 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:05.768 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1cd5f26b-7e58-073d-da4d-d62eb831414f +34:05.768 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1cd5f26b-7e58-073d-da4d-d62eb831414f +34:05.769 [INFO] TestAgent.server: federation state anti-entropy synced +34:05.947 [INFO] TestAgent: Requesting shutdown +34:05.947 [INFO] TestAgent.server: shutting down server +34:05.948 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:05.948 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:05.948 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:05.948 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:05.948 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:05.948 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:05.948 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:05.948 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:05.948 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:05.948 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:05.948 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:05.948 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:05.948Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:05.948Z [INFO] agent.router.manager: shutting down +34:05.948 [INFO] TestAgent: consul server down +34:05.948 [INFO] TestAgent: shutdown complete +34:05.948 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24880 network=tcp +34:05.948 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24880 network=udp +34:05.948 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24881 network=tcp +34:06.449 [INFO] TestAgent: Waiting for endpoints to shut down +34:06.449 [INFO] TestAgent: Endpoints down +--- PASS: TestStatusLeader (0.76s) +=== CONT TestSessionDeleteDestroy +2021-08-13T15:34:05.534Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:05.566Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:05.569 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:505c2a26-09b4-689e-2618-fe7fdb3d4ea3 Address:127.0.0.1:24878}]" +34:05.569 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24878 [Follower]" leader= +34:05.570 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-505c2a26-09b4-689e-2618-fe7fdb3d4ea3.dc1 127.0.0.1 +34:05.571 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-505c2a26-09b4-689e-2618-fe7fdb3d4ea3 127.0.0.1 +2021-08-13T15:34:05.571Z [INFO] agent.router: Initializing LAN area manager +34:05.571 [INFO] TestAgent.server: Adding LAN server: server="Node-505c2a26-09b4-689e-2618-fe7fdb3d4ea3 (Addr: tcp/127.0.0.1:24878) (DC: dc1)" +34:05.571 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-505c2a26-09b4-689e-2618-fe7fdb3d4ea3.dc1 area=wan +34:05.571 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24901 network=tcp +34:05.571 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24901 network=udp +34:05.572 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24902 network=tcp +34:05.572 [INFO] TestAgent: started state syncer +34:05.572 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24879 network=tcp +34:05.630 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:05.630 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24878 [Candidate]" term=2 +34:05.630 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:05.630 [DEBUG] TestAgent.server.raft: vote granted: from=505c2a26-09b4-689e-2618-fe7fdb3d4ea3 term=2 tally=1 +34:05.630 [INFO] TestAgent.server.raft: election won: tally=1 +34:05.630 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24878 [Leader]" +34:05.630 [INFO] TestAgent.server: cluster leadership acquired +34:05.630 [INFO] TestAgent.server: New leader elected: payload=Node-505c2a26-09b4-689e-2618-fe7fdb3d4ea3 +34:05.630 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24878 +34:05.632 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:05.634 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:05.634 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:05.634 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:05.634 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:05.634 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:05.634 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-505c2a26-09b4-689e-2618-fe7fdb3d4ea3 +34:05.634 [INFO] TestAgent.server: member joined, marking health alive: member=Node-505c2a26-09b4-689e-2618-fe7fdb3d4ea3 +34:05.635 [INFO] TestAgent.server: federation state anti-entropy synced +34:05.751 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:05.752 [INFO] TestAgent: Synced node info +34:05.752 [DEBUG] TestAgent: Node info in sync +34:05.972 [INFO] TestAgent: Requesting shutdown +34:05.972 [INFO] TestAgent.server: shutting down server +34:05.972 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:05.972 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:05.972 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:05.972 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:05.972 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:05.972 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:05.973 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:05.973 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:05.973 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:05.973 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:05.973 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:05.973Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:05.973Z [INFO] agent.router.manager: shutting down +34:05.973 [INFO] TestAgent: consul server down +34:05.973 [INFO] TestAgent: shutdown complete +34:05.973 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24901 network=tcp +34:05.973 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24901 network=udp +34:05.973 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24902 network=tcp +34:06.474 [INFO] TestAgent: Waiting for endpoints to shut down +34:06.474 [INFO] TestAgent: Endpoints down +=== RUN TestTxnEndpoint_Bad_Size_Item/allowed +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:05.772Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:05.790Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:05.793 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:aedc5c38-9f9e-b3cf-0f10-ef7192c6b6cb Address:127.0.0.1:24913}]" +34:05.794 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-aedc5c38-9f9e-b3cf-0f10-ef7192c6b6cb.dc1 127.0.0.1 +34:05.795 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24913 [Follower]" leader= +34:05.795 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-aedc5c38-9f9e-b3cf-0f10-ef7192c6b6cb 127.0.0.1 +2021-08-13T15:34:05.795Z [INFO] agent.router: Initializing LAN area manager +34:05.795 [INFO] TestAgent.server: Adding LAN server: server="Node-aedc5c38-9f9e-b3cf-0f10-ef7192c6b6cb (Addr: tcp/127.0.0.1:24913) (DC: dc1)" +34:05.795 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-aedc5c38-9f9e-b3cf-0f10-ef7192c6b6cb.dc1 area=wan +34:05.795 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24908 network=udp +34:05.796 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24908 network=tcp +34:05.796 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24909 network=tcp +34:05.796 [INFO] TestAgent: started state syncer +34:05.797 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24914 network=tcp +34:05.859 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:05.859 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24913 [Candidate]" term=2 +34:05.860 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:05.860 [DEBUG] TestAgent.server.raft: vote granted: from=aedc5c38-9f9e-b3cf-0f10-ef7192c6b6cb term=2 tally=1 +34:05.860 [INFO] TestAgent.server.raft: election won: tally=1 +34:05.860 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24913 [Leader]" +34:05.860 [INFO] TestAgent.server: cluster leadership acquired +34:05.860 [INFO] TestAgent.server: New leader elected: payload=Node-aedc5c38-9f9e-b3cf-0f10-ef7192c6b6cb +34:05.860 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24913 +34:05.861 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:05.864 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:05.864 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:05.864 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:05.864 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:05.864 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:05.864 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-aedc5c38-9f9e-b3cf-0f10-ef7192c6b6cb +34:05.864 [INFO] TestAgent.server: member joined, marking health alive: member=Node-aedc5c38-9f9e-b3cf-0f10-ef7192c6b6cb +34:05.865 [INFO] TestAgent.server: federation state anti-entropy synced +34:06.001 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:06.001 [INFO] TestAgent: Synced node info +34:06.001 [DEBUG] TestAgent: Node info in sync +34:06.133 [INFO] TestAgent: Requesting shutdown +34:06.133 [INFO] TestAgent.server: shutting down server +34:06.133 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:06.133 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:06.133 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:06.133 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:06.133 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:06.133 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:06.133 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:06.133 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:06.134 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:06.134 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:06.134 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:06.134Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:06.134Z [INFO] agent.router.manager: shutting down +34:06.134 [INFO] TestAgent: consul server down +34:06.134 [INFO] TestAgent: shutdown complete +34:06.134 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24908 network=tcp +34:06.134 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24908 network=udp +34:06.134 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24909 network=tcp +34:06.634 [INFO] TestAgent: Waiting for endpoints to shut down +34:06.634 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_AddCheck_ExecRemoteDisable (0.88s) === CONT TestAgent_AddCheck_RestoreState -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._udp_(qType:=1)_compressed=true#01 -2022-09-15T21:42:46.186Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:46.204Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:46.217 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1bec5485-9479-4e4f-70ce-66372039849d Address:127.0.0.1:13772}]" -42:46.218 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13772 [Follower]" leader= -42:46.218 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1bec5485-9479-4e4f-70ce-66372039849d.dc1 127.0.0.1 -42:46.227 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1bec5485-9479-4e4f-70ce-66372039849d 127.0.0.1 -2022-09-15T21:42:46.227Z [INFO] agent.router: Initializing LAN area manager -42:46.227 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13795 network=udp -42:46.227 [INFO] TestAgent.server: Adding LAN server: server="Node-1bec5485-9479-4e4f-70ce-66372039849d (Addr: tcp/127.0.0.1:13772) (DC: dc1)" -42:46.228 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1bec5485-9479-4e4f-70ce-66372039849d.dc1 area=wan -42:46.228 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13795 network=tcp -42:46.229 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13796 network=tcp -42:46.229 [INFO] TestAgent: started state syncer -42:46.230 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13773 network=tcp -42:46.290 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:46.290 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13772 [Candidate]" term=2 -42:46.290 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:46.290 [DEBUG] TestAgent.server.raft: vote granted: from=1bec5485-9479-4e4f-70ce-66372039849d term=2 tally=1 -42:46.290 [INFO] TestAgent.server.raft: election won: tally=1 -42:46.290 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13772 [Leader]" -42:46.290 [INFO] TestAgent.server: cluster leadership acquired -42:46.290 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13772 -42:46.292 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:46.294 [INFO] TestAgent.server: New leader elected: payload=Node-1bec5485-9479-4e4f-70ce-66372039849d -42:46.295 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:46.295 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:46.295 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:46.295 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:46.295 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:46.296 [INFO] TestAgent.server: federation state anti-entropy synced -42:46.296 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1bec5485-9479-4e4f-70ce-66372039849d -42:46.296 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1bec5485-9479-4e4f-70ce-66372039849d -42:46.454 [INFO] TestAgent: Requesting shutdown -42:46.454 [INFO] TestAgent.server: shutting down server -42:46.454 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:46.454 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:46.454 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:46.454 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:46.454 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:46.455 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:46.455 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:46.455 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:46.455 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:46.455 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:46.455 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:46.455 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:46.455Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:46.455Z [INFO] agent.router.manager: shutting down -42:46.455 [INFO] TestAgent: consul server down -42:46.455 [INFO] TestAgent: shutdown complete -42:46.455 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13795 network=tcp -42:46.456 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13795 network=udp -42:46.456 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13796 network=tcp -42:46.956 [INFO] TestAgent: Waiting for endpoints to shut down -42:46.956 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_AddCheck_GRPC (0.79s) +2021-08-13T15:34:05.501Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:05.517Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:05.520 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:460ba7d8-b412-eccb-0f7a-9332bc3c1157 Address:127.0.0.1:24899}]" +34:05.520 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24899 [Follower]" leader= +34:05.521 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-460ba7d8-b412-eccb-0f7a-9332bc3c1157.secondary 127.0.0.1 +34:05.522 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-460ba7d8-b412-eccb-0f7a-9332bc3c1157 127.0.0.1 +2021-08-13T15:34:05.522Z [INFO] agent.router: Initializing LAN area manager +34:05.522 [INFO] TestAgent.server: Adding LAN server: server="Node-460ba7d8-b412-eccb-0f7a-9332bc3c1157 (Addr: tcp/127.0.0.1:24899) (DC: secondary)" +34:05.522 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-460ba7d8-b412-eccb-0f7a-9332bc3c1157.secondary area=wan +34:05.522 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24887 network=udp +34:05.522 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24887 network=tcp +34:05.523 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24888 network=tcp +34:05.523 [INFO] TestAgent: started state syncer +34:05.523 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24900 network=tcp +34:05.589 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:05.589 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24899 [Candidate]" term=2 +34:05.589 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:05.589 [DEBUG] TestAgent.server.raft: vote granted: from=460ba7d8-b412-eccb-0f7a-9332bc3c1157 term=2 tally=1 +34:05.589 [INFO] TestAgent.server.raft: election won: tally=1 +34:05.589 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24899 [Leader]" +34:05.589 [INFO] TestAgent.server: cluster leadership acquired +34:05.589 [INFO] TestAgent.server: New leader elected: payload=Node-460ba7d8-b412-eccb-0f7a-9332bc3c1157 +34:05.589 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24899 +34:05.591 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:05.593 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:05.593 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:05.593 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:05.593 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:05.593 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:05.593 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-460ba7d8-b412-eccb-0f7a-9332bc3c1157 +34:05.593 [INFO] TestAgent.server: member joined, marking health alive: member=Node-460ba7d8-b412-eccb-0f7a-9332bc3c1157 +34:05.594 [INFO] TestAgent.server: federation state anti-entropy synced +34:05.662 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:24891] +34:05.663 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:24891 +34:05.664 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-579888cf-02f3-ecdf-6c0c-06f53031865e.primary 127.0.0.1 +34:05.664 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 +34:05.664 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-579888cf-02f3-ecdf-6c0c-06f53031865e.primary area=wan +34:05.668 [INFO] TestAgent: Requesting shutdown +34:05.668 [INFO] TestAgent.server: shutting down server +34:05.668 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:05.668 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:05.668 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:05.668 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:05.668 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:05.668 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:05.668 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:05.668 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:05.668 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:05.668 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:05.668 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:05.668 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:05.668Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:05.668Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:05.668Z [INFO] agent.router.manager: shutting down +34:05.668 [INFO] TestAgent: consul server down +34:05.668 [INFO] TestAgent: shutdown complete +34:05.668 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24887 network=tcp +34:05.668 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24887 network=udp +34:05.669 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24888 network=tcp +34:06.169 [INFO] TestAgent: Waiting for endpoints to shut down +34:06.169 [INFO] TestAgent: Endpoints down +2021-08-13T15:34:05.123Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:05.141Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:05.144 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:579888cf-02f3-ecdf-6c0c-06f53031865e Address:127.0.0.1:24892}]" +34:05.144 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24892 [Follower]" leader= +34:05.145 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-579888cf-02f3-ecdf-6c0c-06f53031865e.primary 127.0.0.1 +34:05.146 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-579888cf-02f3-ecdf-6c0c-06f53031865e 127.0.0.1 +2021-08-13T15:34:05.146Z [INFO] agent.router: Initializing LAN area manager +34:05.146 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-579888cf-02f3-ecdf-6c0c-06f53031865e.primary area=wan +34:05.146 [INFO] TestAgent.server: Adding LAN server: server="Node-579888cf-02f3-ecdf-6c0c-06f53031865e (Addr: tcp/127.0.0.1:24892) (DC: primary)" +34:05.146 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24873 network=tcp +34:05.146 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24873 network=udp +34:05.147 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24874 network=tcp +34:05.147 [INFO] TestAgent: started state syncer +34:05.147 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24893 network=tcp +34:05.184 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:05.184 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24892 [Candidate]" term=2 +34:05.184 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:05.184 [DEBUG] TestAgent.server.raft: vote granted: from=579888cf-02f3-ecdf-6c0c-06f53031865e term=2 tally=1 +34:05.184 [INFO] TestAgent.server.raft: election won: tally=1 +34:05.184 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24892 [Leader]" +34:05.184 [INFO] TestAgent.server: cluster leadership acquired +34:05.184 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24892 +34:05.185 [INFO] TestAgent.server: New leader elected: payload=Node-579888cf-02f3-ecdf-6c0c-06f53031865e +34:05.186 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:05.188 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:05.188 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:05.188 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:05.188 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:05.188 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:05.188 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-579888cf-02f3-ecdf-6c0c-06f53031865e +34:05.188 [INFO] TestAgent.server: member joined, marking health alive: member=Node-579888cf-02f3-ecdf-6c0c-06f53031865e +34:05.189 [INFO] TestAgent.server: federation state anti-entropy synced +34:05.214 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:05.215 [INFO] TestAgent: Synced node info +34:05.663 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:34520 +34:05.664 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-460ba7d8-b412-eccb-0f7a-9332bc3c1157.secondary 127.0.0.1 +34:05.664 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-460ba7d8-b412-eccb-0f7a-9332bc3c1157.secondary area=wan +34:06.169 [INFO] TestAgent: Requesting shutdown +34:06.169 [INFO] TestAgent.server: shutting down server +34:06.169 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:06.169 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:06.169 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:06.169 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:06.169 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:06.169 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:06.169 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:06.169 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:06.169 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:06.169 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:06.169 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:06.169Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:06.169Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:06.169Z [INFO] agent.router.manager: shutting down +34:06.169 [INFO] TestAgent: consul server down +34:06.169 [INFO] TestAgent: shutdown complete +34:06.169 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24873 network=tcp +34:06.170 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24873 network=udp +34:06.170 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24874 network=tcp +34:06.670 [INFO] TestAgent: Waiting for endpoints to shut down +34:06.670 [INFO] TestAgent: Endpoints down +--- PASS: TestStatusLeaderSecondary (1.57s) +=== CONT TestSessionGet +=== RUN TestSessionGet/#00 +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:05.956Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:05.956Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:05.973Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:05.973Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:05.975 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5ff3860f-9637-81d3-a095-cbbd4479a79b Address:127.0.0.1:24920}]" +34:05.975 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24920 [Follower]" leader= +34:05.976 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5ff3860f-9637-81d3-a095-cbbd4479a79b.dc1 127.0.0.1 +34:05.977 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5ff3860f-9637-81d3-a095-cbbd4479a79b 127.0.0.1 +2021-08-13T15:34:05.977Z [INFO] agent.router: Initializing LAN area manager +34:05.977 [INFO] TestAgent.server: Adding LAN server: server="Node-5ff3860f-9637-81d3-a095-cbbd4479a79b (Addr: tcp/127.0.0.1:24920) (DC: dc1)" +34:05.977 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5ff3860f-9637-81d3-a095-cbbd4479a79b.dc1 area=wan +34:05.977 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24915 network=udp +34:05.977 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24915 network=tcp +34:05.978 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24916 network=tcp +34:05.978 [INFO] TestAgent: started state syncer +34:05.978 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24921 network=tcp +34:06.027 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:06.044 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:06.044 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24920 [Candidate]" term=2 +34:06.044 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:06.044 [DEBUG] TestAgent.server.raft: vote granted: from=5ff3860f-9637-81d3-a095-cbbd4479a79b term=2 tally=1 +34:06.044 [INFO] TestAgent.server.raft: election won: tally=1 +34:06.044 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24920 [Leader]" +34:06.044 [INFO] TestAgent.server: cluster leadership acquired +34:06.045 [INFO] TestAgent.server: New leader elected: payload=Node-5ff3860f-9637-81d3-a095-cbbd4479a79b +34:06.045 [INFO] TestAgent.server: initializing acls +34:06.045 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:06.045 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:06.046 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:06.046 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:06.046 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:06.046 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:06.047 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-5ff3860f-9637-81d3-a095-cbbd4479a79b +34:06.047 [INFO] TestAgent.server: Updating LAN server: server="Node-5ff3860f-9637-81d3-a095-cbbd4479a79b (Addr: tcp/127.0.0.1:24920) (DC: dc1)" +34:06.047 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-5ff3860f-9637-81d3-a095-cbbd4479a79b.dc1 +34:06.047 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-5ff3860f-9637-81d3-a095-cbbd4479a79b.dc1 area=wan +34:06.048 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:06.050 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:06.050 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:06.050 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:06.050 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:06.050 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:06.050 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5ff3860f-9637-81d3-a095-cbbd4479a79b +34:06.051 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5ff3860f-9637-81d3-a095-cbbd4479a79b +34:06.051 [INFO] TestAgent.server: federation state anti-entropy synced +34:06.052 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5ff3860f-9637-81d3-a095-cbbd4479a79b +34:06.221 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:06.221 [INFO] TestAgent: Synced node info +34:06.337 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-5ff3860f-9637-81d3-a095-cbbd4479a79b +34:06.337 [INFO] TestAgent: Requesting shutdown +34:06.337 [INFO] TestAgent.server: shutting down server +34:06.337 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:06.337 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:06.337 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:06.337 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:06.337 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:06.337 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:06.337 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:06.337 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:06.337 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:06.337 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:06.337 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:06.337 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:06.337 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:06.337 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:06.337 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:06.338Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:06.338Z [INFO] agent.router.manager: shutting down +34:06.338 [INFO] TestAgent: consul server down +34:06.338 [INFO] TestAgent: shutdown complete +34:06.338 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24915 network=tcp +34:06.338 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24915 network=udp +34:06.338 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24916 network=tcp +34:06.839 [INFO] TestAgent: Waiting for endpoints to shut down +34:06.839 [INFO] TestAgent: Endpoints down +=== RUN TestSnapshot_Options/GET#01 +2021-08-13T15:34:06.193Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:06.209Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:06.211 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:04f7be7a-636b-0491-d889-df12880507b0 Address:127.0.0.1:24948}]" +34:06.211 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24948 [Follower]" leader= +34:06.212 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-04f7be7a-636b-0491-d889-df12880507b0.dc1 127.0.0.1 +34:06.213 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-04f7be7a-636b-0491-d889-df12880507b0 127.0.0.1 +2021-08-13T15:34:06.213Z [INFO] agent.router: Initializing LAN area manager +34:06.213 [INFO] TestAgent.server: Adding LAN server: server="Node-04f7be7a-636b-0491-d889-df12880507b0 (Addr: tcp/127.0.0.1:24948) (DC: dc1)" +34:06.213 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-04f7be7a-636b-0491-d889-df12880507b0.dc1 area=wan +34:06.213 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24929 network=tcp +34:06.213 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24929 network=udp +34:06.214 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24930 network=tcp +34:06.214 [INFO] TestAgent: started state syncer +34:06.214 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24949 network=tcp +34:06.267 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:06.267 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24948 [Candidate]" term=2 +34:06.268 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:06.268 [DEBUG] TestAgent.server.raft: vote granted: from=04f7be7a-636b-0491-d889-df12880507b0 term=2 tally=1 +34:06.268 [INFO] TestAgent.server.raft: election won: tally=1 +34:06.268 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24948 [Leader]" +34:06.268 [INFO] TestAgent.server: cluster leadership acquired +34:06.268 [INFO] TestAgent.server: New leader elected: payload=Node-04f7be7a-636b-0491-d889-df12880507b0 +34:06.268 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24948 +34:06.269 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:06.272 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:06.272 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:06.272 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:06.272 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:06.272 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:06.272 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-04f7be7a-636b-0491-d889-df12880507b0 +34:06.272 [INFO] TestAgent.server: member joined, marking health alive: member=Node-04f7be7a-636b-0491-d889-df12880507b0 +34:06.272 [INFO] TestAgent.server: federation state anti-entropy synced +34:06.288 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:06.288 [INFO] TestAgent: Synced node info +34:06.385 [INFO] TestAgent: Requesting shutdown +34:06.385 [INFO] TestAgent.server: shutting down server +34:06.385 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:06.385 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:06.385 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:06.385 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:06.385 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:06.385 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:06.385 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:06.385 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:06.385 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:06.385 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:06.386 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:06.386Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:06.386Z [INFO] agent.router.manager: shutting down +34:06.386 [INFO] TestAgent: consul server down +34:06.386 [INFO] TestAgent: shutdown complete +34:06.386 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24929 network=tcp +34:06.386 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24929 network=udp +34:06.386 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24930 network=tcp +34:06.886 [INFO] TestAgent: Waiting for endpoints to shut down +34:06.886 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_AddCheck_ExecDisable (0.71s) === CONT TestAgent_AddCheck_MissingService -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._udp_(qType:=1)_compressed=false#01 -2022-09-15T21:42:46.097Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:46.124Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:46.130 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a6c16ed4-2c8b-1266-0d87-a61e262e3cdf Address:127.0.0.1:13758}]" -42:46.130 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13758 [Follower]" leader= -42:46.131 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a6c16ed4-2c8b-1266-0d87-a61e262e3cdf.dc1 127.0.0.1 -42:46.132 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a6c16ed4-2c8b-1266-0d87-a61e262e3cdf 127.0.0.1 -2022-09-15T21:42:46.132Z [INFO] agent.router: Initializing LAN area manager -42:46.133 [INFO] TestAgent.server: Adding LAN server: server="Node-a6c16ed4-2c8b-1266-0d87-a61e262e3cdf (Addr: tcp/127.0.0.1:13758) (DC: dc1)" -42:46.133 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a6c16ed4-2c8b-1266-0d87-a61e262e3cdf.dc1 area=wan -42:46.133 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13767 network=udp -42:46.133 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13767 network=tcp -42:46.134 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13768 network=tcp -42:46.135 [INFO] TestAgent: started state syncer -42:46.135 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13759 network=tcp -42:46.202 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:46.202 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13758 [Candidate]" term=2 -42:46.202 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:46.202 [DEBUG] TestAgent.server.raft: vote granted: from=a6c16ed4-2c8b-1266-0d87-a61e262e3cdf term=2 tally=1 -42:46.202 [INFO] TestAgent.server.raft: election won: tally=1 -42:46.202 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13758 [Leader]" -42:46.203 [INFO] TestAgent.server: cluster leadership acquired -42:46.209 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13758 -42:46.213 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:46.215 [INFO] TestAgent.server: New leader elected: payload=Node-a6c16ed4-2c8b-1266-0d87-a61e262e3cdf -42:46.217 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:46.217 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:46.217 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:46.217 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:46.217 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:46.217 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a6c16ed4-2c8b-1266-0d87-a61e262e3cdf -42:46.217 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a6c16ed4-2c8b-1266-0d87-a61e262e3cdf -42:46.230 [INFO] TestAgent.server: federation state anti-entropy synced -42:46.363 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:46.365 [INFO] TestAgent: Synced node info -42:46.482 [INFO] TestAgent: Requesting shutdown -42:46.482 [INFO] TestAgent.server: shutting down server -42:46.482 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:46.482 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:46.482 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:46.482 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:46.482 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:46.483 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:46.483 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:46.483 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:46.483 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:46.483 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:46.483 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:46.483Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:46.483Z [INFO] agent.router.manager: shutting down -42:46.484 [INFO] TestAgent: consul server down -42:46.484 [INFO] TestAgent: shutdown complete -42:46.484 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13767 network=tcp -42:46.484 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13767 network=udp -42:46.484 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13768 network=tcp -42:46.990 [INFO] TestAgent: Waiting for endpoints to shut down -42:46.990 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_AddCheck_Alias_setToken (1.01s) +=== RUN TestAgent_JoinWAN_viaMeshGateway/dc1_to_dc2 +=== RUN TestAgent_JoinWAN_viaMeshGateway/dc1_to_dc3 +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestAgent_JoinWAN_viaMeshGateway/dc2_to_dc1 +=== RUN TestAgent_JoinWAN_viaMeshGateway/dc2_to_dc3 +=== RUN TestAgent_JoinWAN_viaMeshGateway/dc3_to_dc1 +=== RUN TestAgent_JoinWAN_viaMeshGateway/dc3_to_dc2 +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:05.956Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:05.973Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:05.975 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e1b49189-eacf-8e98-35b0-3ad476157587 Address:127.0.0.1:24927}]" +34:05.975 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24927 [Follower]" leader= +34:05.976 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e1b49189-eacf-8e98-35b0-3ad476157587.dc1 127.0.0.1 +34:05.977 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e1b49189-eacf-8e98-35b0-3ad476157587 127.0.0.1 +2021-08-13T15:34:05.977Z [INFO] agent.router: Initializing LAN area manager +34:05.977 [INFO] TestAgent.server: Adding LAN server: server="Node-e1b49189-eacf-8e98-35b0-3ad476157587 (Addr: tcp/127.0.0.1:24927) (DC: dc1)" +34:05.977 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e1b49189-eacf-8e98-35b0-3ad476157587.dc1 area=wan +34:05.977 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24922 network=udp +34:05.977 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24922 network=tcp +34:05.978 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24923 network=tcp +34:05.978 [INFO] TestAgent: started state syncer +34:05.978 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24928 network=tcp +34:06.023 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:06.023 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24927 [Candidate]" term=2 +34:06.023 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:06.023 [DEBUG] TestAgent.server.raft: vote granted: from=e1b49189-eacf-8e98-35b0-3ad476157587 term=2 tally=1 +34:06.023 [INFO] TestAgent.server.raft: election won: tally=1 +34:06.023 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24927 [Leader]" +34:06.023 [INFO] TestAgent.server: cluster leadership acquired +34:06.024 [INFO] TestAgent.server: New leader elected: payload=Node-e1b49189-eacf-8e98-35b0-3ad476157587 +34:06.024 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24927 +34:06.025 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:06.027 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:06.027 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:06.027 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:06.027 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:06.027 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:06.027 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e1b49189-eacf-8e98-35b0-3ad476157587 +34:06.028 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e1b49189-eacf-8e98-35b0-3ad476157587 +34:06.028 [INFO] TestAgent.server: federation state anti-entropy synced +34:06.178 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:06.178 [INFO] TestAgent: Synced node info +34:06.178 [DEBUG] TestAgent: Node info in sync +34:06.591 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:06.591 [DEBUG] TestAgent: Node info in sync +34:06.807 [INFO] TestAgent: Requesting shutdown +34:06.807 [INFO] TestAgent.server: shutting down server +34:06.807 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:06.807 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:06.807 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:06.807 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:06.807 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:06.808 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:06.808 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:06.808 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:06.808 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:06.808 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:06.808 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:06.808Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:06.808Z [INFO] agent.router.manager: shutting down +34:06.808 [INFO] TestAgent: consul server down +34:06.808 [INFO] TestAgent: shutdown complete +34:06.808 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24922 network=tcp +34:06.808 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24922 network=udp +34:06.808 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24923 network=tcp +34:07.309 [INFO] TestAgent: Waiting for endpoints to shut down +34:07.309 [INFO] TestAgent: Endpoints down +=== RUN TestTxnEndpoint_Bad_Size_Net/allowed +2021-08-13T15:34:06.704Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:06.721Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:06.725 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b7829b31-19b4-ad4f-ee87-a39e3ee4c063 Address:127.0.0.1:24969}]" +34:06.725 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24969 [Follower]" leader= +34:06.726 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b7829b31-19b4-ad4f-ee87-a39e3ee4c063.dc1 127.0.0.1 +34:06.727 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b7829b31-19b4-ad4f-ee87-a39e3ee4c063 127.0.0.1 +2021-08-13T15:34:06.727Z [INFO] agent.router: Initializing LAN area manager +34:06.727 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24943 network=udp +34:06.728 [INFO] TestAgent.server: Adding LAN server: server="Node-b7829b31-19b4-ad4f-ee87-a39e3ee4c063 (Addr: tcp/127.0.0.1:24969) (DC: dc1)" +34:06.728 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b7829b31-19b4-ad4f-ee87-a39e3ee4c063.dc1 area=wan +34:06.728 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24943 network=tcp +34:06.729 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24944 network=tcp +34:06.729 [INFO] TestAgent: started state syncer +34:06.729 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24970 network=tcp +34:06.787 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:06.787 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24969 [Candidate]" term=2 +34:06.787 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:06.787 [DEBUG] TestAgent.server.raft: vote granted: from=b7829b31-19b4-ad4f-ee87-a39e3ee4c063 term=2 tally=1 +34:06.787 [INFO] TestAgent.server.raft: election won: tally=1 +34:06.787 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24969 [Leader]" +34:06.787 [INFO] TestAgent.server: cluster leadership acquired +34:06.787 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24969 +34:06.788 [INFO] TestAgent.server: New leader elected: payload=Node-b7829b31-19b4-ad4f-ee87-a39e3ee4c063 +34:06.789 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:06.791 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:06.791 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:06.791 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:06.791 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:06.791 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:06.791 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b7829b31-19b4-ad4f-ee87-a39e3ee4c063 +34:06.791 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b7829b31-19b4-ad4f-ee87-a39e3ee4c063 +34:06.792 [INFO] TestAgent.server: federation state anti-entropy synced +34:06.812 [INFO] TestAgent: Requesting shutdown +34:06.812 [INFO] TestAgent.server: shutting down server +34:06.812 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:06.812 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:06.812 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:06.812 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:06.812 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:06.812 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:06.812 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:06.812 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:06.812 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:06.812 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:06.812 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:06.812 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:06.812Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:06.812Z [INFO] agent.router.manager: shutting down +34:06.812 [INFO] TestAgent: consul server down +34:06.812 [INFO] TestAgent: shutdown complete +34:06.812 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24943 network=tcp +34:06.812 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24943 network=udp +34:06.812 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24944 network=tcp +34:07.312 [INFO] TestAgent: Waiting for endpoints to shut down +34:07.312 [INFO] TestAgent: Endpoints down +=== RUN TestSessionGet/#01 +2021-08-13T15:34:06.467Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:06.484Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:06.487 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3a613caf-abb6-45a6-ba4b-2d019613af17 Address:127.0.0.1:24934}]" +34:06.487 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24934 [Follower]" leader= +34:06.488 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3a613caf-abb6-45a6-ba4b-2d019613af17.dc1 127.0.0.1 +34:06.489 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3a613caf-abb6-45a6-ba4b-2d019613af17 127.0.0.1 +2021-08-13T15:34:06.489Z [INFO] agent.router: Initializing LAN area manager +34:06.489 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24950 network=udp +34:06.490 [INFO] TestAgent.server: Adding LAN server: server="Node-3a613caf-abb6-45a6-ba4b-2d019613af17 (Addr: tcp/127.0.0.1:24934) (DC: dc1)" +34:06.490 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3a613caf-abb6-45a6-ba4b-2d019613af17.dc1 area=wan +34:06.490 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24950 network=tcp +34:06.491 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24951 network=tcp +34:06.491 [INFO] TestAgent: started state syncer +34:06.491 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24935 network=tcp +34:06.528 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:06.528 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24934 [Candidate]" term=2 +34:06.528 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:06.528 [DEBUG] TestAgent.server.raft: vote granted: from=3a613caf-abb6-45a6-ba4b-2d019613af17 term=2 tally=1 +34:06.528 [INFO] TestAgent.server.raft: election won: tally=1 +34:06.528 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24934 [Leader]" +34:06.528 [INFO] TestAgent.server: cluster leadership acquired +34:06.529 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24934 +34:06.529 [INFO] TestAgent.server: New leader elected: payload=Node-3a613caf-abb6-45a6-ba4b-2d019613af17 +34:06.530 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:06.533 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:06.533 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:06.533 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:06.533 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:06.533 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:06.533 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3a613caf-abb6-45a6-ba4b-2d019613af17 +34:06.533 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3a613caf-abb6-45a6-ba4b-2d019613af17 +34:06.534 [INFO] TestAgent.server: federation state anti-entropy synced +34:06.838 [INFO] TestAgent: Requesting shutdown +34:06.838 [INFO] TestAgent.server: shutting down server +34:06.838 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:06.838 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:06.838 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:06.838 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:06.838 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:06.838 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:06.838 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:06.838 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:06.838 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:06.838 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:06.838 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:06.838 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:06.839Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:06.839Z [INFO] agent.router.manager: shutting down +34:06.839 [INFO] TestAgent: consul server down +34:06.839 [INFO] TestAgent: shutdown complete +34:06.839 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24950 network=tcp +34:06.839 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24950 network=udp +34:06.839 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24951 network=tcp +34:07.340 [INFO] TestAgent: Waiting for endpoints to shut down +34:07.340 [INFO] TestAgent: Endpoints down +--- PASS: TestSessionDeleteDestroy (0.89s) === CONT TestAgent_AddCheck_MinInterval -2022-09-15T21:42:46.053Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:46.119Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:46.122 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1cd9e6dc-a272-bd02-4cc3-0286a19e2716 Address:127.0.0.1:13793}]" -42:46.124 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1cd9e6dc-a272-bd02-4cc3-0286a19e2716.dc1 127.0.0.1 -42:46.124 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1cd9e6dc-a272-bd02-4cc3-0286a19e2716 127.0.0.1 -2022-09-15T21:42:46.124Z [INFO] agent.router: Initializing LAN area manager -42:46.125 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13793 [Follower]" leader= -42:46.125 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13760 network=tcp -42:46.125 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1cd9e6dc-a272-bd02-4cc3-0286a19e2716.dc1 area=wan -42:46.125 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13760 network=udp -42:46.126 [INFO] TestAgent.server: Adding LAN server: server="Node-1cd9e6dc-a272-bd02-4cc3-0286a19e2716 (Addr: tcp/127.0.0.1:13793) (DC: dc1)" -42:46.127 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13761 network=tcp -42:46.127 [INFO] TestAgent: started state syncer -42:46.127 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13794 network=tcp -42:46.184 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:46.184 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13793 [Candidate]" term=2 -42:46.184 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:46.184 [DEBUG] TestAgent.server.raft: vote granted: from=1cd9e6dc-a272-bd02-4cc3-0286a19e2716 term=2 tally=1 -42:46.184 [INFO] TestAgent.server.raft: election won: tally=1 -42:46.184 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13793 [Leader]" -42:46.185 [INFO] TestAgent.server: cluster leadership acquired -42:46.185 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13793 -42:46.186 [INFO] TestAgent.server: New leader elected: payload=Node-1cd9e6dc-a272-bd02-4cc3-0286a19e2716 -42:46.186 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:46.189 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:46.189 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:46.189 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:46.189 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:46.189 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:46.189 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1cd9e6dc-a272-bd02-4cc3-0286a19e2716 -42:46.189 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1cd9e6dc-a272-bd02-4cc3-0286a19e2716 -42:46.193 [INFO] TestAgent.server: federation state anti-entropy synced -42:46.406 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:46.408 [INFO] TestAgent: Synced node info -42:46.408 [DEBUG] TestAgent: Node info in sync -42:46.501 [INFO] TestAgent: Requesting shutdown -42:46.502 [INFO] TestAgent.server: shutting down server -42:46.502 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:46.502 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:46.502 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:46.502 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:46.502 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:46.503 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:46.504 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:46.504 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:46.504 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:46.504 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:46.504 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:46.505Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:46.505Z [INFO] agent.router.manager: shutting down -42:46.505 [INFO] TestAgent: consul server down -42:46.505 [INFO] TestAgent: shutdown complete -42:46.505 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13760 network=tcp -42:46.505 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13760 network=udp -42:46.505 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13761 network=tcp -42:47.006 [INFO] TestAgent: Waiting for endpoints to shut down -42:47.006 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_AddCheck_Alias (1.02s) +2021-08-13T15:34:06.673Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:06.697Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:06.711 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d53a3c8f-d4f7-608a-1041-317ecb0e7b01 Address:127.0.0.1:24941}]" +34:06.712 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24941 [Follower]" leader= +34:06.713 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d53a3c8f-d4f7-608a-1041-317ecb0e7b01.dc1 127.0.0.1 +34:06.714 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d53a3c8f-d4f7-608a-1041-317ecb0e7b01 127.0.0.1 +2021-08-13T15:34:06.714Z [INFO] agent.router: Initializing LAN area manager +34:06.714 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24957 network=udp +34:06.715 [INFO] TestAgent.server: Adding LAN server: server="Node-d53a3c8f-d4f7-608a-1041-317ecb0e7b01 (Addr: tcp/127.0.0.1:24941) (DC: dc1)" +34:06.715 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d53a3c8f-d4f7-608a-1041-317ecb0e7b01.dc1 area=wan +34:06.715 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24957 network=tcp +34:06.716 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24958 network=tcp +34:06.716 [INFO] TestAgent: started state syncer +34:06.716 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24942 network=tcp +34:06.775 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:06.775 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24941 [Candidate]" term=2 +34:06.775 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:06.775 [DEBUG] TestAgent.server.raft: vote granted: from=d53a3c8f-d4f7-608a-1041-317ecb0e7b01 term=2 tally=1 +34:06.775 [INFO] TestAgent.server.raft: election won: tally=1 +34:06.776 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24941 [Leader]" +34:06.776 [INFO] TestAgent.server: cluster leadership acquired +34:06.776 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24941 +34:06.776 [INFO] TestAgent.server: New leader elected: payload=Node-d53a3c8f-d4f7-608a-1041-317ecb0e7b01 +34:06.780 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:06.784 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:06.784 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:06.784 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:06.784 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:06.784 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:06.784 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d53a3c8f-d4f7-608a-1041-317ecb0e7b01 +34:06.784 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d53a3c8f-d4f7-608a-1041-317ecb0e7b01 +34:06.785 [INFO] TestAgent.server: federation state anti-entropy synced +34:06.852 [INFO] TestAgent: Requesting shutdown +34:06.852 [INFO] TestAgent.server: shutting down server +34:06.852 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:06.852 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:06.852 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:06.852 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:06.852 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:06.852 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:06.852 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:06.852 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:06.852 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:06.852 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:06.852 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:06.852 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:06.853Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:06.853Z [INFO] agent.router.manager: shutting down +34:06.853 [INFO] TestAgent: consul server down +34:06.853 [INFO] TestAgent: shutdown complete +34:06.853 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24957 network=tcp +34:06.853 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24957 network=udp +34:06.853 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24958 network=tcp +34:07.354 [INFO] TestAgent: Waiting for endpoints to shut down +34:07.354 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_AddCheck_RestoreState (0.72s) +=== CONT TestSessionCustomTTL +2021-08-13T15:34:06.501Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:06.517Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:06.520 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:acd96dd4-790f-5d2b-0297-5eed0a7c8916 Address:127.0.0.1:24955}]" +34:06.520 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24955 [Follower]" leader= +34:06.521 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-acd96dd4-790f-5d2b-0297-5eed0a7c8916.dc1 127.0.0.1 +34:06.522 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-acd96dd4-790f-5d2b-0297-5eed0a7c8916 127.0.0.1 +2021-08-13T15:34:06.522Z [INFO] agent.router: Initializing LAN area manager +34:06.522 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-acd96dd4-790f-5d2b-0297-5eed0a7c8916.dc1 area=wan +34:06.522 [INFO] TestAgent.server: Adding LAN server: server="Node-acd96dd4-790f-5d2b-0297-5eed0a7c8916 (Addr: tcp/127.0.0.1:24955) (DC: dc1)" +34:06.522 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24936 network=udp +34:06.522 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24936 network=tcp +34:06.523 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24937 network=tcp +34:06.523 [INFO] TestAgent: started state syncer +34:06.523 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24956 network=tcp +34:06.560 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:06.560 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24955 [Candidate]" term=2 +34:06.560 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:06.560 [DEBUG] TestAgent.server.raft: vote granted: from=acd96dd4-790f-5d2b-0297-5eed0a7c8916 term=2 tally=1 +34:06.560 [INFO] TestAgent.server.raft: election won: tally=1 +34:06.560 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24955 [Leader]" +34:06.560 [INFO] TestAgent.server: cluster leadership acquired +34:06.561 [INFO] TestAgent.server: New leader elected: payload=Node-acd96dd4-790f-5d2b-0297-5eed0a7c8916 +34:06.561 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24955 +34:06.562 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:06.565 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:06.565 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:06.565 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:06.565 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:06.565 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:06.565 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-acd96dd4-790f-5d2b-0297-5eed0a7c8916 +34:06.565 [INFO] TestAgent.server: member joined, marking health alive: member=Node-acd96dd4-790f-5d2b-0297-5eed0a7c8916 +34:06.566 [INFO] TestAgent.server: federation state anti-entropy synced +34:06.827 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:06.828 [INFO] TestAgent: Synced node info +34:06.872 [WARN] TestAgent.server.rpc: Attempting to apply large raft entry: size_in_bytes=1573029 +34:06.878 [INFO] TestAgent: Requesting shutdown +34:06.878 [INFO] TestAgent.server: shutting down server +34:06.878 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:06.878 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:06.878 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:06.878 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:06.878 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:06.878 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:06.878 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:06.878 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +34:06.878 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:06.878 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:06.878 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2021-08-13T15:34:06.879Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:06.879Z [INFO] agent.router.manager: shutting down +34:06.879 [INFO] TestAgent: consul server down +34:06.879 [INFO] TestAgent: shutdown complete +34:06.879 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24936 network=tcp +34:06.879 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24936 network=udp +34:06.879 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24937 network=tcp +34:07.379 [INFO] TestAgent: Waiting for endpoints to shut down +34:07.379 [INFO] TestAgent: Endpoints down +--- PASS: TestTxnEndpoint_Bad_Size_Item (3.67s) + --- PASS: TestTxnEndpoint_Bad_Size_Item/exceeds_default_limits (0.78s) + --- PASS: TestTxnEndpoint_Bad_Size_Item/exceeds_configured_max_txn_len (1.03s) + --- PASS: TestTxnEndpoint_Bad_Size_Item/exceeds_default_max_kv_value_size (0.96s) + --- PASS: TestTxnEndpoint_Bad_Size_Item/allowed (0.91s) === CONT TestAgent_AddCheck_StartPassing -[WARN] freeport: 4 out of 25 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:46.322Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:46.341Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:46.345 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d46d2d19-b0bd-c9d8-0724-f331519475d7 Address:127.0.0.1:13786}]" -42:46.345 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13786 [Follower]" leader= -42:46.347 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d46d2d19-b0bd-c9d8-0724-f331519475d7.dc1 127.0.0.1 -42:46.351 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d46d2d19-b0bd-c9d8-0724-f331519475d7 127.0.0.1 -2022-09-15T21:42:46.352Z [INFO] agent.router: Initializing LAN area manager -42:46.357 [INFO] TestAgent.server: Adding LAN server: server="Node-d46d2d19-b0bd-c9d8-0724-f331519475d7 (Addr: tcp/127.0.0.1:13786) (DC: dc1)" -42:46.360 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13774 network=tcp -42:46.361 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13774 network=udp -42:46.368 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13775 network=tcp -42:46.368 [INFO] TestAgent: started state syncer -42:46.368 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13787 network=tcp -42:46.370 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d46d2d19-b0bd-c9d8-0724-f331519475d7.dc1 area=wan -42:46.406 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:46.410 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13786 [Candidate]" term=2 -42:46.410 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:46.410 [DEBUG] TestAgent.server.raft: vote granted: from=d46d2d19-b0bd-c9d8-0724-f331519475d7 term=2 tally=1 -42:46.410 [INFO] TestAgent.server.raft: election won: tally=1 -42:46.410 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13786 [Leader]" -42:46.411 [INFO] TestAgent.server: cluster leadership acquired -42:46.411 [INFO] TestAgent.server: New leader elected: payload=Node-d46d2d19-b0bd-c9d8-0724-f331519475d7 -42:46.411 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13786 -42:46.414 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:46.425 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:46.425 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:46.425 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:46.425 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:46.425 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:46.425 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d46d2d19-b0bd-c9d8-0724-f331519475d7 -42:46.425 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d46d2d19-b0bd-c9d8-0724-f331519475d7 -42:46.430 [INFO] TestAgent.server: federation state anti-entropy synced -42:46.590 [INFO] TestAgent: Requesting shutdown -42:46.590 [INFO] TestAgent.server: shutting down server -42:46.590 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:46.590 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:46.590 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:46.590 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:46.590 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:46.590 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:46.590 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:46.590 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:46.590 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:46.590 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:46.590 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:46.590 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:46.591Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:46.591Z [INFO] agent.router.manager: shutting down -42:46.591 [INFO] TestAgent: consul server down -42:46.591 [INFO] TestAgent: shutdown complete -42:46.591 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13774 network=tcp -42:46.591 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13774 network=udp -42:46.591 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13775 network=tcp -42:47.091 [INFO] TestAgent: Waiting for endpoints to shut down -42:47.091 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_AddCheck_ExecRemoteDisable (0.79s) +2021-08-13T15:34:06.856Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:06.856Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:06.873Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:06.873Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:06.877 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2046a81a-a62e-e753-85e1-4d61069c12c2 Address:127.0.0.1:24962}]" +34:06.877 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24962 [Follower]" leader= +34:06.878 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2046a81a-a62e-e753-85e1-4d61069c12c2.dc1 127.0.0.1 +34:06.879 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2046a81a-a62e-e753-85e1-4d61069c12c2 127.0.0.1 +2021-08-13T15:34:06.879Z [INFO] agent.router: Initializing LAN area manager +34:06.879 [INFO] TestAgent.server: Adding LAN server: server="Node-2046a81a-a62e-e753-85e1-4d61069c12c2 (Addr: tcp/127.0.0.1:24962) (DC: dc1)" +34:06.879 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2046a81a-a62e-e753-85e1-4d61069c12c2.dc1 area=wan +34:06.879 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24971 network=tcp +34:06.879 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24971 network=udp +34:06.880 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24972 network=tcp +34:06.880 [INFO] TestAgent: started state syncer +34:06.880 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24963 network=tcp +34:06.922 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:06.922 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24962 [Candidate]" term=2 +34:06.922 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:06.922 [DEBUG] TestAgent.server.raft: vote granted: from=2046a81a-a62e-e753-85e1-4d61069c12c2 term=2 tally=1 +34:06.922 [INFO] TestAgent.server.raft: election won: tally=1 +34:06.922 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24962 [Leader]" +34:06.922 [INFO] TestAgent.server: cluster leadership acquired +34:06.922 [INFO] TestAgent.server: New leader elected: payload=Node-2046a81a-a62e-e753-85e1-4d61069c12c2 +34:06.922 [INFO] TestAgent.server: initializing acls +34:06.923 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:06.923 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:06.923 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:06.924 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:06.924 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:06.924 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:06.924 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-2046a81a-a62e-e753-85e1-4d61069c12c2 +34:06.924 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-2046a81a-a62e-e753-85e1-4d61069c12c2.dc1 +34:06.924 [INFO] TestAgent.server: Updating LAN server: server="Node-2046a81a-a62e-e753-85e1-4d61069c12c2 (Addr: tcp/127.0.0.1:24962) (DC: dc1)" +34:06.924 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-2046a81a-a62e-e753-85e1-4d61069c12c2.dc1 area=wan +34:06.925 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:06.927 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:06.927 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:06.927 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:06.927 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:06.927 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:06.927 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2046a81a-a62e-e753-85e1-4d61069c12c2 +34:06.927 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2046a81a-a62e-e753-85e1-4d61069c12c2 +34:06.928 [INFO] TestAgent.server: federation state anti-entropy synced +34:06.928 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2046a81a-a62e-e753-85e1-4d61069c12c2 +34:06.988 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-2046a81a-a62e-e753-85e1-4d61069c12c2 +34:06.989 [INFO] TestAgent: Requesting shutdown +34:06.989 [INFO] TestAgent.server: shutting down server +34:06.989 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:06.989 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:06.989 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:06.989 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:06.989 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:06.989 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:06.989 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:06.989 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:06.989 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:06.989 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:06.989 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:06.989 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:06.989 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:06.989 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:06.989 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:06.989 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:06.989Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:06.989Z [INFO] agent.router.manager: shutting down +34:06.989 [INFO] TestAgent: consul server down +34:06.989 [INFO] TestAgent: shutdown complete +34:06.989 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24971 network=tcp +34:06.989 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24971 network=udp +34:06.989 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24972 network=tcp +34:07.489 [INFO] TestAgent: Waiting for endpoints to shut down +34:07.489 [INFO] TestAgent: Endpoints down +=== RUN TestSnapshot_Options/GET#02 +[WARN] freeport: 10 out of 52 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:06.903Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:06.903Z [WARN] agent: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ +2021-08-13T15:34:06.919Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:06.919Z [WARN] agent.auto_config: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ +34:06.921 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:42c90032-edc4-1ec7-d321-320baa3f3c42 Address:127.0.0.1:24976}]" +34:06.922 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24976 [Follower]" leader= +34:06.922 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-42c90032-edc4-1ec7-d321-320baa3f3c42.dc1 127.0.0.1 +34:06.923 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-42c90032-edc4-1ec7-d321-320baa3f3c42 127.0.0.1 +2021-08-13T15:34:06.923Z [INFO] agent.router: Initializing LAN area manager +34:06.923 [INFO] TestAgent.server: Adding LAN server: server="Node-42c90032-edc4-1ec7-d321-320baa3f3c42 (Addr: tcp/127.0.0.1:24976) (DC: dc1)" +34:06.923 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-42c90032-edc4-1ec7-d321-320baa3f3c42.dc1 area=wan +34:06.924 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24964 network=udp +34:06.924 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24964 network=tcp +34:06.924 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24965 network=tcp +34:06.925 [INFO] TestAgent: started state syncer +34:06.925 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24977 network=tcp +34:06.965 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:06.965 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24976 [Candidate]" term=2 +34:06.965 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:06.965 [DEBUG] TestAgent.server.raft: vote granted: from=42c90032-edc4-1ec7-d321-320baa3f3c42 term=2 tally=1 +34:06.965 [INFO] TestAgent.server.raft: election won: tally=1 +34:06.965 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24976 [Leader]" +34:06.965 [INFO] TestAgent.server: cluster leadership acquired +34:06.965 [INFO] TestAgent.server: New leader elected: payload=Node-42c90032-edc4-1ec7-d321-320baa3f3c42 +34:06.966 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24976 +34:06.967 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:06.969 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:06.969 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:06.969 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:06.969 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:06.969 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:06.969 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-42c90032-edc4-1ec7-d321-320baa3f3c42 +34:06.969 [INFO] TestAgent.server: member joined, marking health alive: member=Node-42c90032-edc4-1ec7-d321-320baa3f3c42 +34:06.970 [INFO] TestAgent.server: federation state anti-entropy synced +34:07.033 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:07.033 [INFO] TestAgent: Synced node info +34:07.046 [INFO] TestAgent: Requesting shutdown +34:07.046 [INFO] TestAgent.server: shutting down server +34:07.046 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:07.046 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:07.046 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:07.046 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:07.046 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:07.046 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:07.046 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:07.046 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:07.046 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:07.046 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:07.046 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:07.047Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:07.047Z [INFO] agent.router.manager: shutting down +34:07.047 [INFO] TestAgent: consul server down +34:07.047 [INFO] TestAgent: shutdown complete +34:07.047 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24964 network=tcp +34:07.047 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24964 network=udp +34:07.047 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24965 network=tcp +34:07.548 [INFO] TestAgent: Waiting for endpoints to shut down +34:07.548 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_AddCheck_MissingService (0.66s) === CONT TestAgent_AddCheck -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=33)_compressed=true#01 -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:46.745Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:46.771Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:46.776 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2bd5a84b-bdb9-ee83-ba85-c7b9b3ffce72 Address:127.0.0.1:13779}]" -42:46.776 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13779 [Follower]" leader= -42:46.777 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2bd5a84b-bdb9-ee83-ba85-c7b9b3ffce72.dc1 127.0.0.1 -42:46.779 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2bd5a84b-bdb9-ee83-ba85-c7b9b3ffce72 127.0.0.1 -2022-09-15T21:42:46.779Z [INFO] agent.router: Initializing LAN area manager -42:46.780 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13788 network=udp -42:46.783 [INFO] TestAgent.server: Adding LAN server: server="Node-2bd5a84b-bdb9-ee83-ba85-c7b9b3ffce72 (Addr: tcp/127.0.0.1:13779) (DC: dc1)" -42:46.783 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2bd5a84b-bdb9-ee83-ba85-c7b9b3ffce72.dc1 area=wan -42:46.784 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13788 network=tcp -42:46.786 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13789 network=tcp -42:46.786 [INFO] TestAgent: started state syncer -42:46.786 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13780 network=tcp -42:46.826 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:46.826 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13779 [Candidate]" term=2 -42:46.827 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:46.827 [DEBUG] TestAgent.server.raft: vote granted: from=2bd5a84b-bdb9-ee83-ba85-c7b9b3ffce72 term=2 tally=1 -42:46.827 [INFO] TestAgent.server.raft: election won: tally=1 -42:46.827 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13779 [Leader]" -42:46.827 [INFO] TestAgent.server: cluster leadership acquired -42:46.827 [INFO] TestAgent.server: New leader elected: payload=Node-2bd5a84b-bdb9-ee83-ba85-c7b9b3ffce72 -42:46.827 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13779 -42:46.829 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:46.836 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:46.836 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:46.836 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:46.836 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:46.836 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:46.836 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2bd5a84b-bdb9-ee83-ba85-c7b9b3ffce72 -42:46.836 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2bd5a84b-bdb9-ee83-ba85-c7b9b3ffce72 -42:46.838 [INFO] TestAgent.server: federation state anti-entropy synced -42:46.907 [INFO] TestAgent: Requesting shutdown -42:46.907 [INFO] TestAgent.server: shutting down server -42:46.907 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:46.907 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:46.907 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:46.907 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:46.907 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:46.907 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:46.907 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:46.907 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:46.907 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -42:46.907 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:46.907 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:46.907 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -2022-09-15T21:42:46.907Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:46.907Z [INFO] agent.router.manager: shutting down -42:46.907 [INFO] TestAgent: consul server down -42:46.907 [INFO] TestAgent: shutdown complete -42:46.907 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13788 network=tcp -42:46.907 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13788 network=udp -42:46.907 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13789 network=tcp -42:47.413 [INFO] TestAgent: Waiting for endpoints to shut down -42:47.413 [INFO] TestAgent: Endpoints down +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:07.346Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:07.362Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:07.365 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:603fc488-00b0-1148-8132-92ce47af4226 Address:127.0.0.1:24990}]" +34:07.365 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24990 [Follower]" leader= +34:07.366 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-603fc488-00b0-1148-8132-92ce47af4226.dc1 127.0.0.1 +34:07.367 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-603fc488-00b0-1148-8132-92ce47af4226 127.0.0.1 +2021-08-13T15:34:07.367Z [INFO] agent.router: Initializing LAN area manager +34:07.367 [INFO] TestAgent.server: Adding LAN server: server="Node-603fc488-00b0-1148-8132-92ce47af4226 (Addr: tcp/127.0.0.1:24990) (DC: dc1)" +34:07.367 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-603fc488-00b0-1148-8132-92ce47af4226.dc1 area=wan +34:07.367 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24985 network=udp +34:07.367 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24985 network=tcp +34:07.368 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24986 network=tcp +34:07.368 [INFO] TestAgent: started state syncer +34:07.368 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24991 network=tcp +34:07.401 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:07.401 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24990 [Candidate]" term=2 +34:07.401 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:07.401 [DEBUG] TestAgent.server.raft: vote granted: from=603fc488-00b0-1148-8132-92ce47af4226 term=2 tally=1 +34:07.401 [INFO] TestAgent.server.raft: election won: tally=1 +34:07.401 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24990 [Leader]" +34:07.401 [INFO] TestAgent.server: cluster leadership acquired +34:07.401 [INFO] TestAgent.server: New leader elected: payload=Node-603fc488-00b0-1148-8132-92ce47af4226 +34:07.401 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24990 +34:07.402 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:07.405 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:07.405 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:07.405 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:07.405 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:07.405 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:07.405 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-603fc488-00b0-1148-8132-92ce47af4226 +34:07.405 [INFO] TestAgent.server: member joined, marking health alive: member=Node-603fc488-00b0-1148-8132-92ce47af4226 +34:07.406 [INFO] TestAgent.server: federation state anti-entropy synced +34:07.630 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:07.631 [INFO] TestAgent: Synced node info +34:07.714 [INFO] TestAgent: Requesting shutdown +34:07.714 [INFO] TestAgent.server: shutting down server +34:07.714 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:07.714 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:07.714 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:07.714 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:07.714 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:07.714 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:07.714 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:07.714 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:07.714 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:07.714 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:07.714 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:07.715Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:07.715Z [INFO] agent.router.manager: shutting down +34:07.715 [INFO] TestAgent: consul server down +34:07.715 [INFO] TestAgent: shutdown complete +34:07.715 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24985 network=tcp +34:07.715 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24985 network=udp +34:07.715 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24986 network=tcp +34:08.215 [INFO] TestAgent: Waiting for endpoints to shut down +34:08.215 [INFO] TestAgent: Endpoints down +--- PASS: TestSessionGet (1.54s) + --- PASS: TestSessionGet/#00 (0.64s) + --- PASS: TestSessionGet/#01 (0.90s) === CONT TestAgent_IndexChurn ---- PASS: TestAgent_AddCheck_ExecDisable (0.72s) === RUN TestAgent_IndexChurn/no_tags -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:46.771Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:46.837Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:46.840 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6180a7bd-e442-41f4-276a-869436da4419 Address:127.0.0.1:13744}]" -42:46.841 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6180a7bd-e442-41f4-276a-869436da4419.dc1 127.0.0.1 -42:46.841 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13744 [Follower]" leader= -42:46.842 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6180a7bd-e442-41f4-276a-869436da4419 127.0.0.1 -2022-09-15T21:42:46.842Z [INFO] agent.router: Initializing LAN area manager -42:46.842 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13781 network=udp -42:46.843 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6180a7bd-e442-41f4-276a-869436da4419.dc1 area=wan -42:46.843 [INFO] TestAgent.server: Adding LAN server: server="Node-6180a7bd-e442-41f4-276a-869436da4419 (Addr: tcp/127.0.0.1:13744) (DC: dc1)" -42:46.843 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13781 network=tcp -42:46.844 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13782 network=tcp -42:46.844 [INFO] TestAgent: started state syncer -42:46.844 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13745 network=tcp -42:46.899 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:46.899 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13744 [Candidate]" term=2 -42:46.899 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:46.899 [DEBUG] TestAgent.server.raft: vote granted: from=6180a7bd-e442-41f4-276a-869436da4419 term=2 tally=1 -42:46.899 [INFO] TestAgent.server.raft: election won: tally=1 -42:46.899 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13744 [Leader]" -42:46.900 [INFO] TestAgent.server: cluster leadership acquired -42:46.900 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13744 -42:46.901 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:46.902 [INFO] TestAgent.server: New leader elected: payload=Node-6180a7bd-e442-41f4-276a-869436da4419 -42:46.904 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:46.905 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:46.905 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:46.905 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:46.905 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:46.905 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6180a7bd-e442-41f4-276a-869436da4419 -42:46.905 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6180a7bd-e442-41f4-276a-869436da4419 -42:46.905 [INFO] TestAgent.server: federation state anti-entropy synced -42:47.088 [INFO] TestAgent: Requesting shutdown -42:47.088 [INFO] TestAgent.server: shutting down server -42:47.088 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:47.088 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:47.088 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:47.088 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:47.088 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:47.088 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:47.088 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:47.088 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:47.088 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:47.088 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:47.088 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:47.088 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:47.089Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:47.089Z [INFO] agent.router.manager: shutting down -42:47.089 [INFO] TestAgent: consul server down -42:47.089 [INFO] TestAgent: shutdown complete -42:47.089 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13781 network=tcp -42:47.089 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13781 network=udp -42:47.089 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13782 network=tcp -42:47.589 [INFO] TestAgent: Waiting for endpoints to shut down -42:47.589 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_AddCheck_RestoreState (0.86s) +2021-08-13T15:34:07.360Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:07.360Z [WARN] agent: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ +2021-08-13T15:34:07.378Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:07.378Z [WARN] agent.auto_config: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ +34:07.381 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7cc21e77-a16e-e743-c608-9d0eea95b57e Address:127.0.0.1:24997}]" +34:07.381 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24997 [Follower]" leader= +34:07.382 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7cc21e77-a16e-e743-c608-9d0eea95b57e.dc1 127.0.0.1 +34:07.383 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7cc21e77-a16e-e743-c608-9d0eea95b57e 127.0.0.1 +2021-08-13T15:34:07.383Z [INFO] agent.router: Initializing LAN area manager +34:07.383 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7cc21e77-a16e-e743-c608-9d0eea95b57e.dc1 area=wan +34:07.383 [INFO] TestAgent.server: Adding LAN server: server="Node-7cc21e77-a16e-e743-c608-9d0eea95b57e (Addr: tcp/127.0.0.1:24997) (DC: dc1)" +34:07.383 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24992 network=tcp +34:07.383 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24992 network=udp +34:07.384 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24993 network=tcp +34:07.384 [INFO] TestAgent: started state syncer +34:07.384 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24998 network=tcp +34:07.439 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:07.439 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24997 [Candidate]" term=2 +34:07.439 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:07.439 [DEBUG] TestAgent.server.raft: vote granted: from=7cc21e77-a16e-e743-c608-9d0eea95b57e term=2 tally=1 +34:07.439 [INFO] TestAgent.server.raft: election won: tally=1 +34:07.439 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24997 [Leader]" +34:07.439 [INFO] TestAgent.server: cluster leadership acquired +34:07.439 [INFO] TestAgent.server: New leader elected: payload=Node-7cc21e77-a16e-e743-c608-9d0eea95b57e +34:07.440 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24997 +34:07.441 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:07.443 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:07.443 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:07.443 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:07.443 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:07.443 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:07.443 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7cc21e77-a16e-e743-c608-9d0eea95b57e +34:07.443 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7cc21e77-a16e-e743-c608-9d0eea95b57e +34:07.444 [INFO] TestAgent.server: federation state anti-entropy synced +34:07.595 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:07.596 [INFO] TestAgent: Synced node info +34:07.596 [DEBUG] TestAgent: Node info in sync +34:07.723 [WARN] TestAgent: check has interval below minimum: check=mem minimum_interval=1s +34:07.723 [INFO] TestAgent: Requesting shutdown +34:07.723 [INFO] TestAgent.server: shutting down server +34:07.723 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:07.723 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:07.723 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:07.723 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:07.723 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:07.723 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:07.723 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:07.724 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:07.724 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:07.724 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:07.724 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:07.724Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:07.724Z [INFO] agent.router.manager: shutting down +34:07.724 [INFO] TestAgent: consul server down +34:07.724 [INFO] TestAgent: shutdown complete +34:07.724 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24992 network=tcp +34:07.724 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24992 network=udp +34:07.724 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24993 network=tcp +34:08.227 [INFO] TestAgent: Waiting for endpoints to shut down +34:08.227 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_AddCheck_MinInterval (0.89s) === CONT TestAgent_ReconnectConfigWanDisabled -2022-09-15T21:42:47.040Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:47.040Z [WARN] agent: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ -2022-09-15T21:42:47.058Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:47.058Z [WARN] agent.auto_config: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ -42:47.061 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a05fc2b1-402d-16fc-8b4f-3d9b4feac06d Address:127.0.0.1:13821}]" -42:47.061 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13821 [Follower]" leader= -42:47.062 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a05fc2b1-402d-16fc-8b4f-3d9b4feac06d.dc1 127.0.0.1 -42:47.062 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a05fc2b1-402d-16fc-8b4f-3d9b4feac06d 127.0.0.1 -2022-09-15T21:42:47.062Z [INFO] agent.router: Initializing LAN area manager -42:47.063 [INFO] TestAgent.server: Adding LAN server: server="Node-a05fc2b1-402d-16fc-8b4f-3d9b4feac06d (Addr: tcp/127.0.0.1:13821) (DC: dc1)" -42:47.063 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a05fc2b1-402d-16fc-8b4f-3d9b4feac06d.dc1 area=wan -42:47.063 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13809 network=udp -42:47.063 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13809 network=tcp -42:47.064 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13810 network=tcp -42:47.064 [INFO] TestAgent: started state syncer -42:47.064 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13822 network=tcp -42:47.105 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:47.105 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13821 [Candidate]" term=2 -42:47.106 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:47.106 [DEBUG] TestAgent.server.raft: vote granted: from=a05fc2b1-402d-16fc-8b4f-3d9b4feac06d term=2 tally=1 -42:47.106 [INFO] TestAgent.server.raft: election won: tally=1 -42:47.106 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13821 [Leader]" -42:47.106 [INFO] TestAgent.server: cluster leadership acquired -42:47.107 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13821 -42:47.107 [INFO] TestAgent.server: New leader elected: payload=Node-a05fc2b1-402d-16fc-8b4f-3d9b4feac06d -42:47.115 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:47.117 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:47.117 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:47.117 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:47.117 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:47.117 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:47.117 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a05fc2b1-402d-16fc-8b4f-3d9b4feac06d -42:47.117 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a05fc2b1-402d-16fc-8b4f-3d9b4feac06d -42:47.119 [INFO] TestAgent.server: federation state anti-entropy synced -42:47.137 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:47.138 [INFO] TestAgent: Synced node info -42:47.138 [DEBUG] TestAgent: Node info in sync -42:47.178 [INFO] TestAgent: Requesting shutdown -42:47.178 [INFO] TestAgent.server: shutting down server -42:47.178 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:47.178 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:47.178 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:47.178 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:47.178 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:47.179 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:47.179 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:47.179 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:47.179 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:47.179 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:47.179 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:47.179Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:47.179Z [INFO] agent.router.manager: shutting down -42:47.179 [INFO] TestAgent: consul server down -42:47.179 [INFO] TestAgent: shutdown complete -42:47.179 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13809 network=tcp -42:47.179 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13809 network=udp -42:47.182 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13810 network=tcp -42:47.683 [INFO] TestAgent: Waiting for endpoints to shut down -42:47.683 [INFO] TestAgent: Endpoints down +2021-08-13T15:34:07.397Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:07.397Z [WARN] agent: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ +2021-08-13T15:34:07.413Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:07.413Z [WARN] agent.auto_config: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ +34:07.415 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ecd8ea86-6dad-539a-7ac0-736cf7bc1c92 Address:127.0.0.1:25004}]" +34:07.415 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25004 [Follower]" leader= +34:07.416 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ecd8ea86-6dad-539a-7ac0-736cf7bc1c92.dc1 127.0.0.1 +34:07.417 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ecd8ea86-6dad-539a-7ac0-736cf7bc1c92 127.0.0.1 +2021-08-13T15:34:07.417Z [INFO] agent.router: Initializing LAN area manager +34:07.417 [INFO] TestAgent.server: Adding LAN server: server="Node-ecd8ea86-6dad-539a-7ac0-736cf7bc1c92 (Addr: tcp/127.0.0.1:25004) (DC: dc1)" +34:07.417 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ecd8ea86-6dad-539a-7ac0-736cf7bc1c92.dc1 area=wan +34:07.417 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25013 network=tcp +34:07.417 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25013 network=udp +34:07.418 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25014 network=tcp +34:07.418 [INFO] TestAgent: started state syncer +34:07.418 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25005 network=tcp +34:07.454 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:07.454 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25004 [Candidate]" term=2 +34:07.454 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:07.454 [DEBUG] TestAgent.server.raft: vote granted: from=ecd8ea86-6dad-539a-7ac0-736cf7bc1c92 term=2 tally=1 +34:07.454 [INFO] TestAgent.server.raft: election won: tally=1 +34:07.454 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25004 [Leader]" +34:07.454 [INFO] TestAgent.server: cluster leadership acquired +34:07.455 [INFO] TestAgent.server: New leader elected: payload=Node-ecd8ea86-6dad-539a-7ac0-736cf7bc1c92 +34:07.455 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25004 +34:07.456 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:07.458 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:07.458 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:07.458 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:07.458 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:07.458 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:07.458 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ecd8ea86-6dad-539a-7ac0-736cf7bc1c92 +34:07.458 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ecd8ea86-6dad-539a-7ac0-736cf7bc1c92 +34:07.459 [INFO] TestAgent.server: federation state anti-entropy synced +34:07.622 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:07.623 [INFO] TestAgent: Synced node info +34:07.623 [DEBUG] TestAgent: Node info in sync +34:07.728 [INFO] TestAgent: Requesting shutdown +34:07.728 [INFO] TestAgent.server: shutting down server +34:07.728 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:07.728 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:07.728 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:07.728 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:07.728 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:07.728 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:07.728 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:07.728 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:07.728 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:07.728 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:07.728 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:07.729Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:07.729Z [INFO] agent.router.manager: shutting down +34:07.729 [INFO] TestAgent: consul server down +34:07.729 [INFO] TestAgent: shutdown complete +34:07.729 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25013 network=tcp +34:07.729 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25013 network=udp +34:07.729 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25014 network=tcp +34:08.231 [INFO] TestAgent: Waiting for endpoints to shut down +34:08.231 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_AddCheck_StartPassing (0.85s) === CONT TestAgent_ReconnectConfigSettings ---- PASS: TestAgent_AddCheck_StartPassing (0.68s) -2022-09-15T21:42:47.112Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:47.112Z [WARN] agent: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ -2022-09-15T21:42:47.130Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:47.130Z [WARN] agent.auto_config: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ -42:47.135 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c0ac59b9-5c0d-0ac6-39b6-f016158ad0cd Address:127.0.0.1:13835}]" -42:47.135 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13835 [Follower]" leader= -42:47.137 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c0ac59b9-5c0d-0ac6-39b6-f016158ad0cd.dc1 127.0.0.1 -42:47.138 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c0ac59b9-5c0d-0ac6-39b6-f016158ad0cd 127.0.0.1 -2022-09-15T21:42:47.138Z [INFO] agent.router: Initializing LAN area manager -42:47.139 [INFO] TestAgent.server: Adding LAN server: server="Node-c0ac59b9-5c0d-0ac6-39b6-f016158ad0cd (Addr: tcp/127.0.0.1:13835) (DC: dc1)" -42:47.139 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c0ac59b9-5c0d-0ac6-39b6-f016158ad0cd.dc1 area=wan -42:47.139 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13823 network=udp -42:47.139 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13823 network=tcp -42:47.140 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13824 network=tcp -42:47.140 [INFO] TestAgent: started state syncer -42:47.140 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13836 network=tcp -42:47.183 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:47.183 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13835 [Candidate]" term=2 -42:47.183 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:47.183 [DEBUG] TestAgent.server.raft: vote granted: from=c0ac59b9-5c0d-0ac6-39b6-f016158ad0cd term=2 tally=1 -42:47.183 [INFO] TestAgent.server.raft: election won: tally=1 -42:47.183 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13835 [Leader]" -42:47.183 [INFO] TestAgent.server: cluster leadership acquired -42:47.183 [INFO] TestAgent.server: New leader elected: payload=Node-c0ac59b9-5c0d-0ac6-39b6-f016158ad0cd -42:47.185 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13835 -42:47.188 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:47.191 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:47.191 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:47.191 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:47.191 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:47.191 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:47.191 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c0ac59b9-5c0d-0ac6-39b6-f016158ad0cd -42:47.191 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c0ac59b9-5c0d-0ac6-39b6-f016158ad0cd -42:47.193 [INFO] TestAgent.server: federation state anti-entropy synced -42:47.216 [INFO] TestAgent: Requesting shutdown -42:47.216 [INFO] TestAgent.server: shutting down server -42:47.216 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:47.216 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:47.216 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:47.216 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:47.216 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:47.222 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:47.222 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:47.216 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:47.217 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:47.222 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:47.216 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:47.222 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:47.222Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:47.223Z [INFO] agent.router.manager: shutting down -42:47.223 [INFO] TestAgent: consul server down -42:47.223 [INFO] TestAgent: shutdown complete -42:47.223 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13823 network=tcp -42:47.223 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13823 network=udp -42:47.223 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13824 network=tcp -42:47.728 [INFO] TestAgent: Waiting for endpoints to shut down -42:47.728 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_AddCheck (0.64s) +2021-08-13T15:34:07.567Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:07.567Z [WARN] agent: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ +2021-08-13T15:34:07.585Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:07.585Z [WARN] agent.auto_config: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ +34:07.588 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:717d0545-f309-fa04-b8d6-1955c9d2f733 Address:127.0.0.1:25018}]" +34:07.588 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25018 [Follower]" leader= +34:07.589 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-717d0545-f309-fa04-b8d6-1955c9d2f733.dc1 127.0.0.1 +34:07.589 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-717d0545-f309-fa04-b8d6-1955c9d2f733 127.0.0.1 +2021-08-13T15:34:07.590Z [INFO] agent.router: Initializing LAN area manager +34:07.590 [INFO] TestAgent.server: Adding LAN server: server="Node-717d0545-f309-fa04-b8d6-1955c9d2f733 (Addr: tcp/127.0.0.1:25018) (DC: dc1)" +34:07.590 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-717d0545-f309-fa04-b8d6-1955c9d2f733.dc1 area=wan +34:07.590 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25027 network=udp +34:07.590 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25027 network=tcp +34:07.591 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25028 network=tcp +34:07.591 [INFO] TestAgent: started state syncer +34:07.591 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25019 network=tcp +34:07.643 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:07.643 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25018 [Candidate]" term=2 +34:07.643 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:07.643 [DEBUG] TestAgent.server.raft: vote granted: from=717d0545-f309-fa04-b8d6-1955c9d2f733 term=2 tally=1 +34:07.643 [INFO] TestAgent.server.raft: election won: tally=1 +34:07.643 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25018 [Leader]" +34:07.643 [INFO] TestAgent.server: cluster leadership acquired +34:07.643 [INFO] TestAgent.server: New leader elected: payload=Node-717d0545-f309-fa04-b8d6-1955c9d2f733 +34:07.643 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25018 +34:07.644 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:07.647 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:07.647 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:07.647 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:07.647 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:07.647 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:07.647 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-717d0545-f309-fa04-b8d6-1955c9d2f733 +34:07.647 [INFO] TestAgent.server: member joined, marking health alive: member=Node-717d0545-f309-fa04-b8d6-1955c9d2f733 +34:07.647 [INFO] TestAgent.server: federation state anti-entropy synced +34:07.740 [INFO] TestAgent: Requesting shutdown +34:07.741 [INFO] TestAgent.server: shutting down server +34:07.741 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:07.741 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:07.741 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:07.741 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:07.741 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:07.741 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:07.741 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:07.741 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:07.741 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:07.741 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:07.741 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:07.741 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:07.741Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:07.741Z [INFO] agent.router.manager: shutting down +34:07.741 [INFO] TestAgent: consul server down +34:07.741 [INFO] TestAgent: shutdown complete +34:07.741 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25027 network=tcp +34:07.741 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25027 network=udp +34:07.741 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25028 network=tcp +34:08.243 [INFO] TestAgent: Waiting for endpoints to shut down +34:08.243 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_AddCheck (0.69s) === CONT TestAgent_TokenStore -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=33)_compressed=false#01 -[WARN] freeport: 4 out of 25 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:47.008Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:47.008Z [WARN] agent: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ -2022-09-15T21:42:47.037Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:47.037Z [WARN] agent.auto_config: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ -42:47.041 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d1406deb-7bc2-e39c-4cee-96052d3d1bc7 Address:127.0.0.1:13800}]" -42:47.042 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13800 [Follower]" leader= -42:47.042 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d1406deb-7bc2-e39c-4cee-96052d3d1bc7.dc1 127.0.0.1 -42:47.043 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d1406deb-7bc2-e39c-4cee-96052d3d1bc7 127.0.0.1 -2022-09-15T21:42:47.043Z [INFO] agent.router: Initializing LAN area manager -42:47.043 [INFO] TestAgent.server: Adding LAN server: server="Node-d1406deb-7bc2-e39c-4cee-96052d3d1bc7 (Addr: tcp/127.0.0.1:13800) (DC: dc1)" -42:47.043 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d1406deb-7bc2-e39c-4cee-96052d3d1bc7.dc1 area=wan -42:47.044 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13746 network=tcp -42:47.044 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13746 network=udp -42:47.045 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13747 network=tcp -42:47.045 [INFO] TestAgent: started state syncer -42:47.045 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13801 network=tcp -42:47.109 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:47.109 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13800 [Candidate]" term=2 -42:47.110 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:47.110 [DEBUG] TestAgent.server.raft: vote granted: from=d1406deb-7bc2-e39c-4cee-96052d3d1bc7 term=2 tally=1 -42:47.110 [INFO] TestAgent.server.raft: election won: tally=1 -42:47.110 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13800 [Leader]" -42:47.110 [INFO] TestAgent.server: cluster leadership acquired -42:47.111 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13800 -42:47.111 [INFO] TestAgent.server: New leader elected: payload=Node-d1406deb-7bc2-e39c-4cee-96052d3d1bc7 -42:47.112 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:47.117 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:47.117 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:47.117 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:47.117 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:47.117 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:47.117 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d1406deb-7bc2-e39c-4cee-96052d3d1bc7 -42:47.117 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d1406deb-7bc2-e39c-4cee-96052d3d1bc7 -42:47.119 [INFO] TestAgent.server: federation state anti-entropy synced -42:47.128 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:47.129 [INFO] TestAgent: Synced node info -42:47.346 [INFO] TestAgent: Requesting shutdown -42:47.346 [INFO] TestAgent.server: shutting down server -42:47.346 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:47.346 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:47.346 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:47.346 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:47.346 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:47.346 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:47.346 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:47.346 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:47.346 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:47.346 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:47.346 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:47.346Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:47.346Z [INFO] agent.router.manager: shutting down -42:47.347 [INFO] TestAgent: consul server down -42:47.347 [INFO] TestAgent: shutdown complete -42:47.347 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13746 network=tcp -42:47.347 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13746 network=udp -42:47.347 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13747 network=tcp -42:47.848 [INFO] TestAgent: Waiting for endpoints to shut down -42:47.848 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_AddCheck_MissingService (0.89s) +[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:07.334Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:07.355Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:07.358 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8e410d0b-1afe-4df0-9539-508ef082f25d Address:127.0.0.1:24983}]" +34:07.358 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24983 [Follower]" leader= +34:07.359 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8e410d0b-1afe-4df0-9539-508ef082f25d.dc1 127.0.0.1 +34:07.360 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8e410d0b-1afe-4df0-9539-508ef082f25d 127.0.0.1 +2021-08-13T15:34:07.360Z [INFO] agent.router: Initializing LAN area manager +34:07.360 [INFO] TestAgent.server: Adding LAN server: server="Node-8e410d0b-1afe-4df0-9539-508ef082f25d (Addr: tcp/127.0.0.1:24983) (DC: dc1)" +34:07.360 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8e410d0b-1afe-4df0-9539-508ef082f25d.dc1 area=wan +34:07.360 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24978 network=udp +34:07.360 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24978 network=tcp +34:07.361 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24979 network=tcp +34:07.361 [INFO] TestAgent: started state syncer +34:07.361 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24984 network=tcp +34:07.421 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:07.421 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24983 [Candidate]" term=2 +34:07.421 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:07.421 [DEBUG] TestAgent.server.raft: vote granted: from=8e410d0b-1afe-4df0-9539-508ef082f25d term=2 tally=1 +34:07.421 [INFO] TestAgent.server.raft: election won: tally=1 +34:07.421 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24983 [Leader]" +34:07.421 [INFO] TestAgent.server: cluster leadership acquired +34:07.422 [INFO] TestAgent.server: New leader elected: payload=Node-8e410d0b-1afe-4df0-9539-508ef082f25d +34:07.422 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24983 +34:07.423 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:07.425 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:07.425 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:07.425 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:07.425 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:07.425 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:07.425 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8e410d0b-1afe-4df0-9539-508ef082f25d +34:07.425 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8e410d0b-1afe-4df0-9539-508ef082f25d +34:07.426 [INFO] TestAgent.server: federation state anti-entropy synced +34:07.788 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:07.790 [INFO] TestAgent: Synced node info +34:07.790 [DEBUG] TestAgent: Node info in sync +34:07.916 [WARN] TestAgent.server.rpc: Attempting to apply large raft entry: size_in_bytes=4719032 +34:07.955 [INFO] TestAgent: Requesting shutdown +34:07.955 [INFO] TestAgent.server: shutting down server +34:07.955 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:07.955 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:07.956 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:07.956 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:07.956 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:07.956 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:07.956 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:07.956 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:07.956 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:07.956 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:07.956 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:07.956Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:07.956Z [INFO] agent.router.manager: shutting down +34:07.956 [INFO] TestAgent: consul server down +34:07.956 [INFO] TestAgent: shutdown complete +34:07.956 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24978 network=tcp +34:07.956 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24978 network=udp +34:07.956 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24979 network=tcp +34:08.456 [INFO] TestAgent: Waiting for endpoints to shut down +34:08.456 [INFO] TestAgent: Endpoints down +=== RUN TestTxnEndpoint_Bad_Size_Net/allowed_kv_max_backward_compatible +2021-08-13T15:34:07.506Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:07.506Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:07.537Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:07.537Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:07.540 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:14d68568-8d94-d796-19db-90f50c09367a Address:127.0.0.1:25025}]" +34:07.540 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25025 [Follower]" leader= +34:07.541 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-14d68568-8d94-d796-19db-90f50c09367a.dc1 127.0.0.1 +34:07.542 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-14d68568-8d94-d796-19db-90f50c09367a 127.0.0.1 +2021-08-13T15:34:07.542Z [INFO] agent.router: Initializing LAN area manager +34:07.542 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25006 network=udp +34:07.542 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-14d68568-8d94-d796-19db-90f50c09367a.dc1 area=wan +34:07.542 [INFO] TestAgent.server: Adding LAN server: server="Node-14d68568-8d94-d796-19db-90f50c09367a (Addr: tcp/127.0.0.1:25025) (DC: dc1)" +34:07.543 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25006 network=tcp +34:07.543 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25007 network=tcp +34:07.543 [INFO] TestAgent: started state syncer +34:07.544 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25026 network=tcp +34:07.590 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:07.590 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25025 [Candidate]" term=2 +34:07.590 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:07.590 [DEBUG] TestAgent.server.raft: vote granted: from=14d68568-8d94-d796-19db-90f50c09367a term=2 tally=1 +34:07.590 [INFO] TestAgent.server.raft: election won: tally=1 +34:07.590 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25025 [Leader]" +34:07.590 [INFO] TestAgent.server: cluster leadership acquired +34:07.591 [INFO] TestAgent.server: initializing acls +34:07.591 [INFO] TestAgent.server: New leader elected: payload=Node-14d68568-8d94-d796-19db-90f50c09367a +34:07.591 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:07.591 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:07.592 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:07.592 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:07.592 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:07.592 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:07.592 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-14d68568-8d94-d796-19db-90f50c09367a +34:07.592 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-14d68568-8d94-d796-19db-90f50c09367a.dc1 +34:07.593 [INFO] TestAgent.server: Updating LAN server: server="Node-14d68568-8d94-d796-19db-90f50c09367a (Addr: tcp/127.0.0.1:25025) (DC: dc1)" +34:07.593 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-14d68568-8d94-d796-19db-90f50c09367a.dc1 area=wan +34:07.593 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:07.596 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:07.596 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:07.596 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:07.596 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:07.596 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:07.596 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-14d68568-8d94-d796-19db-90f50c09367a +34:07.596 [INFO] TestAgent.server: member joined, marking health alive: member=Node-14d68568-8d94-d796-19db-90f50c09367a +34:07.597 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-14d68568-8d94-d796-19db-90f50c09367a +34:07.597 [INFO] TestAgent.server: federation state anti-entropy synced +34:07.956 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:07.957 [INFO] TestAgent: Synced node info +34:07.973 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-14d68568-8d94-d796-19db-90f50c09367a +34:07.973 [INFO] TestAgent.server.fsm: snapshot created: duration=104.32µs +34:07.973 [INFO] TestAgent.server.raft: starting snapshot up to: index=15 +34:07.973 [INFO] TestAgent.server.snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshot_Options_GET#02-TestAgent-agent855151033/raft/snapshots/2-15-1628868847973.tmp +34:07.975 [INFO] TestAgent.server.raft: snapshot complete up to: index=15 +34:07.977 [INFO] TestAgent: Requesting shutdown +34:07.977 [INFO] TestAgent.server: shutting down server +34:07.977 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:07.977 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:07.977 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:07.977 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:07.977 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:07.977 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:07.977 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:07.978 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:07.978 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:07.978 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:07.978 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:07.978 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:07.978 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:07.978 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:07.978 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:07.978Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:07.978Z [INFO] agent.router.manager: shutting down +34:07.978 [INFO] TestAgent: consul server down +34:07.978 [INFO] TestAgent: shutdown complete +34:07.978 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25006 network=tcp +34:07.978 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25006 network=udp +34:07.978 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25007 network=tcp +34:08.479 [INFO] TestAgent: Waiting for endpoints to shut down +34:08.479 [INFO] TestAgent: Endpoints down +=== RUN TestSnapshot_Options/PUT +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:01.794Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:01.812Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:01.814 [INFO] bonnie.server.gateway_locator: will dial the primary datacenter through its mesh gateways +34:01.816 [INFO] bonnie.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b2a4a33e-5953-7bb2-cf0d-027edd9ee04a Address:127.0.0.1:24488}]" +34:01.816 [INFO] bonnie.server.raft: entering follower state: follower="Node at 127.0.0.1:24488 [Follower]" leader= +34:01.817 [INFO] bonnie.server.serf.wan: serf: EventMemberJoin: bonnie.dc3 127.0.0.1 +34:01.818 [INFO] bonnie.server.serf.lan: serf: EventMemberJoin: bonnie 127.0.0.1 +2021-08-13T15:34:01.818Z [INFO] agent.router: Initializing LAN area manager +34:01.818 [INFO] bonnie.server: Adding LAN server: server="bonnie (Addr: tcp/127.0.0.1:24488) (DC: dc3)" +34:01.818 [INFO] bonnie.server: Handled event for server in area: event=member-join server=bonnie.dc3 area=wan +34:01.819 [INFO] bonnie: Started DNS server: address=127.0.0.1:24650 network=udp +34:01.819 [INFO] bonnie: Started DNS server: address=127.0.0.1:24650 network=tcp +34:01.820 [INFO] bonnie: Started HTTP server: address=127.0.0.1:24651 network=tcp +34:01.820 [INFO] bonnie: started state syncer +34:01.820 [INFO] bonnie: Started gRPC server: address=127.0.0.1:24642 network=tcp +34:01.820 [INFO] bonnie: Refreshing mesh gateways is supported for the following discovery methods: discovery_methods="aliyun aws gce mdns os packet" +34:01.820 [INFO] bonnie: Refreshing mesh gateways... +34:01.820 [INFO] bonnie.server.gateway_locator: updated fallback list of primary mesh gateways: mesh_gateways=[127.0.0.1:24486] +34:01.820 [INFO] bonnie: Refreshing mesh gateways completed +34:01.820 [INFO] bonnie: Retry join is supported for the following discovery methods: cluster=WAN discovery_methods="aliyun aws gce mdns os packet" +34:01.820 [INFO] bonnie: Joining cluster...: cluster=WAN +34:01.820 [INFO] bonnie: (WAN) joining: wan_addresses=[*.dc1/192.0.2.2] +34:01.838 [DEBUG] bonnie.server.memberlist.wan: memberlist: Initiating push/pull sync with: *.dc1 127.0.0.1:24486 +34:01.841 [INFO] bonnie.server.serf.wan: serf: EventMemberJoin: betty.dc2 127.0.0.1 +34:01.841 [INFO] bonnie.server.serf.wan: serf: EventMemberJoin: bob.dc1 127.0.0.1 +34:01.841 [INFO] bonnie.server: Handled event for server in area: event=member-join server=betty.dc2 area=wan +34:01.841 [INFO] bonnie: (WAN) joined: number_of_nodes=1 +34:01.841 [INFO] bonnie: Join cluster completed. Synced with initial agents: cluster=WAN num_agents=1 +34:01.841 [INFO] bonnie.server: Handled event for server in area: event=member-join server=bob.dc1 area=wan +34:01.885 [WARN] bonnie.server.raft: heartbeat timeout reached, starting election: last-leader= +34:01.885 [INFO] bonnie.server.raft: entering candidate state: node="Node at 127.0.0.1:24488 [Candidate]" term=2 +34:01.885 [DEBUG] bonnie.server.raft: votes: needed=1 +34:01.885 [DEBUG] bonnie.server.raft: vote granted: from=b2a4a33e-5953-7bb2-cf0d-027edd9ee04a term=2 tally=1 +34:01.885 [INFO] bonnie.server.raft: election won: tally=1 +34:01.885 [INFO] bonnie.server.raft: entering leader state: leader="Node at 127.0.0.1:24488 [Leader]" +34:01.885 [INFO] bonnie.server: cluster leadership acquired +34:01.886 [DEBUG] bonnie.server: Cannot upgrade to new ACLs, servers in acl datacenter are not yet upgraded: ACLDatacenter=dc1 mode=0 found=true +34:01.886 [INFO] bonnie.server: New leader elected: payload=bonnie +34:01.968 [DEBUG] connect.ca.consul: consul CA provider configured: id=ad:4a:c6:ab:ef:63:c9:60:1a:51:7f:19:62:e3:e9:d9:0e:76:55:10:6e:74:24:69:28:a1:6c:b8:b9:8f:fd:89 is_primary=false +34:01.975 [INFO] bonnie.server.connect: received new intermediate certificate from primary datacenter +34:01.975 [INFO] bonnie.server.connect: updated root certificates from primary datacenter +34:01.975 [INFO] bonnie.server.connect: initialized secondary datacenter CA with provider: provider=consul +34:01.975 [INFO] bonnie.leader: started routine: routine="config entry replication" +34:01.975 [INFO] bonnie.leader: started routine: routine="federation state replication" +34:01.975 [INFO] bonnie.leader: started routine: routine="federation state anti-entropy" +34:01.975 [INFO] bonnie.leader: started routine: routine="secondary CA roots watch" +34:01.975 [INFO] bonnie.leader: started routine: routine="intention replication" +34:01.975 [INFO] bonnie.leader: started routine: routine="intermediate cert renew watch" +34:01.975 [INFO] bonnie.leader: started routine: routine="CA root pruning" +34:01.975 [DEBUG] bonnie.server: Skipping self join check for node since the cluster is too small: node=bonnie +34:01.975 [INFO] bonnie.server: member joined, marking health alive: member=bonnie +34:01.976 [DEBUG] bonnie.server.connect: starting Connect intention replication from primary datacenter: primary=dc1 +34:01.977 [DEBUG] bonnie.server.connect: starting Connect CA root replication from primary datacenter: primary=dc1 +34:01.979 [DEBUG] bonnie.server.replication.config_entry: finished fetching config entries: amount=0 +34:01.980 [DEBUG] bonnie.server.replication.config_entry: Config Entry replication: local=0 remote=0 +34:01.980 [DEBUG] bonnie.server.replication.config_entry: Config Entry replication: deletions=0 updates=0 +34:01.980 [DEBUG] bonnie.server.replication.config_entry: replication completed through remote index: index=1 +34:01.981 [INFO] bonnie.server.gateway_locator: will dial the primary datacenter using our local mesh gateways if possible +34:01.981 [DEBUG] bonnie.server: finished fetching remote objects: amount=2 +34:01.981 [DEBUG] bonnie.server: diffing replication state: local_amount=0 remote_amount=2 +34:01.981 [DEBUG] bonnie.server: diffed replication state: deletions=0 updates=2 +34:01.981 [DEBUG] bonnie.server: performing updates: updates=2 +34:01.982 [INFO] bonnie.server.connect: updated root certificates from primary datacenter +34:01.989 [INFO] bonnie.server: federation state anti-entropy synced +34:01.991 [DEBUG] bonnie.server: finished updates +34:01.991 [DEBUG] bonnie.replication.federation_state: replication completed through remote index: index=17 +34:01.991 [INFO] bonnie.server: federation state anti-entropy synced +34:01.992 [INFO] bonnie.server: federation state anti-entropy synced +34:01.993 [DEBUG] bonnie.server: finished fetching remote objects: amount=3 +34:01.993 [DEBUG] bonnie.server: diffing replication state: local_amount=2 remote_amount=3 +34:01.993 [DEBUG] bonnie.server: diffed replication state: deletions=0 updates=1 +34:01.993 [DEBUG] bonnie.server: performing updates: updates=1 +34:01.993 [DEBUG] bonnie.server: finished updates +34:01.993 [DEBUG] bonnie.replication.federation_state: replication completed through remote index: index=20 +34:01.994 [DEBUG] bonnie.server: finished fetching remote objects: amount=3 +34:01.994 [DEBUG] bonnie.server: diffing replication state: local_amount=3 remote_amount=3 +34:01.994 [DEBUG] bonnie.server: diffed replication state: deletions=0 updates=1 +34:01.994 [DEBUG] bonnie.server: performing updates: updates=1 +34:01.994 [DEBUG] bonnie.server: finished updates +34:01.994 [DEBUG] bonnie.replication.federation_state: replication completed through remote index: index=21 +34:02.094 [INFO] bonnie: Synced node info +34:02.095 [INFO] bonnie: Synced service: service=mesh-gateway +34:02.096 [DEBUG] bonnie.server: finished fetching remote objects: amount=3 +34:02.096 [DEBUG] bonnie.server: diffing replication state: local_amount=3 remote_amount=3 +34:02.096 [DEBUG] bonnie.server: diffed replication state: deletions=0 updates=1 +34:02.096 [DEBUG] bonnie.server: performing updates: updates=1 +34:02.096 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:24487 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" +34:02.097 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:24487 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" +34:02.097 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:24487 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" +34:02.097 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:24487 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" +34:02.097 [ERROR] bonnie.proxycfg: watch error: service_id=mesh-gateway id=mesh-gateway:dc2 error="error filling agent cache: rpc error getting client: failed to get conn: Remote DC has no server currently reachable" +34:02.097 [DEBUG] bonnie.server: finished updates +34:02.097 [DEBUG] bonnie.replication.federation_state: replication completed through remote index: index=22 +34:02.097 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:24487 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" +34:02.097 [ERROR] bonnie.proxycfg: watch error: service_id=mesh-gateway id=mesh-gateway:dc2 error="error filling agent cache: rpc error getting client: failed to get conn: Remote DC has no server currently reachable" +34:02.097 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:24487 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" +34:02.097 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:24487 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" +34:02.097 [ERROR] bonnie.proxycfg: watch error: service_id=mesh-gateway id=mesh-gateway:dc2 error="error filling agent cache: rpc error getting client: failed to get conn: Remote DC has no server currently reachable" +34:02.097 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:24487 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" +34:02.097 [ERROR] bonnie.proxycfg: watch error: service_id=mesh-gateway id=mesh-gateway:dc2 error="error filling agent cache: rpc error getting client: failed to get conn: Remote DC has no server currently reachable" +34:02.107 [DEBUG] bonnie: Skipping remote check since it is managed automatically: check=serfHealth +34:02.107 [DEBUG] bonnie: Node info in sync +34:02.107 [DEBUG] bonnie: Service in sync: service=mesh-gateway +34:02.318 [ERROR] bonnie.server.memberlist.wan: memberlist: Failed to send gossip to 127.0.0.1:24633: Remote DC has no server currently reachable +34:02.576 [DEBUG] bonnie.server.serf.wan: serf: messageJoinType: betty.dc2 +34:02.576 [DEBUG] bonnie.server.serf.wan: serf: messageJoinType: bonnie.dc3 +34:03.075 [DEBUG] bonnie.server.serf.wan: serf: messageJoinType: betty.dc2 +34:03.075 [DEBUG] bonnie.server.serf.wan: serf: messageJoinType: bonnie.dc3 +34:03.319 [ERROR] bonnie.server.memberlist.wan: memberlist: Failed to send gossip to 127.0.0.1:24633: Remote DC has no server currently reachable +34:03.817 [INFO] bonnie.server.gateway_locator: new cached locations of mesh gateways: primary=[127.0.0.1:24486] local=[] +34:03.818 [ERROR] bonnie.server.memberlist.wan: memberlist: Failed to send gossip to 127.0.0.1:24633: Remote DC has no server currently reachable +34:04.116 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:24487 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" +34:04.678 [DEBUG] bonnie: Skipping remote check since it is managed automatically: check=serfHealth +34:04.679 [DEBUG] bonnie: Node info in sync +34:04.679 [DEBUG] bonnie: Service in sync: service=mesh-gateway +34:04.679 [DEBUG] bonnie: Node info in sync +34:04.679 [DEBUG] bonnie: Service in sync: service=mesh-gateway +34:04.681 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:24487 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" +34:04.681 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:24487 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" +34:04.681 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:24487 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" +34:04.681 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:24487 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" +34:04.685 [ERROR] bonnie.proxycfg: watch error: service_id=mesh-gateway id=mesh-gateway:dc2 error="error filling agent cache: rpc error getting client: failed to get conn: Remote DC has no server currently reachable" +34:05.605 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:24487 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" +34:06.708 [ERROR] bonnie.server.memberlist.wan: memberlist: Failed to send ack: Remote DC has no server currently reachable from=127.0.0.1:50976 +34:06.818 [ERROR] bonnie.server.memberlist.wan: memberlist: Failed to send ping: Remote DC has no server currently reachable +34:06.979 [INFO] bonnie.server: federation state anti-entropy synced +34:06.979 [DEBUG] bonnie.server: finished fetching remote objects: amount=3 +34:06.979 [DEBUG] bonnie.server: diffing replication state: local_amount=3 remote_amount=3 +34:06.979 [DEBUG] bonnie.server: diffed replication state: deletions=0 updates=1 +34:06.979 [DEBUG] bonnie.server: performing updates: updates=1 +34:06.980 [DEBUG] bonnie.server: finished updates +34:06.980 [DEBUG] bonnie.replication.federation_state: replication completed through remote index: index=23 +34:06.980 [INFO] bonnie.server.gateway_locator: new cached locations of mesh gateways: primary=[127.0.0.1:24486] local=[127.0.0.1:24486] +34:07.070 [INFO] bonnie: Requesting shutdown +34:07.070 [INFO] bonnie.server: shutting down server +34:07.070 [DEBUG] bonnie.leader: stopping routine: routine="intermediate cert renew watch" +34:07.070 [DEBUG] bonnie.leader: stopping routine: routine="CA root pruning" +34:07.070 [DEBUG] bonnie.leader: stopping routine: routine="config entry replication" +34:07.070 [DEBUG] bonnie.leader: stopping routine: routine="federation state replication" +34:07.070 [DEBUG] bonnie.leader: stopping routine: routine="federation state anti-entropy" +34:07.070 [DEBUG] bonnie.leader: stopping routine: routine="secondary CA roots watch" +34:07.070 [DEBUG] bonnie.leader: stopping routine: routine="intention replication" +34:07.070 [WARN] bonnie.server.serf.lan: serf: Shutdown without a Leave +34:07.070 [DEBUG] bonnie.leader: stopped routine: routine="intermediate cert renew watch" +34:07.070 [DEBUG] bonnie.leader: stopped routine: routine="CA root pruning" +34:07.070 [DEBUG] bonnie.leader: stopped routine: routine="federation state anti-entropy" +34:07.070 [WARN] bonnie.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:07.070Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:07.070Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:07.070Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:07.070Z [INFO] agent.router.manager: shutting down +34:07.071 [INFO] bonnie: consul server down +34:07.071 [INFO] bonnie: shutdown complete +34:07.071 [INFO] bonnie: Stopping server: protocol=DNS address=127.0.0.1:24650 network=tcp +34:07.071 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:24508 datacenter=dc1 method=ConfigEntry.ListAll error="rpc error making call: EOF" +34:07.071 [INFO] bonnie: Stopping server: protocol=DNS address=127.0.0.1:24650 network=udp +34:07.071 [INFO] bonnie.server.replication.config_entry: stopped replication +34:07.071 [DEBUG] bonnie.leader: stopped routine: routine="config entry replication" +34:07.071 [INFO] bonnie: Stopping server: protocol=HTTP address=127.0.0.1:24651 network=tcp +34:07.071 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:24508 datacenter=dc1 method=FederationState.List error="rpc error making call: EOF" +34:07.071 [WARN] bonnie.replication.federation_state: replication error (will retry if still leader): error="failed to retrieve federation states: rpc error making call: EOF" +34:07.071 [INFO] bonnie.replication.federation_state: stopped replication +34:07.071 [DEBUG] bonnie.leader: stopped routine: routine="federation state replication" +34:07.071 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:24508 datacenter=dc1 method=Internal.ServiceDump error="rpc error making call: EOF" +34:07.071 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:24508 datacenter=dc1 method=ConnectCA.Roots error="rpc error making call: EOF" +34:07.071 [ERROR] bonnie.server.connect: CA root replication failed, will retry: routine="secondary CA roots watch" error="Error retrieving the primary datacenter's roots: rpc error making call: EOF" +34:07.071 [DEBUG] bonnie.leader: stopped routine: routine="secondary CA roots watch" +34:07.071 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:24508 datacenter=dc1 method=Intention.List error="rpc error making call: EOF" +34:07.071 [ERROR] bonnie.server.connect: error replicating intentions: routine="intention replication" error="rpc error making call: EOF" +34:07.071 [DEBUG] bonnie.leader: stopped routine: routine="intention replication" +34:07.572 [INFO] bonnie: Waiting for endpoints to shut down +34:07.572 [INFO] bonnie: Endpoints down +2021-08-13T15:34:01.596Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:01.674Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:01.675 [INFO] betty.server.gateway_locator: will dial the primary datacenter through its mesh gateways +34:01.678 [INFO] betty.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1fdd1e65-738c-b4d3-298a-bc3e9748c438 Address:127.0.0.1:24487}]" +34:01.679 [INFO] betty.server.serf.wan: serf: EventMemberJoin: betty.dc2 127.0.0.1 +34:01.679 [INFO] betty.server.raft: entering follower state: follower="Node at 127.0.0.1:24487 [Follower]" leader= +34:01.680 [INFO] betty.server.serf.lan: serf: EventMemberJoin: betty 127.0.0.1 +2021-08-13T15:34:01.680Z [INFO] agent.router: Initializing LAN area manager +34:01.680 [INFO] betty: Started DNS server: address=127.0.0.1:24629 network=udp +34:01.680 [INFO] betty.server: Adding LAN server: server="betty (Addr: tcp/127.0.0.1:24487) (DC: dc2)" +34:01.680 [INFO] betty.server: Handled event for server in area: event=member-join server=betty.dc2 area=wan +34:01.681 [INFO] betty: Started DNS server: address=127.0.0.1:24629 network=tcp +34:01.682 [INFO] betty: Started HTTP server: address=127.0.0.1:24630 network=tcp +34:01.682 [INFO] betty: started state syncer +34:01.682 [INFO] betty: Started gRPC server: address=127.0.0.1:24635 network=tcp +34:01.682 [INFO] betty: Refreshing mesh gateways is supported for the following discovery methods: discovery_methods="aliyun aws gce mdns os packet" +34:01.682 [INFO] betty: Refreshing mesh gateways... +34:01.682 [INFO] betty.server.gateway_locator: updated fallback list of primary mesh gateways: mesh_gateways=[127.0.0.1:24486] +34:01.682 [INFO] betty: Refreshing mesh gateways completed +34:01.682 [INFO] betty: Retry join is supported for the following discovery methods: cluster=WAN discovery_methods="aliyun aws gce mdns os packet" +34:01.682 [INFO] betty: Joining cluster...: cluster=WAN +34:01.682 [INFO] betty: (WAN) joining: wan_addresses=[*.dc1/192.0.2.2] +34:01.700 [DEBUG] betty.server.memberlist.wan: memberlist: Initiating push/pull sync with: *.dc1 127.0.0.1:24486 +34:01.702 [INFO] betty.server.serf.wan: serf: EventMemberJoin: bob.dc1 127.0.0.1 +34:01.702 [INFO] betty: (WAN) joined: number_of_nodes=1 +34:01.702 [INFO] betty: Join cluster completed. Synced with initial agents: cluster=WAN num_agents=1 +34:01.703 [INFO] betty.server: Handled event for server in area: event=member-join server=bob.dc1 area=wan +34:01.715 [WARN] betty.server.raft: heartbeat timeout reached, starting election: last-leader= +34:01.715 [INFO] betty.server.raft: entering candidate state: node="Node at 127.0.0.1:24487 [Candidate]" term=2 +34:01.715 [DEBUG] betty.server.raft: votes: needed=1 +34:01.715 [DEBUG] betty.server.raft: vote granted: from=1fdd1e65-738c-b4d3-298a-bc3e9748c438 term=2 tally=1 +34:01.715 [INFO] betty.server.raft: election won: tally=1 +34:01.715 [INFO] betty.server.raft: entering leader state: leader="Node at 127.0.0.1:24487 [Leader]" +34:01.715 [INFO] betty.server: cluster leadership acquired +34:01.715 [INFO] betty.server: New leader elected: payload=betty +34:01.715 [DEBUG] betty.server: Cannot upgrade to new ACLs, servers in acl datacenter are not yet upgraded: ACLDatacenter=dc1 mode=0 found=true +34:01.736 [DEBUG] connect.ca.consul: consul CA provider configured: id=ad:4a:c6:ab:ef:63:c9:60:1a:51:7f:19:62:e3:e9:d9:0e:76:55:10:6e:74:24:69:28:a1:6c:b8:b9:8f:fd:89 is_primary=false +34:01.741 [INFO] betty.server.connect: received new intermediate certificate from primary datacenter +34:01.741 [INFO] betty.server.connect: updated root certificates from primary datacenter +34:01.741 [INFO] betty.server.connect: initialized secondary datacenter CA with provider: provider=consul +34:01.741 [INFO] betty.leader: started routine: routine="config entry replication" +34:01.741 [INFO] betty.leader: started routine: routine="federation state replication" +34:01.741 [INFO] betty.leader: started routine: routine="federation state anti-entropy" +34:01.741 [INFO] betty.leader: started routine: routine="secondary CA roots watch" +34:01.741 [INFO] betty.leader: started routine: routine="intention replication" +34:01.741 [INFO] betty.leader: started routine: routine="intermediate cert renew watch" +34:01.741 [INFO] betty.leader: started routine: routine="CA root pruning" +34:01.741 [DEBUG] betty.server.connect: starting Connect intention replication from primary datacenter: primary=dc1 +34:01.741 [DEBUG] betty.server: Skipping self join check for node since the cluster is too small: node=betty +34:01.741 [INFO] betty.server: member joined, marking health alive: member=betty +34:01.741 [DEBUG] betty.server.connect: starting Connect CA root replication from primary datacenter: primary=dc1 +34:01.742 [INFO] betty.server: federation state anti-entropy synced +34:01.743 [DEBUG] betty.server.replication.config_entry: finished fetching config entries: amount=0 +34:01.743 [DEBUG] betty.server.replication.config_entry: Config Entry replication: local=0 remote=0 +34:01.743 [DEBUG] betty.server.replication.config_entry: Config Entry replication: deletions=0 updates=0 +34:01.743 [DEBUG] betty.server.replication.config_entry: replication completed through remote index: index=1 +34:01.744 [INFO] betty.server.connect: updated root certificates from primary datacenter +34:01.744 [INFO] betty.server.gateway_locator: will dial the primary datacenter using our local mesh gateways if possible +34:01.744 [DEBUG] betty.server: finished fetching remote objects: amount=2 +34:01.744 [DEBUG] betty.server: diffing replication state: local_amount=0 remote_amount=2 +34:01.744 [DEBUG] betty.server: diffed replication state: deletions=0 updates=2 +34:01.744 [DEBUG] betty.server: performing updates: updates=2 +34:01.746 [INFO] betty.server: federation state anti-entropy synced +34:01.755 [DEBUG] betty.server: finished updates +34:01.755 [DEBUG] betty.replication.federation_state: replication completed through remote index: index=16 +34:01.756 [DEBUG] betty.server: finished fetching remote objects: amount=2 +34:01.756 [DEBUG] betty.server: diffing replication state: local_amount=2 remote_amount=2 +34:01.756 [DEBUG] betty.server: diffed replication state: deletions=0 updates=1 +34:01.756 [DEBUG] betty.server: performing updates: updates=1 +34:01.756 [DEBUG] betty.server: finished updates +34:01.756 [DEBUG] betty.replication.federation_state: replication completed through remote index: index=17 +34:01.984 [DEBUG] betty: Skipping remote check since it is managed automatically: check=serfHealth +34:01.985 [INFO] betty: Synced node info +34:01.985 [DEBUG] betty: Node info in sync +34:01.989 [DEBUG] betty.server: finished fetching remote objects: amount=3 +34:01.989 [DEBUG] betty.server: diffing replication state: local_amount=2 remote_amount=3 +34:01.989 [DEBUG] betty.server: diffed replication state: deletions=0 updates=1 +34:01.989 [DEBUG] betty.server: performing updates: updates=1 +34:01.989 [DEBUG] betty.server: finished updates +34:01.989 [DEBUG] betty.replication.federation_state: replication completed through remote index: index=19 +34:01.991 [DEBUG] betty.server: finished fetching remote objects: amount=3 +34:01.991 [DEBUG] betty.server: diffing replication state: local_amount=3 remote_amount=3 +34:01.991 [DEBUG] betty.server: diffed replication state: deletions=0 updates=1 +34:01.991 [DEBUG] betty.server: performing updates: updates=1 +34:01.991 [DEBUG] betty.server: finished updates +34:01.991 [DEBUG] betty.replication.federation_state: replication completed through remote index: index=20 +34:01.993 [DEBUG] betty.server: finished fetching remote objects: amount=3 +34:01.993 [DEBUG] betty.server: diffing replication state: local_amount=3 remote_amount=3 +34:01.993 [DEBUG] betty.server: diffed replication state: deletions=0 updates=1 +34:01.993 [DEBUG] betty.server: performing updates: updates=1 +34:01.993 [DEBUG] betty.server: finished updates +34:01.993 [DEBUG] betty.replication.federation_state: replication completed through remote index: index=21 +34:02.093 [DEBUG] betty: Node info in sync +34:02.093 [INFO] betty: Synced service: service=mesh-gateway +34:02.096 [INFO] betty.server: federation state anti-entropy synced +34:02.096 [INFO] betty.server.serf.wan: serf: EventMemberJoin: bonnie.dc3 127.0.0.1 +34:02.096 [INFO] betty.server: Handled event for server in area: event=member-join server=bonnie.dc3 area=wan +34:02.197 [ERROR] betty.server.memberlist.wan: memberlist: Failed to send gossip to 127.0.0.1:24640: Remote DC has no server currently reachable +34:02.576 [DEBUG] betty.server.serf.wan: serf: messageJoinType: betty.dc2 +34:02.576 [DEBUG] betty.server.serf.wan: serf: messageJoinType: bonnie.dc3 +34:02.679 [ERROR] betty.server.memberlist.wan: memberlist: Failed to send gossip to 127.0.0.1:24640: Remote DC has no server currently reachable +34:02.747 [DEBUG] betty.server: finished fetching remote objects: amount=3 +34:02.748 [DEBUG] betty.server: diffing replication state: local_amount=3 remote_amount=3 +34:02.748 [DEBUG] betty.server: diffed replication state: deletions=0 updates=1 +34:02.748 [DEBUG] betty.server: performing updates: updates=1 +34:02.748 [DEBUG] betty.server: finished updates +34:02.748 [DEBUG] betty.replication.federation_state: replication completed through remote index: index=22 +34:02.767 [DEBUG] betty: Skipping remote check since it is managed automatically: check=serfHealth +34:02.767 [DEBUG] betty: Node info in sync +34:02.767 [DEBUG] betty: Service in sync: service=mesh-gateway +34:02.767 [DEBUG] betty: Node info in sync +34:02.767 [DEBUG] betty: Service in sync: service=mesh-gateway +34:03.075 [DEBUG] betty.server.serf.wan: serf: messageJoinType: betty.dc2 +34:03.075 [DEBUG] betty.server.serf.wan: serf: messageJoinType: bonnie.dc3 +34:03.179 [ERROR] betty.server.memberlist.wan: memberlist: Failed to send gossip to 127.0.0.1:24640: Remote DC has no server currently reachable +34:03.679 [INFO] betty.server.gateway_locator: new cached locations of mesh gateways: primary=[127.0.0.1:24486] local=[127.0.0.1:24486] +34:06.979 [DEBUG] betty.server: finished fetching remote objects: amount=3 +34:06.979 [DEBUG] betty.server: diffing replication state: local_amount=3 remote_amount=3 +34:06.979 [DEBUG] betty.server: diffed replication state: deletions=0 updates=1 +34:06.979 [DEBUG] betty.server: performing updates: updates=1 +34:06.980 [DEBUG] betty.server: finished updates +34:06.980 [DEBUG] betty.replication.federation_state: replication completed through remote index: index=23 +34:07.572 [INFO] betty: Requesting shutdown +34:07.572 [INFO] betty.server: shutting down server +34:07.572 [DEBUG] betty.leader: stopping routine: routine="secondary CA roots watch" +34:07.572 [DEBUG] betty.leader: stopping routine: routine="intention replication" +34:07.572 [DEBUG] betty.leader: stopping routine: routine="intermediate cert renew watch" +34:07.572 [DEBUG] betty.leader: stopping routine: routine="CA root pruning" +34:07.572 [DEBUG] betty.leader: stopping routine: routine="config entry replication" +34:07.572 [DEBUG] betty.leader: stopping routine: routine="federation state replication" +34:07.572 [DEBUG] betty.leader: stopping routine: routine="federation state anti-entropy" +34:07.572 [WARN] betty.server.serf.lan: serf: Shutdown without a Leave +34:07.572 [DEBUG] betty.leader: stopped routine: routine="intermediate cert renew watch" +34:07.572 [DEBUG] betty.leader: stopped routine: routine="federation state anti-entropy" +34:07.572 [DEBUG] betty.leader: stopped routine: routine="CA root pruning" +34:07.573 [WARN] betty.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:07.573Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:07.573Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:07.573Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:07.573Z [INFO] agent.router.manager: shutting down +34:07.574 [INFO] betty: consul server down +34:07.574 [INFO] betty: shutdown complete +34:07.574 [INFO] betty: Stopping server: protocol=DNS address=127.0.0.1:24629 network=tcp +34:07.574 [INFO] betty: Stopping server: protocol=DNS address=127.0.0.1:24629 network=udp +34:07.574 [INFO] betty: Stopping server: protocol=HTTP address=127.0.0.1:24630 network=tcp +34:07.574 [ERROR] betty.server.rpc: RPC failed to server in DC: server=127.0.0.1:24508 datacenter=dc1 method=Internal.ServiceDump error="rpc error making call: EOF" +34:07.574 [ERROR] betty.server.rpc: RPC failed to server in DC: server=127.0.0.1:24508 datacenter=dc1 method=Intention.List error="rpc error making call: EOF" +34:07.574 [ERROR] betty.server.connect: error replicating intentions: routine="intention replication" error="rpc error making call: EOF" +34:07.574 [DEBUG] betty.leader: stopped routine: routine="intention replication" +34:07.574 [ERROR] betty.server.rpc: RPC failed to server in DC: server=127.0.0.1:24508 datacenter=dc1 method=FederationState.List error="rpc error making call: EOF" +34:07.574 [WARN] betty.replication.federation_state: replication error (will retry if still leader): error="failed to retrieve federation states: rpc error making call: EOF" +34:07.574 [INFO] betty.replication.federation_state: stopped replication +34:07.574 [DEBUG] betty.leader: stopped routine: routine="federation state replication" +34:07.574 [ERROR] betty.server.rpc: RPC failed to server in DC: server=127.0.0.1:24508 datacenter=dc1 method=ConnectCA.Roots error="rpc error making call: EOF" +34:07.574 [ERROR] betty.server.connect: CA root replication failed, will retry: routine="secondary CA roots watch" error="Error retrieving the primary datacenter's roots: rpc error making call: EOF" +34:07.574 [DEBUG] betty.leader: stopped routine: routine="secondary CA roots watch" +34:07.574 [ERROR] betty.server.rpc: RPC failed to server in DC: server=127.0.0.1:24508 datacenter=dc1 method=ConfigEntry.ListAll error="rpc error making call: EOF" +34:07.574 [INFO] betty.server.replication.config_entry: stopped replication +34:07.574 [DEBUG] betty.leader: stopped routine: routine="config entry replication" +34:08.074 [INFO] betty: Waiting for endpoints to shut down +34:08.074 [INFO] betty: Endpoints down +2021-08-13T15:33:59.553Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:59.570Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:59.574 [INFO] bob.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6204d920-5f7b-8d6c-1159-3c923772aa1b Address:127.0.0.1:24508}]" +33:59.575 [INFO] bob.server.serf.wan: serf: EventMemberJoin: bob.dc1 127.0.0.1 +33:59.575 [INFO] bob.server.raft: entering follower state: follower="Node at 127.0.0.1:24508 [Follower]" leader= +33:59.576 [INFO] bob.server.serf.lan: serf: EventMemberJoin: bob 127.0.0.1 +2021-08-13T15:33:59.576Z [INFO] agent.router: Initializing LAN area manager +33:59.576 [INFO] bob.server: Handled event for server in area: event=member-join server=bob.dc1 area=wan +33:59.576 [INFO] bob.server: Adding LAN server: server="bob (Addr: tcp/127.0.0.1:24508) (DC: dc1)" +33:59.576 [INFO] bob: Started DNS server: address=127.0.0.1:24489 network=udp +33:59.577 [INFO] bob: Started DNS server: address=127.0.0.1:24489 network=tcp +33:59.577 [INFO] bob: Started HTTP server: address=127.0.0.1:24490 network=tcp +33:59.577 [INFO] bob: started state syncer +33:59.577 [INFO] bob: Started gRPC server: address=127.0.0.1:24509 network=tcp +33:59.629 [WARN] bob.server.raft: heartbeat timeout reached, starting election: last-leader= +33:59.629 [INFO] bob.server.raft: entering candidate state: node="Node at 127.0.0.1:24508 [Candidate]" term=2 +33:59.629 [DEBUG] bob.server.raft: votes: needed=1 +33:59.629 [DEBUG] bob.server.raft: vote granted: from=6204d920-5f7b-8d6c-1159-3c923772aa1b term=2 tally=1 +33:59.629 [INFO] bob.server.raft: election won: tally=1 +33:59.629 [INFO] bob.server.raft: entering leader state: leader="Node at 127.0.0.1:24508 [Leader]" +33:59.630 [INFO] bob.server: cluster leadership acquired +33:59.630 [DEBUG] bob.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24508 +33:59.631 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:59.633 [INFO] bob.server.connect: initialized primary datacenter CA with provider: provider=consul +33:59.633 [INFO] bob.leader: started routine: routine="federation state anti-entropy" +33:59.633 [INFO] bob.leader: started routine: routine="federation state pruning" +33:59.633 [INFO] bob.leader: started routine: routine="intermediate cert renew watch" +33:59.633 [INFO] bob.leader: started routine: routine="CA root pruning" +33:59.633 [DEBUG] bob.server: Skipping self join check for node since the cluster is too small: node=bob +33:59.633 [INFO] bob.server: member joined, marking health alive: member=bob +33:59.634 [INFO] bob.server: New leader elected: payload=bob +33:59.635 [INFO] bob.server: federation state anti-entropy synced +33:59.810 [INFO] bob: Synced node info +33:59.811 [INFO] bob: Synced service: service=mesh-gateway +33:59.816 [INFO] bob.server: federation state anti-entropy synced +33:59.880 [DEBUG] bob: Skipping remote check since it is managed automatically: check=serfHealth +33:59.880 [DEBUG] bob: Node info in sync +33:59.880 [DEBUG] bob: Service in sync: service=mesh-gateway +33:59.880 [DEBUG] bob: Node info in sync +33:59.880 [DEBUG] bob: Service in sync: service=mesh-gateway +34:00.763 [DEBUG] bob: Skipping remote check since it is managed automatically: check=serfHealth +34:00.763 [DEBUG] bob: Node info in sync +34:00.763 [DEBUG] bob: Service in sync: service=mesh-gateway +34:01.575 [INFO] bob.server.gateway_locator: new cached locations of mesh gateways: primary=[127.0.0.1:24486] local=[127.0.0.1:24486] +34:01.701 [DEBUG] bob.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:43978 +34:01.702 [INFO] bob.server.serf.wan: serf: EventMemberJoin: betty.dc2 127.0.0.1 +34:01.702 [INFO] bob.server: Handled event for server in area: event=member-join server=betty.dc2 area=wan +34:01.840 [DEBUG] bob.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:43984 +34:01.840 [INFO] bob.server.serf.wan: serf: EventMemberJoin: bonnie.dc3 127.0.0.1 +34:01.841 [INFO] bob.server: Handled event for server in area: event=member-join server=bonnie.dc3 area=wan +34:02.198 [DEBUG] bob.server.serf.wan: serf: messageJoinType: betty.dc2 +34:02.341 [DEBUG] bob.server.serf.wan: serf: messageJoinType: bonnie.dc3 +34:02.680 [DEBUG] bob.server.serf.wan: serf: messageJoinType: betty.dc2 +34:02.680 [DEBUG] bob.server.serf.wan: serf: messageJoinType: bonnie.dc3 +34:02.819 [DEBUG] bob.server.serf.wan: serf: messageJoinType: bonnie.dc3 +34:02.819 [DEBUG] bob.server.serf.wan: serf: messageJoinType: betty.dc2 +34:03.180 [DEBUG] bob.server.serf.wan: serf: messageJoinType: bonnie.dc3 +34:03.319 [DEBUG] bob.server.serf.wan: serf: messageJoinType: bonnie.dc3 +34:03.319 [DEBUG] bob.server.serf.wan: serf: messageJoinType: betty.dc2 +34:08.074 [INFO] bob: Requesting shutdown +34:08.074 [INFO] bob.server: shutting down server +34:08.074 [DEBUG] bob.leader: stopping routine: routine="federation state anti-entropy" +34:08.074 [DEBUG] bob.leader: stopping routine: routine="federation state pruning" +34:08.074 [DEBUG] bob.leader: stopping routine: routine="intermediate cert renew watch" +34:08.074 [DEBUG] bob.leader: stopping routine: routine="CA root pruning" +34:08.074 [WARN] bob.server.serf.lan: serf: Shutdown without a Leave +34:08.075 [WARN] bob.server.serf.wan: serf: Shutdown without a Leave +34:08.075 [ERROR] bob.server: error performing anti-entropy sync of federation state: error="context canceled" +34:08.075 [DEBUG] bob.leader: stopped routine: routine="federation state anti-entropy" +2021-08-13T15:34:08.076Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:08.076Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:08.076Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:08.076Z [INFO] agent.router.manager: shutting down +34:08.076 [DEBUG] bob.leader: stopped routine: routine="federation state pruning" +34:08.076 [DEBUG] bob.leader: stopped routine: routine="intermediate cert renew watch" +34:08.076 [DEBUG] bob.leader: stopped routine: routine="CA root pruning" +34:08.076 [INFO] bob: consul server down +34:08.076 [INFO] bob: shutdown complete +34:08.076 [INFO] bob: Stopping server: protocol=DNS address=127.0.0.1:24489 network=tcp +34:08.076 [INFO] bob: Stopping server: protocol=DNS address=127.0.0.1:24489 network=udp +34:08.076 [INFO] bob: Stopping server: protocol=HTTP address=127.0.0.1:24490 network=tcp +34:08.577 [INFO] bob: Waiting for endpoints to shut down +34:08.577 [INFO] bob: Endpoints down +--- PASS: TestAgent_JoinWAN_viaMeshGateway (9.04s) + --- PASS: TestAgent_JoinWAN_viaMeshGateway/dc1_to_dc2 (0.02s) + --- PASS: TestAgent_JoinWAN_viaMeshGateway/dc1_to_dc3 (0.02s) + --- PASS: TestAgent_JoinWAN_viaMeshGateway/dc2_to_dc1 (0.00s) + --- PASS: TestAgent_JoinWAN_viaMeshGateway/dc2_to_dc3 (0.02s) + --- PASS: TestAgent_JoinWAN_viaMeshGateway/dc3_to_dc1 (0.00s) + --- PASS: TestAgent_JoinWAN_viaMeshGateway/dc3_to_dc2 (0.02s) === CONT TestAgent_RPCPing -2022-09-15T21:42:47.036Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:47.036Z [WARN] agent: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ -2022-09-15T21:42:47.055Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:47.055Z [WARN] agent.auto_config: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ -42:47.059 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0ae948a2-d10e-b071-c4a3-abaf4a45b3e3 Address:127.0.0.1:13807}]" -42:47.059 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13807 [Follower]" leader= -42:47.060 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0ae948a2-d10e-b071-c4a3-abaf4a45b3e3.dc1 127.0.0.1 -42:47.061 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0ae948a2-d10e-b071-c4a3-abaf4a45b3e3 127.0.0.1 -2022-09-15T21:42:47.061Z [INFO] agent.router: Initializing LAN area manager -42:47.061 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13802 network=udp -42:47.061 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0ae948a2-d10e-b071-c4a3-abaf4a45b3e3.dc1 area=wan -42:47.061 [INFO] TestAgent.server: Adding LAN server: server="Node-0ae948a2-d10e-b071-c4a3-abaf4a45b3e3 (Addr: tcp/127.0.0.1:13807) (DC: dc1)" -42:47.062 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13802 network=tcp -42:47.063 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13803 network=tcp -42:47.063 [INFO] TestAgent: started state syncer -42:47.063 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13808 network=tcp -42:47.119 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:47.119 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13807 [Candidate]" term=2 -42:47.120 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:47.120 [DEBUG] TestAgent.server.raft: vote granted: from=0ae948a2-d10e-b071-c4a3-abaf4a45b3e3 term=2 tally=1 -42:47.120 [INFO] TestAgent.server.raft: election won: tally=1 -42:47.120 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13807 [Leader]" -42:47.120 [INFO] TestAgent.server: cluster leadership acquired -42:47.120 [INFO] TestAgent.server: New leader elected: payload=Node-0ae948a2-d10e-b071-c4a3-abaf4a45b3e3 -42:47.120 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13807 -42:47.122 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:47.125 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:47.125 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:47.125 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:47.125 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:47.125 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:47.125 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0ae948a2-d10e-b071-c4a3-abaf4a45b3e3 -42:47.125 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0ae948a2-d10e-b071-c4a3-abaf4a45b3e3 -42:47.127 [INFO] TestAgent.server: federation state anti-entropy synced -42:47.354 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:47.355 [INFO] TestAgent: Synced node info -42:47.375 [WARN] TestAgent: check has interval below minimum: check=mem minimum_interval=1s -42:47.375 [INFO] TestAgent: Requesting shutdown -42:47.376 [INFO] TestAgent.server: shutting down server -42:47.376 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:47.376 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:47.376 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:47.376 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:47.376 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:47.377 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:47.377 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:47.377 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:47.377 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:47.377 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:47.377 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:47.377Z [INFO] agent.router.manager: shutting down -42:47.378 [INFO] TestAgent: consul server down -42:47.378 [INFO] TestAgent: shutdown complete -42:47.378 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13802 network=tcp -42:47.378 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13802 network=udp -42:47.379 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13803 network=tcp -2022-09-15T21:42:47.386Z [INFO] agent.router.manager: shutting down -42:47.882 [INFO] TestAgent: Waiting for endpoints to shut down -42:47.882 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_AddCheck_MinInterval (0.89s) +[WARN] freeport: 10 out of 34 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:07.376Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:07.392Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:07.395 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:18acfec0-6067-1d78-4a43-6bfe93235236 Address:127.0.0.1:25011}]" +34:07.395 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25011 [Follower]" leader= +34:07.396 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-18acfec0-6067-1d78-4a43-6bfe93235236.dc1 127.0.0.1 +34:07.397 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-18acfec0-6067-1d78-4a43-6bfe93235236 127.0.0.1 +2021-08-13T15:34:07.397Z [INFO] agent.router: Initializing LAN area manager +34:07.397 [INFO] TestAgent.server: Adding LAN server: server="Node-18acfec0-6067-1d78-4a43-6bfe93235236 (Addr: tcp/127.0.0.1:25011) (DC: dc1)" +34:07.397 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-18acfec0-6067-1d78-4a43-6bfe93235236.dc1 area=wan +34:07.397 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24999 network=udp +34:07.397 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24999 network=tcp +34:07.398 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25000 network=tcp +34:07.398 [INFO] TestAgent: started state syncer +34:07.398 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25012 network=tcp +34:07.445 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:07.445 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25011 [Candidate]" term=2 +34:07.445 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:07.445 [DEBUG] TestAgent.server.raft: vote granted: from=18acfec0-6067-1d78-4a43-6bfe93235236 term=2 tally=1 +34:07.445 [INFO] TestAgent.server.raft: election won: tally=1 +34:07.445 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25011 [Leader]" +34:07.445 [INFO] TestAgent.server: cluster leadership acquired +34:07.446 [INFO] TestAgent.server: New leader elected: payload=Node-18acfec0-6067-1d78-4a43-6bfe93235236 +34:07.446 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25011 +34:07.447 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:07.449 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:07.449 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:07.449 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:07.449 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:07.449 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:07.449 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-18acfec0-6067-1d78-4a43-6bfe93235236 +34:07.449 [INFO] TestAgent.server: member joined, marking health alive: member=Node-18acfec0-6067-1d78-4a43-6bfe93235236 +34:07.450 [INFO] TestAgent.server: federation state anti-entropy synced +34:07.462 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:07.463 [INFO] TestAgent: Synced node info +34:08.279 [DEBUG] TestAgent.server: Session TTL expired: session=01dda82e-fcdc-ee44-fb5d-64e707faaee5 +34:08.520 [INFO] TestAgent: Requesting shutdown +34:08.521 [INFO] TestAgent.server: shutting down server +34:08.521 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:08.521 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:08.521 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:08.521 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:08.521 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:08.521 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:08.521 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:08.521 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:08.521 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:08.521 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:08.521 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:08.521Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:08.521Z [INFO] agent.router.manager: shutting down +34:08.521 [INFO] TestAgent: consul server down +34:08.521 [INFO] TestAgent: shutdown complete +34:08.521 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24999 network=tcp +34:08.522 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24999 network=udp +34:08.522 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25000 network=tcp +34:09.022 [INFO] TestAgent: Waiting for endpoints to shut down +34:09.022 [INFO] TestAgent: Endpoints down +--- PASS: TestSessionCustomTTL (1.67s) === CONT TestAgent_StartStop -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=33)_compressed=true#01 -[WARN] freeport: 4 out of 4 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:47.653Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:47.699Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:47.705 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:45143ea6-4385-aeeb-5804-fbab98355141 Address:127.0.0.1:13674}]" -42:47.706 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13674 [Follower]" leader= -42:47.715 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-45143ea6-4385-aeeb-5804-fbab98355141 127.0.0.1 -2022-09-15T21:42:47.715Z [INFO] agent.router: Initializing LAN area manager -42:47.716 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13816 network=udp -42:47.716 [INFO] TestAgent.server: Adding LAN server: server="Node-45143ea6-4385-aeeb-5804-fbab98355141 (Addr: tcp/127.0.0.1:13674) (DC: dc1)" -42:47.718 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13816 network=tcp -42:47.727 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13817 network=tcp -42:47.727 [INFO] TestAgent: started state syncer -42:47.727 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13675 network=tcp -42:47.758 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:47.758 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13674 [Candidate]" term=2 -42:47.758 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:47.758 [DEBUG] TestAgent.server.raft: vote granted: from=45143ea6-4385-aeeb-5804-fbab98355141 term=2 tally=1 -42:47.758 [INFO] TestAgent.server.raft: election won: tally=1 -42:47.758 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13674 [Leader]" -42:47.759 [INFO] TestAgent.server: cluster leadership acquired -42:47.759 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13674 -42:47.761 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:47.762 [INFO] TestAgent.server: New leader elected: payload=Node-45143ea6-4385-aeeb-5804-fbab98355141 -42:47.775 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:47.775 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:47.775 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:47.775 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:47.776 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:47.776 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-45143ea6-4385-aeeb-5804-fbab98355141 -42:47.776 [INFO] TestAgent.server: member joined, marking health alive: member=Node-45143ea6-4385-aeeb-5804-fbab98355141 -42:47.779 [INFO] TestAgent.server: federation state anti-entropy synced -42:47.824 [INFO] TestAgent: Requesting shutdown -42:47.824 [INFO] TestAgent.server: shutting down server -42:47.824 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:47.824 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:47.824 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:47.824 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:47.824 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:47.824 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:47.824 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:47.824 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:47.824 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:47.824 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:47.824 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -2022-09-15T21:42:47.824Z [INFO] agent.router.manager: shutting down -42:47.824 [INFO] TestAgent: consul server down -42:47.824 [INFO] TestAgent: shutdown complete -42:47.824 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13816 network=tcp -42:47.824 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13816 network=udp -42:47.825 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13817 network=tcp -42:48.325 [INFO] TestAgent: Waiting for endpoints to shut down -42:48.325 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_ReconnectConfigWanDisabled (0.73s) +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:08.272Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:08.303Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:08.306 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:78cbf96b-a144-832d-9773-65a912624283 Address:127.0.0.1:25032}]" +34:08.306 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25032 [Follower]" leader= +34:08.307 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-78cbf96b-a144-832d-9773-65a912624283 127.0.0.1 +2021-08-13T15:34:08.307Z [INFO] agent.router: Initializing LAN area manager +34:08.307 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25041 network=udp +34:08.307 [INFO] TestAgent.server: Adding LAN server: server="Node-78cbf96b-a144-832d-9773-65a912624283 (Addr: tcp/127.0.0.1:25032) (DC: dc1)" +34:08.307 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25041 network=tcp +34:08.308 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25042 network=tcp +34:08.308 [INFO] TestAgent: started state syncer +34:08.309 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25033 network=tcp +34:08.367 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:08.367 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25032 [Candidate]" term=2 +34:08.367 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:08.367 [DEBUG] TestAgent.server.raft: vote granted: from=78cbf96b-a144-832d-9773-65a912624283 term=2 tally=1 +34:08.367 [INFO] TestAgent.server.raft: election won: tally=1 +34:08.367 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25032 [Leader]" +34:08.367 [INFO] TestAgent.server: cluster leadership acquired +34:08.368 [INFO] TestAgent.server: New leader elected: payload=Node-78cbf96b-a144-832d-9773-65a912624283 +34:08.368 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25032 +34:08.369 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:08.372 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:08.372 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:08.372 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:08.372 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:08.372 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:08.372 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-78cbf96b-a144-832d-9773-65a912624283 +34:08.372 [INFO] TestAgent.server: member joined, marking health alive: member=Node-78cbf96b-a144-832d-9773-65a912624283 +34:08.373 [INFO] TestAgent.server: federation state anti-entropy synced +34:08.544 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:08.545 [INFO] TestAgent: Synced node info +34:08.545 [DEBUG] TestAgent: Node info in sync +34:08.614 [INFO] TestAgent: Requesting shutdown +34:08.615 [INFO] TestAgent.server: shutting down server +34:08.615 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:08.615 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:08.615 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:08.615 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:08.615 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:08.615 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:08.615 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:08.615 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +2021-08-13T15:34:08.615Z [INFO] agent.router.manager: shutting down +34:08.615 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:08.615 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:08.615 [INFO] TestAgent: consul server down +34:08.615 [INFO] TestAgent: shutdown complete +34:08.615 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25041 network=tcp +34:08.615 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25041 network=udp +34:08.615 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25042 network=tcp +34:09.115 [INFO] TestAgent: Waiting for endpoints to shut down +34:09.116 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_ReconnectConfigWanDisabled (0.89s) === CONT TestAgent_Services_ExposeConfig -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=33)_compressed=false#01 -2022-09-15T21:42:47.867Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:47.893Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:47.896 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d9b6a00a-f374-d509-5b93-5ee5c4613154 Address:127.0.0.1:13863}]" -42:47.897 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d9b6a00a-f374-d509-5b93-5ee5c4613154.dc1 127.0.0.1 -42:47.898 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13863 [Follower]" leader= -42:47.899 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d9b6a00a-f374-d509-5b93-5ee5c4613154 127.0.0.1 -2022-09-15T21:42:47.899Z [INFO] agent.router: Initializing LAN area manager -42:47.900 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13830 network=udp -42:47.900 [INFO] TestAgent.server: Adding LAN server: server="Node-d9b6a00a-f374-d509-5b93-5ee5c4613154 (Addr: tcp/127.0.0.1:13863) (DC: dc1)" -42:47.900 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d9b6a00a-f374-d509-5b93-5ee5c4613154.dc1 area=wan -42:47.900 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13830 network=tcp -42:47.901 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13831 network=tcp -42:47.901 [INFO] TestAgent: started state syncer -42:47.902 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13864 network=tcp -42:47.948 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:47.948 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13863 [Candidate]" term=2 -42:47.949 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:47.949 [DEBUG] TestAgent.server.raft: vote granted: from=d9b6a00a-f374-d509-5b93-5ee5c4613154 term=2 tally=1 -42:47.949 [INFO] TestAgent.server.raft: election won: tally=1 -42:47.949 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13863 [Leader]" -42:47.949 [INFO] TestAgent.server: cluster leadership acquired -42:47.949 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13863 -42:47.954 [INFO] TestAgent.server: New leader elected: payload=Node-d9b6a00a-f374-d509-5b93-5ee5c4613154 -42:47.956 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:47.968 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:47.968 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:47.968 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:47.968 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:47.968 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:47.968 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d9b6a00a-f374-d509-5b93-5ee5c4613154 -42:47.968 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d9b6a00a-f374-d509-5b93-5ee5c4613154 -42:47.982 [INFO] TestAgent.server: federation state anti-entropy synced -42:47.999 [INFO] TestAgent: Requesting shutdown -42:47.999 [INFO] TestAgent.server: shutting down server -42:47.999 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:47.999 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:47.999 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:47.999 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:47.999 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:47.999 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:47.999 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:47.999 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:47.999 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:47.999 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:48.000 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:48.000 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:48.000Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:48.000Z [INFO] agent.router.manager: shutting down -42:48.000 [INFO] TestAgent: consul server down -42:48.000 [INFO] TestAgent: shutdown complete -42:48.000 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13830 network=tcp -42:48.000 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13830 network=udp -42:48.000 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13831 network=tcp -42:48.501 [INFO] TestAgent: Waiting for endpoints to shut down -42:48.501 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_RPCPing (0.65s) -=== CONT TestAgent_HostBadACL -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._tcp_(qType:=33)_compressed=true#01 -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:47.459Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:47.518Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:47.521 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9e2e453e-0f76-f7d2-1af7-6ada33501428 Address:127.0.0.1:13814}]" -42:47.522 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13814 [Follower]" leader= -42:47.523 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9e2e453e-0f76-f7d2-1af7-6ada33501428.dc1 127.0.0.1 -42:47.524 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9e2e453e-0f76-f7d2-1af7-6ada33501428 127.0.0.1 -2022-09-15T21:42:47.524Z [INFO] agent.router: Initializing LAN area manager -42:47.525 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13837 network=udp -42:47.525 [INFO] TestAgent.server: Adding LAN server: server="Node-9e2e453e-0f76-f7d2-1af7-6ada33501428 (Addr: tcp/127.0.0.1:13814) (DC: dc1)" -42:47.525 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9e2e453e-0f76-f7d2-1af7-6ada33501428.dc1 area=wan -42:47.525 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13837 network=tcp -42:47.526 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13838 network=tcp -42:47.527 [INFO] TestAgent: started state syncer -42:47.527 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13815 network=tcp -42:47.589 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:47.590 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13814 [Candidate]" term=2 -42:47.591 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:47.591 [DEBUG] TestAgent.server.raft: vote granted: from=9e2e453e-0f76-f7d2-1af7-6ada33501428 term=2 tally=1 -42:47.591 [INFO] TestAgent.server.raft: election won: tally=1 -42:47.591 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13814 [Leader]" -42:47.591 [INFO] TestAgent.server: cluster leadership acquired -42:47.591 [INFO] TestAgent.server: New leader elected: payload=Node-9e2e453e-0f76-f7d2-1af7-6ada33501428 -42:47.592 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13814 -42:47.593 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:47.596 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:47.596 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:47.596 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:47.596 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:47.596 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:47.596 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9e2e453e-0f76-f7d2-1af7-6ada33501428 -42:47.596 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9e2e453e-0f76-f7d2-1af7-6ada33501428 -42:47.598 [INFO] TestAgent.server: federation state anti-entropy synced -42:47.649 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:47.652 [INFO] TestAgent: Synced node info -42:47.653 [INFO] TestAgent: Synced service: service=redis -42:47.653 [DEBUG] TestAgent: Check in sync: check=redis-check -42:47.654 [INFO] TestAgent: Synced check: check=node-check -42:47.930 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:47.930 [DEBUG] TestAgent: Node info in sync -42:47.930 [DEBUG] TestAgent: Service in sync: service=redis -42:47.930 [DEBUG] TestAgent: Check in sync: check=redis-check -42:47.930 [DEBUG] TestAgent: Check in sync: check=node-check -42:48.161 [DEBUG] TestAgent: Registered node: node="Node-level check" -42:48.161 [DEBUG] TestAgent: Registered node: node="Serf Health Status" -42:48.161 [DEBUG] TestAgent: Registered node: node="Service-level check" -42:48.161 [INFO] TestAgent: Sync in progress: iteration=1 -42:48.161 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:48.161 [DEBUG] TestAgent: Node info in sync -42:48.161 [DEBUG] TestAgent: Service in sync: service=redis -42:48.161 [DEBUG] TestAgent: Check in sync: check=redis-check -42:48.161 [DEBUG] TestAgent: Check in sync: check=node-check -42:48.161 [INFO] TestAgent: Sync in progress: iteration=2 -42:48.166 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:48.166 [DEBUG] TestAgent: Node info in sync -42:48.166 [DEBUG] TestAgent: Service in sync: service=redis -42:48.166 [DEBUG] TestAgent: Check in sync: check=redis-check -42:48.166 [DEBUG] TestAgent: Check in sync: check=node-check -42:48.166 [INFO] TestAgent: Sync in progress: iteration=3 -42:48.166 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:48.166 [DEBUG] TestAgent: Node info in sync -42:48.166 [DEBUG] TestAgent: Service in sync: service=redis -42:48.166 [DEBUG] TestAgent: Check in sync: check=redis-check -42:48.166 [DEBUG] TestAgent: Check in sync: check=node-check -42:48.166 [INFO] TestAgent: Sync in progress: iteration=4 -42:48.166 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:48.166 [DEBUG] TestAgent: Node info in sync -42:48.166 [DEBUG] TestAgent: Service in sync: service=redis -42:48.166 [DEBUG] TestAgent: Check in sync: check=redis-check -42:48.166 [DEBUG] TestAgent: Check in sync: check=node-check -42:48.166 [INFO] TestAgent: Sync in progress: iteration=5 -42:48.166 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:48.166 [DEBUG] TestAgent: Node info in sync -42:48.166 [DEBUG] TestAgent: Service in sync: service=redis -42:48.166 [DEBUG] TestAgent: Check in sync: check=redis-check -42:48.166 [DEBUG] TestAgent: Check in sync: check=node-check -42:48.166 [INFO] TestAgent: Sync in progress: iteration=6 -42:48.166 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:48.166 [DEBUG] TestAgent: Node info in sync -42:48.166 [DEBUG] TestAgent: Service in sync: service=redis -42:48.166 [DEBUG] TestAgent: Check in sync: check=redis-check -42:48.166 [DEBUG] TestAgent: Check in sync: check=node-check -42:48.166 [INFO] TestAgent: Sync in progress: iteration=7 -42:48.167 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:48.167 [DEBUG] TestAgent: Node info in sync -42:48.167 [DEBUG] TestAgent: Service in sync: service=redis -42:48.167 [DEBUG] TestAgent: Check in sync: check=redis-check -42:48.167 [DEBUG] TestAgent: Check in sync: check=node-check -42:48.167 [INFO] TestAgent: Sync in progress: iteration=8 -42:48.167 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:48.167 [DEBUG] TestAgent: Node info in sync -42:48.167 [DEBUG] TestAgent: Service in sync: service=redis -42:48.167 [DEBUG] TestAgent: Check in sync: check=node-check -42:48.167 [DEBUG] TestAgent: Check in sync: check=redis-check -42:48.167 [INFO] TestAgent: Sync in progress: iteration=9 -42:48.167 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:48.167 [DEBUG] TestAgent: Node info in sync -42:48.167 [DEBUG] TestAgent: Service in sync: service=redis -42:48.167 [DEBUG] TestAgent: Check in sync: check=redis-check -42:48.167 [DEBUG] TestAgent: Check in sync: check=node-check -42:48.167 [INFO] TestAgent: Sync in progress: iteration=10 -42:48.167 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:48.167 [DEBUG] TestAgent: Node info in sync -42:48.167 [DEBUG] TestAgent: Service in sync: service=redis -42:48.167 [DEBUG] TestAgent: Check in sync: check=redis-check -42:48.167 [DEBUG] TestAgent: Check in sync: check=node-check -42:48.167 [INFO] TestAgent: Requesting shutdown -42:48.167 [INFO] TestAgent.server: shutting down server -42:48.167 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:48.167 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:48.167 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:48.167 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:48.168 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:48.168 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:48.168 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:48.168 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:48.168 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:48.168 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -42:48.168 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2022-09-15T21:42:48.174Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:48.174Z [INFO] agent.router.manager: shutting down -42:48.174 [INFO] TestAgent: consul server down -42:48.174 [INFO] TestAgent: shutdown complete -42:48.174 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13837 network=tcp -42:48.174 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13837 network=udp -42:48.174 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13838 network=tcp -42:48.683 [INFO] TestAgent: Waiting for endpoints to shut down -42:48.683 [INFO] TestAgent: Endpoints down +2021-08-13T15:34:08.280Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:08.299Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:08.301 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e3e1abef-fbc6-1398-65c9-889328d5e557 Address:127.0.0.1:25053}]" +34:08.301 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25053 [Follower]" leader= +34:08.302 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e3e1abef-fbc6-1398-65c9-889328d5e557.dc1 127.0.0.1 +34:08.303 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e3e1abef-fbc6-1398-65c9-889328d5e557 127.0.0.1 +2021-08-13T15:34:08.303Z [INFO] agent.router: Initializing LAN area manager +34:08.303 [INFO] TestAgent.server: Adding LAN server: server="Node-e3e1abef-fbc6-1398-65c9-889328d5e557 (Addr: tcp/127.0.0.1:25053) (DC: dc1)" +34:08.303 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e3e1abef-fbc6-1398-65c9-889328d5e557.dc1 area=wan +34:08.303 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25048 network=udp +34:08.303 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25048 network=tcp +34:08.304 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25049 network=tcp +34:08.304 [INFO] TestAgent: started state syncer +34:08.304 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25054 network=tcp +34:08.369 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:08.369 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25053 [Candidate]" term=2 +34:08.369 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:08.369 [DEBUG] TestAgent.server.raft: vote granted: from=e3e1abef-fbc6-1398-65c9-889328d5e557 term=2 tally=1 +34:08.369 [INFO] TestAgent.server.raft: election won: tally=1 +34:08.369 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25053 [Leader]" +34:08.369 [INFO] TestAgent.server: cluster leadership acquired +34:08.370 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25053 +34:08.371 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:08.371 [INFO] TestAgent.server: New leader elected: payload=Node-e3e1abef-fbc6-1398-65c9-889328d5e557 +34:08.373 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:08.373 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:08.373 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:08.373 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:08.373 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:08.373 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e3e1abef-fbc6-1398-65c9-889328d5e557 +34:08.373 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e3e1abef-fbc6-1398-65c9-889328d5e557 +34:08.374 [INFO] TestAgent.server: federation state anti-entropy synced +34:08.587 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:08.588 [INFO] TestAgent: Synced node info +34:08.632 [INFO] TestAgent: Requesting shutdown +34:08.632 [INFO] TestAgent.server: shutting down server +34:08.632 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:08.632 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:08.632 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:08.632 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:08.632 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:08.632 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:08.632 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:08.632 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:08.632 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:08.632 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:08.632 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:08.632Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:08.632Z [INFO] agent.router.manager: shutting down +34:08.632 [INFO] TestAgent: consul server down +34:08.632 [INFO] TestAgent: shutdown complete +34:08.632 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25048 network=tcp +34:08.632 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25048 network=udp +34:08.632 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25049 network=tcp +34:09.132 [INFO] TestAgent: Waiting for endpoints to shut down +34:09.132 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_TokenStore (0.89s) +=== CONT TestPreparedQuery_Delete +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:08.501Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:08.501Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:08.519Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:08.519Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:08.521 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d4e02c47-cbad-f926-b3b5-5baa707ee239 Address:127.0.0.1:25060}]" +34:08.522 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25060 [Follower]" leader= +34:08.522 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d4e02c47-cbad-f926-b3b5-5baa707ee239.dc1 127.0.0.1 +34:08.523 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d4e02c47-cbad-f926-b3b5-5baa707ee239 127.0.0.1 +2021-08-13T15:34:08.523Z [INFO] agent.router: Initializing LAN area manager +34:08.523 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25069 network=udp +34:08.523 [INFO] TestAgent.server: Adding LAN server: server="Node-d4e02c47-cbad-f926-b3b5-5baa707ee239 (Addr: tcp/127.0.0.1:25060) (DC: dc1)" +34:08.523 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d4e02c47-cbad-f926-b3b5-5baa707ee239.dc1 area=wan +34:08.524 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25069 network=tcp +34:08.525 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25070 network=tcp +34:08.525 [INFO] TestAgent: started state syncer +34:08.525 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25061 network=tcp +34:08.574 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:08.575 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:08.575 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25060 [Candidate]" term=2 +34:08.575 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:08.575 [DEBUG] TestAgent.server.raft: vote granted: from=d4e02c47-cbad-f926-b3b5-5baa707ee239 term=2 tally=1 +34:08.575 [INFO] TestAgent.server.raft: election won: tally=1 +34:08.575 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25060 [Leader]" +34:08.575 [INFO] TestAgent.server: cluster leadership acquired +34:08.576 [INFO] TestAgent.server: New leader elected: payload=Node-d4e02c47-cbad-f926-b3b5-5baa707ee239 +34:08.576 [INFO] TestAgent.server: initializing acls +34:08.576 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:08.576 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:08.577 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:08.577 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:08.577 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:08.577 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:08.578 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-d4e02c47-cbad-f926-b3b5-5baa707ee239 +34:08.578 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-d4e02c47-cbad-f926-b3b5-5baa707ee239.dc1 +34:08.579 [INFO] TestAgent.server: Updating LAN server: server="Node-d4e02c47-cbad-f926-b3b5-5baa707ee239 (Addr: tcp/127.0.0.1:25060) (DC: dc1)" +34:08.579 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-d4e02c47-cbad-f926-b3b5-5baa707ee239.dc1 area=wan +34:08.579 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:08.582 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:08.582 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:08.582 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:08.582 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:08.582 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:08.582 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d4e02c47-cbad-f926-b3b5-5baa707ee239 +34:08.582 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d4e02c47-cbad-f926-b3b5-5baa707ee239 +34:08.583 [INFO] TestAgent.server: federation state anti-entropy synced +34:08.583 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d4e02c47-cbad-f926-b3b5-5baa707ee239 +34:08.850 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-d4e02c47-cbad-f926-b3b5-5baa707ee239 +34:08.850 [INFO] TestAgent: Requesting shutdown +34:08.850 [INFO] TestAgent.server: shutting down server +34:08.850 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:08.850 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:08.850 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:08.850 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:08.850 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:08.850 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:08.850 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:08.850 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:08.850 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:08.850 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:08.850 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:08.850 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:08.850 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:08.850 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:08.851 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:08.851 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:08.851Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:08.851Z [INFO] agent.router.manager: shutting down +34:08.851 [INFO] TestAgent: consul server down +34:08.851 [INFO] TestAgent: shutdown complete +34:08.851 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25069 network=tcp +34:08.851 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25069 network=udp +34:08.851 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25070 network=tcp +34:09.351 [INFO] TestAgent: Waiting for endpoints to shut down +34:09.351 [INFO] TestAgent: Endpoints down +=== RUN TestSnapshot_Options/PUT#01 +2021-08-13T15:34:08.256Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:08.295Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:08.298 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:dbe72962-6dcc-3313-f0fd-320fdc6c05c4 Address:127.0.0.1:25039}]" +34:08.298 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25039 [Follower]" leader= +34:08.303 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-dbe72962-6dcc-3313-f0fd-320fdc6c05c4.dc1 127.0.0.1 +34:08.304 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-dbe72962-6dcc-3313-f0fd-320fdc6c05c4 127.0.0.1 +2021-08-13T15:34:08.304Z [INFO] agent.router: Initializing LAN area manager +34:08.304 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25020 network=udp +34:08.304 [INFO] TestAgent.server: Adding LAN server: server="Node-dbe72962-6dcc-3313-f0fd-320fdc6c05c4 (Addr: tcp/127.0.0.1:25039) (DC: dc1)" +34:08.304 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-dbe72962-6dcc-3313-f0fd-320fdc6c05c4.dc1 area=wan +34:08.304 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25020 network=tcp +34:08.305 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25021 network=tcp +34:08.305 [INFO] TestAgent: started state syncer +34:08.305 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25040 network=tcp +34:08.353 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:08.353 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25039 [Candidate]" term=2 +34:08.354 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:08.354 [DEBUG] TestAgent.server.raft: vote granted: from=dbe72962-6dcc-3313-f0fd-320fdc6c05c4 term=2 tally=1 +34:08.354 [INFO] TestAgent.server.raft: election won: tally=1 +34:08.354 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25039 [Leader]" +34:08.354 [INFO] TestAgent.server: cluster leadership acquired +34:08.354 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25039 +34:08.356 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:08.358 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:08.358 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:08.358 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:08.358 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:08.358 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:08.358 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-dbe72962-6dcc-3313-f0fd-320fdc6c05c4 +34:08.358 [INFO] TestAgent.server: member joined, marking health alive: member=Node-dbe72962-6dcc-3313-f0fd-320fdc6c05c4 +34:08.359 [INFO] TestAgent.server: New leader elected: payload=Node-dbe72962-6dcc-3313-f0fd-320fdc6c05c4 +34:08.360 [INFO] TestAgent.server: federation state anti-entropy synced +34:08.399 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:08.400 [INFO] TestAgent: Synced node info +34:08.425 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:08.425 [DEBUG] TestAgent: Node info in sync +34:08.426 [INFO] TestAgent: Synced service: service=redis +34:08.427 [DEBUG] TestAgent: Check in sync: check=redis-check +34:08.427 [INFO] TestAgent: Synced check: check=node-check +34:08.928 [DEBUG] TestAgent: Registered node: node="Node-level check" +34:08.928 [DEBUG] TestAgent: Registered node: node="Serf Health Status" +34:08.928 [DEBUG] TestAgent: Registered node: node="Service-level check" +34:08.928 [INFO] TestAgent: Sync in progress: iteration=1 +34:08.928 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:08.928 [DEBUG] TestAgent: Node info in sync +34:08.928 [DEBUG] TestAgent: Service in sync: service=redis +34:08.928 [DEBUG] TestAgent: Check in sync: check=redis-check +34:08.928 [DEBUG] TestAgent: Check in sync: check=node-check +34:08.928 [INFO] TestAgent: Sync in progress: iteration=2 +34:08.929 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:08.929 [DEBUG] TestAgent: Node info in sync +34:08.929 [DEBUG] TestAgent: Service in sync: service=redis +34:08.929 [DEBUG] TestAgent: Check in sync: check=redis-check +34:08.929 [DEBUG] TestAgent: Check in sync: check=node-check +34:08.929 [INFO] TestAgent: Sync in progress: iteration=3 +34:08.929 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:08.929 [DEBUG] TestAgent: Node info in sync +34:08.929 [DEBUG] TestAgent: Service in sync: service=redis +34:08.929 [DEBUG] TestAgent: Check in sync: check=node-check +34:08.929 [DEBUG] TestAgent: Check in sync: check=redis-check +34:08.929 [INFO] TestAgent: Sync in progress: iteration=4 +34:08.929 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:08.929 [DEBUG] TestAgent: Node info in sync +34:08.929 [DEBUG] TestAgent: Service in sync: service=redis +34:08.929 [DEBUG] TestAgent: Check in sync: check=redis-check +34:08.929 [DEBUG] TestAgent: Check in sync: check=node-check +34:08.929 [INFO] TestAgent: Sync in progress: iteration=5 +34:08.929 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:08.929 [DEBUG] TestAgent: Node info in sync +34:08.929 [DEBUG] TestAgent: Service in sync: service=redis +34:08.929 [DEBUG] TestAgent: Check in sync: check=redis-check +34:08.929 [DEBUG] TestAgent: Check in sync: check=node-check +34:08.929 [INFO] TestAgent: Sync in progress: iteration=6 +34:08.929 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:08.929 [DEBUG] TestAgent: Node info in sync +34:08.929 [DEBUG] TestAgent: Service in sync: service=redis +34:08.929 [DEBUG] TestAgent: Check in sync: check=redis-check +34:08.929 [DEBUG] TestAgent: Check in sync: check=node-check +34:08.929 [INFO] TestAgent: Sync in progress: iteration=7 +34:08.929 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:08.929 [DEBUG] TestAgent: Node info in sync +34:08.929 [DEBUG] TestAgent: Service in sync: service=redis +34:08.929 [DEBUG] TestAgent: Check in sync: check=redis-check +34:08.929 [DEBUG] TestAgent: Check in sync: check=node-check +34:08.929 [INFO] TestAgent: Sync in progress: iteration=8 +34:08.929 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:08.929 [DEBUG] TestAgent: Node info in sync +34:08.929 [DEBUG] TestAgent: Service in sync: service=redis +34:08.929 [DEBUG] TestAgent: Check in sync: check=redis-check +34:08.929 [DEBUG] TestAgent: Check in sync: check=node-check +34:08.929 [INFO] TestAgent: Sync in progress: iteration=9 +34:08.929 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:08.929 [DEBUG] TestAgent: Node info in sync +34:08.929 [DEBUG] TestAgent: Service in sync: service=redis +34:08.929 [DEBUG] TestAgent: Check in sync: check=node-check +34:08.929 [DEBUG] TestAgent: Check in sync: check=redis-check +34:08.929 [INFO] TestAgent: Sync in progress: iteration=10 +34:08.929 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:08.929 [DEBUG] TestAgent: Node info in sync +34:08.929 [DEBUG] TestAgent: Service in sync: service=redis +34:08.929 [DEBUG] TestAgent: Check in sync: check=redis-check +34:08.929 [DEBUG] TestAgent: Check in sync: check=node-check +34:08.929 [INFO] TestAgent: Requesting shutdown +34:08.930 [INFO] TestAgent.server: shutting down server +34:08.930 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:08.930 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:08.930 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:08.930 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:08.930 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:08.930 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:08.930 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:08.930 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:08.930 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:08.930 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:08.930 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:08.930Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:08.930Z [INFO] agent.router.manager: shutting down +34:08.931 [INFO] TestAgent: consul server down +34:08.931 [INFO] TestAgent: shutdown complete +34:08.931 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25020 network=tcp +34:08.931 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25020 network=udp +34:08.931 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25021 network=tcp +34:09.431 [INFO] TestAgent: Waiting for endpoints to shut down +34:09.431 [INFO] TestAgent: Endpoints down === RUN TestAgent_IndexChurn/with_tags -=== RUN TestAgent_JoinWAN_viaMeshGateway/dc1_to_dc2 -2022-09-15T21:42:47.785Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:47.816Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:47.819 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4cb25108-e175-8167-21f6-abc7ae43d32b Address:127.0.0.1:13828}]" -42:47.819 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13828 [Follower]" leader= -42:47.820 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4cb25108-e175-8167-21f6-abc7ae43d32b.dc1 127.0.0.1 -42:47.821 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4cb25108-e175-8167-21f6-abc7ae43d32b 127.0.0.1 -2022-09-15T21:42:47.821Z [INFO] agent.router: Initializing LAN area manager -42:47.821 [INFO] TestAgent.server: Adding LAN server: server="Node-4cb25108-e175-8167-21f6-abc7ae43d32b (Addr: tcp/127.0.0.1:13828) (DC: dc1)" -42:47.821 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13851 network=udp -42:47.821 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4cb25108-e175-8167-21f6-abc7ae43d32b.dc1 area=wan -42:47.822 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13851 network=tcp -42:47.822 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13852 network=tcp -42:47.823 [INFO] TestAgent: started state syncer -42:47.823 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13829 network=tcp -42:47.870 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:47.870 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13828 [Candidate]" term=2 -42:47.870 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:47.870 [DEBUG] TestAgent.server.raft: vote granted: from=4cb25108-e175-8167-21f6-abc7ae43d32b term=2 tally=1 -42:47.870 [INFO] TestAgent.server.raft: election won: tally=1 -42:47.870 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13828 [Leader]" -42:47.870 [INFO] TestAgent.server: cluster leadership acquired -42:47.870 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13828 -42:47.871 [INFO] TestAgent.server: New leader elected: payload=Node-4cb25108-e175-8167-21f6-abc7ae43d32b -42:47.872 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:47.875 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:47.875 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:47.875 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:47.875 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:47.875 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:47.875 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4cb25108-e175-8167-21f6-abc7ae43d32b -42:47.875 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4cb25108-e175-8167-21f6-abc7ae43d32b -42:47.876 [INFO] TestAgent.server: federation state anti-entropy synced -42:48.109 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:48.110 [INFO] TestAgent: Synced node info -42:48.236 [INFO] TestAgent: Requesting shutdown -42:48.236 [INFO] TestAgent.server: shutting down server -42:48.236 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:48.236 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:48.236 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:48.236 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:48.236 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:48.237 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:48.237 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:48.237 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:48.237 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:48.237 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:48.237 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:48.238Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:48.238Z [INFO] agent.router.manager: shutting down -42:48.238 [INFO] TestAgent: consul server down -42:48.238 [INFO] TestAgent: shutdown complete -42:48.238 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13851 network=tcp -42:48.238 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13851 network=udp -42:48.238 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13852 network=tcp -42:48.740 [INFO] TestAgent: Waiting for endpoints to shut down -42:48.740 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_TokenStore (1.01s) +2021-08-13T15:34:08.607Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:08.623Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:08.625 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5679c3cb-8a6d-4afe-a00b-f446aba830dc Address:127.0.0.1:25081}]" +34:08.626 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5679c3cb-8a6d-4afe-a00b-f446aba830dc.dc1 127.0.0.1 +34:08.627 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25081 [Follower]" leader= +34:08.627 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5679c3cb-8a6d-4afe-a00b-f446aba830dc 127.0.0.1 +2021-08-13T15:34:08.627Z [INFO] agent.router: Initializing LAN area manager +34:08.627 [INFO] TestAgent.server: Adding LAN server: server="Node-5679c3cb-8a6d-4afe-a00b-f446aba830dc (Addr: tcp/127.0.0.1:25081) (DC: dc1)" +34:08.627 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5679c3cb-8a6d-4afe-a00b-f446aba830dc.dc1 area=wan +34:08.627 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25062 network=udp +34:08.627 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25062 network=tcp +34:08.628 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25063 network=tcp +34:08.628 [INFO] TestAgent: started state syncer +34:08.628 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25082 network=tcp +34:08.675 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:08.675 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25081 [Candidate]" term=2 +34:08.675 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:08.675 [DEBUG] TestAgent.server.raft: vote granted: from=5679c3cb-8a6d-4afe-a00b-f446aba830dc term=2 tally=1 +34:08.675 [INFO] TestAgent.server.raft: election won: tally=1 +34:08.675 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25081 [Leader]" +34:08.676 [INFO] TestAgent.server: cluster leadership acquired +34:08.676 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25081 +34:08.677 [INFO] TestAgent.server: New leader elected: payload=Node-5679c3cb-8a6d-4afe-a00b-f446aba830dc +34:08.678 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:08.685 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:08.685 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:08.685 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:08.685 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:08.685 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:08.685 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5679c3cb-8a6d-4afe-a00b-f446aba830dc +34:08.685 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5679c3cb-8a6d-4afe-a00b-f446aba830dc +34:08.687 [INFO] TestAgent.server: federation state anti-entropy synced +34:08.967 [INFO] TestAgent: Requesting shutdown +34:08.967 [INFO] TestAgent.server: shutting down server +34:08.968 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:08.968 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:08.968 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:08.968 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:08.968 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:08.968 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:08.968 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:08.968 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:08.968 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:08.968 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:08.968 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:08.968 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:08.968Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:08.968Z [INFO] agent.router.manager: shutting down +34:08.969 [INFO] TestAgent: consul server down +34:08.969 [INFO] TestAgent: shutdown complete +34:08.969 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25062 network=tcp +34:08.969 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25062 network=udp +34:08.969 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25063 network=tcp +34:09.469 [INFO] TestAgent: Waiting for endpoints to shut down +34:09.469 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_RPCPing (0.89s) +=== CONT TestAgent_HostBadACL +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:08.477Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:08.494Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:08.497 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b41f1a21-c90f-f012-9244-2c80b3164d98 Address:127.0.0.1:25067}]" +34:08.497 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25067 [Follower]" leader= +34:08.498 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b41f1a21-c90f-f012-9244-2c80b3164d98.dc1 127.0.0.1 +34:08.503 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b41f1a21-c90f-f012-9244-2c80b3164d98 127.0.0.1 +2021-08-13T15:34:08.503Z [INFO] agent.router: Initializing LAN area manager +34:08.503 [INFO] TestAgent.server: Adding LAN server: server="Node-b41f1a21-c90f-f012-9244-2c80b3164d98 (Addr: tcp/127.0.0.1:25067) (DC: dc1)" +34:08.504 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b41f1a21-c90f-f012-9244-2c80b3164d98.dc1 area=wan +34:08.504 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25055 network=udp +34:08.504 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25055 network=tcp +34:08.505 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25056 network=tcp +34:08.505 [INFO] TestAgent: started state syncer +34:08.505 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25068 network=tcp +34:08.546 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:08.546 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25067 [Candidate]" term=2 +34:08.547 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:08.547 [DEBUG] TestAgent.server.raft: vote granted: from=b41f1a21-c90f-f012-9244-2c80b3164d98 term=2 tally=1 +34:08.547 [INFO] TestAgent.server.raft: election won: tally=1 +34:08.547 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25067 [Leader]" +34:08.547 [INFO] TestAgent.server: cluster leadership acquired +34:08.547 [INFO] TestAgent.server: New leader elected: payload=Node-b41f1a21-c90f-f012-9244-2c80b3164d98 +34:08.547 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25067 +34:08.549 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:08.551 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:08.552 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:08.552 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:08.552 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:08.552 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:08.552 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b41f1a21-c90f-f012-9244-2c80b3164d98 +34:08.552 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b41f1a21-c90f-f012-9244-2c80b3164d98 +34:08.553 [INFO] TestAgent.server: federation state anti-entropy synced +34:08.752 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:08.752 [INFO] TestAgent: Synced node info +34:09.085 [WARN] TestAgent.server.rpc: Attempting to apply large raft entry: size_in_bytes=4719032 +34:09.098 [INFO] TestAgent: Requesting shutdown +34:09.099 [INFO] TestAgent.server: shutting down server +34:09.099 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:09.099 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:09.099 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:09.099 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:09.099 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:09.099 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:09.099 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:09.099 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:09.099 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:09.099 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:09.099 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:09.099Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:09.099Z [INFO] agent.router.manager: shutting down +34:09.099 [INFO] TestAgent: consul server down +34:09.099 [INFO] TestAgent: shutdown complete +34:09.099 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25055 network=tcp +34:09.099 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25055 network=udp +34:09.099 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25056 network=tcp +34:09.599 [INFO] TestAgent: Waiting for endpoints to shut down +34:09.599 [INFO] TestAgent: Endpoints down +--- PASS: TestTxnEndpoint_Bad_Size_Net (6.04s) + --- PASS: TestTxnEndpoint_Bad_Size_Net/exceeds_default_limits (1.14s) + --- PASS: TestTxnEndpoint_Bad_Size_Net/exceeds_configured_max_txn_len (1.23s) + --- PASS: TestTxnEndpoint_Bad_Size_Net/exceeds_default_max_kv_value_size (1.37s) + --- PASS: TestTxnEndpoint_Bad_Size_Net/allowed (1.15s) + --- PASS: TestTxnEndpoint_Bad_Size_Net/allowed_kv_max_backward_compatible (1.14s) === CONT TestAgent_Host -=== RUN TestAgent_JoinWAN_viaMeshGateway/dc1_to_dc3 -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestAgent_JoinWAN_viaMeshGateway/dc2_to_dc1 -=== RUN TestAgent_JoinWAN_viaMeshGateway/dc2_to_dc3 -=== RUN TestAgent_JoinWAN_viaMeshGateway/dc3_to_dc1 -=== RUN TestAgent_JoinWAN_viaMeshGateway/dc3_to_dc2 -[WARN] freeport: 4 out of 4 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:48.352Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:48.376Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:48.380 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:000582a8-520a-1106-02b1-691a14cb52b0 Address:127.0.0.1:13880}]" -42:48.381 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-000582a8-520a-1106-02b1-691a14cb52b0.dc1 127.0.0.1 -42:48.382 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13880 [Follower]" leader= -42:48.383 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-000582a8-520a-1106-02b1-691a14cb52b0 127.0.0.1 -2022-09-15T21:42:48.383Z [INFO] agent.router: Initializing LAN area manager -42:48.383 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13844 network=udp -42:48.384 [INFO] TestAgent.server: Adding LAN server: server="Node-000582a8-520a-1106-02b1-691a14cb52b0 (Addr: tcp/127.0.0.1:13880) (DC: dc1)" -42:48.384 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-000582a8-520a-1106-02b1-691a14cb52b0.dc1 area=wan -42:48.384 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13844 network=tcp -42:48.385 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13845 network=tcp -42:48.385 [INFO] TestAgent: started state syncer -42:48.385 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13881 network=tcp -42:48.420 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:48.420 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13880 [Candidate]" term=2 -42:48.422 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:48.422 [DEBUG] TestAgent.server.raft: vote granted: from=000582a8-520a-1106-02b1-691a14cb52b0 term=2 tally=1 -42:48.422 [INFO] TestAgent.server.raft: election won: tally=1 -42:48.422 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13880 [Leader]" -42:48.423 [INFO] TestAgent.server: cluster leadership acquired -42:48.423 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13880 -42:48.424 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:48.426 [INFO] TestAgent.server: New leader elected: payload=Node-000582a8-520a-1106-02b1-691a14cb52b0 -42:48.431 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:48.431 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:48.431 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:48.431 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:48.431 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:48.431 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-000582a8-520a-1106-02b1-691a14cb52b0 -42:48.431 [INFO] TestAgent.server: member joined, marking health alive: member=Node-000582a8-520a-1106-02b1-691a14cb52b0 -42:48.433 [INFO] TestAgent.server: federation state anti-entropy synced -42:48.525 [INFO] TestAgent: Requesting shutdown -42:48.525 [INFO] TestAgent.server: shutting down server -42:48.525 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:48.525 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:48.525 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:48.525 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:48.525 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:48.526 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:48.526 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:48.526 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:48.526 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:48.526 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -42:48.526 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:48.526 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2022-09-15T21:42:48.526Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:48.526Z [INFO] agent.router.manager: shutting down -42:48.526 [INFO] TestAgent: consul server down -42:48.526 [INFO] TestAgent: shutdown complete -42:48.526 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13844 network=tcp -42:48.526 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13844 network=udp -42:48.526 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13845 network=tcp -42:49.042 [INFO] TestAgent: Waiting for endpoints to shut down -42:49.042 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Services_ExposeConfig (0.72s) +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:09.190Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:09.210Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:09.213 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:67d84ffc-2a4e-8483-0e75-9eb48f1714d7 Address:127.0.0.1:25112}]" +34:09.213 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25112 [Follower]" leader= +34:09.214 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-67d84ffc-2a4e-8483-0e75-9eb48f1714d7.dc1 127.0.0.1 +34:09.215 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-67d84ffc-2a4e-8483-0e75-9eb48f1714d7 127.0.0.1 +2021-08-13T15:34:09.215Z [INFO] agent.router: Initializing LAN area manager +34:09.215 [INFO] TestAgent.server: Adding LAN server: server="Node-67d84ffc-2a4e-8483-0e75-9eb48f1714d7 (Addr: tcp/127.0.0.1:25112) (DC: dc1)" +34:09.215 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25090 network=udp +34:09.215 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-67d84ffc-2a4e-8483-0e75-9eb48f1714d7.dc1 area=wan +34:09.215 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25090 network=tcp +34:09.216 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25091 network=tcp +34:09.216 [INFO] TestAgent: started state syncer +34:09.217 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25100 network=tcp +34:09.268 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:09.268 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25112 [Candidate]" term=2 +34:09.269 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:09.269 [DEBUG] TestAgent.server.raft: vote granted: from=67d84ffc-2a4e-8483-0e75-9eb48f1714d7 term=2 tally=1 +34:09.269 [INFO] TestAgent.server.raft: election won: tally=1 +34:09.269 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25112 [Leader]" +34:09.269 [INFO] TestAgent.server: cluster leadership acquired +34:09.269 [INFO] TestAgent.server: New leader elected: payload=Node-67d84ffc-2a4e-8483-0e75-9eb48f1714d7 +34:09.269 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25112 +34:09.271 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:09.273 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:09.273 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:09.273 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:09.273 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:09.273 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:09.273 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-67d84ffc-2a4e-8483-0e75-9eb48f1714d7 +34:09.273 [INFO] TestAgent.server: member joined, marking health alive: member=Node-67d84ffc-2a4e-8483-0e75-9eb48f1714d7 +34:09.274 [INFO] TestAgent.server: federation state anti-entropy synced +34:09.298 [INFO] TestAgent: Requesting shutdown +34:09.298 [INFO] TestAgent.server: shutting down server +34:09.298 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:09.298 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:09.298 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:09.298 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:09.298 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:09.298 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:09.298 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:09.298 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:09.298 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:09.298 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:09.298 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:09.298 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:09.299Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:09.299Z [INFO] agent.router.manager: shutting down +34:09.299 [INFO] TestAgent: consul server down +34:09.299 [INFO] TestAgent: shutdown complete +34:09.299 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25090 network=tcp +34:09.299 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25090 network=udp +34:09.299 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25091 network=tcp +34:09.799 [INFO] TestAgent: Waiting for endpoints to shut down +34:09.799 [INFO] TestAgent: Endpoints down +2021-08-13T15:34:08.266Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:08.302Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:08.308 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:822a693f-d15c-eca1-c80b-887d5f40b49d Address:127.0.0.1:25046}]" +34:08.309 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-822a693f-d15c-eca1-c80b-887d5f40b49d.dc1 127.0.0.1 +34:08.310 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-822a693f-d15c-eca1-c80b-887d5f40b49d 127.0.0.1 +2021-08-13T15:34:08.310Z [INFO] agent.router: Initializing LAN area manager +34:08.310 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25034 network=udp +34:08.310 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25046 [Follower]" leader= +34:08.311 [INFO] TestAgent.server: Adding LAN server: server="Node-822a693f-d15c-eca1-c80b-887d5f40b49d (Addr: tcp/127.0.0.1:25046) (DC: dc1)" +34:08.311 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-822a693f-d15c-eca1-c80b-887d5f40b49d.dc1 area=wan +34:08.311 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25034 network=tcp +34:08.312 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25035 network=tcp +34:08.312 [INFO] TestAgent: started state syncer +34:08.312 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25047 network=tcp +34:08.373 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:08.373 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25046 [Candidate]" term=2 +34:08.373 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:08.373 [DEBUG] TestAgent.server.raft: vote granted: from=822a693f-d15c-eca1-c80b-887d5f40b49d term=2 tally=1 +34:08.373 [INFO] TestAgent.server.raft: election won: tally=1 +34:08.373 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25046 [Leader]" +34:08.373 [INFO] TestAgent.server: cluster leadership acquired +34:08.373 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25046 +34:08.374 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:08.375 [INFO] TestAgent.server: New leader elected: payload=Node-822a693f-d15c-eca1-c80b-887d5f40b49d +34:08.377 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:08.377 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:08.377 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:08.377 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:08.377 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:08.377 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-822a693f-d15c-eca1-c80b-887d5f40b49d +34:08.377 [INFO] TestAgent.server: member joined, marking health alive: member=Node-822a693f-d15c-eca1-c80b-887d5f40b49d +34:08.378 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:08.379 [INFO] TestAgent: Synced node info +34:08.379 [INFO] TestAgent.server: federation state anti-entropy synced +34:08.633 [INFO] TestAgent: Requesting shutdown +34:08.633 [INFO] TestAgent.server: shutting down server +34:08.633 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:08.633 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:08.633 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:08.633 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:08.633 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:08.633 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:08.633 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:08.633 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:08.633 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:08.633 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +34:08.633 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2021-08-13T15:34:08.633Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:08.633Z [INFO] agent.router.manager: shutting down +34:08.633 [INFO] TestAgent: consul server down +34:08.633 [INFO] TestAgent: shutdown complete +34:08.633 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25034 network=tcp +34:08.633 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25034 network=udp +34:08.633 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25035 network=tcp +34:09.134 [INFO] TestAgent: Waiting for endpoints to shut down +34:09.134 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_ReconnectConfigSettings (1.57s) +=== CONT TestSessionDestroy +2021-08-13T15:34:09.192Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:09.212Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:09.214 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0ec1fd2b-aa7d-474a-fdde-4f2681c0f873 Address:127.0.0.1:25088}]" +34:09.214 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25088 [Follower]" leader= +34:09.215 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0ec1fd2b-aa7d-474a-fdde-4f2681c0f873.dc1 127.0.0.1 +34:09.216 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0ec1fd2b-aa7d-474a-fdde-4f2681c0f873 127.0.0.1 +2021-08-13T15:34:09.216Z [INFO] agent.router: Initializing LAN area manager +34:09.216 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25097 network=udp +34:09.216 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0ec1fd2b-aa7d-474a-fdde-4f2681c0f873.dc1 area=wan +34:09.216 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25097 network=tcp +34:09.217 [INFO] TestAgent.server: Adding LAN server: server="Node-0ec1fd2b-aa7d-474a-fdde-4f2681c0f873 (Addr: tcp/127.0.0.1:25088) (DC: dc1)" +34:09.217 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25098 network=tcp +34:09.217 [INFO] TestAgent: started state syncer +34:09.218 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25089 network=tcp +34:09.278 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:09.278 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25088 [Candidate]" term=2 +34:09.278 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:09.278 [DEBUG] TestAgent.server.raft: vote granted: from=0ec1fd2b-aa7d-474a-fdde-4f2681c0f873 term=2 tally=1 +34:09.278 [INFO] TestAgent.server.raft: election won: tally=1 +34:09.278 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25088 [Leader]" +34:09.278 [INFO] TestAgent.server: cluster leadership acquired +34:09.278 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25088 +34:09.280 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:09.282 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:09.282 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:09.282 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:09.282 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:09.282 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:09.282 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0ec1fd2b-aa7d-474a-fdde-4f2681c0f873 +34:09.282 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0ec1fd2b-aa7d-474a-fdde-4f2681c0f873 +34:09.283 [INFO] TestAgent.server: New leader elected: payload=Node-0ec1fd2b-aa7d-474a-fdde-4f2681c0f873 +34:09.284 [INFO] TestAgent.server: federation state anti-entropy synced +34:09.311 [WARN] TestAgent.server: endpoint injected; this should only be used for testing +34:09.312 [INFO] TestAgent: Requesting shutdown +34:09.312 [INFO] TestAgent.server: shutting down server +34:09.312 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:09.312 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:09.312 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:09.312 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:09.312 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:09.312 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:09.312 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:09.312 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:09.312 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:09.312 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:09.312 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:09.312 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:09.312Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:09.312Z [INFO] agent.router.manager: shutting down +34:09.313 [INFO] TestAgent: consul server down +34:09.313 [INFO] TestAgent: shutdown complete +34:09.313 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25097 network=tcp +34:09.313 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25097 network=udp +34:09.313 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25098 network=tcp +34:09.813 [INFO] TestAgent: Waiting for endpoints to shut down +34:09.814 [INFO] TestAgent: Endpoints down +--- PASS: TestPreparedQuery_Delete (0.68s) === CONT TestAgentConnectAuthorize_defaultAllow -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:48.670Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:48.711Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:48.716 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4f7a50e5-fd28-a25c-e2db-a16ab6433861 Address:127.0.0.1:13874}]" -42:48.717 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4f7a50e5-fd28-a25c-e2db-a16ab6433861.dc1 127.0.0.1 -42:48.718 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13874 [Follower]" leader= -42:48.718 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4f7a50e5-fd28-a25c-e2db-a16ab6433861 127.0.0.1 -2022-09-15T21:42:48.718Z [INFO] agent.router: Initializing LAN area manager -42:48.719 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13858 network=udp -42:48.719 [INFO] TestAgent.server: Adding LAN server: server="Node-4f7a50e5-fd28-a25c-e2db-a16ab6433861 (Addr: tcp/127.0.0.1:13874) (DC: dc1)" -42:48.719 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4f7a50e5-fd28-a25c-e2db-a16ab6433861.dc1 area=wan -42:48.719 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13858 network=tcp -42:48.721 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13859 network=tcp -42:48.721 [INFO] TestAgent: started state syncer -42:48.721 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13875 network=tcp -42:48.775 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:48.775 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13874 [Candidate]" term=2 -42:48.775 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:48.775 [DEBUG] TestAgent.server.raft: vote granted: from=4f7a50e5-fd28-a25c-e2db-a16ab6433861 term=2 tally=1 -42:48.775 [INFO] TestAgent.server.raft: election won: tally=1 -42:48.775 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13874 [Leader]" -42:48.775 [INFO] TestAgent.server: cluster leadership acquired -42:48.776 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13874 -42:48.776 [INFO] TestAgent.server: New leader elected: payload=Node-4f7a50e5-fd28-a25c-e2db-a16ab6433861 -42:48.778 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:48.785 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:48.785 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:48.785 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:48.785 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:48.785 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:48.785 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4f7a50e5-fd28-a25c-e2db-a16ab6433861 -42:48.785 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4f7a50e5-fd28-a25c-e2db-a16ab6433861 -42:48.786 [INFO] TestAgent.server: federation state anti-entropy synced -42:48.825 [INFO] TestAgent: Requesting shutdown -42:48.825 [INFO] TestAgent.server: shutting down server -42:48.825 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:48.825 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:48.825 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:48.825 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:48.825 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:48.826 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:48.826 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:48.826 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:48.826 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:48.826 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:48.826 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:48.826 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:48.827Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:48.827Z [INFO] agent.router.manager: shutting down -42:48.827 [INFO] TestAgent: consul server down -42:48.827 [INFO] TestAgent: shutdown complete -42:48.827 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13858 network=tcp -42:48.827 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13858 network=udp -42:48.827 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13859 network=tcp -42:49.328 [INFO] TestAgent: Waiting for endpoints to shut down -42:49.328 [INFO] TestAgent: Endpoints down -2022-09-15T21:42:47.725Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:47.779Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:47.783 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:45ccfcfd-47d7-c272-d651-8d2751d6e276 Address:127.0.0.1:13849}]" -42:47.783 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13849 [Follower]" leader= -42:47.784 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-45ccfcfd-47d7-c272-d651-8d2751d6e276.dc1 127.0.0.1 -42:47.785 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-45ccfcfd-47d7-c272-d651-8d2751d6e276 127.0.0.1 -2022-09-15T21:42:47.785Z [INFO] agent.router: Initializing LAN area manager -42:47.786 [INFO] TestAgent.server: Adding LAN server: server="Node-45ccfcfd-47d7-c272-d651-8d2751d6e276 (Addr: tcp/127.0.0.1:13849) (DC: dc1)" -42:47.786 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13676 network=udp -42:47.786 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-45ccfcfd-47d7-c272-d651-8d2751d6e276.dc1 area=wan -42:47.786 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13676 network=tcp -42:47.788 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13677 network=tcp -42:47.788 [INFO] TestAgent: started state syncer -42:47.789 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13850 network=tcp -42:47.826 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:47.826 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13849 [Candidate]" term=2 -42:47.826 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:47.826 [DEBUG] TestAgent.server.raft: vote granted: from=45ccfcfd-47d7-c272-d651-8d2751d6e276 term=2 tally=1 -42:47.826 [INFO] TestAgent.server.raft: election won: tally=1 -42:47.826 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13849 [Leader]" -42:47.826 [INFO] TestAgent.server: cluster leadership acquired -42:47.827 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13849 -42:47.828 [INFO] TestAgent.server: New leader elected: payload=Node-45ccfcfd-47d7-c272-d651-8d2751d6e276 -42:47.828 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:47.832 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:47.832 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:47.832 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:47.832 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:47.832 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:47.834 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-45ccfcfd-47d7-c272-d651-8d2751d6e276 -42:47.834 [INFO] TestAgent.server: member joined, marking health alive: member=Node-45ccfcfd-47d7-c272-d651-8d2751d6e276 -42:47.898 [INFO] TestAgent.server: federation state anti-entropy synced -42:47.967 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:47.968 [INFO] TestAgent: Synced node info -42:48.116 [INFO] TestAgent: Requesting shutdown -42:48.116 [INFO] TestAgent.server: shutting down server -42:48.116 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:48.116 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:48.116 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:48.116 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:48.116 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:48.116 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:48.116 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:48.116 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:48.116 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:48.116 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:48.117 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:48.117Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:48.117Z [INFO] agent.router.manager: shutting down -42:48.117 [INFO] TestAgent: consul server down -42:48.117 [INFO] TestAgent: shutdown complete -42:48.117 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13676 network=tcp -42:48.117 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13676 network=udp -42:48.117 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13677 network=tcp -42:48.619 [INFO] TestAgent: Waiting for endpoints to shut down -42:48.619 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_ReconnectConfigSettings (1.65s) +2021-08-13T15:34:09.132Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:09.154Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:09.162 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b6b663ea-8f4e-9c03-c272-41ae0f36bcf6 Address:127.0.0.1:25095}]" +34:09.163 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25095 [Follower]" leader= +34:09.164 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b6b663ea-8f4e-9c03-c272-41ae0f36bcf6.dc1 127.0.0.1 +34:09.165 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b6b663ea-8f4e-9c03-c272-41ae0f36bcf6 127.0.0.1 +2021-08-13T15:34:09.165Z [INFO] agent.router: Initializing LAN area manager +34:09.166 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25076 network=udp +34:09.166 [INFO] TestAgent.server: Adding LAN server: server="Node-b6b663ea-8f4e-9c03-c272-41ae0f36bcf6 (Addr: tcp/127.0.0.1:25095) (DC: dc1)" +34:09.166 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b6b663ea-8f4e-9c03-c272-41ae0f36bcf6.dc1 area=wan +34:09.167 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25076 network=tcp +34:09.173 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25077 network=tcp +34:09.173 [INFO] TestAgent: started state syncer +34:09.173 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25096 network=tcp +34:09.203 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:09.203 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25095 [Candidate]" term=2 +34:09.203 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:09.203 [DEBUG] TestAgent.server.raft: vote granted: from=b6b663ea-8f4e-9c03-c272-41ae0f36bcf6 term=2 tally=1 +34:09.203 [INFO] TestAgent.server.raft: election won: tally=1 +34:09.204 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25095 [Leader]" +34:09.204 [INFO] TestAgent.server: cluster leadership acquired +34:09.204 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25095 +34:09.205 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:09.207 [INFO] TestAgent.server: New leader elected: payload=Node-b6b663ea-8f4e-9c03-c272-41ae0f36bcf6 +34:09.208 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:09.208 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:09.208 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:09.208 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:09.208 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:09.208 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b6b663ea-8f4e-9c03-c272-41ae0f36bcf6 +34:09.208 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b6b663ea-8f4e-9c03-c272-41ae0f36bcf6 +34:09.209 [INFO] TestAgent.server: federation state anti-entropy synced +34:09.463 [INFO] TestAgent: Requesting shutdown +34:09.463 [INFO] TestAgent.server: shutting down server +34:09.463 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:09.463 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:09.463 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:09.463 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:09.463 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:09.464 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:09.464 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:09.464 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:09.464 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:09.464 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:09.464 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:09.464 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:09.464Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:09.464Z [INFO] agent.router.manager: shutting down +34:09.465 [INFO] TestAgent: consul server down +34:09.465 [INFO] TestAgent: shutdown complete +34:09.465 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25076 network=tcp +34:09.465 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25076 network=udp +34:09.465 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25077 network=tcp +34:09.965 [INFO] TestAgent: Waiting for endpoints to shut down +34:09.965 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Services_ExposeConfig (0.85s) +=== CONT TestSessionCreate_NoCheck +[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:09.371Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:09.371Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:09.388Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:09.388Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:09.391 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5f126f70-1567-82a5-33bd-7509c1ae732d Address:127.0.0.1:25106}]" +34:09.391 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25106 [Follower]" leader= +34:09.392 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5f126f70-1567-82a5-33bd-7509c1ae732d.dc1 127.0.0.1 +34:09.393 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5f126f70-1567-82a5-33bd-7509c1ae732d 127.0.0.1 +2021-08-13T15:34:09.393Z [INFO] agent.router: Initializing LAN area manager +34:09.393 [INFO] TestAgent.server: Adding LAN server: server="Node-5f126f70-1567-82a5-33bd-7509c1ae732d (Addr: tcp/127.0.0.1:25106) (DC: dc1)" +34:09.393 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5f126f70-1567-82a5-33bd-7509c1ae732d.dc1 area=wan +34:09.394 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25101 network=tcp +34:09.394 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25101 network=udp +34:09.394 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25102 network=tcp +34:09.395 [INFO] TestAgent: started state syncer +34:09.395 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25121 network=tcp +34:09.444 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:09.459 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:09.459 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25106 [Candidate]" term=2 +34:09.459 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:09.459 [DEBUG] TestAgent.server.raft: vote granted: from=5f126f70-1567-82a5-33bd-7509c1ae732d term=2 tally=1 +34:09.459 [INFO] TestAgent.server.raft: election won: tally=1 +34:09.459 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25106 [Leader]" +34:09.459 [INFO] TestAgent.server: cluster leadership acquired +34:09.460 [INFO] TestAgent.server: New leader elected: payload=Node-5f126f70-1567-82a5-33bd-7509c1ae732d +34:09.460 [INFO] TestAgent.server: initializing acls +34:09.460 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:09.460 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:09.461 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:09.461 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:09.461 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:09.461 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:09.461 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-5f126f70-1567-82a5-33bd-7509c1ae732d +34:09.462 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-5f126f70-1567-82a5-33bd-7509c1ae732d.dc1 +34:09.463 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:09.463 [INFO] TestAgent.server: Updating LAN server: server="Node-5f126f70-1567-82a5-33bd-7509c1ae732d (Addr: tcp/127.0.0.1:25106) (DC: dc1)" +34:09.463 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-5f126f70-1567-82a5-33bd-7509c1ae732d.dc1 area=wan +34:09.467 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:09.468 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:09.468 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:09.468 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:09.468 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:09.468 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5f126f70-1567-82a5-33bd-7509c1ae732d +34:09.468 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5f126f70-1567-82a5-33bd-7509c1ae732d +34:09.469 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5f126f70-1567-82a5-33bd-7509c1ae732d +34:09.469 [INFO] TestAgent.server: federation state anti-entropy synced +34:09.513 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:09.514 [INFO] TestAgent: Synced node info +34:09.514 [DEBUG] TestAgent: Node info in sync +34:09.560 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-5f126f70-1567-82a5-33bd-7509c1ae732d +34:09.560 [INFO] TestAgent: Requesting shutdown +34:09.560 [INFO] TestAgent.server: shutting down server +34:09.560 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:09.560 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:09.560 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:09.560 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:09.560 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:09.560 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:09.560 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:09.560 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:09.560 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:09.560 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:09.560 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:09.560 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:09.560 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:09.560 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:09.560 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:09.560Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:09.560Z [INFO] agent.router.manager: shutting down +34:09.561 [INFO] TestAgent: consul server down +34:09.561 [INFO] TestAgent: shutdown complete +34:09.561 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25101 network=tcp +34:09.561 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25101 network=udp +34:09.561 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25102 network=tcp +34:10.061 [INFO] TestAgent: Waiting for endpoints to shut down +34:10.061 [INFO] TestAgent: Endpoints down +=== RUN TestSnapshot_Options/PUT#02 +2021-08-13T15:34:09.488Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:09.488Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:09.506Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:09.506Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:09.509 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:44231d3e-0633-e62e-aa09-56feb21e3bc0 Address:127.0.0.1:25120}]" +34:09.509 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25120 [Follower]" leader= +34:09.510 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-44231d3e-0633-e62e-aa09-56feb21e3bc0.dc1 127.0.0.1 +34:09.511 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-44231d3e-0633-e62e-aa09-56feb21e3bc0 127.0.0.1 +2021-08-13T15:34:09.511Z [INFO] agent.router: Initializing LAN area manager +34:09.511 [INFO] TestAgent.server: Adding LAN server: server="Node-44231d3e-0633-e62e-aa09-56feb21e3bc0 (Addr: tcp/127.0.0.1:25120) (DC: dc1)" +34:09.511 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-44231d3e-0633-e62e-aa09-56feb21e3bc0.dc1 area=wan +34:09.511 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25115 network=udp +34:09.511 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25115 network=tcp +34:09.512 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25116 network=tcp +34:09.512 [INFO] TestAgent: started state syncer +34:09.512 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25128 network=tcp +34:09.560 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:09.560 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25120 [Candidate]" term=2 +34:09.560 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:09.560 [DEBUG] TestAgent.server.raft: vote granted: from=44231d3e-0633-e62e-aa09-56feb21e3bc0 term=2 tally=1 +34:09.560 [INFO] TestAgent.server.raft: election won: tally=1 +34:09.560 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25120 [Leader]" +34:09.560 [INFO] TestAgent.server: cluster leadership acquired +34:09.561 [INFO] TestAgent.server: initializing acls +34:09.561 [INFO] TestAgent.server: New leader elected: payload=Node-44231d3e-0633-e62e-aa09-56feb21e3bc0 +34:09.561 [INFO] TestAgent.server: initializing acls +34:09.561 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:09.561 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:09.561 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:09.562 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:09.562 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:09.562 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:09.562 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:09.563 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:09.563 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:09.563 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode +34:09.563 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-44231d3e-0633-e62e-aa09-56feb21e3bc0 +34:09.563 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-44231d3e-0633-e62e-aa09-56feb21e3bc0.dc1 +34:09.563 [INFO] TestAgent.server: Updating LAN server: server="Node-44231d3e-0633-e62e-aa09-56feb21e3bc0 (Addr: tcp/127.0.0.1:25120) (DC: dc1)" +34:09.563 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-44231d3e-0633-e62e-aa09-56feb21e3bc0.dc1 area=wan +34:09.563 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-44231d3e-0633-e62e-aa09-56feb21e3bc0 +34:09.563 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-44231d3e-0633-e62e-aa09-56feb21e3bc0.dc1 +34:09.563 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-44231d3e-0633-e62e-aa09-56feb21e3bc0.dc1 area=wan +34:09.564 [INFO] TestAgent.server: Updating LAN server: server="Node-44231d3e-0633-e62e-aa09-56feb21e3bc0 (Addr: tcp/127.0.0.1:25120) (DC: dc1)" +34:09.564 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:09.566 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:09.566 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:09.566 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:09.566 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:09.566 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:09.566 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-44231d3e-0633-e62e-aa09-56feb21e3bc0 +34:09.566 [INFO] TestAgent.server: member joined, marking health alive: member=Node-44231d3e-0633-e62e-aa09-56feb21e3bc0 +34:09.567 [INFO] TestAgent.server: federation state anti-entropy synced +34:09.568 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-44231d3e-0633-e62e-aa09-56feb21e3bc0 +34:09.568 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-44231d3e-0633-e62e-aa09-56feb21e3bc0 +34:09.584 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-44231d3e-0633-e62e-aa09-56feb21e3bc0 +34:09.584 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-44231d3e-0633-e62e-aa09-56feb21e3bc0 +34:09.585 [INFO] TestAgent: Requesting shutdown +34:09.585 [INFO] TestAgent.server: shutting down server +34:09.585 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:09.585 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:09.585 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:09.585 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:09.585 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:09.585 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:09.585 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:09.585 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:09.585 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:09.585 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:09.585 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:09.585 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:09.585 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:09.585 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:09.585 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:09.585 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:09.585Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:09.585Z [INFO] agent.router.manager: shutting down +34:09.585 [INFO] TestAgent: consul server down +34:09.585 [INFO] TestAgent: shutdown complete +34:09.585 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25115 network=tcp +34:09.585 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25115 network=udp +34:09.585 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25116 network=tcp +34:10.086 [INFO] TestAgent: Waiting for endpoints to shut down +34:10.086 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_HostBadACL (0.62s) +=== CONT TestSessionCreate_DefaultCheck +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:09.621Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:09.621Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:09.639Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:09.639Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:09.642 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3c68305f-8217-64c8-7704-5d50d62e2298 Address:127.0.0.1:25134}]" +34:09.642 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25134 [Follower]" leader= +34:09.643 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3c68305f-8217-64c8-7704-5d50d62e2298.dc1 127.0.0.1 +34:09.644 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3c68305f-8217-64c8-7704-5d50d62e2298 127.0.0.1 +2021-08-13T15:34:09.644Z [INFO] agent.router: Initializing LAN area manager +34:09.644 [INFO] TestAgent.server: Adding LAN server: server="Node-3c68305f-8217-64c8-7704-5d50d62e2298 (Addr: tcp/127.0.0.1:25134) (DC: dc1)" +34:09.644 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3c68305f-8217-64c8-7704-5d50d62e2298.dc1 area=wan +34:09.645 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25129 network=tcp +34:09.645 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25129 network=udp +34:09.645 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25130 network=tcp +34:09.646 [INFO] TestAgent: started state syncer +34:09.646 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25135 network=tcp +34:09.695 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:09.702 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:09.702 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25134 [Candidate]" term=2 +34:09.702 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:09.702 [DEBUG] TestAgent.server.raft: vote granted: from=3c68305f-8217-64c8-7704-5d50d62e2298 term=2 tally=1 +34:09.702 [INFO] TestAgent.server.raft: election won: tally=1 +34:09.702 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25134 [Leader]" +34:09.702 [INFO] TestAgent.server: cluster leadership acquired +34:09.702 [INFO] TestAgent.server: New leader elected: payload=Node-3c68305f-8217-64c8-7704-5d50d62e2298 +34:09.703 [INFO] TestAgent.server: initializing acls +34:09.703 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:09.703 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:09.704 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:09.704 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:09.704 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:09.704 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:09.705 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-3c68305f-8217-64c8-7704-5d50d62e2298 +34:09.705 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-3c68305f-8217-64c8-7704-5d50d62e2298.dc1 +34:09.705 [INFO] TestAgent.server: Updating LAN server: server="Node-3c68305f-8217-64c8-7704-5d50d62e2298 (Addr: tcp/127.0.0.1:25134) (DC: dc1)" +34:09.705 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-3c68305f-8217-64c8-7704-5d50d62e2298.dc1 area=wan +34:09.706 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:09.708 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:09.708 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:09.708 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:09.708 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:09.708 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:09.708 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3c68305f-8217-64c8-7704-5d50d62e2298 +34:09.708 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3c68305f-8217-64c8-7704-5d50d62e2298 +34:09.709 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3c68305f-8217-64c8-7704-5d50d62e2298 +34:09.710 [INFO] TestAgent.server: federation state anti-entropy synced +34:09.790 [INFO] TestAgent: Requesting shutdown +34:09.790 [INFO] TestAgent.server: shutting down server +34:09.790 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:09.790 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:09.790 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:09.790 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:09.790 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:09.790 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:09.790 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:09.790 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:09.790 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:09.791 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:09.791 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:09.791 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:09.791 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:09.791 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:09.791 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +34:09.791 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +2021-08-13T15:34:09.791Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:09.791Z [INFO] agent.router.manager: shutting down +34:09.791 [INFO] TestAgent: consul server down +34:09.791 [INFO] TestAgent: shutdown complete +34:09.791 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25129 network=tcp +34:09.791 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25129 network=udp +34:09.791 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25130 network=tcp +34:10.291 [INFO] TestAgent: Waiting for endpoints to shut down +34:10.291 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Host (0.69s) === CONT TestAgentConnectAuthorize_defaultDeny -2022-09-15T21:42:48.524Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:48.524Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:48.541Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:48.541Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:48.545 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3ccdd56c-b3cb-23d6-6851-f5ee36e95c62 Address:127.0.0.1:13856}]" -42:48.546 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13856 [Follower]" leader= -42:48.546 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3ccdd56c-b3cb-23d6-6851-f5ee36e95c62.dc1 127.0.0.1 -42:48.547 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3ccdd56c-b3cb-23d6-6851-f5ee36e95c62 127.0.0.1 -2022-09-15T21:42:48.547Z [INFO] agent.router: Initializing LAN area manager -42:48.547 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13869 network=udp -42:48.547 [INFO] TestAgent.server: Adding LAN server: server="Node-3ccdd56c-b3cb-23d6-6851-f5ee36e95c62 (Addr: tcp/127.0.0.1:13856) (DC: dc1)" -42:48.548 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3ccdd56c-b3cb-23d6-6851-f5ee36e95c62.dc1 area=wan -42:48.548 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13869 network=tcp -42:48.549 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13870 network=tcp -42:48.549 [INFO] TestAgent: started state syncer -42:48.549 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13857 network=tcp -42:48.600 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -42:48.605 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:48.605 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13856 [Candidate]" term=2 -42:48.605 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:48.605 [DEBUG] TestAgent.server.raft: vote granted: from=3ccdd56c-b3cb-23d6-6851-f5ee36e95c62 term=2 tally=1 -42:48.605 [INFO] TestAgent.server.raft: election won: tally=1 -42:48.605 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13856 [Leader]" -42:48.605 [INFO] TestAgent.server: cluster leadership acquired -42:48.605 [INFO] TestAgent.server: initializing acls -42:48.606 [INFO] TestAgent.server: New leader elected: payload=Node-3ccdd56c-b3cb-23d6-6851-f5ee36e95c62 -42:48.606 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:48.606 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:48.606 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:48.607 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:48.607 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:48.607 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:48.607 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-3ccdd56c-b3cb-23d6-6851-f5ee36e95c62 -42:48.607 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-3ccdd56c-b3cb-23d6-6851-f5ee36e95c62.dc1 -42:48.608 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:48.609 [INFO] TestAgent.server: Updating LAN server: server="Node-3ccdd56c-b3cb-23d6-6851-f5ee36e95c62 (Addr: tcp/127.0.0.1:13856) (DC: dc1)" -42:48.609 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-3ccdd56c-b3cb-23d6-6851-f5ee36e95c62.dc1 area=wan -42:48.611 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:48.611 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:48.611 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:48.611 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:48.611 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:48.611 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3ccdd56c-b3cb-23d6-6851-f5ee36e95c62 -42:48.611 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3ccdd56c-b3cb-23d6-6851-f5ee36e95c62 -42:48.612 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3ccdd56c-b3cb-23d6-6851-f5ee36e95c62 -42:48.615 [INFO] TestAgent.server: federation state anti-entropy synced -42:48.721 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="ACL not found" -42:48.858 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-3ccdd56c-b3cb-23d6-6851-f5ee36e95c62 -42:48.858 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-3ccdd56c-b3cb-23d6-6851-f5ee36e95c62 -42:48.858 [INFO] TestAgent: Requesting shutdown -42:48.858 [INFO] TestAgent.server: shutting down server -42:48.858 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:48.858 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:48.858 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:48.858 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:48.858 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:48.858 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:48.858 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:48.859 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:48.859 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:48.859 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:48.859 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:48.859 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:48.859 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:48.859 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:48.859 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:48.859Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:48.859Z [INFO] agent.router.manager: shutting down -42:48.859 [INFO] TestAgent: consul server down -42:48.859 [INFO] TestAgent: shutdown complete -42:48.859 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13869 network=tcp -42:48.859 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13869 network=udp -42:48.859 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13870 network=tcp -42:49.368 [INFO] TestAgent: Waiting for endpoints to shut down -42:49.368 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_HostBadACL (0.87s) +=== RUN TestSessionCreate_NoCheck/no_check_fields_should_yield_default_serfHealth +=== RUN TestSessionCreate_NoCheck/overwrite_nodechecks_to_associate_with_no_checks +=== RUN TestSessionCreate_NoCheck/overwrite_checks_to_associate_with_no_checks +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:09.449Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:09.467Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:09.470 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f016c27f-c3a7-9d5b-5691-d0dd4b786c31 Address:127.0.0.1:25127}]" +34:09.471 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25127 [Follower]" leader= +34:09.472 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f016c27f-c3a7-9d5b-5691-d0dd4b786c31.dc1 127.0.0.1 +34:09.473 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f016c27f-c3a7-9d5b-5691-d0dd4b786c31 127.0.0.1 +2021-08-13T15:34:09.473Z [INFO] agent.router: Initializing LAN area manager +34:09.473 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25122 network=udp +34:09.473 [INFO] TestAgent.server: Adding LAN server: server="Node-f016c27f-c3a7-9d5b-5691-d0dd4b786c31 (Addr: tcp/127.0.0.1:25127) (DC: dc1)" +34:09.473 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f016c27f-c3a7-9d5b-5691-d0dd4b786c31.dc1 area=wan +34:09.474 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25122 network=tcp +34:09.474 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25123 network=tcp +34:09.475 [INFO] TestAgent: started state syncer +34:09.475 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25114 network=tcp +34:09.513 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:09.514 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25127 [Candidate]" term=2 +34:09.514 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:09.514 [DEBUG] TestAgent.server.raft: vote granted: from=f016c27f-c3a7-9d5b-5691-d0dd4b786c31 term=2 tally=1 +34:09.514 [INFO] TestAgent.server.raft: election won: tally=1 +34:09.514 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25127 [Leader]" +34:09.514 [INFO] TestAgent.server: cluster leadership acquired +34:09.514 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25127 +34:09.514 [INFO] TestAgent.server: New leader elected: payload=Node-f016c27f-c3a7-9d5b-5691-d0dd4b786c31 +34:09.516 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:09.519 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:09.519 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:09.519 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:09.519 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:09.519 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:09.519 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f016c27f-c3a7-9d5b-5691-d0dd4b786c31 +34:09.519 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f016c27f-c3a7-9d5b-5691-d0dd4b786c31 +34:09.520 [INFO] TestAgent.server: federation state anti-entropy synced +34:09.552 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:09.553 [INFO] TestAgent: Synced node info +34:09.554 [INFO] TestAgent: Synced service: service=redis +34:09.554 [DEBUG] TestAgent: Check in sync: check=redis-check +34:09.554 [INFO] TestAgent: Synced check: check=node-check +34:09.644 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:09.644 [DEBUG] TestAgent: Node info in sync +34:09.644 [DEBUG] TestAgent: Service in sync: service=redis +34:09.644 [DEBUG] TestAgent: Check in sync: check=redis-check +34:09.644 [DEBUG] TestAgent: Check in sync: check=node-check +34:09.644 [DEBUG] TestAgent: Node info in sync +34:09.644 [DEBUG] TestAgent: Service in sync: service=redis +34:09.644 [DEBUG] TestAgent: Check in sync: check=redis-check +34:09.644 [DEBUG] TestAgent: Check in sync: check=node-check +34:09.658 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:09.658 [DEBUG] TestAgent: Node info in sync +34:09.658 [DEBUG] TestAgent: Service in sync: service=redis +34:09.658 [DEBUG] TestAgent: Check in sync: check=redis-check +34:09.658 [DEBUG] TestAgent: Check in sync: check=node-check +34:10.054 [DEBUG] TestAgent: Registered node: node="Node-level check" +34:10.054 [DEBUG] TestAgent: Registered node: node="Serf Health Status" +34:10.054 [DEBUG] TestAgent: Registered node: node="Service-level check" +34:10.054 [INFO] TestAgent: Sync in progress: iteration=1 +34:10.055 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:10.055 [DEBUG] TestAgent: Node info in sync +34:10.055 [DEBUG] TestAgent: Service in sync: service=redis +34:10.055 [DEBUG] TestAgent: Check in sync: check=redis-check +34:10.055 [DEBUG] TestAgent: Check in sync: check=node-check +34:10.055 [INFO] TestAgent: Sync in progress: iteration=2 +34:10.055 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:10.055 [DEBUG] TestAgent: Node info in sync +34:10.055 [DEBUG] TestAgent: Service in sync: service=redis +34:10.055 [DEBUG] TestAgent: Check in sync: check=redis-check +34:10.055 [DEBUG] TestAgent: Check in sync: check=node-check +34:10.055 [INFO] TestAgent: Sync in progress: iteration=3 +34:10.055 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:10.055 [DEBUG] TestAgent: Node info in sync +34:10.055 [DEBUG] TestAgent: Service in sync: service=redis +34:10.055 [DEBUG] TestAgent: Check in sync: check=redis-check +34:10.055 [DEBUG] TestAgent: Check in sync: check=node-check +34:10.055 [INFO] TestAgent: Sync in progress: iteration=4 +34:10.055 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:10.055 [DEBUG] TestAgent: Node info in sync +34:10.055 [DEBUG] TestAgent: Service in sync: service=redis +34:10.055 [DEBUG] TestAgent: Check in sync: check=redis-check +34:10.055 [DEBUG] TestAgent: Check in sync: check=node-check +34:10.055 [INFO] TestAgent: Sync in progress: iteration=5 +34:10.055 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:10.055 [DEBUG] TestAgent: Node info in sync +34:10.055 [DEBUG] TestAgent: Service in sync: service=redis +34:10.055 [DEBUG] TestAgent: Check in sync: check=redis-check +34:10.055 [DEBUG] TestAgent: Check in sync: check=node-check +34:10.055 [INFO] TestAgent: Sync in progress: iteration=6 +34:10.055 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:10.055 [DEBUG] TestAgent: Node info in sync +34:10.055 [DEBUG] TestAgent: Service in sync: service=redis +34:10.055 [DEBUG] TestAgent: Check in sync: check=redis-check +34:10.055 [DEBUG] TestAgent: Check in sync: check=node-check +34:10.055 [INFO] TestAgent: Sync in progress: iteration=7 +34:10.055 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:10.055 [DEBUG] TestAgent: Node info in sync +34:10.055 [DEBUG] TestAgent: Service in sync: service=redis +34:10.055 [DEBUG] TestAgent: Check in sync: check=redis-check +34:10.055 [DEBUG] TestAgent: Check in sync: check=node-check +34:10.055 [INFO] TestAgent: Sync in progress: iteration=8 +34:10.056 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:10.056 [DEBUG] TestAgent: Node info in sync +34:10.056 [DEBUG] TestAgent: Service in sync: service=redis +34:10.056 [DEBUG] TestAgent: Check in sync: check=redis-check +34:10.056 [DEBUG] TestAgent: Check in sync: check=node-check +34:10.056 [INFO] TestAgent: Sync in progress: iteration=9 +34:10.056 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:10.056 [DEBUG] TestAgent: Node info in sync +34:10.056 [DEBUG] TestAgent: Service in sync: service=redis +34:10.056 [DEBUG] TestAgent: Check in sync: check=redis-check +34:10.056 [DEBUG] TestAgent: Check in sync: check=node-check +34:10.056 [INFO] TestAgent: Sync in progress: iteration=10 +34:10.056 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:10.056 [DEBUG] TestAgent: Node info in sync +34:10.056 [DEBUG] TestAgent: Service in sync: service=redis +34:10.056 [DEBUG] TestAgent: Check in sync: check=node-check +34:10.056 [DEBUG] TestAgent: Check in sync: check=redis-check +34:10.056 [INFO] TestAgent: Requesting shutdown +34:10.056 [INFO] TestAgent.server: shutting down server +34:10.056 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:10.056 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:10.056 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:10.056 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:10.056 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:10.056 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:10.056 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:10.056 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:10.056 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:10.056 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:10.057 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:10.057Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:10.057Z [INFO] agent.router.manager: shutting down +34:10.057 [INFO] TestAgent: consul server down +34:10.057 [INFO] TestAgent: shutdown complete +34:10.057 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25122 network=tcp +34:10.057 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25122 network=udp +34:10.057 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25123 network=tcp +34:10.557 [INFO] TestAgent: Waiting for endpoints to shut down +34:10.557 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_IndexChurn (2.34s) + --- PASS: TestAgent_IndexChurn/no_tags (1.22s) + --- PASS: TestAgent_IndexChurn/with_tags (1.13s) === CONT TestAgentConnectAuthorize_serviceWrite -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._tcp_(qType:=33)_compressed=false#01 -[WARN] freeport: 4 out of 25 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:48.805Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:48.805Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:48.914Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:48.914Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:48.919 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9ab1a066-0142-fa15-4798-3db0f00847d9 Address:127.0.0.1:13902}]" -42:48.921 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9ab1a066-0142-fa15-4798-3db0f00847d9.dc1 127.0.0.1 -42:48.922 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13902 [Follower]" leader= -42:48.923 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9ab1a066-0142-fa15-4798-3db0f00847d9 127.0.0.1 -2022-09-15T21:42:48.923Z [INFO] agent.router: Initializing LAN area manager -42:48.923 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13897 network=udp -42:48.923 [INFO] TestAgent.server: Adding LAN server: server="Node-9ab1a066-0142-fa15-4798-3db0f00847d9 (Addr: tcp/127.0.0.1:13902) (DC: dc1)" -42:48.923 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9ab1a066-0142-fa15-4798-3db0f00847d9.dc1 area=wan -42:48.924 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13897 network=tcp -42:48.925 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13898 network=tcp -42:48.925 [INFO] TestAgent: started state syncer -42:48.926 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13903 network=tcp -42:48.975 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -42:48.986 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:48.986 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13902 [Candidate]" term=2 -42:48.987 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:48.987 [DEBUG] TestAgent.server.raft: vote granted: from=9ab1a066-0142-fa15-4798-3db0f00847d9 term=2 tally=1 -42:48.987 [INFO] TestAgent.server.raft: election won: tally=1 -42:48.987 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13902 [Leader]" -42:48.987 [INFO] TestAgent.server: cluster leadership acquired -42:48.987 [INFO] TestAgent.server: initializing acls -42:48.988 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:48.988 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:48.988 [INFO] TestAgent.server: New leader elected: payload=Node-9ab1a066-0142-fa15-4798-3db0f00847d9 -42:48.989 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:48.994 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:48.994 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:48.994 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:48.995 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-9ab1a066-0142-fa15-4798-3db0f00847d9 -42:48.996 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-9ab1a066-0142-fa15-4798-3db0f00847d9.dc1 -42:48.996 [INFO] TestAgent.server: Updating LAN server: server="Node-9ab1a066-0142-fa15-4798-3db0f00847d9 (Addr: tcp/127.0.0.1:13902) (DC: dc1)" -42:48.996 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-9ab1a066-0142-fa15-4798-3db0f00847d9.dc1 area=wan -42:48.997 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:49.002 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:49.002 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:49.002 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:49.002 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:49.002 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:49.003 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9ab1a066-0142-fa15-4798-3db0f00847d9 -42:49.003 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9ab1a066-0142-fa15-4798-3db0f00847d9 -42:49.004 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9ab1a066-0142-fa15-4798-3db0f00847d9 -42:49.004 [INFO] TestAgent.server: federation state anti-entropy synced -42:49.054 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="ACL not found" -42:49.143 [INFO] TestAgent: Requesting shutdown -42:49.143 [INFO] TestAgent.server: shutting down server -42:49.143 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:49.143 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:49.143 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:49.143 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:49.143 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:49.143 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:49.143 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:49.143 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:49.143 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:49.144 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:49.144 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:49.144 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:49.144 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:49.144 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:49.144 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:49.144Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:49.144Z [INFO] agent.router.manager: shutting down -42:49.144 [INFO] TestAgent: consul server down -42:49.144 [INFO] TestAgent: shutdown complete -42:49.144 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13897 network=tcp -42:49.144 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13897 network=udp -42:49.145 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13898 network=tcp -42:49.650 [INFO] TestAgent: Waiting for endpoints to shut down -42:49.650 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Host (0.91s) +2021-08-13T15:34:09.832Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:09.886Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:09.889 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7996d183-5025-5683-ce9a-9be7c6e6a57b Address:127.0.0.1:25141}]" +34:09.889 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25141 [Follower]" leader= +34:09.890 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7996d183-5025-5683-ce9a-9be7c6e6a57b.dc1 127.0.0.1 +34:09.891 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7996d183-5025-5683-ce9a-9be7c6e6a57b 127.0.0.1 +2021-08-13T15:34:09.891Z [INFO] agent.router: Initializing LAN area manager +34:09.892 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25136 network=udp +34:09.892 [INFO] TestAgent.server: Adding LAN server: server="Node-7996d183-5025-5683-ce9a-9be7c6e6a57b (Addr: tcp/127.0.0.1:25141) (DC: dc1)" +34:09.892 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7996d183-5025-5683-ce9a-9be7c6e6a57b.dc1 area=wan +34:09.892 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25136 network=tcp +34:09.893 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25137 network=tcp +34:09.893 [INFO] TestAgent: started state syncer +34:09.893 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25142 network=tcp +34:09.949 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:09.949 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25141 [Candidate]" term=2 +34:09.950 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:09.950 [DEBUG] TestAgent.server.raft: vote granted: from=7996d183-5025-5683-ce9a-9be7c6e6a57b term=2 tally=1 +34:09.950 [INFO] TestAgent.server.raft: election won: tally=1 +34:09.950 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25141 [Leader]" +34:09.950 [INFO] TestAgent.server: cluster leadership acquired +34:09.950 [INFO] TestAgent.server: New leader elected: payload=Node-7996d183-5025-5683-ce9a-9be7c6e6a57b +34:09.950 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25141 +34:09.951 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:09.954 [INFO] TestAgent: Synced node info +34:09.954 [DEBUG] TestAgent: Node info in sync +34:09.954 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:09.954 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:09.954 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:09.954 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:09.954 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:09.954 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7996d183-5025-5683-ce9a-9be7c6e6a57b +34:09.954 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7996d183-5025-5683-ce9a-9be7c6e6a57b +34:09.955 [INFO] TestAgent.server: federation state anti-entropy synced +34:10.060 [INFO] TestAgent: Requesting shutdown +34:10.060 [INFO] TestAgent.server: shutting down server +34:10.060 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:10.060 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:10.060 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:10.060 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:10.060 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:10.061 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:10.061 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:10.061 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:10.061 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:10.061 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:10.061 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:10.061Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:10.061Z [INFO] agent.router.manager: shutting down +34:10.061 [INFO] TestAgent: consul server down +34:10.061 [INFO] TestAgent: shutdown complete +34:10.061 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25136 network=tcp +34:10.061 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25136 network=udp +34:10.061 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25137 network=tcp +34:10.561 [INFO] TestAgent: Waiting for endpoints to shut down +34:10.561 [INFO] TestAgent: Endpoints down +--- PASS: TestSessionDestroy (0.76s) +=== CONT TestSessionCreate_Delete +2021-08-13T15:34:09.845Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:09.845Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:09.885Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:09.885Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:09.888 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:087ea0cc-8402-69fb-a0b9-a3d19d6afc91 Address:127.0.0.1:25148}]" +34:09.889 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-087ea0cc-8402-69fb-a0b9-a3d19d6afc91.dc1 127.0.0.1 +34:09.889 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25148 [Follower]" leader= +34:09.890 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-087ea0cc-8402-69fb-a0b9-a3d19d6afc91 127.0.0.1 +2021-08-13T15:34:09.890Z [INFO] agent.router: Initializing LAN area manager +34:09.890 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25143 network=udp +34:09.890 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-087ea0cc-8402-69fb-a0b9-a3d19d6afc91.dc1 area=wan +34:09.890 [INFO] TestAgent.server: Adding LAN server: server="Node-087ea0cc-8402-69fb-a0b9-a3d19d6afc91 (Addr: tcp/127.0.0.1:25148) (DC: dc1)" +34:09.891 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25143 network=tcp +34:09.892 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25144 network=tcp +34:09.892 [INFO] TestAgent: started state syncer +34:09.892 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25149 network=tcp +34:09.941 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:09.950 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:09.950 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25148 [Candidate]" term=2 +34:09.950 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:09.950 [DEBUG] TestAgent.server.raft: vote granted: from=087ea0cc-8402-69fb-a0b9-a3d19d6afc91 term=2 tally=1 +34:09.950 [INFO] TestAgent.server.raft: election won: tally=1 +34:09.950 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25148 [Leader]" +34:09.950 [INFO] TestAgent.server: cluster leadership acquired +34:09.950 [INFO] TestAgent.server: New leader elected: payload=Node-087ea0cc-8402-69fb-a0b9-a3d19d6afc91 +34:09.950 [INFO] TestAgent.server: initializing acls +34:09.951 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:09.951 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:09.951 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:09.952 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:09.952 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:09.952 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:09.952 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-087ea0cc-8402-69fb-a0b9-a3d19d6afc91 +34:09.952 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-087ea0cc-8402-69fb-a0b9-a3d19d6afc91.dc1 +34:09.952 [INFO] TestAgent.server: Updating LAN server: server="Node-087ea0cc-8402-69fb-a0b9-a3d19d6afc91 (Addr: tcp/127.0.0.1:25148) (DC: dc1)" +34:09.952 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-087ea0cc-8402-69fb-a0b9-a3d19d6afc91.dc1 area=wan +34:09.953 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:09.956 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:09.956 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:09.956 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:09.956 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:09.956 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:09.956 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-087ea0cc-8402-69fb-a0b9-a3d19d6afc91 +34:09.956 [INFO] TestAgent.server: member joined, marking health alive: member=Node-087ea0cc-8402-69fb-a0b9-a3d19d6afc91 +34:09.957 [INFO] TestAgent.server: federation state anti-entropy synced +34:09.957 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-087ea0cc-8402-69fb-a0b9-a3d19d6afc91 +34:10.145 [INFO] TestAgent: Requesting shutdown +34:10.145 [INFO] TestAgent.server: shutting down server +34:10.145 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:10.145 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:10.145 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:10.145 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:10.145 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:10.145 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:10.145 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:10.145 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:10.145 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:10.145 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:10.145 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:10.145 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:10.145 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:10.145 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:10.145 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:10.145 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:10.145Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:10.145Z [INFO] agent.router.manager: shutting down +34:10.145 [INFO] TestAgent: consul server down +34:10.145 [INFO] TestAgent: shutdown complete +34:10.145 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25143 network=tcp +34:10.145 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25143 network=udp +34:10.145 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25144 network=tcp +34:10.646 [INFO] TestAgent: Waiting for endpoints to shut down +34:10.646 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentConnectAuthorize_defaultAllow (0.83s) === CONT TestAgentConnectAuthorize_denyWildcard -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:48.740Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:48.781Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:48.789 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:033d3ce0-fdb9-c291-51eb-dbc48502d200 Address:127.0.0.1:13888}]" -42:48.790 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13888 [Follower]" leader= -42:48.799 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-033d3ce0-fdb9-c291-51eb-dbc48502d200.dc1 127.0.0.1 -42:48.800 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-033d3ce0-fdb9-c291-51eb-dbc48502d200 127.0.0.1 -2022-09-15T21:42:48.800Z [INFO] agent.router: Initializing LAN area manager -42:48.801 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-033d3ce0-fdb9-c291-51eb-dbc48502d200.dc1 area=wan -42:48.801 [INFO] TestAgent.server: Adding LAN server: server="Node-033d3ce0-fdb9-c291-51eb-dbc48502d200 (Addr: tcp/127.0.0.1:13888) (DC: dc1)" -42:48.802 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13890 network=udp -42:48.802 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13890 network=tcp -42:48.803 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13891 network=tcp -42:48.803 [INFO] TestAgent: started state syncer -42:48.804 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13889 network=tcp -42:48.854 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:48.854 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13888 [Candidate]" term=2 -42:48.854 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:48.854 [DEBUG] TestAgent.server.raft: vote granted: from=033d3ce0-fdb9-c291-51eb-dbc48502d200 term=2 tally=1 -42:48.854 [INFO] TestAgent.server.raft: election won: tally=1 -42:48.854 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13888 [Leader]" -42:48.855 [INFO] TestAgent.server: cluster leadership acquired -42:48.855 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13888 -42:48.857 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:48.860 [INFO] TestAgent.server: New leader elected: payload=Node-033d3ce0-fdb9-c291-51eb-dbc48502d200 -42:48.873 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:48.873 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:48.873 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:48.873 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:48.873 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:48.873 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-033d3ce0-fdb9-c291-51eb-dbc48502d200 -42:48.873 [INFO] TestAgent.server: member joined, marking health alive: member=Node-033d3ce0-fdb9-c291-51eb-dbc48502d200 -42:48.878 [INFO] TestAgent: Synced node info -42:48.879 [INFO] TestAgent.server: federation state anti-entropy synced -42:48.982 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:48.982 [DEBUG] TestAgent: Node info in sync -42:48.984 [INFO] TestAgent: Synced service: service=redis -42:48.984 [DEBUG] TestAgent: Check in sync: check=redis-check -42:48.985 [INFO] TestAgent: Synced check: check=node-check -42:49.168 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:49.168 [DEBUG] TestAgent: Node info in sync -42:49.168 [DEBUG] TestAgent: Service in sync: service=redis -42:49.168 [DEBUG] TestAgent: Check in sync: check=redis-check -42:49.168 [DEBUG] TestAgent: Check in sync: check=node-check -42:49.168 [DEBUG] TestAgent: Node info in sync -42:49.168 [DEBUG] TestAgent: Service in sync: service=redis -42:49.168 [DEBUG] TestAgent: Check in sync: check=redis-check -42:49.168 [DEBUG] TestAgent: Check in sync: check=node-check -42:49.486 [DEBUG] TestAgent: Registered node: node="Node-level check" -42:49.486 [DEBUG] TestAgent: Registered node: node="Serf Health Status" -42:49.486 [DEBUG] TestAgent: Registered node: node="Service-level check" -42:49.486 [INFO] TestAgent: Sync in progress: iteration=1 -42:49.486 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:49.486 [DEBUG] TestAgent: Node info in sync -42:49.486 [DEBUG] TestAgent: Service in sync: service=redis -42:49.486 [DEBUG] TestAgent: Check in sync: check=redis-check -42:49.486 [DEBUG] TestAgent: Check in sync: check=node-check -42:49.486 [INFO] TestAgent: Sync in progress: iteration=2 -42:49.486 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:49.486 [DEBUG] TestAgent: Node info in sync -42:49.486 [DEBUG] TestAgent: Service in sync: service=redis -42:49.486 [DEBUG] TestAgent: Check in sync: check=node-check -42:49.486 [DEBUG] TestAgent: Check in sync: check=redis-check -42:49.486 [INFO] TestAgent: Sync in progress: iteration=3 -42:49.486 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:49.486 [DEBUG] TestAgent: Node info in sync -42:49.486 [DEBUG] TestAgent: Service in sync: service=redis -42:49.486 [DEBUG] TestAgent: Check in sync: check=redis-check -42:49.486 [DEBUG] TestAgent: Check in sync: check=node-check -42:49.486 [INFO] TestAgent: Sync in progress: iteration=4 -42:49.486 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:49.486 [DEBUG] TestAgent: Node info in sync -42:49.486 [DEBUG] TestAgent: Service in sync: service=redis -42:49.487 [DEBUG] TestAgent: Check in sync: check=redis-check -42:49.487 [DEBUG] TestAgent: Check in sync: check=node-check -42:49.487 [INFO] TestAgent: Sync in progress: iteration=5 -42:49.487 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:49.487 [DEBUG] TestAgent: Node info in sync -42:49.487 [DEBUG] TestAgent: Service in sync: service=redis -42:49.487 [DEBUG] TestAgent: Check in sync: check=redis-check -42:49.487 [DEBUG] TestAgent: Check in sync: check=node-check -42:49.487 [INFO] TestAgent: Sync in progress: iteration=6 -42:49.487 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:49.487 [DEBUG] TestAgent: Node info in sync -42:49.487 [DEBUG] TestAgent: Service in sync: service=redis -42:49.487 [DEBUG] TestAgent: Check in sync: check=redis-check -42:49.487 [DEBUG] TestAgent: Check in sync: check=node-check -42:49.487 [INFO] TestAgent: Sync in progress: iteration=7 -42:49.487 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:49.487 [DEBUG] TestAgent: Node info in sync -42:49.487 [DEBUG] TestAgent: Service in sync: service=redis -42:49.487 [DEBUG] TestAgent: Check in sync: check=redis-check -42:49.487 [DEBUG] TestAgent: Check in sync: check=node-check -42:49.487 [INFO] TestAgent: Sync in progress: iteration=8 -42:49.487 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:49.487 [DEBUG] TestAgent: Node info in sync -42:49.487 [DEBUG] TestAgent: Service in sync: service=redis -42:49.487 [DEBUG] TestAgent: Check in sync: check=redis-check -42:49.487 [DEBUG] TestAgent: Check in sync: check=node-check -42:49.487 [INFO] TestAgent: Sync in progress: iteration=9 -42:49.487 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:49.487 [DEBUG] TestAgent: Node info in sync -42:49.487 [DEBUG] TestAgent: Service in sync: service=redis -42:49.487 [DEBUG] TestAgent: Check in sync: check=redis-check -42:49.487 [DEBUG] TestAgent: Check in sync: check=node-check -42:49.487 [INFO] TestAgent: Sync in progress: iteration=10 -42:49.487 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:49.487 [DEBUG] TestAgent: Node info in sync -42:49.487 [DEBUG] TestAgent: Service in sync: service=redis -42:49.487 [DEBUG] TestAgent: Check in sync: check=redis-check -42:49.487 [DEBUG] TestAgent: Check in sync: check=node-check -42:49.487 [INFO] TestAgent: Requesting shutdown -42:49.487 [INFO] TestAgent.server: shutting down server -42:49.487 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:49.487 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:49.487 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:49.487 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:49.487 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:49.488 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:49.488 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:49.488 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:49.488 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:49.488 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:49.488 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:49.488Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:49.488Z [INFO] agent.router.manager: shutting down -42:49.488 [INFO] TestAgent: consul server down -42:49.488 [INFO] TestAgent: shutdown complete -42:49.488 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13890 network=tcp -42:49.488 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13890 network=udp -42:49.488 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13891 network=tcp -42:49.990 [INFO] TestAgent: Waiting for endpoints to shut down -42:49.990 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_IndexChurn (2.57s) - --- PASS: TestAgent_IndexChurn/no_tags (1.27s) - --- PASS: TestAgent_IndexChurn/with_tags (1.31s) +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:10.080Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:10.080Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:10.097Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:10.097Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:10.100 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bb0d2a31-2817-279b-fb0c-3057f60d02ff Address:127.0.0.1:25162}]" +34:10.100 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25162 [Follower]" leader= +34:10.101 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bb0d2a31-2817-279b-fb0c-3057f60d02ff.dc1 127.0.0.1 +34:10.101 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bb0d2a31-2817-279b-fb0c-3057f60d02ff 127.0.0.1 +2021-08-13T15:34:10.101Z [INFO] agent.router: Initializing LAN area manager +34:10.102 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25157 network=udp +34:10.102 [INFO] TestAgent.server: Adding LAN server: server="Node-bb0d2a31-2817-279b-fb0c-3057f60d02ff (Addr: tcp/127.0.0.1:25162) (DC: dc1)" +34:10.102 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bb0d2a31-2817-279b-fb0c-3057f60d02ff.dc1 area=wan +34:10.102 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25157 network=tcp +34:10.103 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25158 network=tcp +34:10.103 [INFO] TestAgent: started state syncer +34:10.104 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25163 network=tcp +34:10.152 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:10.153 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:10.153 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25162 [Candidate]" term=2 +34:10.153 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:10.153 [DEBUG] TestAgent.server.raft: vote granted: from=bb0d2a31-2817-279b-fb0c-3057f60d02ff term=2 tally=1 +34:10.153 [INFO] TestAgent.server.raft: election won: tally=1 +34:10.153 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25162 [Leader]" +34:10.153 [INFO] TestAgent.server: cluster leadership acquired +34:10.153 [INFO] TestAgent.server: New leader elected: payload=Node-bb0d2a31-2817-279b-fb0c-3057f60d02ff +34:10.153 [INFO] TestAgent.server: initializing acls +34:10.154 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:10.154 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:10.155 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:10.155 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:10.155 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:10.155 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:10.155 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-bb0d2a31-2817-279b-fb0c-3057f60d02ff +34:10.155 [INFO] TestAgent.server: Updating LAN server: server="Node-bb0d2a31-2817-279b-fb0c-3057f60d02ff (Addr: tcp/127.0.0.1:25162) (DC: dc1)" +34:10.155 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-bb0d2a31-2817-279b-fb0c-3057f60d02ff.dc1 +34:10.156 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-bb0d2a31-2817-279b-fb0c-3057f60d02ff.dc1 area=wan +34:10.157 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:10.159 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:10.159 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:10.159 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:10.159 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:10.159 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:10.159 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bb0d2a31-2817-279b-fb0c-3057f60d02ff +34:10.159 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bb0d2a31-2817-279b-fb0c-3057f60d02ff +34:10.159 [INFO] TestAgent.server: federation state anti-entropy synced +34:10.160 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bb0d2a31-2817-279b-fb0c-3057f60d02ff +34:10.326 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:10.327 [INFO] TestAgent: Synced node info +34:10.327 [DEBUG] TestAgent: Node info in sync +34:10.327 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-bb0d2a31-2817-279b-fb0c-3057f60d02ff +34:10.327 [INFO] TestAgent: Requesting shutdown +34:10.327 [INFO] TestAgent.server: shutting down server +34:10.327 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:10.327 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:10.327 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:10.327 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:10.327 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:10.327 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:10.327 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:10.327 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:10.327 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:10.327 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:10.328 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:10.328 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:10.328 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:10.328 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:10.328 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:10.328Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:10.328Z [INFO] agent.router.manager: shutting down +34:10.328 [INFO] TestAgent: consul server down +34:10.328 [INFO] TestAgent: shutdown complete +34:10.328 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25157 network=tcp +34:10.328 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25157 network=udp +34:10.328 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25158 network=tcp +34:10.828 [INFO] TestAgent: Waiting for endpoints to shut down +34:10.828 [INFO] TestAgent: Endpoints down +--- PASS: TestSnapshot_Options (4.89s) + --- PASS: TestSnapshot_Options/GET (0.90s) + --- PASS: TestSnapshot_Options/GET#01 (0.65s) + --- PASS: TestSnapshot_Options/GET#02 (0.99s) + --- PASS: TestSnapshot_Options/PUT (0.87s) + --- PASS: TestSnapshot_Options/PUT#01 (0.71s) + --- PASS: TestSnapshot_Options/PUT#02 (0.77s) +=== CONT TestSessionCreate_NodeChecks +2021-08-13T15:34:09.983Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:09.998Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:10.001 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b2ff4cf7-b297-807a-5e87-4d4ea83d3bbe Address:127.0.0.1:25155}]" +34:10.001 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25155 [Follower]" leader= +34:10.002 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b2ff4cf7-b297-807a-5e87-4d4ea83d3bbe.dc1 127.0.0.1 +34:10.003 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b2ff4cf7-b297-807a-5e87-4d4ea83d3bbe 127.0.0.1 +2021-08-13T15:34:10.003Z [INFO] agent.router: Initializing LAN area manager +34:10.003 [INFO] TestAgent.server: Adding LAN server: server="Node-b2ff4cf7-b297-807a-5e87-4d4ea83d3bbe (Addr: tcp/127.0.0.1:25155) (DC: dc1)" +34:10.003 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b2ff4cf7-b297-807a-5e87-4d4ea83d3bbe.dc1 area=wan +34:10.003 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25150 network=tcp +34:10.003 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25150 network=udp +34:10.004 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25151 network=tcp +34:10.004 [INFO] TestAgent: started state syncer +34:10.004 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25156 network=tcp +34:10.053 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:10.053 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25155 [Candidate]" term=2 +34:10.054 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:10.054 [DEBUG] TestAgent.server.raft: vote granted: from=b2ff4cf7-b297-807a-5e87-4d4ea83d3bbe term=2 tally=1 +34:10.054 [INFO] TestAgent.server.raft: election won: tally=1 +34:10.054 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25155 [Leader]" +34:10.054 [INFO] TestAgent.server: cluster leadership acquired +34:10.054 [INFO] TestAgent.server: New leader elected: payload=Node-b2ff4cf7-b297-807a-5e87-4d4ea83d3bbe +34:10.054 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25155 +34:10.055 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:10.058 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:10.058 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:10.058 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:10.058 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:10.058 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:10.058 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b2ff4cf7-b297-807a-5e87-4d4ea83d3bbe +34:10.058 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b2ff4cf7-b297-807a-5e87-4d4ea83d3bbe +34:10.059 [INFO] TestAgent.server: federation state anti-entropy synced +34:10.349 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:10.350 [INFO] TestAgent: Synced node info +34:10.436 [INFO] TestAgent: Requesting shutdown +34:10.436 [INFO] TestAgent.server: shutting down server +34:10.436 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:10.436 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:10.436 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:10.436 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:10.436 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:10.436 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:10.436 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:10.436 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:10.436 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:10.436 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:10.436 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:10.436Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:10.436Z [INFO] agent.router.manager: shutting down +34:10.436 [INFO] TestAgent: consul server down +34:10.436 [INFO] TestAgent: shutdown complete +34:10.436 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25150 network=tcp +34:10.436 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25150 network=udp +34:10.436 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25151 network=tcp +34:10.937 [INFO] TestAgent: Waiting for endpoints to shut down +34:10.937 [INFO] TestAgent: Endpoints down +--- PASS: TestSessionCreate_NoCheck (0.97s) + --- PASS: TestSessionCreate_NoCheck/no_check_fields_should_yield_default_serfHealth (0.00s) + --- PASS: TestSessionCreate_NoCheck/overwrite_nodechecks_to_associate_with_no_checks (0.00s) + --- PASS: TestSessionCreate_NoCheck/overwrite_checks_to_associate_with_no_checks (0.00s) === CONT TestAgentConnectAuthorize_allowTrustDomain -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:49.117Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:49.117Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:49.180Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:49.180Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:49.191 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f4de2dc3-14ca-5666-f0df-3cf9ae139125 Address:127.0.0.1:13909}]" -42:49.191 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13909 [Follower]" leader= -42:49.192 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f4de2dc3-14ca-5666-f0df-3cf9ae139125.dc1 127.0.0.1 -42:49.194 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f4de2dc3-14ca-5666-f0df-3cf9ae139125 127.0.0.1 -2022-09-15T21:42:49.194Z [INFO] agent.router: Initializing LAN area manager -42:49.194 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f4de2dc3-14ca-5666-f0df-3cf9ae139125.dc1 area=wan -42:49.194 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13904 network=udp -42:49.194 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13904 network=tcp -42:49.196 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13905 network=tcp -42:49.196 [INFO] TestAgent: started state syncer -42:49.196 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13910 network=tcp -42:49.196 [INFO] TestAgent.server: Adding LAN server: server="Node-f4de2dc3-14ca-5666-f0df-3cf9ae139125 (Addr: tcp/127.0.0.1:13909) (DC: dc1)" -42:49.246 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:49.246 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13909 [Candidate]" term=2 -42:49.246 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:49.246 [DEBUG] TestAgent.server.raft: vote granted: from=f4de2dc3-14ca-5666-f0df-3cf9ae139125 term=2 tally=1 -42:49.246 [INFO] TestAgent.server.raft: election won: tally=1 -42:49.246 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13909 [Leader]" -42:49.246 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=2 mode=2 found=true leader=127.0.0.1:13909 -42:49.246 [INFO] TestAgent.server: cluster leadership acquired -42:49.247 [INFO] TestAgent.server: initializing acls -42:49.247 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:49.248 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:49.248 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:49.249 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:49.249 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:49.249 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:49.250 [INFO] TestAgent.server: New leader elected: payload=Node-f4de2dc3-14ca-5666-f0df-3cf9ae139125 -42:49.258 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-f4de2dc3-14ca-5666-f0df-3cf9ae139125 -42:49.258 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-f4de2dc3-14ca-5666-f0df-3cf9ae139125.dc1 -42:49.260 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:49.261 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-f4de2dc3-14ca-5666-f0df-3cf9ae139125.dc1 area=wan -42:49.262 [INFO] TestAgent.server: Updating LAN server: server="Node-f4de2dc3-14ca-5666-f0df-3cf9ae139125 (Addr: tcp/127.0.0.1:13909) (DC: dc1)" -42:49.272 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:49.272 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:49.272 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:49.272 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:49.272 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:49.273 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f4de2dc3-14ca-5666-f0df-3cf9ae139125 -42:49.273 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f4de2dc3-14ca-5666-f0df-3cf9ae139125 -42:49.283 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f4de2dc3-14ca-5666-f0df-3cf9ae139125 -42:49.283 [INFO] TestAgent.server: federation state anti-entropy synced -42:49.583 [INFO] TestAgent: Requesting shutdown -42:49.584 [INFO] TestAgent.server: shutting down server -42:49.584 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:49.584 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:49.584 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:49.584 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:49.584 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:49.584 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:49.584 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:49.584 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:49.584 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:49.584 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:49.584 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:49.584 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:49.584 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:49.584 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:49.584 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:49.584 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:49.584Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:49.584Z [INFO] agent.router.manager: shutting down -42:49.584 [INFO] TestAgent: consul server down -42:49.584 [INFO] TestAgent: shutdown complete -42:49.584 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13904 network=tcp -42:49.585 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13904 network=udp -42:49.585 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13905 network=tcp -42:50.090 [INFO] TestAgent: Waiting for endpoints to shut down -42:50.090 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentConnectAuthorize_defaultAllow (1.05s) -=== CONT TestAgentConnectAuthorize_deny -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._udp_(qType:=33)_compressed=true#01 -2022-09-15T21:42:49.441Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:49.441Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:49.468Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:49.468Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:49.471 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:23c8ac94-0bca-1a86-5a2c-b6ffc3a3364c Address:127.0.0.1:13916}]" -42:49.472 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-23c8ac94-0bca-1a86-5a2c-b6ffc3a3364c.dc1 127.0.0.1 -42:49.473 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-23c8ac94-0bca-1a86-5a2c-b6ffc3a3364c 127.0.0.1 -2022-09-15T21:42:49.473Z [INFO] agent.router: Initializing LAN area manager -42:49.473 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13916 [Follower]" leader= -42:49.473 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13925 network=udp -42:49.473 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-23c8ac94-0bca-1a86-5a2c-b6ffc3a3364c.dc1 area=wan -42:49.473 [INFO] TestAgent.server: Adding LAN server: server="Node-23c8ac94-0bca-1a86-5a2c-b6ffc3a3364c (Addr: tcp/127.0.0.1:13916) (DC: dc1)" -42:49.473 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13925 network=tcp -42:49.474 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13926 network=tcp -42:49.474 [INFO] TestAgent: started state syncer -42:49.475 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13917 network=tcp -42:49.523 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -42:49.532 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:49.532 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13916 [Candidate]" term=2 -42:49.532 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:49.532 [DEBUG] TestAgent.server.raft: vote granted: from=23c8ac94-0bca-1a86-5a2c-b6ffc3a3364c term=2 tally=1 -42:49.532 [INFO] TestAgent.server.raft: election won: tally=1 -42:49.532 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13916 [Leader]" -42:49.533 [INFO] TestAgent.server: cluster leadership acquired -42:49.533 [INFO] TestAgent.server: initializing acls -42:49.533 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:49.533 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:49.534 [INFO] TestAgent.server: New leader elected: payload=Node-23c8ac94-0bca-1a86-5a2c-b6ffc3a3364c -42:49.534 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:49.534 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:49.534 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:49.534 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:49.535 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-23c8ac94-0bca-1a86-5a2c-b6ffc3a3364c -42:49.535 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-23c8ac94-0bca-1a86-5a2c-b6ffc3a3364c.dc1 -42:49.536 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:49.538 [INFO] TestAgent.server: Updating LAN server: server="Node-23c8ac94-0bca-1a86-5a2c-b6ffc3a3364c (Addr: tcp/127.0.0.1:13916) (DC: dc1)" -42:49.538 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-23c8ac94-0bca-1a86-5a2c-b6ffc3a3364c.dc1 area=wan -42:49.539 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:49.539 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:49.539 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:49.539 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:49.539 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:49.539 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-23c8ac94-0bca-1a86-5a2c-b6ffc3a3364c -42:49.539 [INFO] TestAgent.server: member joined, marking health alive: member=Node-23c8ac94-0bca-1a86-5a2c-b6ffc3a3364c -42:49.540 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-23c8ac94-0bca-1a86-5a2c-b6ffc3a3364c -42:49.541 [INFO] TestAgent.server: federation state anti-entropy synced -42:49.741 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-23c8ac94-0bca-1a86-5a2c-b6ffc3a3364c -42:49.741 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-23c8ac94-0bca-1a86-5a2c-b6ffc3a3364c -42:49.744 [INFO] TestAgent: Requesting shutdown -42:49.744 [INFO] TestAgent.server: shutting down server -42:49.744 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:49.744 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:49.744 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:49.744 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:49.744 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:49.744 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:49.744 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:49.744 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:49.744 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:49.744 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:49.744 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:49.744 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:49.744 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:49.744 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:49.744 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:49.745 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:49.745Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:49.745Z [INFO] agent.router.manager: shutting down -42:49.745 [INFO] TestAgent: consul server down -42:49.745 [INFO] TestAgent: shutdown complete -42:49.745 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13925 network=tcp -42:49.745 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13925 network=udp -42:49.745 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13926 network=tcp -42:50.247 [INFO] TestAgent: Waiting for endpoints to shut down -42:50.247 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentConnectAuthorize_serviceWrite (0.88s) -=== CONT TestAgentConnectAuthorize_allow -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._udp_(qType:=33)_compressed=false#01 -2022-09-15T21:42:49.440Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:49.440Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:49.458Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:49.458Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:49.461 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a43b2959-62e0-145d-187f-6c564c4a5e39 Address:127.0.0.1:13923}]" -42:49.462 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13923 [Follower]" leader= -42:49.462 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a43b2959-62e0-145d-187f-6c564c4a5e39.dc1 127.0.0.1 -42:49.463 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a43b2959-62e0-145d-187f-6c564c4a5e39 127.0.0.1 -2022-09-15T21:42:49.463Z [INFO] agent.router: Initializing LAN area manager -42:49.463 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13911 network=udp -42:49.464 [INFO] TestAgent.server: Adding LAN server: server="Node-a43b2959-62e0-145d-187f-6c564c4a5e39 (Addr: tcp/127.0.0.1:13923) (DC: dc1)" -42:49.464 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a43b2959-62e0-145d-187f-6c564c4a5e39.dc1 area=wan -42:49.464 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13911 network=tcp -42:49.465 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13912 network=tcp -42:49.465 [INFO] TestAgent: started state syncer -42:49.465 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13924 network=tcp -42:49.517 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -42:49.522 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:49.522 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13923 [Candidate]" term=2 -42:49.522 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:49.522 [DEBUG] TestAgent.server.raft: vote granted: from=a43b2959-62e0-145d-187f-6c564c4a5e39 term=2 tally=1 -42:49.522 [INFO] TestAgent.server.raft: election won: tally=1 -42:49.522 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13923 [Leader]" -42:49.522 [INFO] TestAgent.server: cluster leadership acquired -42:49.522 [INFO] TestAgent.server: initializing acls -42:49.523 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:49.523 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:49.523 [INFO] TestAgent.server: New leader elected: payload=Node-a43b2959-62e0-145d-187f-6c564c4a5e39 -42:49.524 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:49.524 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:49.524 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:49.524 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:49.525 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-a43b2959-62e0-145d-187f-6c564c4a5e39 -42:49.525 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-a43b2959-62e0-145d-187f-6c564c4a5e39.dc1 -42:49.526 [INFO] TestAgent.server: Updating LAN server: server="Node-a43b2959-62e0-145d-187f-6c564c4a5e39 (Addr: tcp/127.0.0.1:13923) (DC: dc1)" -42:49.526 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-a43b2959-62e0-145d-187f-6c564c4a5e39.dc1 area=wan -42:49.527 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:49.530 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:49.530 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:49.530 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:49.530 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:49.530 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:49.530 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a43b2959-62e0-145d-187f-6c564c4a5e39 -42:49.530 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a43b2959-62e0-145d-187f-6c564c4a5e39 -42:49.532 [INFO] TestAgent.server: federation state anti-entropy synced -42:49.532 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a43b2959-62e0-145d-187f-6c564c4a5e39 -42:49.722 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:49.723 [INFO] TestAgent: Synced node info -42:49.723 [DEBUG] TestAgent: Node info in sync -42:49.910 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-a43b2959-62e0-145d-187f-6c564c4a5e39 -42:49.910 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-a43b2959-62e0-145d-187f-6c564c4a5e39 -42:49.911 [INFO] TestAgent: Requesting shutdown -42:49.911 [INFO] TestAgent.server: shutting down server -42:49.911 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:49.911 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:49.911 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:49.912 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:49.912 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:49.912 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:49.912 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:49.912 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:49.912 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:49.912 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:49.912 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:49.912 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:49.912 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:49.912 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:49.913 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:49.913Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:49.913Z [INFO] agent.router.manager: shutting down -42:49.914 [INFO] TestAgent: consul server down -42:49.914 [INFO] TestAgent: shutdown complete -42:49.914 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13911 network=tcp -42:49.914 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13911 network=udp -42:49.914 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13912 network=tcp -42:50.415 [INFO] TestAgent: Waiting for endpoints to shut down -42:50.415 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentConnectAuthorize_defaultDeny (1.08s) -=== CONT TestAgentConnectAuthorize_idNotService -2022-09-15T21:42:43.419Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:43.497Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:43.511 [INFO] bonnie.server.gateway_locator: will dial the primary datacenter through its mesh gateways -42:43.518 [INFO] bonnie.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:601a4c67-78e5-5602-05c0-0653e8e117a7 Address:127.0.0.1:13554}]" -42:43.519 [INFO] bonnie.server.serf.wan: serf: EventMemberJoin: bonnie.dc3 127.0.0.1 -42:43.521 [INFO] bonnie.server.raft: entering follower state: follower="Node at 127.0.0.1:13554 [Follower]" leader= -42:43.522 [INFO] bonnie.server.serf.lan: serf: EventMemberJoin: bonnie 127.0.0.1 -2022-09-15T21:42:43.522Z [INFO] agent.router: Initializing LAN area manager -42:43.522 [INFO] bonnie.server: Adding LAN server: server="bonnie (Addr: tcp/127.0.0.1:13554) (DC: dc3)" -42:43.522 [INFO] bonnie.server: Handled event for server in area: event=member-join server=bonnie.dc3 area=wan -42:43.523 [INFO] bonnie: Started DNS server: address=127.0.0.1:13620 network=udp -42:43.523 [INFO] bonnie: Started DNS server: address=127.0.0.1:13620 network=tcp -42:43.526 [INFO] bonnie: Started HTTP server: address=127.0.0.1:13621 network=tcp -42:43.526 [INFO] bonnie: started state syncer -42:43.526 [INFO] bonnie: Started gRPC server: address=127.0.0.1:13626 network=tcp -42:43.526 [INFO] bonnie: Refreshing mesh gateways is supported for the following discovery methods: discovery_methods="aliyun aws gce mdns os packet" -42:43.526 [INFO] bonnie: Refreshing mesh gateways... -42:43.526 [INFO] bonnie.server.gateway_locator: updated fallback list of primary mesh gateways: mesh_gateways=[127.0.0.1:13449] -42:43.526 [INFO] bonnie: Refreshing mesh gateways completed -42:43.526 [INFO] bonnie: Retry join is supported for the following discovery methods: cluster=WAN discovery_methods="aliyun aws gce mdns os packet" -42:43.526 [INFO] bonnie: Joining cluster...: cluster=WAN -42:43.526 [INFO] bonnie: (WAN) joining: wan_addresses=[*.dc1/192.0.2.2] -42:43.581 [WARN] bonnie.server.raft: heartbeat timeout reached, starting election: last-leader= -42:43.581 [INFO] bonnie.server.raft: entering candidate state: node="Node at 127.0.0.1:13554 [Candidate]" term=2 -42:43.582 [DEBUG] bonnie.server.raft: votes: needed=1 -42:43.582 [DEBUG] bonnie.server.raft: vote granted: from=601a4c67-78e5-5602-05c0-0653e8e117a7 term=2 tally=1 -42:43.582 [INFO] bonnie.server.raft: election won: tally=1 -42:43.582 [INFO] bonnie.server.raft: entering leader state: leader="Node at 127.0.0.1:13554 [Leader]" -42:43.583 [INFO] bonnie.server: cluster leadership acquired -42:43.584 [DEBUG] bonnie.server: Cannot upgrade to new ACLs, servers in acl datacenter are not yet upgraded: ACLDatacenter=dc1 mode=1 found=false -42:43.584 [INFO] bonnie.server: New leader elected: payload=bonnie -42:43.586 [DEBUG] bonnie.server.memberlist.wan: memberlist: Initiating push/pull sync with: *.dc1 127.0.0.1:13449 -42:43.591 [INFO] bonnie.server.serf.wan: serf: EventMemberJoin: betty.dc2 127.0.0.1 -42:43.592 [INFO] bonnie.server: Handled event for server in area: event=member-join server=betty.dc2 area=wan -42:43.593 [INFO] bonnie.server.serf.wan: serf: EventMemberJoin: bob.dc1 127.0.0.1 -42:43.593 [INFO] bonnie: (WAN) joined: number_of_nodes=1 -42:43.593 [INFO] bonnie: Join cluster completed. Synced with initial agents: cluster=WAN num_agents=1 -42:43.593 [INFO] bonnie.server: Handled event for server in area: event=member-join server=bob.dc1 area=wan -42:43.628 [INFO] bonnie: Synced node info -42:43.628 [DEBUG] bonnie: Node info in sync -42:43.649 [DEBUG] connect.ca.consul: consul CA provider configured: id=ad:4a:c6:ab:ef:63:c9:60:1a:51:7f:19:62:e3:e9:d9:0e:76:55:10:6e:74:24:69:28:a1:6c:b8:b9:8f:fd:89 is_primary=false -42:43.663 [INFO] bonnie.server.connect: received new intermediate certificate from primary datacenter -42:43.664 [INFO] bonnie.server.connect: updated root certificates from primary datacenter -42:43.664 [INFO] bonnie.server.connect: initialized secondary datacenter CA with provider: provider=consul -42:43.664 [INFO] bonnie.leader: started routine: routine="config entry replication" -42:43.664 [INFO] bonnie.leader: started routine: routine="federation state replication" -42:43.664 [INFO] bonnie.leader: started routine: routine="federation state anti-entropy" -42:43.664 [INFO] bonnie.leader: started routine: routine="secondary CA roots watch" -42:43.664 [INFO] bonnie.leader: started routine: routine="intention replication" -42:43.664 [INFO] bonnie.leader: started routine: routine="intermediate cert renew watch" -42:43.664 [INFO] bonnie.leader: started routine: routine="CA root pruning" -42:43.664 [DEBUG] bonnie.server: Skipping self join check for node since the cluster is too small: node=bonnie -42:43.664 [INFO] bonnie.server: member joined, marking health alive: member=bonnie -42:43.665 [DEBUG] bonnie.server.connect: starting Connect CA root replication from primary datacenter: primary=dc1 -42:43.665 [DEBUG] bonnie.server.connect: starting Connect intention replication from primary datacenter: primary=dc1 -42:43.669 [INFO] bonnie.server.connect: updated root certificates from primary datacenter -42:43.671 [INFO] bonnie.server: federation state anti-entropy synced -42:43.674 [INFO] bonnie.server.gateway_locator: will dial the primary datacenter using our local mesh gateways if possible -42:43.674 [DEBUG] bonnie.server: finished fetching remote objects: amount=3 -42:43.674 [DEBUG] bonnie.server: diffing replication state: local_amount=0 remote_amount=3 -42:43.674 [DEBUG] bonnie.server: diffed replication state: deletions=0 updates=3 -42:43.674 [DEBUG] bonnie.server: performing updates: updates=3 -42:43.677 [DEBUG] bonnie.server.replication.config_entry: finished fetching config entries: amount=0 -42:43.677 [DEBUG] bonnie.server.replication.config_entry: Config Entry replication: local=0 remote=0 -42:43.677 [DEBUG] bonnie.server.replication.config_entry: Config Entry replication: deletions=0 updates=0 -42:43.677 [DEBUG] bonnie.server.replication.config_entry: replication completed through remote index: index=1 -42:43.678 [INFO] bonnie.server: federation state anti-entropy synced -42:43.690 [INFO] bonnie.server: federation state anti-entropy synced -42:43.695 [DEBUG] bonnie.server: finished updates -42:43.695 [DEBUG] bonnie.replication.federation_state: replication completed through remote index: index=19 -42:43.697 [DEBUG] bonnie.server: finished fetching remote objects: amount=3 -42:43.697 [DEBUG] bonnie.server: diffing replication state: local_amount=3 remote_amount=3 -42:43.697 [DEBUG] bonnie.server: diffed replication state: deletions=0 updates=1 -42:43.697 [DEBUG] bonnie.server: performing updates: updates=1 -42:43.697 [DEBUG] bonnie.server: finished updates -42:43.697 [DEBUG] bonnie.replication.federation_state: replication completed through remote index: index=21 -42:43.731 [DEBUG] bonnie: Node info in sync -42:43.731 [INFO] bonnie: Synced service: service=mesh-gateway -42:43.733 [DEBUG] bonnie.server: finished fetching remote objects: amount=3 -42:43.733 [DEBUG] bonnie.server: diffing replication state: local_amount=3 remote_amount=3 -42:43.733 [DEBUG] bonnie.server: diffed replication state: deletions=0 updates=1 -42:43.733 [DEBUG] bonnie.server: performing updates: updates=1 -42:43.734 [DEBUG] bonnie.server: finished updates -42:43.734 [DEBUG] bonnie.replication.federation_state: replication completed through remote index: index=22 -42:43.734 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:13450 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -42:43.734 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:13450 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -42:43.734 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:13450 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -42:43.734 [ERROR] bonnie.proxycfg: watch error: service_id=mesh-gateway id=mesh-gateway:dc2 error="error filling agent cache: rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -42:43.735 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:13450 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -42:43.735 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:13450 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -42:43.736 [ERROR] bonnie.proxycfg: watch error: service_id=mesh-gateway id=mesh-gateway:dc2 error="error filling agent cache: rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -42:43.736 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:13450 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -42:43.736 [ERROR] bonnie.proxycfg: watch error: service_id=mesh-gateway id=mesh-gateway:dc2 error="error filling agent cache: rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -42:43.736 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:13450 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -42:43.736 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:13450 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -42:43.736 [ERROR] bonnie.proxycfg: watch error: service_id=mesh-gateway id=mesh-gateway:dc2 error="error filling agent cache: rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -42:44.015 [DEBUG] bonnie: Skipping remote check since it is managed automatically: check=serfHealth -42:44.015 [DEBUG] bonnie: Node info in sync -42:44.015 [DEBUG] bonnie: Service in sync: service=mesh-gateway -42:44.015 [DEBUG] bonnie: Node info in sync -42:44.015 [DEBUG] bonnie: Service in sync: service=mesh-gateway -42:44.039 [DEBUG] bonnie.server.serf.wan: serf: messageJoinType: betty.dc2 -42:44.055 [ERROR] bonnie.server.memberlist.wan: memberlist: Failed to send gossip to 127.0.0.1:13596: Remote DC has no server currently reachable -42:44.469 [DEBUG] bonnie.server.serf.wan: serf: messageJoinType: betty.dc2 -42:44.469 [DEBUG] bonnie.server.serf.wan: serf: messageJoinType: bonnie.dc3 -42:44.530 [ERROR] bonnie.server.memberlist.wan: memberlist: Failed to send gossip to 127.0.0.1:13596: Remote DC has no server currently reachable -42:44.971 [DEBUG] bonnie.server.serf.wan: serf: messageJoinType: bonnie.dc3 -42:45.149 [DEBUG] bonnie.server.serf.wan: serf: messageJoinType: bonnie.dc3 -42:45.525 [INFO] bonnie.server.gateway_locator: new cached locations of mesh gateways: primary=[127.0.0.1:13449] local=[] -42:46.521 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:13450 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -42:47.009 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:13450 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -42:47.125 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:13450 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -42:47.125 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:13450 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -42:47.125 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:13450 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -42:47.125 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:13450 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -42:47.125 [ERROR] bonnie.proxycfg: watch error: service_id=mesh-gateway id=mesh-gateway:dc2 error="error filling agent cache: rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -42:48.119 [ERROR] bonnie.server.memberlist.wan: memberlist: Failed to send ack: Remote DC has no server currently reachable from=127.0.0.1:57450 -42:48.522 [ERROR] bonnie.server.memberlist.wan: memberlist: Failed to send ping: Remote DC has no server currently reachable -42:48.681 [DEBUG] bonnie.server: finished fetching remote objects: amount=3 -42:48.681 [DEBUG] bonnie.server: diffing replication state: local_amount=3 remote_amount=3 -42:48.681 [DEBUG] bonnie.server: diffed replication state: deletions=0 updates=1 -42:48.681 [DEBUG] bonnie.server: performing updates: updates=1 -42:48.682 [DEBUG] bonnie.server: finished updates -42:48.682 [DEBUG] bonnie.replication.federation_state: replication completed through remote index: index=23 -42:48.683 [INFO] bonnie.server.gateway_locator: new cached locations of mesh gateways: primary=[127.0.0.1:13449] local=[127.0.0.1:13449] -42:48.683 [INFO] bonnie.server: federation state anti-entropy synced -42:48.944 [INFO] bonnie: Requesting shutdown -42:48.944 [INFO] bonnie.server: shutting down server -42:48.944 [DEBUG] bonnie.leader: stopping routine: routine="secondary CA roots watch" -42:48.944 [DEBUG] bonnie.leader: stopping routine: routine="intention replication" -42:48.944 [DEBUG] bonnie.leader: stopping routine: routine="intermediate cert renew watch" -42:48.944 [DEBUG] bonnie.leader: stopping routine: routine="CA root pruning" -42:48.944 [DEBUG] bonnie.leader: stopping routine: routine="config entry replication" -42:48.944 [DEBUG] bonnie.leader: stopping routine: routine="federation state replication" -42:48.944 [DEBUG] bonnie.leader: stopping routine: routine="federation state anti-entropy" -42:48.944 [WARN] bonnie.server.serf.lan: serf: Shutdown without a Leave -42:48.946 [DEBUG] bonnie.leader: stopped routine: routine="intermediate cert renew watch" -42:48.946 [DEBUG] bonnie.leader: stopped routine: routine="CA root pruning" -42:48.946 [DEBUG] bonnie.leader: stopped routine: routine="federation state anti-entropy" -42:48.946 [WARN] bonnie.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:48.947Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:48.947Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:48.947Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:48.947Z [INFO] agent.router.manager: shutting down -42:48.947 [INFO] bonnie: consul server down -42:48.948 [INFO] bonnie: shutdown complete -42:48.948 [INFO] bonnie: Stopping server: protocol=DNS address=127.0.0.1:13620 network=tcp -42:48.948 [INFO] bonnie: Stopping server: protocol=DNS address=127.0.0.1:13620 network=udp -42:48.948 [INFO] bonnie: Stopping server: protocol=HTTP address=127.0.0.1:13621 network=tcp -42:48.948 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:13451 datacenter=dc1 method=Internal.ServiceDump error="rpc error making call: EOF" -42:48.948 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:13451 datacenter=dc1 method=ConfigEntry.ListAll error="rpc error making call: EOF" -42:48.948 [INFO] bonnie.server.replication.config_entry: stopped replication -42:48.948 [DEBUG] bonnie.leader: stopped routine: routine="config entry replication" -42:48.948 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:13451 datacenter=dc1 method=ConnectCA.Roots error="rpc error making call: EOF" -42:48.948 [ERROR] bonnie.server.connect: CA root replication failed, will retry: routine="secondary CA roots watch" error="Error retrieving the primary datacenter's roots: rpc error making call: EOF" -42:48.948 [DEBUG] bonnie.leader: stopped routine: routine="secondary CA roots watch" -42:48.948 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:13451 datacenter=dc1 method=FederationState.List error="rpc error making call: EOF" -42:48.948 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:13451 datacenter=dc1 method=Intention.List error="rpc error making call: EOF" -42:48.948 [ERROR] bonnie.server.connect: error replicating intentions: routine="intention replication" error="rpc error making call: EOF" -42:48.948 [DEBUG] bonnie.leader: stopped routine: routine="intention replication" -42:48.948 [WARN] bonnie.replication.federation_state: replication error (will retry if still leader): error="failed to retrieve federation states: rpc error making call: EOF" -42:48.948 [INFO] bonnie.replication.federation_state: stopped replication -42:48.948 [DEBUG] bonnie.leader: stopped routine: routine="federation state replication" -42:49.450 [INFO] bonnie: Waiting for endpoints to shut down -42:49.450 [INFO] bonnie: Endpoints down -2022-09-15T21:42:43.084Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:43.110Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:43.112 [INFO] betty.server.gateway_locator: will dial the primary datacenter through its mesh gateways -42:43.115 [INFO] betty.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c0a596ae-d119-2e6d-1950-20184ce983c5 Address:127.0.0.1:13450}]" -42:43.115 [INFO] betty.server.raft: entering follower state: follower="Node at 127.0.0.1:13450 [Follower]" leader= -42:43.116 [INFO] betty.server.serf.wan: serf: EventMemberJoin: betty.dc2 127.0.0.1 -42:43.116 [INFO] betty.server.serf.lan: serf: EventMemberJoin: betty 127.0.0.1 -2022-09-15T21:42:43.117Z [INFO] agent.router: Initializing LAN area manager -42:43.117 [INFO] betty: Started DNS server: address=127.0.0.1:13592 network=udp -42:43.117 [INFO] betty.server: Adding LAN server: server="betty (Addr: tcp/127.0.0.1:13450) (DC: dc2)" -42:43.117 [INFO] betty.server: Handled event for server in area: event=member-join server=betty.dc2 area=wan -42:43.118 [INFO] betty: Started DNS server: address=127.0.0.1:13592 network=tcp -42:43.119 [INFO] betty: Started HTTP server: address=127.0.0.1:13593 network=tcp -42:43.119 [INFO] betty: started state syncer -42:43.119 [INFO] betty: Started gRPC server: address=127.0.0.1:13598 network=tcp -42:43.119 [INFO] betty: Refreshing mesh gateways is supported for the following discovery methods: discovery_methods="aliyun aws gce mdns os packet" -42:43.119 [INFO] betty: Refreshing mesh gateways... -42:43.119 [INFO] betty.server.gateway_locator: updated fallback list of primary mesh gateways: mesh_gateways=[127.0.0.1:13449] -42:43.119 [INFO] betty: Refreshing mesh gateways completed -42:43.119 [INFO] betty: Retry join is supported for the following discovery methods: cluster=WAN discovery_methods="aliyun aws gce mdns os packet" -42:43.119 [INFO] betty: Joining cluster...: cluster=WAN -42:43.119 [INFO] betty: (WAN) joining: wan_addresses=[*.dc1/192.0.2.2] -42:43.143 [DEBUG] betty.server.memberlist.wan: memberlist: Initiating push/pull sync with: *.dc1 127.0.0.1:13449 -42:43.145 [INFO] betty.server.serf.wan: serf: EventMemberJoin: bob.dc1 127.0.0.1 -42:43.145 [INFO] betty: (WAN) joined: number_of_nodes=1 -42:43.145 [INFO] betty: Join cluster completed. Synced with initial agents: cluster=WAN num_agents=1 -42:43.145 [INFO] betty.server: Handled event for server in area: event=member-join server=bob.dc1 area=wan -42:43.170 [WARN] betty.server.raft: heartbeat timeout reached, starting election: last-leader= -42:43.170 [INFO] betty.server.raft: entering candidate state: node="Node at 127.0.0.1:13450 [Candidate]" term=2 -42:43.170 [DEBUG] betty.server.raft: votes: needed=1 -42:43.170 [DEBUG] betty.server.raft: vote granted: from=c0a596ae-d119-2e6d-1950-20184ce983c5 term=2 tally=1 -42:43.170 [INFO] betty.server.raft: election won: tally=1 -42:43.170 [INFO] betty.server.raft: entering leader state: leader="Node at 127.0.0.1:13450 [Leader]" -42:43.170 [INFO] betty.server: cluster leadership acquired -42:43.170 [DEBUG] betty.server: Cannot upgrade to new ACLs, servers in acl datacenter are not yet upgraded: ACLDatacenter=dc1 mode=0 found=true -42:43.171 [INFO] betty.server: New leader elected: payload=betty -42:43.197 [DEBUG] connect.ca.consul: consul CA provider configured: id=ad:4a:c6:ab:ef:63:c9:60:1a:51:7f:19:62:e3:e9:d9:0e:76:55:10:6e:74:24:69:28:a1:6c:b8:b9:8f:fd:89 is_primary=false -42:43.202 [INFO] betty.server.connect: received new intermediate certificate from primary datacenter -42:43.203 [INFO] betty.server.connect: updated root certificates from primary datacenter -42:43.203 [INFO] betty.server.connect: initialized secondary datacenter CA with provider: provider=consul -42:43.203 [INFO] betty.leader: started routine: routine="config entry replication" -42:43.203 [INFO] betty.leader: started routine: routine="federation state replication" -42:43.203 [INFO] betty.leader: started routine: routine="federation state anti-entropy" -42:43.203 [INFO] betty.leader: started routine: routine="secondary CA roots watch" -42:43.203 [INFO] betty.leader: started routine: routine="intention replication" -42:43.203 [INFO] betty.leader: started routine: routine="intermediate cert renew watch" -42:43.203 [INFO] betty.leader: started routine: routine="CA root pruning" -42:43.203 [DEBUG] betty.server: Skipping self join check for node since the cluster is too small: node=betty -42:43.203 [INFO] betty.server: member joined, marking health alive: member=betty -42:43.204 [DEBUG] betty.server.connect: starting Connect CA root replication from primary datacenter: primary=dc1 -42:43.204 [DEBUG] betty.server.connect: starting Connect intention replication from primary datacenter: primary=dc1 -42:43.205 [DEBUG] betty.server.replication.config_entry: finished fetching config entries: amount=0 -42:43.205 [DEBUG] betty.server.replication.config_entry: Config Entry replication: local=0 remote=0 -42:43.205 [DEBUG] betty.server.replication.config_entry: Config Entry replication: deletions=0 updates=0 -42:43.205 [DEBUG] betty.server.replication.config_entry: replication completed through remote index: index=1 -42:43.205 [INFO] betty.server: federation state anti-entropy synced -42:43.205 [INFO] betty.server.gateway_locator: will dial the primary datacenter using our local mesh gateways if possible -42:43.205 [DEBUG] betty.server: finished fetching remote objects: amount=1 -42:43.205 [DEBUG] betty.server: diffing replication state: local_amount=0 remote_amount=1 -42:43.205 [DEBUG] betty.server: diffed replication state: deletions=0 updates=1 -42:43.205 [DEBUG] betty.server: performing updates: updates=1 -42:43.206 [DEBUG] betty.server: finished updates -42:43.206 [DEBUG] betty.replication.federation_state: replication completed through remote index: index=14 -42:43.206 [INFO] betty.server.connect: updated root certificates from primary datacenter -42:43.207 [INFO] betty.server: federation state anti-entropy synced -42:43.207 [DEBUG] betty.server: finished fetching remote objects: amount=2 -42:43.207 [DEBUG] betty.server: diffing replication state: local_amount=1 remote_amount=2 -42:43.207 [DEBUG] betty.server: diffed replication state: deletions=0 updates=1 -42:43.207 [DEBUG] betty.server: performing updates: updates=1 -42:43.208 [DEBUG] betty.server: finished updates -42:43.208 [DEBUG] betty.replication.federation_state: replication completed through remote index: index=17 -42:43.508 [DEBUG] betty: Skipping remote check since it is managed automatically: check=serfHealth -42:43.518 [INFO] betty: Synced node info -42:43.673 [DEBUG] betty.server: finished fetching remote objects: amount=3 -42:43.673 [DEBUG] betty.server: diffing replication state: local_amount=2 remote_amount=3 -42:43.673 [DEBUG] betty.server: diffed replication state: deletions=0 updates=1 -42:43.673 [DEBUG] betty.server: performing updates: updates=1 -42:43.675 [DEBUG] betty.server: finished updates -42:43.675 [DEBUG] betty.replication.federation_state: replication completed through remote index: index=19 -42:43.679 [DEBUG] betty.server: finished fetching remote objects: amount=3 -42:43.679 [DEBUG] betty.server: diffing replication state: local_amount=3 remote_amount=3 -42:43.679 [DEBUG] betty.server: diffed replication state: deletions=0 updates=1 -42:43.679 [DEBUG] betty.server: performing updates: updates=1 -42:43.679 [DEBUG] betty.server: finished updates -42:43.679 [DEBUG] betty.replication.federation_state: replication completed through remote index: index=20 -42:43.690 [DEBUG] betty.server: finished fetching remote objects: amount=3 -42:43.690 [DEBUG] betty.server: diffing replication state: local_amount=3 remote_amount=3 -42:43.690 [DEBUG] betty.server: diffed replication state: deletions=0 updates=1 -42:43.690 [DEBUG] betty.server: performing updates: updates=1 -42:43.691 [DEBUG] betty.server: finished updates -42:43.691 [DEBUG] betty.replication.federation_state: replication completed through remote index: index=21 -42:43.729 [DEBUG] betty: Node info in sync -42:43.730 [INFO] betty: Synced service: service=mesh-gateway -42:43.733 [INFO] betty.server: federation state anti-entropy synced -42:44.033 [INFO] betty.server.serf.wan: serf: EventMemberJoin: bonnie.dc3 127.0.0.1 -42:44.033 [DEBUG] betty.server.serf.wan: serf: messageJoinType: betty.dc2 -42:44.033 [INFO] betty.server: Handled event for server in area: event=member-join server=bonnie.dc3 area=wan -42:44.117 [ERROR] betty.server.memberlist.wan: memberlist: Failed to send gossip to 127.0.0.1:13638: Remote DC has no server currently reachable -42:44.207 [DEBUG] betty.server: finished fetching remote objects: amount=3 -42:44.207 [DEBUG] betty.server: diffing replication state: local_amount=3 remote_amount=3 -42:44.207 [DEBUG] betty.server: diffed replication state: deletions=0 updates=1 -42:44.207 [DEBUG] betty.server: performing updates: updates=1 -42:44.208 [DEBUG] betty.server: finished updates -42:44.208 [DEBUG] betty.replication.federation_state: replication completed through remote index: index=22 -42:44.469 [DEBUG] betty.server.serf.wan: serf: messageJoinType: betty.dc2 -42:44.469 [DEBUG] betty.server.serf.wan: serf: messageJoinType: bonnie.dc3 -42:44.617 [ERROR] betty.server.memberlist.wan: memberlist: Failed to send gossip to 127.0.0.1:13638: Remote DC has no server currently reachable -42:44.970 [DEBUG] betty.server.serf.wan: serf: messageJoinType: bonnie.dc3 -42:45.115 [INFO] betty.server.gateway_locator: new cached locations of mesh gateways: primary=[127.0.0.1:13449] local=[127.0.0.1:13449] -42:45.930 [DEBUG] betty: Skipping remote check since it is managed automatically: check=serfHealth -42:45.930 [DEBUG] betty: Node info in sync -42:45.930 [DEBUG] betty: Service in sync: service=mesh-gateway -42:45.930 [DEBUG] betty: Node info in sync -42:45.930 [DEBUG] betty: Service in sync: service=mesh-gateway -42:48.683 [DEBUG] betty.server: finished fetching remote objects: amount=3 -42:48.683 [DEBUG] betty.server: diffing replication state: local_amount=3 remote_amount=3 -42:48.683 [DEBUG] betty.server: diffed replication state: deletions=0 updates=1 -42:48.683 [DEBUG] betty.server: performing updates: updates=1 -42:48.684 [DEBUG] betty.server: finished updates -42:48.684 [DEBUG] betty.replication.federation_state: replication completed through remote index: index=23 -42:49.450 [INFO] betty: Requesting shutdown -42:49.450 [INFO] betty.server: shutting down server -42:49.450 [DEBUG] betty.leader: stopping routine: routine="secondary CA roots watch" -42:49.450 [DEBUG] betty.leader: stopping routine: routine="intention replication" -42:49.450 [DEBUG] betty.leader: stopping routine: routine="intermediate cert renew watch" -42:49.450 [DEBUG] betty.leader: stopping routine: routine="CA root pruning" -42:49.450 [DEBUG] betty.leader: stopping routine: routine="config entry replication" -42:49.450 [DEBUG] betty.leader: stopping routine: routine="federation state replication" -42:49.450 [DEBUG] betty.leader: stopping routine: routine="federation state anti-entropy" -42:49.450 [WARN] betty.server.serf.lan: serf: Shutdown without a Leave -42:49.450 [ERROR] betty.server: error performing anti-entropy sync of federation state: error="context canceled" -42:49.450 [DEBUG] betty.leader: stopped routine: routine="federation state anti-entropy" -42:49.450 [DEBUG] betty.leader: stopped routine: routine="intermediate cert renew watch" -42:49.450 [DEBUG] betty.leader: stopped routine: routine="CA root pruning" -42:49.450 [WARN] betty.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:49.451Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:49.451Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:49.451Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:49.451Z [INFO] agent.router.manager: shutting down -42:49.451 [INFO] betty: consul server down -42:49.451 [INFO] betty: shutdown complete -42:49.451 [INFO] betty: Stopping server: protocol=DNS address=127.0.0.1:13592 network=tcp -42:49.452 [INFO] betty: Stopping server: protocol=DNS address=127.0.0.1:13592 network=udp -42:49.452 [INFO] betty: Stopping server: protocol=HTTP address=127.0.0.1:13593 network=tcp -42:49.452 [ERROR] betty.server.rpc: RPC failed to server in DC: server=127.0.0.1:13451 datacenter=dc1 method=ConnectCA.Roots error="rpc error making call: EOF" -42:49.452 [ERROR] betty.server.connect: CA root replication failed, will retry: routine="secondary CA roots watch" error="Error retrieving the primary datacenter's roots: rpc error making call: EOF" -42:49.452 [DEBUG] betty.leader: stopped routine: routine="secondary CA roots watch" -42:49.452 [ERROR] betty.server.rpc: RPC failed to server in DC: server=127.0.0.1:13451 datacenter=dc1 method=FederationState.List error="rpc error making call: EOF" -42:49.452 [WARN] betty.replication.federation_state: replication error (will retry if still leader): error="failed to retrieve federation states: rpc error making call: EOF" -42:49.452 [INFO] betty.replication.federation_state: stopped replication -42:49.452 [DEBUG] betty.leader: stopped routine: routine="federation state replication" -42:49.452 [ERROR] betty.server.rpc: RPC failed to server in DC: server=127.0.0.1:13451 datacenter=dc1 method=ConfigEntry.ListAll error="rpc error making call: EOF" -42:49.452 [INFO] betty.server.replication.config_entry: stopped replication -42:49.452 [DEBUG] betty.leader: stopped routine: routine="config entry replication" -42:49.452 [ERROR] betty.server.rpc: RPC failed to server in DC: server=127.0.0.1:13451 datacenter=dc1 method=Internal.ServiceDump error="rpc error making call: EOF" -42:49.452 [ERROR] betty.server.rpc: RPC failed to server in DC: server=127.0.0.1:13451 datacenter=dc1 method=Intention.List error="rpc error making call: EOF" -42:49.452 [ERROR] betty.server.connect: error replicating intentions: routine="intention replication" error="rpc error making call: EOF" -42:49.452 [DEBUG] betty.leader: stopped routine: routine="intention replication" -42:49.954 [INFO] betty: Waiting for endpoints to shut down -42:49.954 [INFO] betty: Endpoints down -2022-09-15T21:42:40.939Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:40.960Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:40.964 [INFO] bob.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a0e1bec4-fb15-1ffc-cb1e-3b12db7f4cae Address:127.0.0.1:13451}]" -42:40.966 [INFO] bob.server.raft: entering follower state: follower="Node at 127.0.0.1:13451 [Follower]" leader= -42:40.967 [INFO] bob.server.serf.wan: serf: EventMemberJoin: bob.dc1 127.0.0.1 -42:40.968 [INFO] bob.server.serf.lan: serf: EventMemberJoin: bob 127.0.0.1 -2022-09-15T21:42:40.968Z [INFO] agent.router: Initializing LAN area manager -42:40.968 [INFO] bob.server: Handled event for server in area: event=member-join server=bob.dc1 area=wan -42:40.968 [INFO] bob.server: Adding LAN server: server="bob (Addr: tcp/127.0.0.1:13451) (DC: dc1)" -42:40.968 [INFO] bob: Started DNS server: address=127.0.0.1:13555 network=udp -42:40.968 [INFO] bob: Started DNS server: address=127.0.0.1:13555 network=tcp -42:40.969 [INFO] bob: Started HTTP server: address=127.0.0.1:13557 network=tcp -42:40.969 [INFO] bob: started state syncer -42:40.970 [INFO] bob: Started gRPC server: address=127.0.0.1:13452 network=tcp -42:41.031 [WARN] bob.server.raft: heartbeat timeout reached, starting election: last-leader= -42:41.031 [INFO] bob.server.raft: entering candidate state: node="Node at 127.0.0.1:13451 [Candidate]" term=2 -42:41.031 [DEBUG] bob.server.raft: votes: needed=1 -42:41.031 [DEBUG] bob.server.raft: vote granted: from=a0e1bec4-fb15-1ffc-cb1e-3b12db7f4cae term=2 tally=1 -42:41.031 [INFO] bob.server.raft: election won: tally=1 -42:41.031 [INFO] bob.server.raft: entering leader state: leader="Node at 127.0.0.1:13451 [Leader]" -42:41.031 [INFO] bob.server: cluster leadership acquired -42:41.032 [DEBUG] bob.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13451 -42:41.033 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:41.033 [INFO] bob.server: New leader elected: payload=bob -42:41.036 [INFO] bob.server.connect: initialized primary datacenter CA with provider: provider=consul -42:41.036 [INFO] bob.leader: started routine: routine="federation state anti-entropy" -42:41.036 [INFO] bob.leader: started routine: routine="federation state pruning" -42:41.036 [INFO] bob.leader: started routine: routine="intermediate cert renew watch" -42:41.036 [INFO] bob.leader: started routine: routine="CA root pruning" -42:41.036 [DEBUG] bob.server: Skipping self join check for node since the cluster is too small: node=bob -42:41.036 [INFO] bob.server: member joined, marking health alive: member=bob -42:41.037 [INFO] bob.server: federation state anti-entropy synced -42:41.167 [DEBUG] bob: Skipping remote check since it is managed automatically: check=serfHealth -42:41.169 [INFO] bob: Synced node info -42:41.413 [DEBUG] bob: Node info in sync -42:41.415 [INFO] bob: Synced service: service=mesh-gateway -42:41.435 [INFO] bob.server: federation state anti-entropy synced -42:42.616 [DEBUG] bob: Skipping remote check since it is managed automatically: check=serfHealth -42:42.616 [DEBUG] bob: Node info in sync -42:42.616 [DEBUG] bob: Service in sync: service=mesh-gateway -42:42.616 [DEBUG] bob: Node info in sync -42:42.616 [DEBUG] bob: Service in sync: service=mesh-gateway -42:42.970 [INFO] bob.server.gateway_locator: new cached locations of mesh gateways: primary=[127.0.0.1:13449] local=[127.0.0.1:13449] -42:43.144 [DEBUG] bob.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:50346 -42:43.145 [INFO] bob.server.serf.wan: serf: EventMemberJoin: betty.dc2 127.0.0.1 -42:43.145 [INFO] bob.server: Handled event for server in area: event=member-join server=betty.dc2 area=wan -42:43.589 [DEBUG] bob.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:50372 -42:43.594 [INFO] bob.server.serf.wan: serf: EventMemberJoin: bonnie.dc3 127.0.0.1 -42:43.598 [INFO] bob.server: Handled event for server in area: event=member-join server=bonnie.dc3 area=wan -42:43.675 [DEBUG] bob.server.serf.wan: serf: messageJoinType: betty.dc2 -42:44.057 [DEBUG] bob.server.serf.wan: serf: messageJoinType: bonnie.dc3 -42:44.117 [DEBUG] bob.server.serf.wan: serf: messageJoinType: betty.dc2 -42:44.531 [DEBUG] bob.server.serf.wan: serf: messageJoinType: bonnie.dc3 -42:44.531 [DEBUG] bob.server.serf.wan: serf: messageJoinType: betty.dc2 -42:44.618 [DEBUG] bob.server.serf.wan: serf: messageJoinType: bonnie.dc3 -42:45.021 [DEBUG] bob.server.serf.wan: serf: messageJoinType: betty.dc2 -42:45.148 [DEBUG] bob.server.serf.wan: serf: messageJoinType: bonnie.dc3 -42:49.954 [INFO] bob: Requesting shutdown -42:49.954 [INFO] bob.server: shutting down server -42:49.954 [DEBUG] bob.leader: stopping routine: routine="federation state anti-entropy" -42:49.954 [DEBUG] bob.leader: stopping routine: routine="federation state pruning" -42:49.954 [DEBUG] bob.leader: stopping routine: routine="intermediate cert renew watch" -42:49.954 [DEBUG] bob.leader: stopping routine: routine="CA root pruning" -42:49.954 [WARN] bob.server.serf.lan: serf: Shutdown without a Leave -42:49.954 [DEBUG] bob.leader: stopped routine: routine="federation state pruning" -42:49.954 [DEBUG] bob.leader: stopped routine: routine="intermediate cert renew watch" -42:49.954 [DEBUG] bob.leader: stopped routine: routine="CA root pruning" -42:49.954 [WARN] bob.server.serf.wan: serf: Shutdown without a Leave -42:49.955 [ERROR] bob.server: error performing anti-entropy sync of federation state: error="context canceled" -42:49.955 [DEBUG] bob.leader: stopped routine: routine="federation state anti-entropy" -2022-09-15T21:42:49.955Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:49.955Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:49.955Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:49.955Z [INFO] agent.router.manager: shutting down -42:49.956 [INFO] bob: consul server down -42:49.956 [INFO] bob: shutdown complete -42:49.956 [INFO] bob: Stopping server: protocol=DNS address=127.0.0.1:13555 network=tcp -42:49.956 [INFO] bob: Stopping server: protocol=DNS address=127.0.0.1:13555 network=udp -42:49.956 [INFO] bob: Stopping server: protocol=HTTP address=127.0.0.1:13557 network=tcp -42:50.456 [INFO] bob: Waiting for endpoints to shut down -42:50.456 [INFO] bob: Endpoints down ---- PASS: TestAgent_JoinWAN_viaMeshGateway (9.59s) - --- PASS: TestAgent_JoinWAN_viaMeshGateway/dc1_to_dc2 (0.06s) - --- PASS: TestAgent_JoinWAN_viaMeshGateway/dc1_to_dc3 (0.08s) - --- PASS: TestAgent_JoinWAN_viaMeshGateway/dc2_to_dc1 (0.00s) - --- PASS: TestAgent_JoinWAN_viaMeshGateway/dc2_to_dc3 (0.08s) - --- PASS: TestAgent_JoinWAN_viaMeshGateway/dc3_to_dc1 (0.00s) - --- PASS: TestAgent_JoinWAN_viaMeshGateway/dc3_to_dc2 (0.03s) -=== CONT TestAgentConnectAuthorize_idInvalidFormat -2022-09-15T21:42:49.749Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:49.824Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:49.833 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2ff3c9bb-4fad-20ee-e66d-c5837eb3186d Address:127.0.0.1:13930}]" -42:49.834 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2ff3c9bb-4fad-20ee-e66d-c5837eb3186d.dc1 127.0.0.1 -42:49.835 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2ff3c9bb-4fad-20ee-e66d-c5837eb3186d 127.0.0.1 -2022-09-15T21:42:49.835Z [INFO] agent.router: Initializing LAN area manager -42:49.836 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13918 network=udp -42:49.836 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13930 [Follower]" leader= -42:49.837 [INFO] TestAgent.server: Adding LAN server: server="Node-2ff3c9bb-4fad-20ee-e66d-c5837eb3186d (Addr: tcp/127.0.0.1:13930) (DC: dc1)" -42:49.837 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2ff3c9bb-4fad-20ee-e66d-c5837eb3186d.dc1 area=wan -42:49.839 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13918 network=tcp -42:49.841 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13919 network=tcp -42:49.841 [INFO] TestAgent: started state syncer -42:49.843 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13931 network=tcp -42:49.902 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:49.902 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13930 [Candidate]" term=2 -42:49.903 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:49.903 [DEBUG] TestAgent.server.raft: vote granted: from=2ff3c9bb-4fad-20ee-e66d-c5837eb3186d term=2 tally=1 -42:49.903 [INFO] TestAgent.server.raft: election won: tally=1 -42:49.903 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13930 [Leader]" -42:49.903 [INFO] TestAgent.server: cluster leadership acquired -42:49.903 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13930 -42:49.905 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:49.906 [INFO] TestAgent.server: New leader elected: payload=Node-2ff3c9bb-4fad-20ee-e66d-c5837eb3186d -42:49.917 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:49.917 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:49.917 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:49.917 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:49.917 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:49.919 [INFO] TestAgent.server: federation state anti-entropy synced -42:49.922 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2ff3c9bb-4fad-20ee-e66d-c5837eb3186d -42:49.922 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2ff3c9bb-4fad-20ee-e66d-c5837eb3186d -42:49.966 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:49.967 [INFO] TestAgent: Synced node info -42:49.983 [INFO] TestAgent: Requesting shutdown -42:49.983 [INFO] TestAgent.server: shutting down server -42:49.983 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:49.983 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:49.983 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:49.983 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:49.983 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:49.983 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:49.983 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:49.983 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:49.983 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:49.983 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:49.983 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -42:49.984 [INFO] TestAgent: consul server down -42:49.984 [INFO] TestAgent: shutdown complete -42:49.984 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13918 network=tcp -42:49.984 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13918 network=udp -42:49.984 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13919 network=tcp -2022-09-15T21:42:49.984Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:49.984Z [INFO] agent.router.manager: shutting down -42:50.485 [INFO] TestAgent: Waiting for endpoints to shut down -42:50.485 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentConnectAuthorize_denyWildcard (0.84s) -=== CONT TestAgentConnectAuthorize_noTarget -[WARN] freeport: 4 out of 42 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 4 out of 4 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 4 out of 4 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:50.025Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:50.132Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:50.137 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1728a168-6cd9-7fb8-f296-0dfb6f8d3dd3 Address:127.0.0.1:13937}]" -42:50.137 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13937 [Follower]" leader= -42:50.138 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1728a168-6cd9-7fb8-f296-0dfb6f8d3dd3.dc1 127.0.0.1 -42:50.139 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1728a168-6cd9-7fb8-f296-0dfb6f8d3dd3 127.0.0.1 -2022-09-15T21:42:50.139Z [INFO] agent.router: Initializing LAN area manager -42:50.140 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13932 network=udp -42:50.140 [INFO] TestAgent.server: Adding LAN server: server="Node-1728a168-6cd9-7fb8-f296-0dfb6f8d3dd3 (Addr: tcp/127.0.0.1:13937) (DC: dc1)" -42:50.140 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1728a168-6cd9-7fb8-f296-0dfb6f8d3dd3.dc1 area=wan -42:50.140 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13932 network=tcp -42:50.141 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13933 network=tcp -42:50.141 [INFO] TestAgent: started state syncer -42:50.141 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13938 network=tcp -42:50.190 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:50.190 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13937 [Candidate]" term=2 -42:50.191 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:50.191 [DEBUG] TestAgent.server.raft: vote granted: from=1728a168-6cd9-7fb8-f296-0dfb6f8d3dd3 term=2 tally=1 -42:50.191 [INFO] TestAgent.server.raft: election won: tally=1 -42:50.191 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13937 [Leader]" -42:50.191 [INFO] TestAgent.server: cluster leadership acquired -42:50.191 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13937 -42:50.193 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:50.194 [INFO] TestAgent.server: New leader elected: payload=Node-1728a168-6cd9-7fb8-f296-0dfb6f8d3dd3 -42:50.208 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:50.208 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:50.208 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:50.208 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:50.208 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:50.208 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1728a168-6cd9-7fb8-f296-0dfb6f8d3dd3 -42:50.208 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1728a168-6cd9-7fb8-f296-0dfb6f8d3dd3 -42:50.210 [INFO] TestAgent.server: federation state anti-entropy synced -42:50.231 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:50.238 [INFO] TestAgent: Synced node info -42:50.238 [DEBUG] TestAgent: Node info in sync -42:50.573 [INFO] TestAgent: Requesting shutdown -42:50.573 [INFO] TestAgent.server: shutting down server -42:50.573 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:50.573 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:50.573 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:50.573 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:50.573 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:50.574 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:50.574 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:50.574 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:50.574 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:50.574 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:50.574 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:50.574Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:50.574Z [INFO] agent.router.manager: shutting down -42:50.574 [INFO] TestAgent: consul server down -42:50.574 [INFO] TestAgent: shutdown complete -42:50.574 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13932 network=tcp -42:50.574 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13932 network=udp -42:50.574 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13933 network=tcp -42:51.075 [INFO] TestAgent: Waiting for endpoints to shut down -42:51.075 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentConnectAuthorize_allowTrustDomain (1.08s) -=== CONT TestPreparedQuery_parseLimit ---- PASS: TestPreparedQuery_parseLimit (0.00s) +2021-08-13T15:34:10.309Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:10.309Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:10.325Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:10.325Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:10.327 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d7435343-15eb-08f6-8798-0d574dbe8555 Address:127.0.0.1:25176}]" +34:10.328 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d7435343-15eb-08f6-8798-0d574dbe8555.dc1 127.0.0.1 +34:10.328 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25176 [Follower]" leader= +34:10.329 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d7435343-15eb-08f6-8798-0d574dbe8555 127.0.0.1 +2021-08-13T15:34:10.329Z [INFO] agent.router: Initializing LAN area manager +34:10.329 [INFO] TestAgent.server: Adding LAN server: server="Node-d7435343-15eb-08f6-8798-0d574dbe8555 (Addr: tcp/127.0.0.1:25176) (DC: dc1)" +34:10.329 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d7435343-15eb-08f6-8798-0d574dbe8555.dc1 area=wan +34:10.329 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25171 network=udp +34:10.329 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25171 network=tcp +34:10.330 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25172 network=tcp +34:10.330 [INFO] TestAgent: started state syncer +34:10.330 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25177 network=tcp +34:10.368 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:10.368 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25176 [Candidate]" term=2 +34:10.369 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:10.369 [DEBUG] TestAgent.server.raft: vote granted: from=d7435343-15eb-08f6-8798-0d574dbe8555 term=2 tally=1 +34:10.369 [INFO] TestAgent.server.raft: election won: tally=1 +34:10.369 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25176 [Leader]" +34:10.369 [INFO] TestAgent.server: cluster leadership acquired +34:10.369 [INFO] TestAgent.server: New leader elected: payload=Node-d7435343-15eb-08f6-8798-0d574dbe8555 +34:10.369 [INFO] TestAgent.server: initializing acls +34:10.370 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:10.370 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:10.370 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:10.371 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:10.371 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:10.371 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:10.371 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-d7435343-15eb-08f6-8798-0d574dbe8555 +34:10.371 [INFO] TestAgent.server: Updating LAN server: server="Node-d7435343-15eb-08f6-8798-0d574dbe8555 (Addr: tcp/127.0.0.1:25176) (DC: dc1)" +34:10.371 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-d7435343-15eb-08f6-8798-0d574dbe8555.dc1 +34:10.371 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-d7435343-15eb-08f6-8798-0d574dbe8555.dc1 area=wan +34:10.372 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:10.375 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:10.375 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:10.375 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:10.375 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:10.375 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:10.375 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d7435343-15eb-08f6-8798-0d574dbe8555 +34:10.375 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d7435343-15eb-08f6-8798-0d574dbe8555 +34:10.376 [INFO] TestAgent.server: federation state anti-entropy synced +34:10.376 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d7435343-15eb-08f6-8798-0d574dbe8555 +34:10.378 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:10.379 [INFO] TestAgent: Synced node info +34:10.379 [DEBUG] TestAgent: Node info in sync +34:10.512 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-d7435343-15eb-08f6-8798-0d574dbe8555 +34:10.512 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-d7435343-15eb-08f6-8798-0d574dbe8555 +34:10.513 [INFO] TestAgent: Requesting shutdown +34:10.513 [INFO] TestAgent.server: shutting down server +34:10.513 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:10.513 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:10.513 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:10.513 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:10.513 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:10.513 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:10.513 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:10.513 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:10.513 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:10.513 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:10.513 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:10.513 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:10.513 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:10.513 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:10.513 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:10.513Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:10.513Z [INFO] agent.router.manager: shutting down +34:10.514 [INFO] TestAgent: consul server down +34:10.514 [INFO] TestAgent: shutdown complete +34:10.514 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25171 network=tcp +34:10.514 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25171 network=udp +34:10.514 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25172 network=tcp +34:11.014 [INFO] TestAgent: Waiting for endpoints to shut down +34:11.014 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentConnectAuthorize_defaultDeny (0.72s) === CONT TestSessionCreate -2022-09-15T21:42:25.378Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:25.396Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:25.399 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5a1b3e87-7b4e-bed6-f6ba-94fc4534fa3c Address:127.0.0.1:13637}]" -42:25.400 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5a1b3e87-7b4e-bed6-f6ba-94fc4534fa3c.dc1 127.0.0.1 -42:25.401 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5a1b3e87-7b4e-bed6-f6ba-94fc4534fa3c 127.0.0.1 -2022-09-15T21:42:25.401Z [INFO] agent.router: Initializing LAN area manager -42:25.402 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13637 [Follower]" leader= -42:25.402 [INFO] TestAgent.server: Adding LAN server: server="Node-5a1b3e87-7b4e-bed6-f6ba-94fc4534fa3c (Addr: tcp/127.0.0.1:13637) (DC: dc1)" -42:25.402 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5a1b3e87-7b4e-bed6-f6ba-94fc4534fa3c.dc1 area=wan -42:25.403 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13632 network=udp -42:25.403 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13632 network=tcp -42:25.404 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13633 network=tcp -42:25.404 [INFO] TestAgent: started state syncer -42:25.404 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13666 network=tcp -42:25.447 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:25.447 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13637 [Candidate]" term=2 -42:25.447 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:25.447 [DEBUG] TestAgent.server.raft: vote granted: from=5a1b3e87-7b4e-bed6-f6ba-94fc4534fa3c term=2 tally=1 -42:25.447 [INFO] TestAgent.server.raft: election won: tally=1 -42:25.447 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13637 [Leader]" -42:25.448 [INFO] TestAgent.server: cluster leadership acquired -42:25.448 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13637 -42:25.448 [INFO] TestAgent.server: New leader elected: payload=Node-5a1b3e87-7b4e-bed6-f6ba-94fc4534fa3c -42:25.449 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:25.452 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:25.452 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:25.452 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:25.452 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:25.452 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:25.452 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5a1b3e87-7b4e-bed6-f6ba-94fc4534fa3c -42:25.452 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5a1b3e87-7b4e-bed6-f6ba-94fc4534fa3c -42:25.453 [INFO] TestAgent.server: federation state anti-entropy synced -42:25.608 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:25.609 [INFO] TestAgent: Synced node info -42:25.609 [DEBUG] TestAgent: Node info in sync -42:25.796 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=359.72µs client=127.0.0.1:39126 client_network=tcp -42:25.797 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=265.66µs client=127.0.0.1:39128 client_network=tcp -42:25.798 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=234.3µs client=127.0.0.1:53380 client_network=udp -42:25.799 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=190.46µs client=127.0.0.1:58455 client_network=udp -42:25.800 [DEBUG] TestAgent.dns: request served from client: name=4596e128-44c5-77a6-1e88-84b434bd7419.query.consul. type=ANY class=IN latency=216.36µs client=127.0.0.1:39130 client_network=tcp -42:25.801 [DEBUG] TestAgent.dns: request served from client: name=4596e128-44c5-77a6-1e88-84b434bd7419.query.consul. type=ANY class=IN latency=259.18µs client=127.0.0.1:39132 client_network=tcp -42:25.801 [DEBUG] TestAgent.dns: request served from client: name=4596e128-44c5-77a6-1e88-84b434bd7419.query.consul. type=ANY class=IN latency=202.54µs client=127.0.0.1:46701 client_network=udp -42:25.802 [DEBUG] TestAgent.dns: request served from client: name=4596e128-44c5-77a6-1e88-84b434bd7419.query.consul. type=ANY class=IN latency=165.32µs client=127.0.0.1:52064 client_network=udp -42:25.803 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=281.64µs client=127.0.0.1:39134 client_network=tcp -42:25.804 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=266.04µs client=127.0.0.1:39136 client_network=tcp -42:25.805 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=218.06µs client=127.0.0.1:54817 client_network=udp -42:25.805 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=205.16µs client=127.0.0.1:40372 client_network=udp -42:25.806 [DEBUG] TestAgent.dns: request served from client: name=4596e128-44c5-77a6-1e88-84b434bd7419.query.consul. type=ANY class=IN latency=272.38µs client=127.0.0.1:39138 client_network=tcp -42:25.807 [DEBUG] TestAgent.dns: request served from client: name=4596e128-44c5-77a6-1e88-84b434bd7419.query.consul. type=ANY class=IN latency=250.02µs client=127.0.0.1:39140 client_network=tcp -42:25.808 [DEBUG] TestAgent.dns: request served from client: name=4596e128-44c5-77a6-1e88-84b434bd7419.query.consul. type=ANY class=IN latency=209.18µs client=127.0.0.1:44844 client_network=udp -42:25.809 [DEBUG] TestAgent.dns: request served from client: name=4596e128-44c5-77a6-1e88-84b434bd7419.query.consul. type=ANY class=IN latency=200.1µs client=127.0.0.1:55257 client_network=udp -42:25.810 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=391.62µs client=127.0.0.1:39142 client_network=tcp -42:25.811 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=312.74µs client=127.0.0.1:39144 client_network=tcp -42:25.812 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=245.68µs client=127.0.0.1:36693 client_network=udp -42:25.813 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=212.46µs client=127.0.0.1:37990 client_network=udp -42:25.813 [DEBUG] TestAgent.dns: request served from client: name=4596e128-44c5-77a6-1e88-84b434bd7419.query.consul. type=ANY class=IN latency=268.26µs client=127.0.0.1:39146 client_network=tcp -42:25.814 [DEBUG] TestAgent.dns: request served from client: name=4596e128-44c5-77a6-1e88-84b434bd7419.query.consul. type=ANY class=IN latency=215.66µs client=127.0.0.1:39148 client_network=tcp -42:25.815 [DEBUG] TestAgent.dns: request served from client: name=4596e128-44c5-77a6-1e88-84b434bd7419.query.consul. type=ANY class=IN latency=170.84µs client=127.0.0.1:50029 client_network=udp -42:25.815 [DEBUG] TestAgent.dns: request served from client: name=4596e128-44c5-77a6-1e88-84b434bd7419.query.consul. type=ANY class=IN latency=168.82µs client=127.0.0.1:38065 client_network=udp -42:25.816 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=285.94µs client=127.0.0.1:39150 client_network=tcp -42:25.817 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=235.26µs client=127.0.0.1:39152 client_network=tcp -42:25.817 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=225.8µs client=127.0.0.1:59983 client_network=udp -42:25.818 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=214.38µs client=127.0.0.1:58961 client_network=udp -42:25.819 [DEBUG] TestAgent.dns: request served from client: name=4596e128-44c5-77a6-1e88-84b434bd7419.query.consul. type=ANY class=IN latency=231.92µs client=127.0.0.1:39154 client_network=tcp -42:25.820 [DEBUG] TestAgent.dns: request served from client: name=4596e128-44c5-77a6-1e88-84b434bd7419.query.consul. type=ANY class=IN latency=232.74µs client=127.0.0.1:39156 client_network=tcp -42:25.821 [DEBUG] TestAgent.dns: request served from client: name=4596e128-44c5-77a6-1e88-84b434bd7419.query.consul. type=ANY class=IN latency=188.24µs client=127.0.0.1:33667 client_network=udp -42:25.821 [DEBUG] TestAgent.dns: request served from client: name=4596e128-44c5-77a6-1e88-84b434bd7419.query.consul. type=ANY class=IN latency=179.98µs client=127.0.0.1:59804 client_network=udp -42:25.822 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=323.14µs client=127.0.0.1:39158 client_network=tcp -42:25.823 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=301.36µs client=127.0.0.1:39160 client_network=tcp -42:25.824 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=228.1µs client=127.0.0.1:56347 client_network=udp -42:25.825 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=211.54µs client=127.0.0.1:37007 client_network=udp -42:25.830 [DEBUG] TestAgent.dns: request served from client: name=4596e128-44c5-77a6-1e88-84b434bd7419.query.consul. type=ANY class=IN latency=344.68µs client=127.0.0.1:39162 client_network=tcp -42:25.831 [DEBUG] TestAgent.dns: request served from client: name=4596e128-44c5-77a6-1e88-84b434bd7419.query.consul. type=ANY class=IN latency=272.16µs client=127.0.0.1:39164 client_network=tcp -42:25.832 [DEBUG] TestAgent.dns: request served from client: name=4596e128-44c5-77a6-1e88-84b434bd7419.query.consul. type=ANY class=IN latency=232.5µs client=127.0.0.1:54018 client_network=udp -42:25.833 [DEBUG] TestAgent.dns: request served from client: name=4596e128-44c5-77a6-1e88-84b434bd7419.query.consul. type=ANY class=IN latency=183.78µs client=127.0.0.1:46757 client_network=udp -42:25.833 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=287.04µs client=127.0.0.1:39166 client_network=tcp -42:25.834 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=274.88µs client=127.0.0.1:39168 client_network=tcp -42:25.835 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=211.44µs client=127.0.0.1:54563 client_network=udp -42:25.836 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=202.36µs client=127.0.0.1:36779 client_network=udp -42:25.836 [DEBUG] TestAgent.dns: request served from client: name=4596e128-44c5-77a6-1e88-84b434bd7419.query.consul. type=ANY class=IN latency=257.54µs client=127.0.0.1:39170 client_network=tcp -42:25.839 [DEBUG] TestAgent.dns: request served from client: name=4596e128-44c5-77a6-1e88-84b434bd7419.query.consul. type=ANY class=IN latency=315.7µs client=127.0.0.1:39172 client_network=tcp -42:25.840 [DEBUG] TestAgent.dns: request served from client: name=4596e128-44c5-77a6-1e88-84b434bd7419.query.consul. type=ANY class=IN latency=250.78µs client=127.0.0.1:59215 client_network=udp -42:25.840 [DEBUG] TestAgent.dns: request served from client: name=4596e128-44c5-77a6-1e88-84b434bd7419.query.consul. type=ANY class=IN latency=169.48µs client=127.0.0.1:52705 client_network=udp -42:28.228 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:28.228 [DEBUG] TestAgent: Node info in sync -42:30.588 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 -42:30.590 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=101.57866ms client=127.0.0.1:39198 client_network=tcp -42:31.489 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 -42:31.492 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=114.86874ms client=127.0.0.1:39200 client_network=tcp -42:32.071 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=169.57082ms client=127.0.0.1:39616 client_network=udp -42:32.181 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=104.36122ms client=127.0.0.1:50808 client_network=udp -42:32.356 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{95038981-2f31-418d-8a88-78efac6ec227.query.consul. 255 1}"] records=4091/5001 size=65523/80083 -42:32.359 [DEBUG] TestAgent.dns: request served from client: name=95038981-2f31-418d-8a88-78efac6ec227.query.consul. type=ANY class=IN latency=167.27302ms client=127.0.0.1:39202 client_network=tcp -42:32.976 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{95038981-2f31-418d-8a88-78efac6ec227.query.consul. 255 1}"] records=4091/5001 size=65523/80083 -42:32.979 [DEBUG] TestAgent.dns: request served from client: name=95038981-2f31-418d-8a88-78efac6ec227.query.consul. type=ANY class=IN latency=147.0809ms client=127.0.0.1:39214 client_network=tcp -42:33.554 [DEBUG] TestAgent.dns: request served from client: name=95038981-2f31-418d-8a88-78efac6ec227.query.consul. type=ANY class=IN latency=107.51744ms client=127.0.0.1:42360 client_network=udp -42:33.708 [DEBUG] TestAgent.dns: request served from client: name=95038981-2f31-418d-8a88-78efac6ec227.query.consul. type=ANY class=IN latency=128.60342ms client=127.0.0.1:46594 client_network=udp -42:33.848 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 -42:33.858 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=142.11306ms client=127.0.0.1:39216 client_network=tcp -42:34.495 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 -42:34.498 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=149.92422ms client=127.0.0.1:39218 client_network=tcp -42:35.087 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=105.95404ms client=127.0.0.1:44723 client_network=udp -42:35.233 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=122.7143ms client=127.0.0.1:34933 client_network=udp -42:35.423 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{95038981-2f31-418d-8a88-78efac6ec227.query.consul. 255 1}"] records=4091/5001 size=65523/80083 -42:35.426 [DEBUG] TestAgent.dns: request served from client: name=95038981-2f31-418d-8a88-78efac6ec227.query.consul. type=ANY class=IN latency=187.56448ms client=127.0.0.1:39228 client_network=tcp -42:36.301 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{95038981-2f31-418d-8a88-78efac6ec227.query.consul. 255 1}"] records=4091/5001 size=65523/80083 -42:36.304 [DEBUG] TestAgent.dns: request served from client: name=95038981-2f31-418d-8a88-78efac6ec227.query.consul. type=ANY class=IN latency=155.02494ms client=127.0.0.1:39238 client_network=tcp -42:37.088 [DEBUG] TestAgent.dns: request served from client: name=95038981-2f31-418d-8a88-78efac6ec227.query.consul. type=ANY class=IN latency=145.37238ms client=127.0.0.1:57575 client_network=udp -42:37.206 [DEBUG] TestAgent.dns: request served from client: name=95038981-2f31-418d-8a88-78efac6ec227.query.consul. type=ANY class=IN latency=110.8977ms client=127.0.0.1:43534 client_network=udp -42:37.383 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 -42:37.385 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=165.76394ms client=127.0.0.1:39240 client_network=tcp -42:38.067 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 -42:38.070 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=113.68068ms client=127.0.0.1:39244 client_network=tcp -42:38.682 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=199.42632ms client=127.0.0.1:43914 client_network=udp -42:38.838 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=150.78234ms client=127.0.0.1:41989 client_network=udp -42:38.967 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{95038981-2f31-418d-8a88-78efac6ec227.query.consul. 255 1}"] records=4091/5001 size=65523/80083 -42:38.970 [DEBUG] TestAgent.dns: request served from client: name=95038981-2f31-418d-8a88-78efac6ec227.query.consul. type=ANY class=IN latency=126.94592ms client=127.0.0.1:39246 client_network=tcp -42:39.904 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{95038981-2f31-418d-8a88-78efac6ec227.query.consul. 255 1}"] records=4091/5001 size=65523/80083 -42:39.907 [DEBUG] TestAgent.dns: request served from client: name=95038981-2f31-418d-8a88-78efac6ec227.query.consul. type=ANY class=IN latency=118.19226ms client=127.0.0.1:39250 client_network=tcp -42:40.696 [DEBUG] TestAgent.dns: request served from client: name=95038981-2f31-418d-8a88-78efac6ec227.query.consul. type=ANY class=IN latency=133.34638ms client=127.0.0.1:52395 client_network=udp -42:40.814 [DEBUG] TestAgent.dns: request served from client: name=95038981-2f31-418d-8a88-78efac6ec227.query.consul. type=ANY class=IN latency=109.83674ms client=127.0.0.1:45407 client_network=udp -42:41.029 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 -42:41.031 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=180.86606ms client=127.0.0.1:39254 client_network=tcp -42:41.684 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 -42:41.686 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=148.173ms client=127.0.0.1:39256 client_network=tcp -42:42.266 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=176.7889ms client=127.0.0.1:40638 client_network=udp -42:42.377 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=103.21258ms client=127.0.0.1:57069 client_network=udp -42:42.487 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{95038981-2f31-418d-8a88-78efac6ec227.query.consul. 255 1}"] records=4091/5001 size=65523/80083 -42:42.489 [DEBUG] TestAgent.dns: request served from client: name=95038981-2f31-418d-8a88-78efac6ec227.query.consul. type=ANY class=IN latency=106.94056ms client=127.0.0.1:39258 client_network=tcp -42:43.212 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{95038981-2f31-418d-8a88-78efac6ec227.query.consul. 255 1}"] records=4091/5001 size=65523/80083 -42:43.214 [DEBUG] TestAgent.dns: request served from client: name=95038981-2f31-418d-8a88-78efac6ec227.query.consul. type=ANY class=IN latency=113.5949ms client=127.0.0.1:39260 client_network=tcp -42:43.839 [DEBUG] TestAgent.dns: request served from client: name=95038981-2f31-418d-8a88-78efac6ec227.query.consul. type=ANY class=IN latency=145.09152ms client=127.0.0.1:49218 client_network=udp -42:43.987 [DEBUG] TestAgent.dns: request served from client: name=95038981-2f31-418d-8a88-78efac6ec227.query.consul. type=ANY class=IN latency=141.03086ms client=127.0.0.1:49704 client_network=udp -42:44.216 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 -42:44.222 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=139.10202ms client=127.0.0.1:39306 client_network=tcp -42:44.717 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 -42:44.719 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=118.06758ms client=127.0.0.1:39326 client_network=tcp -42:45.143 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=99.04194ms client=127.0.0.1:34306 client_network=udp -42:45.287 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=139.18882ms client=127.0.0.1:56158 client_network=udp -42:45.430 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{95038981-2f31-418d-8a88-78efac6ec227.query.consul. 255 1}"] records=4091/5001 size=65523/80083 -42:45.433 [DEBUG] TestAgent.dns: request served from client: name=95038981-2f31-418d-8a88-78efac6ec227.query.consul. type=ANY class=IN latency=139.86806ms client=127.0.0.1:39346 client_network=tcp -42:46.323 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{95038981-2f31-418d-8a88-78efac6ec227.query.consul. 255 1}"] records=4091/5001 size=65523/80083 -42:46.326 [DEBUG] TestAgent.dns: request served from client: name=95038981-2f31-418d-8a88-78efac6ec227.query.consul. type=ANY class=IN latency=214.283ms client=127.0.0.1:39350 client_network=tcp -42:46.957 [DEBUG] TestAgent.dns: request served from client: name=95038981-2f31-418d-8a88-78efac6ec227.query.consul. type=ANY class=IN latency=119.01276ms client=127.0.0.1:60414 client_network=udp -42:47.125 [DEBUG] TestAgent.dns: request served from client: name=95038981-2f31-418d-8a88-78efac6ec227.query.consul. type=ANY class=IN latency=144.24888ms client=127.0.0.1:53132 client_network=udp -42:47.343 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 -42:47.345 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=212.14946ms client=127.0.0.1:39356 client_network=tcp -42:47.885 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 -42:47.888 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=143.1444ms client=127.0.0.1:39358 client_network=tcp -42:48.394 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=126.89588ms client=127.0.0.1:39041 client_network=udp -42:48.504 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=101.62998ms client=127.0.0.1:52519 client_network=udp -42:48.709 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{95038981-2f31-418d-8a88-78efac6ec227.query.consul. 255 1}"] records=4091/5001 size=65523/80083 -42:48.720 [DEBUG] TestAgent.dns: request served from client: name=95038981-2f31-418d-8a88-78efac6ec227.query.consul. type=ANY class=IN latency=211.08848ms client=127.0.0.1:39360 client_network=tcp -42:49.620 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{95038981-2f31-418d-8a88-78efac6ec227.query.consul. 255 1}"] records=4091/5001 size=65523/80083 -42:49.622 [DEBUG] TestAgent.dns: request served from client: name=95038981-2f31-418d-8a88-78efac6ec227.query.consul. type=ANY class=IN latency=179.50132ms client=127.0.0.1:39370 client_network=tcp -42:50.409 [DEBUG] TestAgent.dns: request served from client: name=95038981-2f31-418d-8a88-78efac6ec227.query.consul. type=ANY class=IN latency=168.46802ms client=127.0.0.1:58206 client_network=udp -42:50.580 [DEBUG] TestAgent.dns: request served from client: name=95038981-2f31-418d-8a88-78efac6ec227.query.consul. type=ANY class=IN latency=157.05062ms client=127.0.0.1:40859 client_network=udp -42:50.586 [INFO] TestAgent: Requesting shutdown -42:50.586 [INFO] TestAgent.server: shutting down server -42:50.586 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:50.586 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:50.586 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:50.586 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:50.587 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:50.587 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:50.587 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:50.587 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:50.587 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:50.587 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:50.587 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:50.587Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:50.587Z [INFO] agent.router.manager: shutting down -42:50.588 [INFO] TestAgent: consul server down -42:50.588 [INFO] TestAgent: shutdown complete -42:50.588 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13632 network=tcp -42:50.588 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13632 network=udp -42:50.588 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13633 network=tcp -42:51.089 [INFO] TestAgent: Waiting for endpoints to shut down -42:51.089 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_TCP_and_UDP_Truncate (25.73s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=255)_compressed=true (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=255)_compressed=false (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=255)_compressed=true (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=255)_compressed=false (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._tcp_(qType:=255)_compressed=true (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._tcp_(qType:=255)_compressed=false (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._udp_(qType:=255)_compressed=true (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._udp_(qType:=255)_compressed=false (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=1)_compressed=true (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=1)_compressed=false (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=1)_compressed=true (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=1)_compressed=false (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._tcp_(qType:=1)_compressed=true (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._tcp_(qType:=1)_compressed=false (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._udp_(qType:=1)_compressed=true (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._udp_(qType:=1)_compressed=false (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=33)_compressed=true (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=33)_compressed=false (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=33)_compressed=true (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=33)_compressed=false (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._tcp_(qType:=33)_compressed=true (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._tcp_(qType:=33)_compressed=false (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._udp_(qType:=33)_compressed=true (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._udp_(qType:=33)_compressed=false (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=255)_compressed=true#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=255)_compressed=false#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=255)_compressed=true#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=255)_compressed=false#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._tcp_(qType:=255)_compressed=true#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._tcp_(qType:=255)_compressed=false#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._udp_(qType:=255)_compressed=true#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._udp_(qType:=255)_compressed=false#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=1)_compressed=true#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=1)_compressed=false#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=1)_compressed=true#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=1)_compressed=false#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._tcp_(qType:=1)_compressed=true#01 (0.01s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._tcp_(qType:=1)_compressed=false#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._udp_(qType:=1)_compressed=true#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._udp_(qType:=1)_compressed=false#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=33)_compressed=true#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=33)_compressed=false#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=33)_compressed=true#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=33)_compressed=false#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._tcp_(qType:=33)_compressed=true#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._tcp_(qType:=33)_compressed=false#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._udp_(qType:=33)_compressed=true#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_4596e128-44c5-77a6-1e88-84b434bd7419.query.consul._udp_(qType:=33)_compressed=false#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=255)_compressed=true (0.89s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=255)_compressed=false (0.53s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=255)_compressed=true (0.17s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=255)_compressed=false (0.11s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._tcp_(qType:=255)_compressed=true (0.64s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._tcp_(qType:=255)_compressed=false (0.61s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._udp_(qType:=255)_compressed=true (0.13s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._udp_(qType:=255)_compressed=false (0.14s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=1)_compressed=true (0.63s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=1)_compressed=false (0.63s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=1)_compressed=true (0.13s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=1)_compressed=false (0.13s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._tcp_(qType:=1)_compressed=true (0.91s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._tcp_(qType:=1)_compressed=false (0.79s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._udp_(qType:=1)_compressed=true (0.15s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._udp_(qType:=1)_compressed=false (0.12s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=33)_compressed=true (0.74s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=33)_compressed=false (0.53s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=33)_compressed=true (0.20s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=33)_compressed=false (0.16s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._tcp_(qType:=33)_compressed=true (0.94s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._tcp_(qType:=33)_compressed=false (0.78s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._udp_(qType:=33)_compressed=true (0.14s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._udp_(qType:=33)_compressed=false (0.12s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=255)_compressed=true#01 (0.71s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=255)_compressed=false#01 (0.55s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=255)_compressed=true#01 (0.18s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=255)_compressed=false#01 (0.11s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._tcp_(qType:=255)_compressed=true#01 (0.72s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._tcp_(qType:=255)_compressed=false#01 (0.59s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._udp_(qType:=255)_compressed=true#01 (0.15s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._udp_(qType:=255)_compressed=false#01 (0.23s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=1)_compressed=true#01 (0.52s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=1)_compressed=false#01 (0.44s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=1)_compressed=true#01 (0.10s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=1)_compressed=false#01 (0.14s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._tcp_(qType:=1)_compressed=true#01 (0.82s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._tcp_(qType:=1)_compressed=false#01 (0.73s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._udp_(qType:=1)_compressed=true#01 (0.14s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._udp_(qType:=1)_compressed=false#01 (0.15s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=33)_compressed=true#01 (0.61s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=33)_compressed=false#01 (0.52s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=33)_compressed=true#01 (0.13s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=33)_compressed=false#01 (0.11s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._tcp_(qType:=33)_compressed=true#01 (0.92s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._tcp_(qType:=33)_compressed=false#01 (0.81s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._udp_(qType:=33)_compressed=true#01 (0.17s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_95038981-2f31-418d-8a88-78efac6ec227.query.consul._udp_(qType:=33)_compressed=false#01 (0.17s) +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:10.103Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:10.118Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:10.121 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:54d42e97-a348-90db-f210-3a1d1a2779d8 Address:127.0.0.1:25169}]" +34:10.121 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25169 [Follower]" leader= +34:10.122 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-54d42e97-a348-90db-f210-3a1d1a2779d8.dc1 127.0.0.1 +34:10.123 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-54d42e97-a348-90db-f210-3a1d1a2779d8 127.0.0.1 +2021-08-13T15:34:10.123Z [INFO] agent.router: Initializing LAN area manager +34:10.123 [INFO] TestAgent.server: Adding LAN server: server="Node-54d42e97-a348-90db-f210-3a1d1a2779d8 (Addr: tcp/127.0.0.1:25169) (DC: dc1)" +34:10.123 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-54d42e97-a348-90db-f210-3a1d1a2779d8.dc1 area=wan +34:10.123 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25164 network=tcp +34:10.123 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25164 network=udp +34:10.124 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25165 network=tcp +34:10.124 [INFO] TestAgent: started state syncer +34:10.124 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25170 network=tcp +34:10.165 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:10.165 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25169 [Candidate]" term=2 +34:10.165 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:10.165 [DEBUG] TestAgent.server.raft: vote granted: from=54d42e97-a348-90db-f210-3a1d1a2779d8 term=2 tally=1 +34:10.165 [INFO] TestAgent.server.raft: election won: tally=1 +34:10.165 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25169 [Leader]" +34:10.165 [INFO] TestAgent.server: cluster leadership acquired +34:10.165 [INFO] TestAgent.server: New leader elected: payload=Node-54d42e97-a348-90db-f210-3a1d1a2779d8 +34:10.165 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25169 +34:10.167 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:10.169 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:10.169 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:10.169 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:10.169 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:10.169 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:10.169 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-54d42e97-a348-90db-f210-3a1d1a2779d8 +34:10.169 [INFO] TestAgent.server: member joined, marking health alive: member=Node-54d42e97-a348-90db-f210-3a1d1a2779d8 +34:10.170 [INFO] TestAgent.server: federation state anti-entropy synced +34:10.254 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:10.254 [INFO] TestAgent: Synced node info +34:10.254 [DEBUG] TestAgent: Node info in sync +34:10.554 [INFO] TestAgent: Requesting shutdown +34:10.554 [INFO] TestAgent.server: shutting down server +34:10.554 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:10.554 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:10.554 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:10.554 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:10.554 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:10.554 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:10.554 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:10.554 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:10.554 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:10.554 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:10.554 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:10.554Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:10.554Z [INFO] agent.router.manager: shutting down +34:10.554 [INFO] TestAgent: consul server down +34:10.554 [INFO] TestAgent: shutdown complete +34:10.555 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25164 network=tcp +34:10.555 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25164 network=udp +34:10.555 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25165 network=tcp +34:11.055 [INFO] TestAgent: Waiting for endpoints to shut down +34:11.055 [INFO] TestAgent: Endpoints down +--- PASS: TestSessionCreate_DefaultCheck (0.97s) +=== CONT TestAgentConnectAuthorize_deny +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:10.610Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:10.642Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:10.645 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8244b7c5-9d8a-4860-a940-e71ced8dbbd8 Address:127.0.0.1:25190}]" +34:10.645 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25190 [Follower]" leader= +34:10.646 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8244b7c5-9d8a-4860-a940-e71ced8dbbd8.dc1 127.0.0.1 +34:10.646 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8244b7c5-9d8a-4860-a940-e71ced8dbbd8 127.0.0.1 +2021-08-13T15:34:10.646Z [INFO] agent.router: Initializing LAN area manager +34:10.647 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25185 network=udp +34:10.647 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8244b7c5-9d8a-4860-a940-e71ced8dbbd8.dc1 area=wan +34:10.647 [INFO] TestAgent.server: Adding LAN server: server="Node-8244b7c5-9d8a-4860-a940-e71ced8dbbd8 (Addr: tcp/127.0.0.1:25190) (DC: dc1)" +34:10.647 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25185 network=tcp +34:10.648 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25186 network=tcp +34:10.648 [INFO] TestAgent: started state syncer +34:10.648 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25191 network=tcp +34:10.709 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:10.709 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25190 [Candidate]" term=2 +34:10.709 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:10.709 [DEBUG] TestAgent.server.raft: vote granted: from=8244b7c5-9d8a-4860-a940-e71ced8dbbd8 term=2 tally=1 +34:10.709 [INFO] TestAgent.server.raft: election won: tally=1 +34:10.709 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25190 [Leader]" +34:10.709 [INFO] TestAgent.server: cluster leadership acquired +34:10.709 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25190 +34:10.710 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:10.711 [INFO] TestAgent.server: New leader elected: payload=Node-8244b7c5-9d8a-4860-a940-e71ced8dbbd8 +34:10.713 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:10.713 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:10.713 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:10.713 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:10.713 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:10.713 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8244b7c5-9d8a-4860-a940-e71ced8dbbd8 +34:10.713 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8244b7c5-9d8a-4860-a940-e71ced8dbbd8 +34:10.714 [INFO] TestAgent.server: federation state anti-entropy synced +34:10.792 [INFO] TestAgent: Requesting shutdown +34:10.792 [INFO] TestAgent.server: shutting down server +34:10.792 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:10.792 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:10.792 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:10.792 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:10.792 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:10.792 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:10.792 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:10.792 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:10.792 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:10.792 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:10.792 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:10.792 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:10.792Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:10.792Z [INFO] agent.router.manager: shutting down +34:10.792 [INFO] TestAgent: consul server down +34:10.792 [INFO] TestAgent: shutdown complete +34:10.792 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25185 network=tcp +34:10.792 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25185 network=udp +34:10.792 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25186 network=tcp +34:11.293 [INFO] TestAgent: Waiting for endpoints to shut down +34:11.293 [INFO] TestAgent: Endpoints down +--- PASS: TestSessionCreate_Delete (0.73s) === CONT TestServiceManager_PersistService_ConfigFiles -2022-09-15T21:42:50.476Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:50.492Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:50.500 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e3b3060a-5032-151d-084a-41793d5347b5 Address:127.0.0.1:13972}]" -42:50.502 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13972 [Follower]" leader= -42:50.502 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e3b3060a-5032-151d-084a-41793d5347b5.dc1 127.0.0.1 -42:50.504 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e3b3060a-5032-151d-084a-41793d5347b5 127.0.0.1 -2022-09-15T21:42:50.504Z [INFO] agent.router: Initializing LAN area manager -42:50.505 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13967 network=udp -42:50.506 [INFO] TestAgent.server: Adding LAN server: server="Node-e3b3060a-5032-151d-084a-41793d5347b5 (Addr: tcp/127.0.0.1:13972) (DC: dc1)" -42:50.506 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e3b3060a-5032-151d-084a-41793d5347b5.dc1 area=wan -42:50.506 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13967 network=tcp -42:50.508 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13968 network=tcp -42:50.508 [INFO] TestAgent: started state syncer -42:50.508 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13973 network=tcp -42:50.558 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:50.558 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13972 [Candidate]" term=2 -42:50.558 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:50.558 [DEBUG] TestAgent.server.raft: vote granted: from=e3b3060a-5032-151d-084a-41793d5347b5 term=2 tally=1 -42:50.558 [INFO] TestAgent.server.raft: election won: tally=1 -42:50.558 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13972 [Leader]" -42:50.558 [INFO] TestAgent.server: cluster leadership acquired -42:50.559 [INFO] TestAgent.server: New leader elected: payload=Node-e3b3060a-5032-151d-084a-41793d5347b5 -42:50.559 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13972 -42:50.561 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:50.565 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:50.565 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:50.565 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:50.565 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:50.565 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:50.565 [INFO] TestAgent.server: federation state anti-entropy synced -42:50.567 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e3b3060a-5032-151d-084a-41793d5347b5 -42:50.567 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e3b3060a-5032-151d-084a-41793d5347b5 -42:50.592 [INFO] TestAgent: Requesting shutdown -42:50.592 [INFO] TestAgent.server: shutting down server -42:50.592 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:50.592 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:50.592 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:50.592 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:50.592 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:50.593 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:50.593 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:50.593 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:50.593 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:50.593 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:50.593 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:50.593 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:50.593Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:50.593Z [INFO] agent.router.manager: shutting down -42:50.593 [INFO] TestAgent: consul server down -42:50.593 [INFO] TestAgent: shutdown complete -42:50.593 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13967 network=tcp -42:50.593 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13967 network=udp -42:50.593 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13968 network=tcp -42:51.093 [INFO] TestAgent: Waiting for endpoints to shut down -42:51.093 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentConnectAuthorize_idInvalidFormat (0.64s) +2021-08-13T15:34:10.662Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:10.677Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:10.680 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9d27ac55-0276-b719-5224-8c1b459b6ef7 Address:127.0.0.1:25197}]" +34:10.680 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25197 [Follower]" leader= +34:10.681 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9d27ac55-0276-b719-5224-8c1b459b6ef7.dc1 127.0.0.1 +34:10.682 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9d27ac55-0276-b719-5224-8c1b459b6ef7 127.0.0.1 +2021-08-13T15:34:10.682Z [INFO] agent.router: Initializing LAN area manager +34:10.682 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9d27ac55-0276-b719-5224-8c1b459b6ef7.dc1 area=wan +34:10.682 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25192 network=tcp +34:10.682 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25192 network=udp +34:10.682 [INFO] TestAgent.server: Adding LAN server: server="Node-9d27ac55-0276-b719-5224-8c1b459b6ef7 (Addr: tcp/127.0.0.1:25197) (DC: dc1)" +34:10.683 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25193 network=tcp +34:10.683 [INFO] TestAgent: started state syncer +34:10.683 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25198 network=tcp +34:10.731 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:10.731 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25197 [Candidate]" term=2 +34:10.731 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:10.731 [DEBUG] TestAgent.server.raft: vote granted: from=9d27ac55-0276-b719-5224-8c1b459b6ef7 term=2 tally=1 +34:10.731 [INFO] TestAgent.server.raft: election won: tally=1 +34:10.731 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25197 [Leader]" +34:10.731 [INFO] TestAgent.server: cluster leadership acquired +34:10.732 [INFO] TestAgent.server: New leader elected: payload=Node-9d27ac55-0276-b719-5224-8c1b459b6ef7 +34:10.732 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25197 +34:10.733 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:10.735 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:10.735 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:10.735 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:10.735 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:10.735 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:10.735 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9d27ac55-0276-b719-5224-8c1b459b6ef7 +34:10.735 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9d27ac55-0276-b719-5224-8c1b459b6ef7 +34:10.736 [INFO] TestAgent.server: federation state anti-entropy synced +34:10.811 [INFO] TestAgent: Requesting shutdown +34:10.811 [INFO] TestAgent.server: shutting down server +34:10.811 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:10.811 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:10.811 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:10.811 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:10.811 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:10.811 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:10.811 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:10.811 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:10.811 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:10.811 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:10.811 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:10.811 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:10.811Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:10.811Z [INFO] agent.router.manager: shutting down +34:10.811 [INFO] TestAgent: consul server down +34:10.811 [INFO] TestAgent: shutdown complete +34:10.811 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25192 network=tcp +34:10.811 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25192 network=udp +34:10.811 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25193 network=tcp +34:11.313 [INFO] TestAgent: Waiting for endpoints to shut down +34:11.313 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentConnectAuthorize_denyWildcard (0.67s) +=== CONT TestAgentConnectAuthorize_allow +2021-08-13T15:34:10.616Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:10.616Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:10.638Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:10.638Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:10.641 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:20f8ccad-15d2-fe8c-9802-ba117ad18384 Address:127.0.0.1:25183}]" +34:10.641 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25183 [Follower]" leader= +34:10.642 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-20f8ccad-15d2-fe8c-9802-ba117ad18384.dc1 127.0.0.1 +34:10.643 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-20f8ccad-15d2-fe8c-9802-ba117ad18384 127.0.0.1 +2021-08-13T15:34:10.643Z [INFO] agent.router: Initializing LAN area manager +34:10.643 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25178 network=udp +34:10.643 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-20f8ccad-15d2-fe8c-9802-ba117ad18384.dc1 area=wan +34:10.644 [INFO] TestAgent.server: Adding LAN server: server="Node-20f8ccad-15d2-fe8c-9802-ba117ad18384 (Addr: tcp/127.0.0.1:25183) (DC: dc1)" +34:10.644 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25178 network=tcp +34:10.645 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25179 network=tcp +34:10.645 [INFO] TestAgent: started state syncer +34:10.645 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25184 network=tcp +34:10.693 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:10.693 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25183 [Candidate]" term=2 +34:10.693 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:10.693 [DEBUG] TestAgent.server.raft: vote granted: from=20f8ccad-15d2-fe8c-9802-ba117ad18384 term=2 tally=1 +34:10.693 [INFO] TestAgent.server.raft: election won: tally=1 +34:10.693 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25183 [Leader]" +34:10.693 [INFO] TestAgent.server: cluster leadership acquired +34:10.693 [INFO] TestAgent.server: initializing acls +34:10.693 [INFO] TestAgent.server: New leader elected: payload=Node-20f8ccad-15d2-fe8c-9802-ba117ad18384 +34:10.694 [INFO] TestAgent.server: initializing acls +34:10.694 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:10.694 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:10.694 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:10.694 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:10.694 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:10.695 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:10.695 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:10.695 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:10.695 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:10.695 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode +34:10.695 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:10.695 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-20f8ccad-15d2-fe8c-9802-ba117ad18384 +34:10.695 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-20f8ccad-15d2-fe8c-9802-ba117ad18384.dc1 +34:10.695 [INFO] TestAgent.server: Updating LAN server: server="Node-20f8ccad-15d2-fe8c-9802-ba117ad18384 (Addr: tcp/127.0.0.1:25183) (DC: dc1)" +34:10.695 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-20f8ccad-15d2-fe8c-9802-ba117ad18384.dc1 area=wan +34:10.696 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-20f8ccad-15d2-fe8c-9802-ba117ad18384 +34:10.696 [INFO] TestAgent.server: Updating LAN server: server="Node-20f8ccad-15d2-fe8c-9802-ba117ad18384 (Addr: tcp/127.0.0.1:25183) (DC: dc1)" +34:10.696 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-20f8ccad-15d2-fe8c-9802-ba117ad18384.dc1 +34:10.696 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-20f8ccad-15d2-fe8c-9802-ba117ad18384.dc1 area=wan +34:10.697 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:10.699 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:10.699 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:10.699 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:10.699 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:10.699 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:10.699 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-20f8ccad-15d2-fe8c-9802-ba117ad18384 +34:10.699 [INFO] TestAgent.server: member joined, marking health alive: member=Node-20f8ccad-15d2-fe8c-9802-ba117ad18384 +34:10.700 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-20f8ccad-15d2-fe8c-9802-ba117ad18384 +34:10.700 [INFO] TestAgent.server: federation state anti-entropy synced +34:10.700 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-20f8ccad-15d2-fe8c-9802-ba117ad18384 +34:10.824 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:10.825 [INFO] TestAgent: Synced node info +34:10.825 [DEBUG] TestAgent: Node info in sync +34:10.876 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-20f8ccad-15d2-fe8c-9802-ba117ad18384 +34:10.876 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-20f8ccad-15d2-fe8c-9802-ba117ad18384 +34:10.877 [INFO] TestAgent: Requesting shutdown +34:10.877 [INFO] TestAgent.server: shutting down server +34:10.877 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:10.877 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:10.877 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:10.877 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:10.877 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:10.877 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:10.877 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:10.877 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:10.877 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:10.877 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:10.877 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:10.877 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:10.877 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:10.877 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:10.878 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:10.878Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:10.878Z [INFO] agent.router.manager: shutting down +34:10.878 [INFO] TestAgent: consul server down +34:10.878 [INFO] TestAgent: shutdown complete +34:10.878 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25178 network=tcp +34:10.878 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25178 network=udp +34:10.878 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25179 network=tcp +34:11.378 [INFO] TestAgent: Waiting for endpoints to shut down +34:11.378 [INFO] TestAgent: Endpoints down === CONT TestHandleRemoteExecFailed -2022-09-15T21:42:50.144Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:50.204Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:50.209 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ef93449c-4d68-c26e-7f6f-1b454b882915 Address:127.0.0.1:13958}]" -42:50.210 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ef93449c-4d68-c26e-7f6f-1b454b882915.dc1 127.0.0.1 -42:50.213 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13958 [Follower]" leader= -42:50.216 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ef93449c-4d68-c26e-7f6f-1b454b882915 127.0.0.1 -2022-09-15T21:42:50.216Z [INFO] agent.router: Initializing LAN area manager -42:50.216 [INFO] TestAgent.server: Adding LAN server: server="Node-ef93449c-4d68-c26e-7f6f-1b454b882915 (Addr: tcp/127.0.0.1:13958) (DC: dc1)" -42:50.216 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ef93449c-4d68-c26e-7f6f-1b454b882915.dc1 area=wan -42:50.216 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13939 network=udp -42:50.216 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13939 network=tcp -42:50.218 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13940 network=tcp -42:50.218 [INFO] TestAgent: started state syncer -42:50.218 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13959 network=tcp -42:50.266 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:50.267 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13958 [Candidate]" term=2 -42:50.267 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:50.267 [DEBUG] TestAgent.server.raft: vote granted: from=ef93449c-4d68-c26e-7f6f-1b454b882915 term=2 tally=1 -42:50.267 [INFO] TestAgent.server.raft: election won: tally=1 -42:50.267 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13958 [Leader]" -42:50.267 [INFO] TestAgent.server: cluster leadership acquired -42:50.267 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13958 -42:50.269 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:50.270 [INFO] TestAgent.server: New leader elected: payload=Node-ef93449c-4d68-c26e-7f6f-1b454b882915 -42:50.282 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:50.282 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:50.282 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:50.282 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:50.282 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:50.282 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ef93449c-4d68-c26e-7f6f-1b454b882915 -42:50.282 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ef93449c-4d68-c26e-7f6f-1b454b882915 -42:50.283 [INFO] TestAgent: Synced node info -42:50.284 [INFO] TestAgent.server: federation state anti-entropy synced -42:50.422 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:50.422 [DEBUG] TestAgent: Node info in sync -42:50.422 [DEBUG] TestAgent: Node info in sync -42:50.653 [INFO] TestAgent: Requesting shutdown -42:50.654 [INFO] TestAgent.server: shutting down server -42:50.654 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:50.654 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:50.654 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:50.654 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:50.654 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:50.654 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:50.654 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:50.654 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:50.654 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:50.654 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:50.654 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:50.655Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:50.655Z [INFO] agent.router.manager: shutting down -42:50.655 [INFO] TestAgent: consul server down -42:50.655 [INFO] TestAgent: shutdown complete -42:50.655 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13939 network=tcp -42:50.655 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13939 network=udp -42:50.655 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13940 network=tcp -42:51.156 [INFO] TestAgent: Waiting for endpoints to shut down -42:51.156 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentConnectAuthorize_deny (1.07s) +--- PASS: TestAgentConnectAuthorize_serviceWrite (0.82s) +2021-08-13T15:34:10.846Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:10.862Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:10.865 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4663a44c-d606-f1f9-83c8-df29a64cefaa Address:127.0.0.1:25204}]" +34:10.865 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25204 [Follower]" leader= +34:10.866 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4663a44c-d606-f1f9-83c8-df29a64cefaa.dc1 127.0.0.1 +34:10.866 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4663a44c-d606-f1f9-83c8-df29a64cefaa 127.0.0.1 +2021-08-13T15:34:10.867Z [INFO] agent.router: Initializing LAN area manager +34:10.867 [INFO] TestAgent.server: Adding LAN server: server="Node-4663a44c-d606-f1f9-83c8-df29a64cefaa (Addr: tcp/127.0.0.1:25204) (DC: dc1)" +34:10.867 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4663a44c-d606-f1f9-83c8-df29a64cefaa.dc1 area=wan +34:10.867 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25199 network=udp +34:10.867 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25199 network=tcp +34:10.868 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25200 network=tcp +34:10.868 [INFO] TestAgent: started state syncer +34:10.868 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25205 network=tcp +34:10.911 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:10.911 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25204 [Candidate]" term=2 +34:10.912 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:10.912 [DEBUG] TestAgent.server.raft: vote granted: from=4663a44c-d606-f1f9-83c8-df29a64cefaa term=2 tally=1 +34:10.912 [INFO] TestAgent.server.raft: election won: tally=1 +34:10.912 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25204 [Leader]" +34:10.912 [INFO] TestAgent.server: cluster leadership acquired +34:10.912 [INFO] TestAgent.server: New leader elected: payload=Node-4663a44c-d606-f1f9-83c8-df29a64cefaa +34:10.912 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25204 +34:10.913 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:10.916 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:10.916 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:10.916 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:10.916 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:10.916 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:10.916 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4663a44c-d606-f1f9-83c8-df29a64cefaa +34:10.916 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4663a44c-d606-f1f9-83c8-df29a64cefaa +34:10.917 [INFO] TestAgent.server: federation state anti-entropy synced +34:10.969 [INFO] TestAgent: Requesting shutdown +34:10.969 [INFO] TestAgent.server: shutting down server +34:10.970 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:10.970 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:10.970 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:10.970 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:10.970 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:10.970 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:10.970 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:10.970 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:10.970 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:10.970 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:10.970 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:10.970 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:10.970Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:10.970Z [INFO] agent.router.manager: shutting down +34:10.970 [INFO] TestAgent: consul server down +34:10.970 [INFO] TestAgent: shutdown complete +34:10.970 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25199 network=tcp +34:10.970 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25199 network=udp +34:10.970 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25200 network=tcp +34:11.470 [INFO] TestAgent: Waiting for endpoints to shut down +34:11.470 [INFO] TestAgent: Endpoints down +--- PASS: TestSessionCreate_NodeChecks (0.64s) +=== CONT TestAgentConnectAuthorize_idNotService +[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:11.030Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:11.046Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:11.048 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2c1090f6-45a1-268c-5e4b-b9fe1626c3fd Address:127.0.0.1:25218}]" +34:11.048 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25218 [Follower]" leader= +34:11.049 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2c1090f6-45a1-268c-5e4b-b9fe1626c3fd.dc1 127.0.0.1 +34:11.050 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2c1090f6-45a1-268c-5e4b-b9fe1626c3fd 127.0.0.1 +2021-08-13T15:34:11.050Z [INFO] agent.router: Initializing LAN area manager +34:11.050 [INFO] TestAgent.server: Adding LAN server: server="Node-2c1090f6-45a1-268c-5e4b-b9fe1626c3fd (Addr: tcp/127.0.0.1:25218) (DC: dc1)" +34:11.050 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2c1090f6-45a1-268c-5e4b-b9fe1626c3fd.dc1 area=wan +34:11.050 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25213 network=udp +34:11.050 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25213 network=tcp +34:11.051 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25214 network=tcp +34:11.051 [INFO] TestAgent: started state syncer +34:11.051 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25219 network=tcp +34:11.096 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:11.096 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25218 [Candidate]" term=2 +34:11.096 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:11.096 [DEBUG] TestAgent.server.raft: vote granted: from=2c1090f6-45a1-268c-5e4b-b9fe1626c3fd term=2 tally=1 +34:11.096 [INFO] TestAgent.server.raft: election won: tally=1 +34:11.096 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25218 [Leader]" +34:11.096 [INFO] TestAgent.server: cluster leadership acquired +34:11.097 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25218 +34:11.097 [INFO] TestAgent.server: New leader elected: payload=Node-2c1090f6-45a1-268c-5e4b-b9fe1626c3fd +34:11.098 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:11.104 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:11.104 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:11.104 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:11.104 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:11.104 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:11.105 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2c1090f6-45a1-268c-5e4b-b9fe1626c3fd +34:11.105 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2c1090f6-45a1-268c-5e4b-b9fe1626c3fd +34:11.106 [INFO] TestAgent.server: federation state anti-entropy synced +34:11.182 [INFO] TestAgent: Requesting shutdown +34:11.182 [INFO] TestAgent.server: shutting down server +34:11.182 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:11.182 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:11.182 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:11.182 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:11.182 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:11.182 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:11.182 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:11.182 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:11.182 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:11.183 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:11.183 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:11.183 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:11.183Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:11.183Z [INFO] agent.router.manager: shutting down +34:11.183 [INFO] TestAgent: consul server down +34:11.183 [INFO] TestAgent: shutdown complete +34:11.183 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25213 network=tcp +34:11.183 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25213 network=udp +34:11.183 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25214 network=tcp +34:11.685 [INFO] TestAgent: Waiting for endpoints to shut down +34:11.685 [INFO] TestAgent: Endpoints down +--- PASS: TestSessionCreate (0.67s) === CONT TestHandleRemoteExec -2022-09-15T21:42:50.306Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:50.331Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:50.336 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d1830e2c-172c-f41d-d351-dd382b9212c6 Address:127.0.0.1:13944}]" -42:50.336 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13944 [Follower]" leader= -42:50.337 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d1830e2c-172c-f41d-d351-dd382b9212c6.dc1 127.0.0.1 -42:50.338 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d1830e2c-172c-f41d-d351-dd382b9212c6 127.0.0.1 -2022-09-15T21:42:50.338Z [INFO] agent.router: Initializing LAN area manager -42:50.339 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13960 network=udp -42:50.339 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d1830e2c-172c-f41d-d351-dd382b9212c6.dc1 area=wan -42:50.339 [INFO] TestAgent.server: Adding LAN server: server="Node-d1830e2c-172c-f41d-d351-dd382b9212c6 (Addr: tcp/127.0.0.1:13944) (DC: dc1)" -42:50.339 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13960 network=tcp -42:50.340 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13961 network=tcp -42:50.340 [INFO] TestAgent: started state syncer -42:50.340 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13945 network=tcp -42:50.390 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:50.390 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13944 [Candidate]" term=2 -42:50.390 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:50.390 [DEBUG] TestAgent.server.raft: vote granted: from=d1830e2c-172c-f41d-d351-dd382b9212c6 term=2 tally=1 -42:50.390 [INFO] TestAgent.server.raft: election won: tally=1 -42:50.390 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13944 [Leader]" -42:50.390 [INFO] TestAgent.server: cluster leadership acquired -42:50.391 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13944 -42:50.392 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:50.394 [INFO] TestAgent.server: New leader elected: payload=Node-d1830e2c-172c-f41d-d351-dd382b9212c6 -42:50.395 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:50.395 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:50.395 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:50.395 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:50.395 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:50.395 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d1830e2c-172c-f41d-d351-dd382b9212c6 -42:50.395 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d1830e2c-172c-f41d-d351-dd382b9212c6 -42:50.395 [INFO] TestAgent.server: federation state anti-entropy synced -42:50.606 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:50.606 [INFO] TestAgent: Synced node info -42:50.606 [DEBUG] TestAgent: Node info in sync -42:50.637 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:50.637 [DEBUG] TestAgent: Node info in sync -42:50.662 [INFO] TestAgent: Requesting shutdown -42:50.662 [INFO] TestAgent.server: shutting down server -42:50.662 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:50.662 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:50.662 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:50.662 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:50.662 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:50.662 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:50.662 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:50.662 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:50.662 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:50.662 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -42:50.662 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -2022-09-15T21:42:50.663Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:50.663Z [INFO] agent.router.manager: shutting down -42:50.663 [INFO] TestAgent: consul server down -42:50.663 [INFO] TestAgent: shutdown complete -42:50.663 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13960 network=tcp -42:50.663 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13960 network=udp -42:50.663 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13961 network=tcp -42:51.164 [INFO] TestAgent: Waiting for endpoints to shut down -42:51.164 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentConnectAuthorize_allow (0.92s) +2021-08-13T15:34:10.953Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:10.969Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:10.971 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:75de4774-9fe7-ef44-b455-5d01094b0ab6 Address:127.0.0.1:25211}]" +34:10.971 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25211 [Follower]" leader= +34:10.972 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-75de4774-9fe7-ef44-b455-5d01094b0ab6.dc1 127.0.0.1 +34:10.973 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-75de4774-9fe7-ef44-b455-5d01094b0ab6 127.0.0.1 +2021-08-13T15:34:10.973Z [INFO] agent.router: Initializing LAN area manager +34:10.973 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25206 network=udp +34:10.973 [INFO] TestAgent.server: Adding LAN server: server="Node-75de4774-9fe7-ef44-b455-5d01094b0ab6 (Addr: tcp/127.0.0.1:25211) (DC: dc1)" +34:10.973 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-75de4774-9fe7-ef44-b455-5d01094b0ab6.dc1 area=wan +34:10.974 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25206 network=tcp +34:10.974 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25207 network=tcp +34:10.974 [INFO] TestAgent: started state syncer +34:10.974 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25212 network=tcp +34:11.039 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:11.039 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25211 [Candidate]" term=2 +34:11.039 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:11.039 [DEBUG] TestAgent.server.raft: vote granted: from=75de4774-9fe7-ef44-b455-5d01094b0ab6 term=2 tally=1 +34:11.039 [INFO] TestAgent.server.raft: election won: tally=1 +34:11.039 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25211 [Leader]" +34:11.039 [INFO] TestAgent.server: cluster leadership acquired +34:11.039 [INFO] TestAgent.server: New leader elected: payload=Node-75de4774-9fe7-ef44-b455-5d01094b0ab6 +34:11.039 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25211 +34:11.040 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:11.043 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:11.043 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:11.043 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:11.043 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:11.043 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:11.043 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-75de4774-9fe7-ef44-b455-5d01094b0ab6 +34:11.043 [INFO] TestAgent.server: member joined, marking health alive: member=Node-75de4774-9fe7-ef44-b455-5d01094b0ab6 +34:11.044 [INFO] TestAgent.server: federation state anti-entropy synced +34:11.118 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:11.119 [INFO] TestAgent: Synced node info +34:11.119 [DEBUG] TestAgent: Node info in sync +34:11.183 [INFO] TestAgent: Requesting shutdown +34:11.183 [INFO] TestAgent.server: shutting down server +34:11.183 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:11.183 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:11.183 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:11.183 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:11.184 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:11.184 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:11.184 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:11.184 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:11.184 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:11.184 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:11.184 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:11.184Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:11.184Z [INFO] agent.router.manager: shutting down +34:11.184 [INFO] TestAgent: consul server down +34:11.184 [INFO] TestAgent: shutdown complete +34:11.184 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25206 network=tcp +34:11.184 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25206 network=udp +34:11.184 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25207 network=tcp +34:11.685 [INFO] TestAgent: Waiting for endpoints to shut down +34:11.685 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentConnectAuthorize_allowTrustDomain (0.75s) +=== CONT TestAgentConnectAuthorize_idInvalidFormat +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:11.097Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:11.120Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:11.123 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c22ae7b3-7572-851b-7116-5b6196334033 Address:127.0.0.1:25225}]" +34:11.123 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25225 [Follower]" leader= +34:11.124 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c22ae7b3-7572-851b-7116-5b6196334033.dc1 127.0.0.1 +34:11.124 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c22ae7b3-7572-851b-7116-5b6196334033 127.0.0.1 +2021-08-13T15:34:11.124Z [INFO] agent.router: Initializing LAN area manager +34:11.125 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c22ae7b3-7572-851b-7116-5b6196334033.dc1 area=wan +34:11.125 [INFO] TestAgent.server: Adding LAN server: server="Node-c22ae7b3-7572-851b-7116-5b6196334033 (Addr: tcp/127.0.0.1:25225) (DC: dc1)" +34:11.125 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25220 network=tcp +34:11.125 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25220 network=udp +34:11.126 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25221 network=tcp +34:11.126 [INFO] TestAgent: started state syncer +34:11.126 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25226 network=tcp +34:11.168 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:11.168 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25225 [Candidate]" term=2 +34:11.169 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:11.169 [DEBUG] TestAgent.server.raft: vote granted: from=c22ae7b3-7572-851b-7116-5b6196334033 term=2 tally=1 +34:11.169 [INFO] TestAgent.server.raft: election won: tally=1 +34:11.169 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25225 [Leader]" +34:11.169 [INFO] TestAgent.server: cluster leadership acquired +34:11.169 [INFO] TestAgent.server: New leader elected: payload=Node-c22ae7b3-7572-851b-7116-5b6196334033 +34:11.169 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25225 +34:11.170 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:11.173 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:11.173 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:11.173 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:11.173 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:11.173 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:11.173 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c22ae7b3-7572-851b-7116-5b6196334033 +34:11.173 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c22ae7b3-7572-851b-7116-5b6196334033 +34:11.174 [INFO] TestAgent.server: federation state anti-entropy synced +34:11.338 [INFO] TestAgent: Requesting shutdown +34:11.338 [INFO] TestAgent.server: shutting down server +34:11.338 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:11.338 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:11.338 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:11.338 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:11.338 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:11.338 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:11.338 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:11.338 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:11.338 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:11.338 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:11.338 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:11.338 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:11.338Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:11.338Z [INFO] agent.router.manager: shutting down +34:11.338 [INFO] TestAgent: consul server down +34:11.338 [INFO] TestAgent: shutdown complete +34:11.338 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25220 network=tcp +34:11.338 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25220 network=udp +34:11.338 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25221 network=tcp +34:11.839 [INFO] TestAgent: Waiting for endpoints to shut down +34:11.839 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentConnectAuthorize_deny (0.78s) === CONT TestRemoteExecWrites_ACLDeny -[WARN] freeport: 4 out of 39 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:50.436Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:50.454Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:50.457 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2188b21c-c996-5b39-1714-74e65a891eb1 Address:127.0.0.1:13965}]" -42:50.458 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13965 [Follower]" leader= -42:50.458 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2188b21c-c996-5b39-1714-74e65a891eb1.dc1 127.0.0.1 -42:50.459 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2188b21c-c996-5b39-1714-74e65a891eb1 127.0.0.1 -2022-09-15T21:42:50.459Z [INFO] agent.router: Initializing LAN area manager -42:50.459 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13946 network=udp -42:50.460 [INFO] TestAgent.server: Adding LAN server: server="Node-2188b21c-c996-5b39-1714-74e65a891eb1 (Addr: tcp/127.0.0.1:13965) (DC: dc1)" -42:50.460 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2188b21c-c996-5b39-1714-74e65a891eb1.dc1 area=wan -42:50.460 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13946 network=tcp -42:50.461 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13947 network=tcp -42:50.461 [INFO] TestAgent: started state syncer -42:50.461 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13966 network=tcp -42:50.532 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:50.532 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13965 [Candidate]" term=2 -42:50.532 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:50.532 [DEBUG] TestAgent.server.raft: vote granted: from=2188b21c-c996-5b39-1714-74e65a891eb1 term=2 tally=1 -42:50.532 [INFO] TestAgent.server.raft: election won: tally=1 -42:50.532 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13965 [Leader]" -42:50.534 [INFO] TestAgent.server: cluster leadership acquired -42:50.534 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13965 -42:50.536 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:50.538 [INFO] TestAgent.server: New leader elected: payload=Node-2188b21c-c996-5b39-1714-74e65a891eb1 -42:50.545 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:50.545 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:50.545 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:50.545 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:50.545 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:50.545 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2188b21c-c996-5b39-1714-74e65a891eb1 -42:50.545 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2188b21c-c996-5b39-1714-74e65a891eb1 -42:50.547 [INFO] TestAgent.server: federation state anti-entropy synced -42:50.657 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:50.658 [INFO] TestAgent: Synced node info -42:50.886 [INFO] TestAgent: Requesting shutdown -42:50.886 [INFO] TestAgent.server: shutting down server -42:50.886 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:50.886 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:50.886 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:50.886 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:50.886 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:50.887 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:50.887 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:50.887 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:50.887 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:50.887 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:50.887 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:50.887Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:50.887Z [INFO] agent.router.manager: shutting down -42:50.887 [INFO] TestAgent: consul server down -42:50.887 [INFO] TestAgent: shutdown complete -42:50.887 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13946 network=tcp -42:50.887 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13946 network=udp -42:50.887 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13947 network=tcp -42:51.388 [INFO] TestAgent: Waiting for endpoints to shut down -42:51.388 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentConnectAuthorize_idNotService (0.97s) +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:11.489Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:11.505Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:11.507 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e80e8306-0f1c-cb11-d47d-b236488d6610 Address:127.0.0.1:25253}]" +34:11.508 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e80e8306-0f1c-cb11-d47d-b236488d6610.dc1 127.0.0.1 +34:11.509 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e80e8306-0f1c-cb11-d47d-b236488d6610 127.0.0.1 +2021-08-13T15:34:11.509Z [INFO] agent.router: Initializing LAN area manager +34:11.509 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25253 [Follower]" leader= +34:11.510 [INFO] TestAgent.server: Adding LAN server: server="Node-e80e8306-0f1c-cb11-d47d-b236488d6610 (Addr: tcp/127.0.0.1:25253) (DC: dc1)" +34:11.510 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e80e8306-0f1c-cb11-d47d-b236488d6610.dc1 area=wan +34:11.510 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25248 network=tcp +34:11.510 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25248 network=udp +34:11.511 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25249 network=tcp +34:11.511 [INFO] TestAgent: started state syncer +34:11.511 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25254 network=tcp +34:11.558 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:11.558 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25253 [Candidate]" term=2 +34:11.563 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:11.563 [DEBUG] TestAgent.server.raft: vote granted: from=e80e8306-0f1c-cb11-d47d-b236488d6610 term=2 tally=1 +34:11.563 [INFO] TestAgent.server.raft: election won: tally=1 +34:11.563 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25253 [Leader]" +34:11.563 [INFO] TestAgent.server: cluster leadership acquired +34:11.563 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25253 +34:11.564 [INFO] TestAgent.server: New leader elected: payload=Node-e80e8306-0f1c-cb11-d47d-b236488d6610 +34:11.565 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:11.572 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:11.572 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:11.572 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:11.572 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:11.573 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:11.573 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e80e8306-0f1c-cb11-d47d-b236488d6610 +34:11.573 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e80e8306-0f1c-cb11-d47d-b236488d6610 +34:11.575 [INFO] TestAgent.server: federation state anti-entropy synced +34:11.587 [INFO] TestAgent: Requesting shutdown +34:11.588 [INFO] TestAgent.server: shutting down server +34:11.588 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:11.588 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:11.588 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:11.588 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:11.588 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:11.588 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:11.588 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:11.588 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:11.588 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:11.588 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:11.588 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:11.588 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:11.588Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:11.588Z [INFO] agent.router.manager: shutting down +34:11.588 [INFO] TestAgent: consul server down +34:11.588 [INFO] TestAgent: shutdown complete +34:11.588 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25248 network=tcp +34:11.588 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25248 network=udp +34:11.588 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25249 network=tcp +34:12.089 [INFO] TestAgent: Waiting for endpoints to shut down +34:12.089 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentConnectAuthorize_idNotService (0.62s) +=== CONT TestAgentConnectAuthorize_noTarget +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:11.330Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:11.347Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:11.350 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:49a68a4a-ce0a-a0ae-4cec-d1dc85c97f61 Address:127.0.0.1:25239}]" +34:11.350 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25239 [Follower]" leader= +34:11.351 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-49a68a4a-ce0a-a0ae-4cec-d1dc85c97f61.dc1 127.0.0.1 +34:11.352 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-49a68a4a-ce0a-a0ae-4cec-d1dc85c97f61 127.0.0.1 +2021-08-13T15:34:11.352Z [INFO] agent.router: Initializing LAN area manager +34:11.352 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-49a68a4a-ce0a-a0ae-4cec-d1dc85c97f61.dc1 area=wan +34:11.352 [INFO] TestAgent.server: Adding LAN server: server="Node-49a68a4a-ce0a-a0ae-4cec-d1dc85c97f61 (Addr: tcp/127.0.0.1:25239) (DC: dc1)" +34:11.352 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25234 network=udp +34:11.352 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25234 network=tcp +34:11.353 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25235 network=tcp +34:11.353 [INFO] TestAgent: started state syncer +34:11.353 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25240 network=tcp +34:11.396 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:11.396 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25239 [Candidate]" term=2 +34:11.397 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:11.397 [DEBUG] TestAgent.server.raft: vote granted: from=49a68a4a-ce0a-a0ae-4cec-d1dc85c97f61 term=2 tally=1 +34:11.397 [INFO] TestAgent.server.raft: election won: tally=1 +34:11.397 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25239 [Leader]" +34:11.397 [INFO] TestAgent.server: cluster leadership acquired +34:11.397 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25239 +34:11.398 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:11.398 [INFO] TestAgent.server: New leader elected: payload=Node-49a68a4a-ce0a-a0ae-4cec-d1dc85c97f61 +34:11.400 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:11.400 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:11.400 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:11.400 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:11.400 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:11.400 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-49a68a4a-ce0a-a0ae-4cec-d1dc85c97f61 +34:11.400 [INFO] TestAgent.server: member joined, marking health alive: member=Node-49a68a4a-ce0a-a0ae-4cec-d1dc85c97f61 +34:11.402 [INFO] TestAgent.server: federation state anti-entropy synced +34:11.751 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:11.752 [INFO] TestAgent: Synced node info +34:11.752 [DEBUG] TestAgent: Node info in sync +34:11.785 [INFO] TestAgent: Requesting shutdown +34:11.785 [INFO] TestAgent.server: shutting down server +34:11.785 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:11.785 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:11.785 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:11.785 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:11.785 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:11.785 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:11.785 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:11.785 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:11.785 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:11.785 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:11.785 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:11.785Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:11.785Z [INFO] agent.router.manager: shutting down +34:11.786 [INFO] TestAgent: consul server down +34:11.786 [INFO] TestAgent: shutdown complete +34:11.786 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25234 network=tcp +34:11.786 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25234 network=udp +34:11.786 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25235 network=tcp +34:12.286 [INFO] TestAgent: Waiting for endpoints to shut down +34:12.286 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentConnectAuthorize_allow (0.97s) === CONT TestRemoteExecWrites_ACLAgentToken -2022-09-15T21:42:50.542Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:50.594Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:50.597 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:73291a19-377f-5871-cc5c-0e8b86cca152 Address:127.0.0.1:13979}]" -42:50.597 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13979 [Follower]" leader= -42:50.599 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-73291a19-377f-5871-cc5c-0e8b86cca152.dc1 127.0.0.1 -42:50.602 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-73291a19-377f-5871-cc5c-0e8b86cca152 127.0.0.1 -2022-09-15T21:42:50.602Z [INFO] agent.router: Initializing LAN area manager -42:50.603 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13974 network=udp -42:50.603 [INFO] TestAgent.server: Adding LAN server: server="Node-73291a19-377f-5871-cc5c-0e8b86cca152 (Addr: tcp/127.0.0.1:13979) (DC: dc1)" -42:50.603 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-73291a19-377f-5871-cc5c-0e8b86cca152.dc1 area=wan -42:50.603 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13974 network=tcp -42:50.604 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13975 network=tcp -42:50.604 [INFO] TestAgent: started state syncer -42:50.604 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13980 network=tcp -42:50.638 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:50.638 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13979 [Candidate]" term=2 -42:50.638 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:50.638 [DEBUG] TestAgent.server.raft: vote granted: from=73291a19-377f-5871-cc5c-0e8b86cca152 term=2 tally=1 -42:50.638 [INFO] TestAgent.server.raft: election won: tally=1 -42:50.638 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13979 [Leader]" -42:50.638 [INFO] TestAgent.server: cluster leadership acquired -42:50.639 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13979 -42:50.640 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:50.641 [INFO] TestAgent.server: New leader elected: payload=Node-73291a19-377f-5871-cc5c-0e8b86cca152 -42:50.644 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:50.644 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:50.644 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:50.644 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:50.644 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:50.644 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-73291a19-377f-5871-cc5c-0e8b86cca152 -42:50.644 [INFO] TestAgent.server: member joined, marking health alive: member=Node-73291a19-377f-5871-cc5c-0e8b86cca152 -42:50.645 [INFO] TestAgent.server: federation state anti-entropy synced -42:50.831 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:50.832 [INFO] TestAgent: Synced node info -42:50.976 [INFO] TestAgent: Requesting shutdown -42:50.977 [INFO] TestAgent.server: shutting down server -42:50.977 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:50.977 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:50.977 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:50.977 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:50.977 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:50.977 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:50.977 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:50.977 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:50.977 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:50.977 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:50.977 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:50.977Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:50.977Z [INFO] agent.router.manager: shutting down -42:50.977 [INFO] TestAgent: consul server down -42:50.977 [INFO] TestAgent: shutdown complete -42:50.977 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13974 network=tcp -42:50.977 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13974 network=udp -42:50.978 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13975 network=tcp -42:51.480 [INFO] TestAgent: Waiting for endpoints to shut down -42:51.481 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentConnectAuthorize_noTarget (1.00s) +2021-08-13T15:34:11.395Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:11.411Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:11.414 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9eaa6dea-96b6-4d05-e5df-738262a5db40 Address:127.0.0.1:25246}]" +34:11.415 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9eaa6dea-96b6-4d05-e5df-738262a5db40.dc1 127.0.0.1 +34:11.415 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25246 [Follower]" leader= +34:11.436 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9eaa6dea-96b6-4d05-e5df-738262a5db40 127.0.0.1 +2021-08-13T15:34:11.436Z [INFO] agent.router: Initializing LAN area manager +34:11.437 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25241 network=udp +34:11.437 [INFO] TestAgent.server: Adding LAN server: server="Node-9eaa6dea-96b6-4d05-e5df-738262a5db40 (Addr: tcp/127.0.0.1:25246) (DC: dc1)" +34:11.437 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9eaa6dea-96b6-4d05-e5df-738262a5db40.dc1 area=wan +34:11.437 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25241 network=tcp +34:11.438 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25242 network=tcp +34:11.438 [INFO] TestAgent: started state syncer +34:11.438 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25247 network=tcp +34:11.458 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:11.458 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25246 [Candidate]" term=2 +34:11.458 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:11.458 [DEBUG] TestAgent.server.raft: vote granted: from=9eaa6dea-96b6-4d05-e5df-738262a5db40 term=2 tally=1 +34:11.458 [INFO] TestAgent.server.raft: election won: tally=1 +34:11.458 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25246 [Leader]" +34:11.458 [INFO] TestAgent.server: cluster leadership acquired +34:11.458 [INFO] TestAgent.server: New leader elected: payload=Node-9eaa6dea-96b6-4d05-e5df-738262a5db40 +34:11.459 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25246 +34:11.460 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:11.463 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:11.463 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:11.463 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:11.463 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:11.463 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:11.463 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9eaa6dea-96b6-4d05-e5df-738262a5db40 +34:11.463 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9eaa6dea-96b6-4d05-e5df-738262a5db40 +34:11.463 [INFO] TestAgent.server: federation state anti-entropy synced +34:11.850 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:11.851 [INFO] TestAgent: Synced node info +34:11.865 [DEBUG] TestAgent: received remote exec event: id=02399a24-d03b-d0f7-ff17-f1ac420391c2 +34:11.865 [INFO] TestAgent: remote exec script: script="echo failing;exit 2" +34:11.870 [INFO] TestAgent: Requesting shutdown +34:11.870 [INFO] TestAgent.server: shutting down server +34:11.870 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:11.870 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:11.870 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:11.870 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:11.870 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:11.870 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:11.870 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:11.870 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:11.870 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:11.870 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:11.870 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:11.870Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:11.870Z [INFO] agent.router.manager: shutting down +34:11.871 [INFO] TestAgent: consul server down +34:11.871 [INFO] TestAgent: shutdown complete +34:11.871 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25241 network=tcp +34:11.871 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25241 network=udp +34:11.871 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25242 network=tcp +34:12.371 [INFO] TestAgent: Waiting for endpoints to shut down +34:12.371 [INFO] TestAgent: Endpoints down +--- PASS: TestHandleRemoteExecFailed (0.99s) === CONT TestRemoteExecWrites_ACLToken -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 4 out of 4 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:51.097Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:51.119Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:51.124 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e99f5410-0bd4-4b0e-3c29-102784a68efa Address:127.0.0.1:14007}]" -42:51.125 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e99f5410-0bd4-4b0e-3c29-102784a68efa.dc1 127.0.0.1 -42:51.126 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14007 [Follower]" leader= -42:51.126 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e99f5410-0bd4-4b0e-3c29-102784a68efa 127.0.0.1 -2022-09-15T21:42:51.127Z [INFO] agent.router: Initializing LAN area manager -42:51.127 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13981 network=udp -42:51.127 [INFO] TestAgent.server: Adding LAN server: server="Node-e99f5410-0bd4-4b0e-3c29-102784a68efa (Addr: tcp/127.0.0.1:14007) (DC: dc1)" -42:51.127 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e99f5410-0bd4-4b0e-3c29-102784a68efa.dc1 area=wan -42:51.127 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13981 network=tcp -42:51.128 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13982 network=tcp -42:51.128 [INFO] TestAgent: started state syncer -42:51.129 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14008 network=tcp -42:51.175 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:51.175 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14007 [Candidate]" term=2 -42:51.175 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:51.175 [DEBUG] TestAgent.server.raft: vote granted: from=e99f5410-0bd4-4b0e-3c29-102784a68efa term=2 tally=1 -42:51.176 [INFO] TestAgent.server.raft: election won: tally=1 -42:51.176 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14007 [Leader]" -42:51.177 [INFO] TestAgent.server: cluster leadership acquired -42:51.177 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14007 -42:51.178 [INFO] TestAgent.server: New leader elected: payload=Node-e99f5410-0bd4-4b0e-3c29-102784a68efa -42:51.178 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:51.181 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:51.182 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:51.182 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:51.182 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:51.182 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:51.182 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e99f5410-0bd4-4b0e-3c29-102784a68efa -42:51.182 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e99f5410-0bd4-4b0e-3c29-102784a68efa -42:51.183 [INFO] TestAgent.server: federation state anti-entropy synced -42:51.300 [INFO] TestAgent: Requesting shutdown -42:51.300 [INFO] TestAgent.server: shutting down server -42:51.300 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:51.300 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:51.300 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:51.300 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:51.301 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:51.301 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:51.301 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:51.301 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:51.301 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:51.301 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:51.301 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:51.301 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:51.302Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:51.302Z [INFO] agent.router.manager: shutting down -42:51.302 [INFO] TestAgent: consul server down -42:51.302 [INFO] TestAgent: shutdown complete -42:51.302 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13981 network=tcp -42:51.302 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13981 network=udp -42:51.302 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13982 network=tcp -42:51.803 [INFO] TestAgent: Waiting for endpoints to shut down -42:51.803 [INFO] TestAgent: Endpoints down ---- PASS: TestSessionCreate (0.73s) -=== CONT TestRemoteExecWrites -2022-09-15T21:42:51.182Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:51.182Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:51.199Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:51.200Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:51.202 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:362b2e45-2681-9ed3-fe22-21e801f0f858 Address:127.0.0.1:14021}]" -42:51.203 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-362b2e45-2681-9ed3-fe22-21e801f0f858.dc1 127.0.0.1 -42:51.204 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-362b2e45-2681-9ed3-fe22-21e801f0f858 127.0.0.1 -2022-09-15T21:42:51.204Z [INFO] agent.router: Initializing LAN area manager -42:51.205 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14016 network=udp -42:51.205 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14021 [Follower]" leader= -42:51.205 [INFO] TestAgent.server: Adding LAN server: server="Node-362b2e45-2681-9ed3-fe22-21e801f0f858 (Addr: tcp/127.0.0.1:14021) (DC: dc1)" -42:51.205 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-362b2e45-2681-9ed3-fe22-21e801f0f858.dc1 area=wan -42:51.205 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14016 network=tcp -42:51.206 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14017 network=tcp -42:51.206 [INFO] TestAgent: started state syncer -42:51.206 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14022 network=tcp -42:51.256 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -42:51.262 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:51.262 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14021 [Candidate]" term=2 -42:51.263 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:51.263 [DEBUG] TestAgent.server.raft: vote granted: from=362b2e45-2681-9ed3-fe22-21e801f0f858 term=2 tally=1 -42:51.263 [INFO] TestAgent.server.raft: election won: tally=1 -42:51.263 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14021 [Leader]" -42:51.263 [INFO] TestAgent.server: cluster leadership acquired -42:51.263 [INFO] TestAgent.server: New leader elected: payload=Node-362b2e45-2681-9ed3-fe22-21e801f0f858 -42:51.263 [INFO] TestAgent.server: initializing acls -42:51.263 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:51.264 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:51.264 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:51.265 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:51.265 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:51.265 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:51.265 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-362b2e45-2681-9ed3-fe22-21e801f0f858 -42:51.265 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-362b2e45-2681-9ed3-fe22-21e801f0f858.dc1 -42:51.266 [INFO] TestAgent.server: Updating LAN server: server="Node-362b2e45-2681-9ed3-fe22-21e801f0f858 (Addr: tcp/127.0.0.1:14021) (DC: dc1)" -42:51.266 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-362b2e45-2681-9ed3-fe22-21e801f0f858.dc1 area=wan -42:51.267 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:51.269 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:51.269 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:51.269 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:51.269 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:51.269 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:51.269 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-362b2e45-2681-9ed3-fe22-21e801f0f858 -42:51.269 [INFO] TestAgent.server: member joined, marking health alive: member=Node-362b2e45-2681-9ed3-fe22-21e801f0f858 -42:51.271 [INFO] TestAgent.server: federation state anti-entropy synced -42:51.271 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-362b2e45-2681-9ed3-fe22-21e801f0f858 -42:51.340 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-362b2e45-2681-9ed3-fe22-21e801f0f858 -42:51.340 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-362b2e45-2681-9ed3-fe22-21e801f0f858 -42:51.342 [ERROR] TestAgent: failed to ack remote exec job: error="Permission denied" -42:51.342 [ERROR] TestAgent: failed to write output for remote exec job: error="Permission denied" -42:51.342 [ERROR] TestAgent: failed to write output for remote exec job: error="Permission denied" -42:51.342 [INFO] TestAgent: Requesting shutdown -42:51.343 [INFO] TestAgent.server: shutting down server -42:51.343 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:51.343 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:51.343 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:51.343 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:51.343 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:51.343 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:51.343 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:51.343 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:51.343 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:51.343 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:51.343 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:51.343 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:51.343 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:51.343 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:51.343 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:51.343 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:51.343Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:51.343Z [INFO] agent.router.manager: shutting down -42:51.344 [INFO] TestAgent: consul server down -42:51.344 [INFO] TestAgent: shutdown complete -42:51.344 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14016 network=tcp -42:51.344 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14016 network=udp -42:51.344 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14017 network=tcp -42:51.845 [INFO] TestAgent: Waiting for endpoints to shut down -42:51.845 [INFO] TestAgent: Endpoints down ---- PASS: TestRemoteExecWrites_ACLDeny (0.68s) -=== CONT TestRemoteExecGetSpec_ACLDeny -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:51.121Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:51.139Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:51.142 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b84a2e46-bd55-7861-5f2d-20be8e47c96f Address:127.0.0.1:13993}]" -42:51.143 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b84a2e46-bd55-7861-5f2d-20be8e47c96f.dc1 127.0.0.1 -42:51.144 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b84a2e46-bd55-7861-5f2d-20be8e47c96f 127.0.0.1 -2022-09-15T21:42:51.144Z [INFO] agent.router: Initializing LAN area manager -42:51.144 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13988 network=udp -42:51.144 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13993 [Follower]" leader= -42:51.145 [INFO] TestAgent.server: Adding LAN server: server="Node-b84a2e46-bd55-7861-5f2d-20be8e47c96f (Addr: tcp/127.0.0.1:13993) (DC: dc1)" -42:51.145 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b84a2e46-bd55-7861-5f2d-20be8e47c96f.dc1 area=wan -42:51.145 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13988 network=tcp -42:51.146 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13989 network=tcp -42:51.146 [INFO] TestAgent: started state syncer -42:51.146 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13994 network=tcp -42:51.205 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:51.205 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13993 [Candidate]" term=2 -42:51.205 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:51.205 [DEBUG] TestAgent.server.raft: vote granted: from=b84a2e46-bd55-7861-5f2d-20be8e47c96f term=2 tally=1 -42:51.205 [INFO] TestAgent.server.raft: election won: tally=1 -42:51.205 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13993 [Leader]" -42:51.206 [INFO] TestAgent.server: cluster leadership acquired -42:51.206 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13993 -42:51.207 [INFO] TestAgent.server: New leader elected: payload=Node-b84a2e46-bd55-7861-5f2d-20be8e47c96f -42:51.207 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:51.209 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:51.209 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:51.209 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:51.209 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:51.209 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:51.210 [INFO] TestAgent.server: federation state anti-entropy synced -42:51.213 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b84a2e46-bd55-7861-5f2d-20be8e47c96f -42:51.213 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b84a2e46-bd55-7861-5f2d-20be8e47c96f -42:51.457 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:51.458 [INFO] TestAgent: Synced node info -42:51.458 [DEBUG] TestAgent: Node info in sync -42:51.511 [DEBUG] TestAgent: received remote exec event: id=b0945982-7b79-9de8-1425-a740634e7651 -42:51.511 [INFO] TestAgent: remote exec script: script="echo failing;exit 2" -42:51.526 [INFO] TestAgent: Requesting shutdown -42:51.526 [INFO] TestAgent.server: shutting down server -42:51.526 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:51.526 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:51.526 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:51.526 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:51.526 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:51.526 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:51.526 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:51.526 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:51.526 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:51.526 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:51.526 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:51.527Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:51.527Z [INFO] agent.router.manager: shutting down -42:51.527 [INFO] TestAgent: consul server down -42:51.527 [INFO] TestAgent: shutdown complete -42:51.527 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13988 network=tcp -42:51.527 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13988 network=udp -42:51.527 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13989 network=tcp -42:52.030 [INFO] TestAgent: Waiting for endpoints to shut down -42:52.030 [INFO] TestAgent: Endpoints down ---- PASS: TestHandleRemoteExecFailed (0.94s) -=== CONT TestRemoteExecGetSpec_ACLAgentToken -2022-09-15T21:42:51.176Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:51.199Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:51.209 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fc98cb7b-f1a5-0ddb-05f1-46cf378fdb24 Address:127.0.0.1:14014}]" -42:51.210 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14014 [Follower]" leader= -42:51.211 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fc98cb7b-f1a5-0ddb-05f1-46cf378fdb24.dc1 127.0.0.1 -42:51.214 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fc98cb7b-f1a5-0ddb-05f1-46cf378fdb24 127.0.0.1 -2022-09-15T21:42:51.214Z [INFO] agent.router: Initializing LAN area manager -42:51.214 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fc98cb7b-f1a5-0ddb-05f1-46cf378fdb24.dc1 area=wan -42:51.214 [INFO] TestAgent.server: Adding LAN server: server="Node-fc98cb7b-f1a5-0ddb-05f1-46cf378fdb24 (Addr: tcp/127.0.0.1:14014) (DC: dc1)" -42:51.215 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13995 network=udp -42:51.215 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13995 network=tcp -42:51.216 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13996 network=tcp -42:51.216 [INFO] TestAgent: started state syncer -42:51.216 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14015 network=tcp -42:51.256 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:51.256 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14014 [Candidate]" term=2 -42:51.256 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:51.256 [DEBUG] TestAgent.server.raft: vote granted: from=fc98cb7b-f1a5-0ddb-05f1-46cf378fdb24 term=2 tally=1 -42:51.256 [INFO] TestAgent.server.raft: election won: tally=1 -42:51.256 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14014 [Leader]" -42:51.256 [INFO] TestAgent.server: cluster leadership acquired -42:51.256 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14014 -42:51.258 [INFO] TestAgent.server: New leader elected: payload=Node-fc98cb7b-f1a5-0ddb-05f1-46cf378fdb24 -42:51.258 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:51.261 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:51.261 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:51.261 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:51.261 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:51.261 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:51.261 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fc98cb7b-f1a5-0ddb-05f1-46cf378fdb24 -42:51.261 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fc98cb7b-f1a5-0ddb-05f1-46cf378fdb24 -42:51.262 [INFO] TestAgent.server: federation state anti-entropy synced -42:51.479 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:51.480 [INFO] TestAgent: Synced node info -42:51.480 [DEBUG] TestAgent: Node info in sync -42:51.576 [DEBUG] TestAgent: received remote exec event: id=671223f3-49a0-7218-dc4b-f815f7012400 -42:51.576 [INFO] TestAgent: remote exec script: script=uptime -42:51.590 [INFO] TestAgent: Requesting shutdown -42:51.590 [INFO] TestAgent.server: shutting down server -42:51.590 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:51.590 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:51.590 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:51.590 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:51.590 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:51.590 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:51.590 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:51.590 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:51.590 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:51.590 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:51.590 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:51.591Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:51.591Z [INFO] agent.router.manager: shutting down -42:51.591 [INFO] TestAgent: consul server down -42:51.591 [INFO] TestAgent: shutdown complete -42:51.591 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13995 network=tcp -42:51.591 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13995 network=udp -42:51.591 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13996 network=tcp -42:52.093 [INFO] TestAgent: Waiting for endpoints to shut down -42:52.093 [INFO] TestAgent: Endpoints down ---- PASS: TestHandleRemoteExec (0.94s) -=== CONT TestRemoteExecGetSpec_ACLToken -2022-09-15T21:42:51.409Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:51.409Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:51.445Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:51.445Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:51.449 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:caf665f6-962b-bb99-5ea6-eefa4ab5f956 Address:127.0.0.1:14028}]" -42:51.452 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-caf665f6-962b-bb99-5ea6-eefa4ab5f956.dc1 127.0.0.1 -42:51.453 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-caf665f6-962b-bb99-5ea6-eefa4ab5f956 127.0.0.1 -2022-09-15T21:42:51.453Z [INFO] agent.router: Initializing LAN area manager -42:51.453 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14028 [Follower]" leader= -42:51.454 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-caf665f6-962b-bb99-5ea6-eefa4ab5f956.dc1 area=wan -42:51.454 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14023 network=tcp -42:51.455 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14023 network=udp -42:51.456 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14024 network=tcp -42:51.456 [INFO] TestAgent: started state syncer -42:51.457 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14029 network=tcp -42:51.458 [INFO] TestAgent.server: Adding LAN server: server="Node-caf665f6-962b-bb99-5ea6-eefa4ab5f956 (Addr: tcp/127.0.0.1:14028) (DC: dc1)" -42:51.504 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -42:51.518 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:51.518 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14028 [Candidate]" term=2 -42:51.518 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:51.518 [DEBUG] TestAgent.server.raft: vote granted: from=caf665f6-962b-bb99-5ea6-eefa4ab5f956 term=2 tally=1 -42:51.518 [INFO] TestAgent.server.raft: election won: tally=1 -42:51.518 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14028 [Leader]" -42:51.518 [INFO] TestAgent.server: cluster leadership acquired -42:51.519 [INFO] TestAgent.server: initializing acls -42:51.519 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:51.519 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:51.520 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:51.520 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:51.520 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:51.520 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:51.521 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-caf665f6-962b-bb99-5ea6-eefa4ab5f956 -42:51.521 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-caf665f6-962b-bb99-5ea6-eefa4ab5f956.dc1 -42:51.522 [INFO] TestAgent.server: New leader elected: payload=Node-caf665f6-962b-bb99-5ea6-eefa4ab5f956 -42:51.522 [INFO] TestAgent.server: Updating LAN server: server="Node-caf665f6-962b-bb99-5ea6-eefa4ab5f956 (Addr: tcp/127.0.0.1:14028) (DC: dc1)" -42:51.522 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-caf665f6-962b-bb99-5ea6-eefa4ab5f956.dc1 area=wan -42:51.522 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:51.525 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:51.525 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:51.525 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:51.525 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:51.525 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:51.525 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-caf665f6-962b-bb99-5ea6-eefa4ab5f956 -42:51.525 [INFO] TestAgent.server: member joined, marking health alive: member=Node-caf665f6-962b-bb99-5ea6-eefa4ab5f956 -42:51.528 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-caf665f6-962b-bb99-5ea6-eefa4ab5f956 -42:51.528 [INFO] TestAgent.server: federation state anti-entropy synced -42:51.664 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-caf665f6-962b-bb99-5ea6-eefa4ab5f956 -42:51.665 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-caf665f6-962b-bb99-5ea6-eefa4ab5f956 -42:51.669 [INFO] TestAgent: Requesting shutdown -42:51.669 [INFO] TestAgent.server: shutting down server -42:51.669 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:51.669 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:51.669 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:51.669 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:51.669 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:51.669 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:51.669 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:51.669 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:51.669 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:51.669 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:51.669 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:51.669 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:51.669 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:51.669 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:51.669 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:51.669 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:51.669Z [INFO] agent.router.manager: shutting down -42:51.670 [INFO] TestAgent: consul server down -42:51.670 [INFO] TestAgent: shutdown complete -42:51.670 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14023 network=tcp -42:51.670 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14023 network=udp -42:51.670 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14024 network=tcp -2022-09-15T21:42:51.670Z [INFO] agent.router.manager: shutting down -42:52.170 [INFO] TestAgent: Waiting for endpoints to shut down -42:52.170 [INFO] TestAgent: Endpoints down ---- PASS: TestRemoteExecWrites_ACLAgentToken (0.78s) -=== CONT TestRemoteExecGetSpec -[WARN] freeport: 4 out of 25 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:51.514Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:51.514Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:51.541Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:51.542Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:51.545 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0ffacf93-ddf9-a979-36a2-4c0652a6b976 Address:127.0.0.1:14035}]" -42:51.546 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14035 [Follower]" leader= -42:51.550 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0ffacf93-ddf9-a979-36a2-4c0652a6b976.dc1 127.0.0.1 -42:51.551 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0ffacf93-ddf9-a979-36a2-4c0652a6b976 127.0.0.1 -2022-09-15T21:42:51.551Z [INFO] agent.router: Initializing LAN area manager -42:51.552 [INFO] TestAgent.server: Adding LAN server: server="Node-0ffacf93-ddf9-a979-36a2-4c0652a6b976 (Addr: tcp/127.0.0.1:14035) (DC: dc1)" -42:51.552 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0ffacf93-ddf9-a979-36a2-4c0652a6b976.dc1 area=wan -42:51.552 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14030 network=tcp -42:51.552 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14030 network=udp -42:51.553 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14031 network=tcp -42:51.553 [INFO] TestAgent: started state syncer -42:51.553 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14036 network=tcp -42:51.598 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:51.598 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14035 [Candidate]" term=2 -42:51.598 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:51.598 [DEBUG] TestAgent.server.raft: vote granted: from=0ffacf93-ddf9-a979-36a2-4c0652a6b976 term=2 tally=1 -42:51.598 [INFO] TestAgent.server.raft: election won: tally=1 -42:51.598 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14035 [Leader]" -42:51.599 [INFO] TestAgent.server: cluster leadership acquired -42:51.599 [INFO] TestAgent.server: initializing acls -42:51.599 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:51.599 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:51.600 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:51.600 [INFO] TestAgent.server: New leader elected: payload=Node-0ffacf93-ddf9-a979-36a2-4c0652a6b976 -42:51.601 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:51.601 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:51.601 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:51.601 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-0ffacf93-ddf9-a979-36a2-4c0652a6b976 -42:51.601 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-0ffacf93-ddf9-a979-36a2-4c0652a6b976.dc1 -42:51.602 [INFO] TestAgent.server: Updating LAN server: server="Node-0ffacf93-ddf9-a979-36a2-4c0652a6b976 (Addr: tcp/127.0.0.1:14035) (DC: dc1)" -42:51.602 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-0ffacf93-ddf9-a979-36a2-4c0652a6b976.dc1 area=wan -42:51.603 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:51.606 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:51.606 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:51.606 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:51.606 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:51.606 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:51.606 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0ffacf93-ddf9-a979-36a2-4c0652a6b976 -42:51.606 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0ffacf93-ddf9-a979-36a2-4c0652a6b976 -42:51.607 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0ffacf93-ddf9-a979-36a2-4c0652a6b976 -42:51.608 [INFO] TestAgent.server: federation state anti-entropy synced -42:51.663 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:51.664 [INFO] TestAgent: Synced node info -42:51.664 [DEBUG] TestAgent: Node info in sync -42:51.779 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-0ffacf93-ddf9-a979-36a2-4c0652a6b976 -42:51.779 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-0ffacf93-ddf9-a979-36a2-4c0652a6b976 -42:51.783 [INFO] TestAgent: Requesting shutdown -42:51.783 [INFO] TestAgent.server: shutting down server -42:51.783 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:51.783 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:51.783 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:51.783 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:51.783 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:51.783 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:51.783 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:51.783 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:51.783 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:51.783 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:51.783 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:51.783 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:51.783 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:51.783 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:51.783 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:51.783Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:51.783Z [INFO] agent.router.manager: shutting down -42:51.783 [INFO] TestAgent: consul server down -42:51.783 [INFO] TestAgent: shutdown complete -42:51.783 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14030 network=tcp -42:51.784 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14030 network=udp -42:51.784 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14031 network=tcp -42:52.284 [INFO] TestAgent: Waiting for endpoints to shut down -42:52.284 [INFO] TestAgent: Endpoints down ---- PASS: TestRemoteExecWrites_ACLToken (0.80s) -=== CONT TestPreparedQuery_Integration -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:51.875Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:51.875Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:51.894Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:51.894Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:51.897 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d37908cd-09f2-2b1d-c3f5-2be6820b9bc5 Address:127.0.0.1:13951}]" -42:51.897 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13951 [Follower]" leader= -42:51.898 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d37908cd-09f2-2b1d-c3f5-2be6820b9bc5.dc1 127.0.0.1 -42:51.899 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d37908cd-09f2-2b1d-c3f5-2be6820b9bc5 127.0.0.1 -2022-09-15T21:42:51.899Z [INFO] agent.router: Initializing LAN area manager -42:51.899 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d37908cd-09f2-2b1d-c3f5-2be6820b9bc5.dc1 area=wan -42:51.899 [INFO] TestAgent.server: Adding LAN server: server="Node-d37908cd-09f2-2b1d-c3f5-2be6820b9bc5 (Addr: tcp/127.0.0.1:13951) (DC: dc1)" -42:51.899 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14002 network=tcp -42:51.899 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14002 network=udp -42:51.900 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14003 network=tcp -42:51.900 [INFO] TestAgent: started state syncer -42:51.901 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13952 network=tcp -42:51.948 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:51.948 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13951 [Candidate]" term=2 -42:51.948 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:51.948 [DEBUG] TestAgent.server.raft: vote granted: from=d37908cd-09f2-2b1d-c3f5-2be6820b9bc5 term=2 tally=1 -42:51.948 [INFO] TestAgent.server.raft: election won: tally=1 -42:51.948 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13951 [Leader]" -42:51.948 [INFO] TestAgent.server: cluster leadership acquired -42:51.948 [INFO] TestAgent.server: initializing acls -42:51.949 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:51.949 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:51.950 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:51.950 [INFO] TestAgent.server: New leader elected: payload=Node-d37908cd-09f2-2b1d-c3f5-2be6820b9bc5 -42:51.950 [INFO] TestAgent.server: initializing acls -42:51.950 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:51.951 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:51.951 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:51.951 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:51.951 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode -42:51.951 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-d37908cd-09f2-2b1d-c3f5-2be6820b9bc5 -42:51.951 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-d37908cd-09f2-2b1d-c3f5-2be6820b9bc5.dc1 -42:51.951 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-d37908cd-09f2-2b1d-c3f5-2be6820b9bc5.dc1 area=wan -42:51.951 [INFO] TestAgent.server: Updating LAN server: server="Node-d37908cd-09f2-2b1d-c3f5-2be6820b9bc5 (Addr: tcp/127.0.0.1:13951) (DC: dc1)" -42:51.953 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-d37908cd-09f2-2b1d-c3f5-2be6820b9bc5 -42:51.953 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-d37908cd-09f2-2b1d-c3f5-2be6820b9bc5.dc1 -42:51.954 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-d37908cd-09f2-2b1d-c3f5-2be6820b9bc5.dc1 area=wan -42:51.954 [INFO] TestAgent.server: Updating LAN server: server="Node-d37908cd-09f2-2b1d-c3f5-2be6820b9bc5 (Addr: tcp/127.0.0.1:13951) (DC: dc1)" -42:51.954 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:51.957 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:51.957 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:51.957 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:51.957 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:51.957 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:51.958 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d37908cd-09f2-2b1d-c3f5-2be6820b9bc5 -42:51.958 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d37908cd-09f2-2b1d-c3f5-2be6820b9bc5 -42:51.958 [INFO] TestAgent.server: federation state anti-entropy synced -42:51.959 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d37908cd-09f2-2b1d-c3f5-2be6820b9bc5 -42:51.959 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d37908cd-09f2-2b1d-c3f5-2be6820b9bc5 -42:52.078 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-d37908cd-09f2-2b1d-c3f5-2be6820b9bc5 -42:52.078 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-d37908cd-09f2-2b1d-c3f5-2be6820b9bc5 -42:52.080 [ERROR] TestAgent: failed to get remote exec job: error="Permission denied" -42:52.081 [INFO] TestAgent: Requesting shutdown -42:52.081 [INFO] TestAgent.server: shutting down server -42:52.081 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:52.081 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:52.081 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:52.081 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:52.081 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:52.081 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:52.081 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:52.081 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:52.081 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:52.081 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:52.081 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:52.081 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:52.081 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:52.081 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:52.081 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:52.081 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:52.082Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:52.082Z [INFO] agent.router.manager: shutting down -42:52.082 [INFO] TestAgent: consul server down -42:52.082 [INFO] TestAgent: shutdown complete -42:52.082 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14002 network=tcp -42:52.082 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14002 network=udp -42:52.082 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14003 network=tcp -42:52.583 [INFO] TestAgent: Waiting for endpoints to shut down -42:52.583 [INFO] TestAgent: Endpoints down ---- PASS: TestRemoteExecGetSpec_ACLDeny (0.74s) -=== CONT TestAgent_Join -2022-09-15T21:42:51.825Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:51.844Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:51.847 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:df7f19c4-76f9-999a-2dc7-02fac97a33ac Address:127.0.0.1:14000}]" -42:51.848 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-df7f19c4-76f9-999a-2dc7-02fac97a33ac.dc1 127.0.0.1 -42:51.849 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-df7f19c4-76f9-999a-2dc7-02fac97a33ac 127.0.0.1 -2022-09-15T21:42:51.849Z [INFO] agent.router: Initializing LAN area manager -42:51.849 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14051 network=udp -42:51.849 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14000 [Follower]" leader= -42:51.850 [INFO] TestAgent.server: Adding LAN server: server="Node-df7f19c4-76f9-999a-2dc7-02fac97a33ac (Addr: tcp/127.0.0.1:14000) (DC: dc1)" -42:51.850 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-df7f19c4-76f9-999a-2dc7-02fac97a33ac.dc1 area=wan -42:51.850 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14051 network=tcp -42:51.851 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14052 network=tcp -42:51.851 [INFO] TestAgent: started state syncer -42:51.851 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14001 network=tcp -42:51.902 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:51.902 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14000 [Candidate]" term=2 -42:51.902 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:51.902 [DEBUG] TestAgent.server.raft: vote granted: from=df7f19c4-76f9-999a-2dc7-02fac97a33ac term=2 tally=1 -42:51.902 [INFO] TestAgent.server.raft: election won: tally=1 -42:51.902 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14000 [Leader]" -42:51.902 [INFO] TestAgent.server: cluster leadership acquired -42:51.902 [INFO] TestAgent.server: New leader elected: payload=Node-df7f19c4-76f9-999a-2dc7-02fac97a33ac -42:51.902 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14000 -42:51.904 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:51.907 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:51.907 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:51.907 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:51.907 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:51.907 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:51.908 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-df7f19c4-76f9-999a-2dc7-02fac97a33ac -42:51.908 [INFO] TestAgent.server: member joined, marking health alive: member=Node-df7f19c4-76f9-999a-2dc7-02fac97a33ac -42:51.908 [INFO] TestAgent.server: federation state anti-entropy synced -42:52.014 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:52.015 [INFO] TestAgent: Synced node info -42:52.015 [DEBUG] TestAgent: Node info in sync -42:52.093 [INFO] TestAgent: Requesting shutdown -42:52.093 [INFO] TestAgent.server: shutting down server -42:52.093 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:52.093 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:52.093 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:52.093 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:52.093 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:52.093 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:52.093 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:52.093 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:52.093 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:52.093 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:52.093 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:52.094Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:52.094Z [INFO] agent.router.manager: shutting down -42:52.094 [INFO] TestAgent: consul server down -42:52.094 [INFO] TestAgent: shutdown complete -42:52.094 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14051 network=tcp -42:52.094 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14051 network=udp -42:52.094 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14052 network=tcp -42:52.598 [INFO] TestAgent: Waiting for endpoints to shut down -42:52.598 [INFO] TestAgent: Endpoints down ---- PASS: TestRemoteExecWrites (0.79s) +2021-08-13T15:34:11.706Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:11.725Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:11.729 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5d23bdb4-bc29-3d80-d4e8-8d6de221b8ba Address:127.0.0.1:25267}]" +34:11.729 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25267 [Follower]" leader= +34:11.730 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5d23bdb4-bc29-3d80-d4e8-8d6de221b8ba.dc1 127.0.0.1 +34:11.731 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5d23bdb4-bc29-3d80-d4e8-8d6de221b8ba 127.0.0.1 +2021-08-13T15:34:11.731Z [INFO] agent.router: Initializing LAN area manager +34:11.731 [INFO] TestAgent.server: Adding LAN server: server="Node-5d23bdb4-bc29-3d80-d4e8-8d6de221b8ba (Addr: tcp/127.0.0.1:25267) (DC: dc1)" +34:11.732 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25262 network=udp +34:11.732 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5d23bdb4-bc29-3d80-d4e8-8d6de221b8ba.dc1 area=wan +34:11.732 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25262 network=tcp +34:11.733 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25263 network=tcp +34:11.733 [INFO] TestAgent: started state syncer +34:11.733 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25268 network=tcp +34:11.766 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:11.766 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25267 [Candidate]" term=2 +34:11.766 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:11.766 [DEBUG] TestAgent.server.raft: vote granted: from=5d23bdb4-bc29-3d80-d4e8-8d6de221b8ba term=2 tally=1 +34:11.766 [INFO] TestAgent.server.raft: election won: tally=1 +34:11.766 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25267 [Leader]" +34:11.767 [INFO] TestAgent.server: cluster leadership acquired +34:11.767 [INFO] TestAgent.server: New leader elected: payload=Node-5d23bdb4-bc29-3d80-d4e8-8d6de221b8ba +34:11.767 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25267 +34:11.768 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:11.771 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:11.771 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:11.771 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:11.771 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:11.771 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:11.771 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5d23bdb4-bc29-3d80-d4e8-8d6de221b8ba +34:11.771 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5d23bdb4-bc29-3d80-d4e8-8d6de221b8ba +34:11.772 [INFO] TestAgent.server: federation state anti-entropy synced +34:11.866 [DEBUG] TestAgent: received remote exec event: id=d58cd19a-19d9-92e1-ac5a-6d97cef3e36d +34:11.867 [INFO] TestAgent: remote exec script: script=uptime +34:11.874 [INFO] TestAgent: Requesting shutdown +34:11.874 [INFO] TestAgent.server: shutting down server +34:11.874 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:11.874 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:11.874 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:11.874 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:11.874 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:11.875 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:11.875 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:11.875 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:11.875 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:11.875 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:11.875 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:11.875 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:11.875Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:11.875Z [INFO] agent.router.manager: shutting down +34:11.875 [INFO] TestAgent: consul server down +34:11.875 [INFO] TestAgent: shutdown complete +34:11.875 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25262 network=tcp +34:11.875 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25262 network=udp +34:11.875 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25263 network=tcp +34:12.376 [INFO] TestAgent: Waiting for endpoints to shut down +34:12.376 [INFO] TestAgent: Endpoints down +--- PASS: TestHandleRemoteExec (0.69s) +=== CONT TestAgentConnectAuthorize_badBody +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:11.717Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:11.738Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:11.741 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:77a87fa5-ce9e-55d3-794a-584c504c7452 Address:127.0.0.1:25274}]" +34:11.741 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25274 [Follower]" leader= +34:11.742 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-77a87fa5-ce9e-55d3-794a-584c504c7452.dc1 127.0.0.1 +34:11.743 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-77a87fa5-ce9e-55d3-794a-584c504c7452 127.0.0.1 +2021-08-13T15:34:11.743Z [INFO] agent.router: Initializing LAN area manager +34:11.743 [INFO] TestAgent.server: Adding LAN server: server="Node-77a87fa5-ce9e-55d3-794a-584c504c7452 (Addr: tcp/127.0.0.1:25274) (DC: dc1)" +34:11.743 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-77a87fa5-ce9e-55d3-794a-584c504c7452.dc1 area=wan +34:11.744 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25269 network=tcp +34:11.744 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25269 network=udp +34:11.745 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25270 network=tcp +34:11.745 [INFO] TestAgent: started state syncer +34:11.745 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25275 network=tcp +34:11.808 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:11.808 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25274 [Candidate]" term=2 +34:11.809 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:11.809 [DEBUG] TestAgent.server.raft: vote granted: from=77a87fa5-ce9e-55d3-794a-584c504c7452 term=2 tally=1 +34:11.809 [INFO] TestAgent.server.raft: election won: tally=1 +34:11.809 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25274 [Leader]" +34:11.809 [INFO] TestAgent.server: cluster leadership acquired +34:11.809 [INFO] TestAgent.server: New leader elected: payload=Node-77a87fa5-ce9e-55d3-794a-584c504c7452 +34:11.809 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25274 +34:11.811 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:11.813 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:11.813 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:11.813 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:11.813 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:11.813 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:11.813 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-77a87fa5-ce9e-55d3-794a-584c504c7452 +34:11.814 [INFO] TestAgent.server: member joined, marking health alive: member=Node-77a87fa5-ce9e-55d3-794a-584c504c7452 +34:11.814 [INFO] TestAgent.server: federation state anti-entropy synced +34:11.966 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:11.966 [INFO] TestAgent: Synced node info +34:12.067 [INFO] TestAgent: Requesting shutdown +34:12.067 [INFO] TestAgent.server: shutting down server +34:12.067 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:12.067 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:12.067 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:12.067 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:12.067 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:12.067 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:12.067 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:12.067 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:12.067 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:12.067 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:12.067 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:12.067Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:12.068Z [INFO] agent.router.manager: shutting down +34:12.068 [INFO] TestAgent: consul server down +34:12.068 [INFO] TestAgent: shutdown complete +34:12.068 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25269 network=tcp +34:12.068 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25269 network=udp +34:12.068 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25270 network=tcp +34:12.568 [INFO] TestAgent: Waiting for endpoints to shut down +34:12.568 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentConnectAuthorize_idInvalidFormat (0.88s) === CONT TestAgentConnectCALeafCert_secondaryDC_good -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:52.116Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:52.116Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:52.134Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:52.134Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:52.138 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d28cfa52-6369-8b1c-1d80-d71dfc817fae Address:127.0.0.1:14056}]" -42:52.139 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d28cfa52-6369-8b1c-1d80-d71dfc817fae.dc1 127.0.0.1 -42:52.139 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d28cfa52-6369-8b1c-1d80-d71dfc817fae 127.0.0.1 -2022-09-15T21:42:52.140Z [INFO] agent.router: Initializing LAN area manager -42:52.139 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14056 [Follower]" leader= -42:52.140 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d28cfa52-6369-8b1c-1d80-d71dfc817fae.dc1 area=wan -42:52.140 [INFO] TestAgent.server: Adding LAN server: server="Node-d28cfa52-6369-8b1c-1d80-d71dfc817fae (Addr: tcp/127.0.0.1:14056) (DC: dc1)" -42:52.140 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13883 network=tcp -42:52.140 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13883 network=udp -42:52.141 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13884 network=tcp -42:52.142 [INFO] TestAgent: started state syncer -42:52.142 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14057 network=tcp -42:52.187 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:52.187 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14056 [Candidate]" term=2 -42:52.187 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:52.187 [DEBUG] TestAgent.server.raft: vote granted: from=d28cfa52-6369-8b1c-1d80-d71dfc817fae term=2 tally=1 -42:52.187 [INFO] TestAgent.server.raft: election won: tally=1 -42:52.187 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14056 [Leader]" -42:52.187 [INFO] TestAgent.server: cluster leadership acquired -42:52.187 [INFO] TestAgent.server: initializing acls -42:52.188 [INFO] TestAgent.server: New leader elected: payload=Node-d28cfa52-6369-8b1c-1d80-d71dfc817fae -42:52.188 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:52.188 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:52.189 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:52.189 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:52.189 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:52.189 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:52.189 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-d28cfa52-6369-8b1c-1d80-d71dfc817fae -42:52.190 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-d28cfa52-6369-8b1c-1d80-d71dfc817fae.dc1 -42:52.191 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:52.193 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:52.193 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:52.193 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:52.193 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:52.194 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:52.194 [INFO] TestAgent.server: Updating LAN server: server="Node-d28cfa52-6369-8b1c-1d80-d71dfc817fae (Addr: tcp/127.0.0.1:14056) (DC: dc1)" -42:52.194 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-d28cfa52-6369-8b1c-1d80-d71dfc817fae.dc1 area=wan -42:52.194 [INFO] TestAgent.server: federation state anti-entropy synced -42:52.195 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d28cfa52-6369-8b1c-1d80-d71dfc817fae -42:52.195 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d28cfa52-6369-8b1c-1d80-d71dfc817fae -42:52.196 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d28cfa52-6369-8b1c-1d80-d71dfc817fae -42:52.396 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-d28cfa52-6369-8b1c-1d80-d71dfc817fae -42:52.397 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-d28cfa52-6369-8b1c-1d80-d71dfc817fae -42:52.399 [INFO] TestAgent: Requesting shutdown -42:52.399 [INFO] TestAgent.server: shutting down server -42:52.399 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:52.399 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:52.399 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:52.399 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:52.399 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:52.399 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:52.399 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:52.399 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:52.399 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:52.399 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:52.399 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:52.399 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:52.399 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:52.399 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:52.399 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:52.399 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:52.400Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:52.400Z [INFO] agent.router.manager: shutting down -42:52.400 [INFO] TestAgent: consul server down -42:52.400 [INFO] TestAgent: shutdown complete -42:52.400 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13883 network=tcp -42:52.400 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13883 network=udp -42:52.400 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13884 network=tcp -42:52.902 [INFO] TestAgent: Waiting for endpoints to shut down -42:52.902 [INFO] TestAgent: Endpoints down ---- PASS: TestRemoteExecGetSpec_ACLToken (0.81s) +2021-08-13T15:34:11.858Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:11.858Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:11.876Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:11.876Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:11.879 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:39be513c-5871-ea8c-a3ab-1e92704e9a5a Address:127.0.0.1:25281}]" +34:11.880 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25281 [Follower]" leader= +34:11.881 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-39be513c-5871-ea8c-a3ab-1e92704e9a5a.dc1 127.0.0.1 +34:11.882 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-39be513c-5871-ea8c-a3ab-1e92704e9a5a 127.0.0.1 +2021-08-13T15:34:11.882Z [INFO] agent.router: Initializing LAN area manager +34:11.882 [INFO] TestAgent.server: Adding LAN server: server="Node-39be513c-5871-ea8c-a3ab-1e92704e9a5a (Addr: tcp/127.0.0.1:25281) (DC: dc1)" +34:11.882 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-39be513c-5871-ea8c-a3ab-1e92704e9a5a.dc1 area=wan +34:11.882 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25276 network=tcp +34:11.882 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25276 network=udp +34:11.883 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25277 network=tcp +34:11.883 [INFO] TestAgent: started state syncer +34:11.883 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25282 network=tcp +34:11.932 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:11.941 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:11.941 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25281 [Candidate]" term=2 +34:11.941 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:11.941 [DEBUG] TestAgent.server.raft: vote granted: from=39be513c-5871-ea8c-a3ab-1e92704e9a5a term=2 tally=1 +34:11.941 [INFO] TestAgent.server.raft: election won: tally=1 +34:11.941 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25281 [Leader]" +34:11.942 [INFO] TestAgent.server: cluster leadership acquired +34:11.942 [INFO] TestAgent.server: initializing acls +34:11.942 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:11.942 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:11.943 [INFO] TestAgent.server: New leader elected: payload=Node-39be513c-5871-ea8c-a3ab-1e92704e9a5a +34:11.943 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:11.944 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:11.944 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:11.944 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:11.944 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-39be513c-5871-ea8c-a3ab-1e92704e9a5a +34:11.944 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-39be513c-5871-ea8c-a3ab-1e92704e9a5a.dc1 +34:11.944 [INFO] TestAgent.server: Updating LAN server: server="Node-39be513c-5871-ea8c-a3ab-1e92704e9a5a (Addr: tcp/127.0.0.1:25281) (DC: dc1)" +34:11.944 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-39be513c-5871-ea8c-a3ab-1e92704e9a5a.dc1 area=wan +34:11.945 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:11.948 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:11.948 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:11.948 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:11.948 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:11.948 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:11.948 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-39be513c-5871-ea8c-a3ab-1e92704e9a5a +34:11.948 [INFO] TestAgent.server: member joined, marking health alive: member=Node-39be513c-5871-ea8c-a3ab-1e92704e9a5a +34:11.949 [INFO] TestAgent.server: federation state anti-entropy synced +34:11.949 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-39be513c-5871-ea8c-a3ab-1e92704e9a5a +34:12.067 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth +34:12.067 [WARN] TestAgent: Node info update blocked by ACLs: node=39be513c-5871-ea8c-a3ab-1e92704e9a5a accessorID=00000000-0000-0000-0000-000000000002 +34:12.103 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-39be513c-5871-ea8c-a3ab-1e92704e9a5a +34:12.103 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-39be513c-5871-ea8c-a3ab-1e92704e9a5a +34:12.104 [ERROR] TestAgent: failed to ack remote exec job: error="Permission denied" +34:12.104 [ERROR] TestAgent: failed to write output for remote exec job: error="Permission denied" +34:12.104 [ERROR] TestAgent: failed to write output for remote exec job: error="Permission denied" +34:12.105 [INFO] TestAgent: Requesting shutdown +34:12.105 [INFO] TestAgent.server: shutting down server +34:12.105 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:12.105 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:12.105 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:12.105 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:12.105 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:12.105 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:12.105 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:12.105 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:12.105 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:12.105 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:12.105 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:12.105 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:12.105 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:12.105 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:12.105 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:12.105Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:12.105Z [INFO] agent.router.manager: shutting down +34:12.105 [INFO] TestAgent: consul server down +34:12.105 [INFO] TestAgent: shutdown complete +34:12.105 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25276 network=tcp +34:12.106 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25276 network=udp +34:12.106 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25277 network=tcp +34:12.610 [INFO] TestAgent: Waiting for endpoints to shut down +34:12.610 [INFO] TestAgent: Endpoints down +--- PASS: TestRemoteExecWrites_ACLDeny (0.77s) +=== CONT TestRemoteExecWrites +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:12.105Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:12.120Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:12.123 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d87e2563-73a0-845d-8ee3-68b005af5f98 Address:127.0.0.1:25288}]" +34:12.123 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25288 [Follower]" leader= +34:12.124 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d87e2563-73a0-845d-8ee3-68b005af5f98.dc1 127.0.0.1 +34:12.125 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d87e2563-73a0-845d-8ee3-68b005af5f98 127.0.0.1 +2021-08-13T15:34:12.125Z [INFO] agent.router: Initializing LAN area manager +34:12.125 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d87e2563-73a0-845d-8ee3-68b005af5f98.dc1 area=wan +34:12.125 [INFO] TestAgent.server: Adding LAN server: server="Node-d87e2563-73a0-845d-8ee3-68b005af5f98 (Addr: tcp/127.0.0.1:25288) (DC: dc1)" +34:12.125 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25283 network=udp +34:12.125 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25283 network=tcp +34:12.126 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25284 network=tcp +34:12.126 [INFO] TestAgent: started state syncer +34:12.126 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25289 network=tcp +34:12.169 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:12.169 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25288 [Candidate]" term=2 +34:12.169 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:12.169 [DEBUG] TestAgent.server.raft: vote granted: from=d87e2563-73a0-845d-8ee3-68b005af5f98 term=2 tally=1 +34:12.169 [INFO] TestAgent.server.raft: election won: tally=1 +34:12.169 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25288 [Leader]" +34:12.169 [INFO] TestAgent.server: cluster leadership acquired +34:12.170 [INFO] TestAgent.server: New leader elected: payload=Node-d87e2563-73a0-845d-8ee3-68b005af5f98 +34:12.170 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25288 +34:12.171 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:12.173 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:12.173 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:12.173 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:12.173 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:12.173 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:12.173 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d87e2563-73a0-845d-8ee3-68b005af5f98 +34:12.173 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d87e2563-73a0-845d-8ee3-68b005af5f98 +34:12.174 [INFO] TestAgent.server: federation state anti-entropy synced +34:12.214 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:12.215 [INFO] TestAgent: Synced node info +34:12.215 [DEBUG] TestAgent: Node info in sync +34:12.332 [INFO] TestAgent: Requesting shutdown +34:12.332 [INFO] TestAgent.server: shutting down server +34:12.332 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:12.332 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:12.332 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:12.332 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:12.332 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:12.333 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:12.333 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:12.333 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:12.333 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:12.333 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:12.333 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:12.334Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:12.334Z [INFO] agent.router.manager: shutting down +34:12.334 [INFO] TestAgent: consul server down +34:12.334 [INFO] TestAgent: shutdown complete +34:12.334 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25283 network=tcp +34:12.334 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25283 network=udp +34:12.334 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25284 network=tcp +34:12.834 [INFO] TestAgent: Waiting for endpoints to shut down +34:12.834 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentConnectAuthorize_noTarget (0.75s) === CONT TestAgentConnectCALeafCert_aclServiceReadDeny -2022-09-15T21:42:52.190Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:52.207Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:52.210 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6f9ac8cf-868d-8889-5d31-af422bdd02ab Address:127.0.0.1:14042}]" -42:52.211 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6f9ac8cf-868d-8889-5d31-af422bdd02ab.dc1 127.0.0.1 -42:52.211 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14042 [Follower]" leader= -42:52.212 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6f9ac8cf-868d-8889-5d31-af422bdd02ab 127.0.0.1 -2022-09-15T21:42:52.212Z [INFO] agent.router: Initializing LAN area manager -42:52.213 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14058 network=udp -42:52.213 [INFO] TestAgent.server: Adding LAN server: server="Node-6f9ac8cf-868d-8889-5d31-af422bdd02ab (Addr: tcp/127.0.0.1:14042) (DC: dc1)" -42:52.213 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6f9ac8cf-868d-8889-5d31-af422bdd02ab.dc1 area=wan -42:52.213 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14058 network=tcp -42:52.216 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14059 network=tcp -42:52.217 [INFO] TestAgent: started state syncer -42:52.217 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14043 network=tcp -42:52.281 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:52.281 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14042 [Candidate]" term=2 -42:52.281 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:52.281 [DEBUG] TestAgent.server.raft: vote granted: from=6f9ac8cf-868d-8889-5d31-af422bdd02ab term=2 tally=1 -42:52.281 [INFO] TestAgent.server.raft: election won: tally=1 -42:52.281 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14042 [Leader]" -42:52.281 [INFO] TestAgent.server: cluster leadership acquired -42:52.282 [INFO] TestAgent.server: New leader elected: payload=Node-6f9ac8cf-868d-8889-5d31-af422bdd02ab -42:52.282 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14042 -42:52.283 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:52.287 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:52.287 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:52.287 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:52.287 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:52.287 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:52.287 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6f9ac8cf-868d-8889-5d31-af422bdd02ab -42:52.287 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6f9ac8cf-868d-8889-5d31-af422bdd02ab -42:52.288 [INFO] TestAgent.server: federation state anti-entropy synced -42:52.462 [INFO] TestAgent: Requesting shutdown -42:52.463 [INFO] TestAgent.server: shutting down server -42:52.463 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:52.463 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:52.463 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:52.463 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:52.463 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:52.463 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:52.463 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:52.463 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:52.463 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:52.463 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:52.463 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:52.463 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:52.463Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:52.463Z [INFO] agent.router.manager: shutting down -42:52.463 [INFO] TestAgent: consul server down -42:52.463 [INFO] TestAgent: shutdown complete -42:52.463 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14058 network=tcp -42:52.463 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14058 network=udp -42:52.463 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14059 network=tcp -42:52.966 [INFO] TestAgent: Waiting for endpoints to shut down -42:52.966 [INFO] TestAgent: Endpoints down ---- PASS: TestRemoteExecGetSpec (0.80s) +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:12.388Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:12.388Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:12.405Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:12.405Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:12.407 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:73d40216-6e4b-b6e8-8924-46514fd5d58e Address:127.0.0.1:25302}]" +34:12.408 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-73d40216-6e4b-b6e8-8924-46514fd5d58e.dc1 127.0.0.1 +34:12.409 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-73d40216-6e4b-b6e8-8924-46514fd5d58e 127.0.0.1 +2021-08-13T15:34:12.409Z [INFO] agent.router: Initializing LAN area manager +34:12.409 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25302 [Follower]" leader= +34:12.409 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25297 network=udp +34:12.410 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-73d40216-6e4b-b6e8-8924-46514fd5d58e.dc1 area=wan +34:12.410 [INFO] TestAgent.server: Adding LAN server: server="Node-73d40216-6e4b-b6e8-8924-46514fd5d58e (Addr: tcp/127.0.0.1:25302) (DC: dc1)" +34:12.410 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25297 network=tcp +34:12.411 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25298 network=tcp +34:12.411 [INFO] TestAgent: started state syncer +34:12.411 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25303 network=tcp +34:12.460 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:12.475 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:12.475 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25302 [Candidate]" term=2 +34:12.475 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:12.475 [DEBUG] TestAgent.server.raft: vote granted: from=73d40216-6e4b-b6e8-8924-46514fd5d58e term=2 tally=1 +34:12.475 [INFO] TestAgent.server.raft: election won: tally=1 +34:12.475 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25302 [Leader]" +34:12.475 [INFO] TestAgent.server: cluster leadership acquired +34:12.476 [INFO] TestAgent.server: initializing acls +34:12.476 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:12.476 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:12.477 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:12.477 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:12.477 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:12.477 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:12.477 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-73d40216-6e4b-b6e8-8924-46514fd5d58e +34:12.477 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-73d40216-6e4b-b6e8-8924-46514fd5d58e.dc1 +34:12.478 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:12.479 [INFO] TestAgent.server: New leader elected: payload=Node-73d40216-6e4b-b6e8-8924-46514fd5d58e +34:12.479 [INFO] TestAgent.server: Updating LAN server: server="Node-73d40216-6e4b-b6e8-8924-46514fd5d58e (Addr: tcp/127.0.0.1:25302) (DC: dc1)" +34:12.479 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-73d40216-6e4b-b6e8-8924-46514fd5d58e.dc1 area=wan +34:12.485 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:12.485 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:12.485 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:12.485 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:12.485 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:12.485 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-73d40216-6e4b-b6e8-8924-46514fd5d58e +34:12.485 [INFO] TestAgent.server: member joined, marking health alive: member=Node-73d40216-6e4b-b6e8-8924-46514fd5d58e +34:12.486 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-73d40216-6e4b-b6e8-8924-46514fd5d58e +34:12.486 [INFO] TestAgent.server: federation state anti-entropy synced +34:12.554 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-73d40216-6e4b-b6e8-8924-46514fd5d58e +34:12.554 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-73d40216-6e4b-b6e8-8924-46514fd5d58e +34:12.558 [INFO] TestAgent: Requesting shutdown +34:12.558 [INFO] TestAgent.server: shutting down server +34:12.558 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:12.558 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:12.558 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:12.558 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:12.558 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:12.558 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:12.558 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:12.558 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:12.558 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:12.558 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:12.558 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:12.558 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:12.558 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:12.558 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:12.558 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:12.558 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:12.558Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:12.558Z [INFO] agent.router.manager: shutting down +34:12.558 [INFO] TestAgent: consul server down +34:12.558 [INFO] TestAgent: shutdown complete +34:12.558 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25297 network=tcp +34:12.559 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25297 network=udp +34:12.559 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25298 network=tcp +34:13.059 [INFO] TestAgent: Waiting for endpoints to shut down +34:13.059 [INFO] TestAgent: Endpoints down +--- PASS: TestRemoteExecWrites_ACLToken (0.69s) +=== CONT TestRemoteExecGetSpec_ACLDeny +2021-08-13T15:34:12.304Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:12.304Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:12.367Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:12.367Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:12.370 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5120c4d2-fa7b-7e02-a81e-0ca796032d1f Address:127.0.0.1:25295}]" +34:12.370 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25295 [Follower]" leader= +34:12.371 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5120c4d2-fa7b-7e02-a81e-0ca796032d1f.dc1 127.0.0.1 +34:12.372 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5120c4d2-fa7b-7e02-a81e-0ca796032d1f 127.0.0.1 +2021-08-13T15:34:12.372Z [INFO] agent.router: Initializing LAN area manager +34:12.372 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25290 network=udp +34:12.372 [INFO] TestAgent.server: Adding LAN server: server="Node-5120c4d2-fa7b-7e02-a81e-0ca796032d1f (Addr: tcp/127.0.0.1:25295) (DC: dc1)" +34:12.372 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5120c4d2-fa7b-7e02-a81e-0ca796032d1f.dc1 area=wan +34:12.372 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25290 network=tcp +34:12.373 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25291 network=tcp +34:12.373 [INFO] TestAgent: started state syncer +34:12.373 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25296 network=tcp +34:12.406 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:12.406 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25295 [Candidate]" term=2 +34:12.406 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:12.407 [DEBUG] TestAgent.server.raft: vote granted: from=5120c4d2-fa7b-7e02-a81e-0ca796032d1f term=2 tally=1 +34:12.407 [INFO] TestAgent.server.raft: election won: tally=1 +34:12.407 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25295 [Leader]" +34:12.407 [INFO] TestAgent.server: cluster leadership acquired +34:12.407 [INFO] TestAgent.server: initializing acls +34:12.407 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:12.407 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:12.408 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:12.408 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:12.408 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:12.409 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:12.409 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-5120c4d2-fa7b-7e02-a81e-0ca796032d1f +34:12.409 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-5120c4d2-fa7b-7e02-a81e-0ca796032d1f.dc1 +34:12.410 [INFO] TestAgent.server: New leader elected: payload=Node-5120c4d2-fa7b-7e02-a81e-0ca796032d1f +34:12.410 [INFO] TestAgent.server: Updating LAN server: server="Node-5120c4d2-fa7b-7e02-a81e-0ca796032d1f (Addr: tcp/127.0.0.1:25295) (DC: dc1)" +34:12.410 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-5120c4d2-fa7b-7e02-a81e-0ca796032d1f.dc1 area=wan +34:12.411 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:12.413 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:12.413 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:12.413 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:12.413 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:12.413 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:12.413 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5120c4d2-fa7b-7e02-a81e-0ca796032d1f +34:12.413 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5120c4d2-fa7b-7e02-a81e-0ca796032d1f +34:12.415 [INFO] TestAgent.server: federation state anti-entropy synced +34:12.415 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5120c4d2-fa7b-7e02-a81e-0ca796032d1f +34:12.600 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-5120c4d2-fa7b-7e02-a81e-0ca796032d1f +34:12.600 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-5120c4d2-fa7b-7e02-a81e-0ca796032d1f +34:12.605 [INFO] TestAgent: Requesting shutdown +34:12.605 [INFO] TestAgent.server: shutting down server +34:12.605 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:12.605 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:12.605 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:12.605 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:12.605 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:12.605 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:12.605 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:12.605 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:12.605 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:12.605 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:12.605 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:12.605 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:12.605 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:12.605 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:12.605 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:12.605 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:12.605Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:12.605Z [INFO] agent.router.manager: shutting down +34:12.606 [INFO] TestAgent: consul server down +34:12.606 [INFO] TestAgent: shutdown complete +34:12.606 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25290 network=tcp +34:12.606 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25290 network=udp +34:12.606 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25291 network=tcp +34:13.106 [INFO] TestAgent: Waiting for endpoints to shut down +34:13.106 [INFO] TestAgent: Endpoints down +--- PASS: TestRemoteExecWrites_ACLAgentToken (0.82s) === CONT TestAgentConnectCALeafCert_aclServiceWrite -2022-09-15T21:42:52.053Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:52.053Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:52.087Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:52.087Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:52.090 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bca6e5c9-42fb-57a1-d6c9-519df6cd8203 Address:127.0.0.1:13895}]" -42:52.090 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13895 [Follower]" leader= -42:52.091 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bca6e5c9-42fb-57a1-d6c9-519df6cd8203.dc1 127.0.0.1 -42:52.092 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bca6e5c9-42fb-57a1-d6c9-519df6cd8203 127.0.0.1 -2022-09-15T21:42:52.092Z [INFO] agent.router: Initializing LAN area manager -42:52.092 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13953 network=udp -42:52.092 [INFO] TestAgent.server: Adding LAN server: server="Node-bca6e5c9-42fb-57a1-d6c9-519df6cd8203 (Addr: tcp/127.0.0.1:13895) (DC: dc1)" -42:52.092 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bca6e5c9-42fb-57a1-d6c9-519df6cd8203.dc1 area=wan -42:52.093 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13953 network=tcp -42:52.093 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13954 network=tcp -42:52.094 [INFO] TestAgent: started state syncer -42:52.094 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13896 network=tcp -42:52.140 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:52.140 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13895 [Candidate]" term=2 -42:52.140 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:52.141 [DEBUG] TestAgent.server.raft: vote granted: from=bca6e5c9-42fb-57a1-d6c9-519df6cd8203 term=2 tally=1 -42:52.141 [INFO] TestAgent.server.raft: election won: tally=1 -42:52.141 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13895 [Leader]" -42:52.141 [INFO] TestAgent.server: cluster leadership acquired -42:52.141 [INFO] TestAgent.server: initializing acls -42:52.141 [INFO] TestAgent.server: New leader elected: payload=Node-bca6e5c9-42fb-57a1-d6c9-519df6cd8203 -42:52.141 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:52.141 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:52.142 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:52.142 [INFO] TestAgent.server: initializing acls -42:52.142 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:52.143 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:52.143 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:52.143 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:52.143 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode -42:52.143 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-bca6e5c9-42fb-57a1-d6c9-519df6cd8203 -42:52.144 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-bca6e5c9-42fb-57a1-d6c9-519df6cd8203.dc1 -42:52.144 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-bca6e5c9-42fb-57a1-d6c9-519df6cd8203.dc1 area=wan -42:52.144 [INFO] TestAgent.server: Updating LAN server: server="Node-bca6e5c9-42fb-57a1-d6c9-519df6cd8203 (Addr: tcp/127.0.0.1:13895) (DC: dc1)" -42:52.144 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:52.144 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-bca6e5c9-42fb-57a1-d6c9-519df6cd8203 -42:52.144 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-bca6e5c9-42fb-57a1-d6c9-519df6cd8203.dc1 -42:52.145 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:52.146 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-bca6e5c9-42fb-57a1-d6c9-519df6cd8203.dc1 area=wan -42:52.146 [INFO] TestAgent.server: Updating LAN server: server="Node-bca6e5c9-42fb-57a1-d6c9-519df6cd8203 (Addr: tcp/127.0.0.1:13895) (DC: dc1)" -42:52.148 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:52.148 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:52.148 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:52.148 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:52.148 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:52.148 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bca6e5c9-42fb-57a1-d6c9-519df6cd8203 -42:52.149 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bca6e5c9-42fb-57a1-d6c9-519df6cd8203 -42:52.150 [INFO] TestAgent.server: federation state anti-entropy synced -42:52.150 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bca6e5c9-42fb-57a1-d6c9-519df6cd8203 -42:52.150 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bca6e5c9-42fb-57a1-d6c9-519df6cd8203 -42:52.343 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:52.344 [INFO] TestAgent: Synced node info -42:52.344 [DEBUG] TestAgent: Node info in sync -42:52.480 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-bca6e5c9-42fb-57a1-d6c9-519df6cd8203 -42:52.481 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-bca6e5c9-42fb-57a1-d6c9-519df6cd8203 -42:52.482 [INFO] TestAgent: Requesting shutdown -42:52.483 [INFO] TestAgent.server: shutting down server -42:52.483 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:52.483 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:52.483 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:52.483 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:52.483 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:52.483 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:52.483 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:52.483 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:52.483 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:52.483 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:52.483 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:52.483 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:52.483 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:52.483 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:52.483 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:52.483Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:52.483Z [INFO] agent.router.manager: shutting down -42:52.484 [INFO] TestAgent: consul server down -42:52.484 [INFO] TestAgent: shutdown complete -42:52.484 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13953 network=tcp -42:52.484 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13953 network=udp -42:52.484 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13954 network=tcp -42:52.984 [INFO] TestAgent: Waiting for endpoints to shut down -42:52.984 [INFO] TestAgent: Endpoints down ---- PASS: TestRemoteExecGetSpec_ACLAgentToken (0.95s) +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:12.393Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:12.409Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:12.411 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:458bddce-3c84-5262-0ab0-453b9fa8d738 Address:127.0.0.1:25309}]" +34:12.411 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25309 [Follower]" leader= +34:12.412 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-458bddce-3c84-5262-0ab0-453b9fa8d738.dc1 127.0.0.1 +34:12.413 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-458bddce-3c84-5262-0ab0-453b9fa8d738 127.0.0.1 +2021-08-13T15:34:12.413Z [INFO] agent.router: Initializing LAN area manager +34:12.413 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25304 network=udp +34:12.413 [INFO] TestAgent.server: Adding LAN server: server="Node-458bddce-3c84-5262-0ab0-453b9fa8d738 (Addr: tcp/127.0.0.1:25309) (DC: dc1)" +34:12.413 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-458bddce-3c84-5262-0ab0-453b9fa8d738.dc1 area=wan +34:12.413 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25304 network=tcp +34:12.414 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25305 network=tcp +34:12.414 [INFO] TestAgent: started state syncer +34:12.415 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25310 network=tcp +34:12.452 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:12.452 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25309 [Candidate]" term=2 +34:12.452 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:12.452 [DEBUG] TestAgent.server.raft: vote granted: from=458bddce-3c84-5262-0ab0-453b9fa8d738 term=2 tally=1 +34:12.452 [INFO] TestAgent.server.raft: election won: tally=1 +34:12.452 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25309 [Leader]" +34:12.453 [INFO] TestAgent.server: cluster leadership acquired +34:12.453 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25309 +34:12.454 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:12.455 [INFO] TestAgent.server: New leader elected: payload=Node-458bddce-3c84-5262-0ab0-453b9fa8d738 +34:12.457 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:12.457 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:12.457 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:12.457 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:12.457 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:12.457 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-458bddce-3c84-5262-0ab0-453b9fa8d738 +34:12.457 [INFO] TestAgent.server: member joined, marking health alive: member=Node-458bddce-3c84-5262-0ab0-453b9fa8d738 +34:12.458 [INFO] TestAgent.server: federation state anti-entropy synced +34:12.604 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:12.604 [INFO] TestAgent: Synced node info +34:12.604 [DEBUG] TestAgent: Node info in sync +34:12.816 [INFO] TestAgent: Requesting shutdown +34:12.816 [INFO] TestAgent.server: shutting down server +34:12.816 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:12.816 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:12.816 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:12.816 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:12.816 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:12.817 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:12.817 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:12.817 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:12.817 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:12.817 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:12.817 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:12.817Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:12.817Z [INFO] agent.router.manager: shutting down +34:12.817 [INFO] TestAgent: consul server down +34:12.817 [INFO] TestAgent: shutdown complete +34:12.817 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25304 network=tcp +34:12.817 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25304 network=udp +34:12.817 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25305 network=tcp +34:13.317 [INFO] TestAgent: Waiting for endpoints to shut down +34:13.317 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentConnectAuthorize_badBody (0.94s) +=== CONT TestRemoteExecGetSpec_ACLAgentToken +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:12.626Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:12.642Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:12.644 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f1c19c41-90db-8454-2577-690effe7bd2c Address:127.0.0.1:24303}]" +34:12.645 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24303 [Follower]" leader= +34:12.645 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f1c19c41-90db-8454-2577-690effe7bd2c.dc1 127.0.0.1 +34:12.646 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f1c19c41-90db-8454-2577-690effe7bd2c 127.0.0.1 +2021-08-13T15:34:12.646Z [INFO] agent.router: Initializing LAN area manager +34:12.647 [INFO] TestAgent.server: Adding LAN server: server="Node-f1c19c41-90db-8454-2577-690effe7bd2c (Addr: tcp/127.0.0.1:24303) (DC: dc1)" +34:12.647 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f1c19c41-90db-8454-2577-690effe7bd2c.dc1 area=wan +34:12.647 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24298 network=tcp +34:12.647 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24298 network=udp +34:12.648 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24299 network=tcp +34:12.648 [INFO] TestAgent: started state syncer +34:12.648 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24304 network=tcp +34:12.700 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:12.700 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24303 [Candidate]" term=2 +34:12.701 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:12.701 [DEBUG] TestAgent.server.raft: vote granted: from=f1c19c41-90db-8454-2577-690effe7bd2c term=2 tally=1 +34:12.701 [INFO] TestAgent.server.raft: election won: tally=1 +34:12.701 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24303 [Leader]" +34:12.701 [INFO] TestAgent.server: cluster leadership acquired +34:12.701 [INFO] TestAgent.server: New leader elected: payload=Node-f1c19c41-90db-8454-2577-690effe7bd2c +34:12.701 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24303 +34:12.702 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:12.705 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:12.705 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:12.705 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:12.705 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:12.705 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:12.705 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f1c19c41-90db-8454-2577-690effe7bd2c +34:12.705 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f1c19c41-90db-8454-2577-690effe7bd2c +34:12.706 [INFO] TestAgent.server: federation state anti-entropy synced +34:13.045 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:13.045 [INFO] TestAgent: Synced node info +34:13.050 [INFO] TestAgent: Requesting shutdown +34:13.051 [INFO] TestAgent.server: shutting down server +34:13.051 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:13.051 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:13.051 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:13.051 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:13.051 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:13.051 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:13.051 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:13.051 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:13.051 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:13.051 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:13.051 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:13.051Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:13.051Z [INFO] agent.router.manager: shutting down +34:13.051 [INFO] TestAgent: consul server down +34:13.051 [INFO] TestAgent: shutdown complete +34:13.051 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24298 network=tcp +34:13.051 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24298 network=udp +34:13.051 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24299 network=tcp +34:13.551 [INFO] TestAgent: Waiting for endpoints to shut down +34:13.551 [INFO] TestAgent: Endpoints down +--- PASS: TestRemoteExecWrites (0.94s) === CONT TestAgentConnectCALeafCert_aclDefaultDeny -[WARN] freeport: 4 out of 25 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:52.335Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:52.373Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:52.376 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4b4de75e-4fe1-cb83-3a52-835e6192a0d3 Address:127.0.0.1:14070}]" -42:52.376 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14070 [Follower]" leader= -42:52.377 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4b4de75e-4fe1-cb83-3a52-835e6192a0d3.dc1 127.0.0.1 -42:52.377 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4b4de75e-4fe1-cb83-3a52-835e6192a0d3 127.0.0.1 -2022-09-15T21:42:52.377Z [INFO] agent.router: Initializing LAN area manager -42:52.378 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14044 network=udp -42:52.378 [INFO] TestAgent.server: Adding LAN server: server="Node-4b4de75e-4fe1-cb83-3a52-835e6192a0d3 (Addr: tcp/127.0.0.1:14070) (DC: dc1)" -42:52.378 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4b4de75e-4fe1-cb83-3a52-835e6192a0d3.dc1 area=wan -42:52.378 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14044 network=tcp -42:52.379 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14045 network=tcp -42:52.379 [INFO] TestAgent: started state syncer -42:52.379 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14071 network=tcp -42:52.443 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:52.443 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14070 [Candidate]" term=2 -42:52.444 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:52.444 [DEBUG] TestAgent.server.raft: vote granted: from=4b4de75e-4fe1-cb83-3a52-835e6192a0d3 term=2 tally=1 -42:52.444 [INFO] TestAgent.server.raft: election won: tally=1 -42:52.444 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14070 [Leader]" -42:52.444 [INFO] TestAgent.server: cluster leadership acquired -42:52.444 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14070 -42:52.445 [INFO] TestAgent.server: New leader elected: payload=Node-4b4de75e-4fe1-cb83-3a52-835e6192a0d3 -42:52.446 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:52.448 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:52.448 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:52.448 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:52.448 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:52.448 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:52.448 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4b4de75e-4fe1-cb83-3a52-835e6192a0d3 -42:52.448 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4b4de75e-4fe1-cb83-3a52-835e6192a0d3 -42:52.449 [INFO] TestAgent.server: federation state anti-entropy synced -42:52.570 [INFO] TestAgent: Requesting shutdown -42:52.570 [INFO] TestAgent.server: shutting down server -42:52.570 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:52.570 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:52.570 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:52.570 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:52.570 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:52.570 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:52.570 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:52.570 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:52.570 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:52.570 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:52.570 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:52.570 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:52.570Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:52.571Z [INFO] agent.router.manager: shutting down -42:52.571 [INFO] TestAgent: consul server down -42:52.571 [INFO] TestAgent: shutdown complete -42:52.571 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14044 network=tcp -42:52.571 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14044 network=udp -42:52.571 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14045 network=tcp -42:53.071 [INFO] TestAgent: Waiting for endpoints to shut down -42:53.084 [INFO] TestAgent: Endpoints down ---- PASS: TestPreparedQuery_Integration (0.80s) +2021-08-13T15:34:12.851Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:12.851Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:12.867Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:12.867Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:12.870 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ecb034b9-e2fd-24f5-7900-933043b93263 Address:127.0.0.1:24310}]" +34:12.870 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24310 [Follower]" leader= +34:12.871 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ecb034b9-e2fd-24f5-7900-933043b93263.dc1 127.0.0.1 +34:12.871 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ecb034b9-e2fd-24f5-7900-933043b93263 127.0.0.1 +2021-08-13T15:34:12.871Z [INFO] agent.router: Initializing LAN area manager +34:12.872 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24305 network=udp +34:12.872 [INFO] TestAgent.server: Adding LAN server: server="Node-ecb034b9-e2fd-24f5-7900-933043b93263 (Addr: tcp/127.0.0.1:24310) (DC: dc1)" +34:12.872 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ecb034b9-e2fd-24f5-7900-933043b93263.dc1 area=wan +34:12.872 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24305 network=tcp +34:12.873 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24306 network=tcp +34:12.873 [INFO] TestAgent: started state syncer +34:12.873 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24339 network=tcp +34:12.923 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:12.936 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:12.936 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24310 [Candidate]" term=2 +34:12.937 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:12.937 [DEBUG] TestAgent.server.raft: vote granted: from=ecb034b9-e2fd-24f5-7900-933043b93263 term=2 tally=1 +34:12.937 [INFO] TestAgent.server.raft: election won: tally=1 +34:12.937 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24310 [Leader]" +34:12.937 [INFO] TestAgent.server: cluster leadership acquired +34:12.937 [INFO] TestAgent.server: initializing acls +34:12.938 [INFO] TestAgent.server: New leader elected: payload=Node-ecb034b9-e2fd-24f5-7900-933043b93263 +34:12.938 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:12.938 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:12.939 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:12.949 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:12.950 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:12.950 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:12.950 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-ecb034b9-e2fd-24f5-7900-933043b93263 +34:12.950 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-ecb034b9-e2fd-24f5-7900-933043b93263.dc1 +34:12.951 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-ecb034b9-e2fd-24f5-7900-933043b93263.dc1 area=wan +34:12.951 [INFO] TestAgent.server: Updating LAN server: server="Node-ecb034b9-e2fd-24f5-7900-933043b93263 (Addr: tcp/127.0.0.1:24310) (DC: dc1)" +34:12.952 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:12.955 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:12.955 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:12.955 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:12.955 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:12.955 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:12.956 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ecb034b9-e2fd-24f5-7900-933043b93263 +34:12.956 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ecb034b9-e2fd-24f5-7900-933043b93263 +34:12.957 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ecb034b9-e2fd-24f5-7900-933043b93263 +34:12.957 [INFO] TestAgent.server: federation state anti-entropy synced +34:13.155 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-ecb034b9-e2fd-24f5-7900-933043b93263 +34:13.155 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-ecb034b9-e2fd-24f5-7900-933043b93263 +34:13.158 [INFO] TestAgent: Synced node info +34:13.159 [INFO] TestAgent: Synced service: service=test-id +34:13.159 [DEBUG] TestAgent: Check in sync: check=service:test-id +34:13.161 [INFO] TestAgent: Requesting shutdown +34:13.161 [INFO] TestAgent.server: shutting down server +34:13.161 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:13.161 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:13.161 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:13.161 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:13.161 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:13.161 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:13.161 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:13.161 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:13.161 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:13.161 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:13.161 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:13.161 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:13.161 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:13.161 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:13.161 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:13.161 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:13.161Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:13.161Z [INFO] agent.router.manager: shutting down +34:13.161 [INFO] TestAgent: consul server down +34:13.161 [INFO] TestAgent: shutdown complete +34:13.161 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24305 network=tcp +34:13.161 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24305 network=udp +34:13.161 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24306 network=tcp +34:13.662 [INFO] TestAgent: Waiting for endpoints to shut down +34:13.662 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentConnectCALeafCert_aclServiceReadDeny (0.83s) +=== CONT TestRemoteExecGetSpec_ACLToken +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:13.123Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:13.123Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:13.139Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:13.139Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:13.142 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a16fe06f-4110-6e4b-b886-e37c0b4e18f7 Address:127.0.0.1:24366}]" +34:13.142 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24366 [Follower]" leader= +34:13.143 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a16fe06f-4110-6e4b-b886-e37c0b4e18f7.dc1 127.0.0.1 +34:13.144 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a16fe06f-4110-6e4b-b886-e37c0b4e18f7 127.0.0.1 +2021-08-13T15:34:13.144Z [INFO] agent.router: Initializing LAN area manager +34:13.144 [INFO] TestAgent.server: Adding LAN server: server="Node-a16fe06f-4110-6e4b-b886-e37c0b4e18f7 (Addr: tcp/127.0.0.1:24366) (DC: dc1)" +34:13.144 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a16fe06f-4110-6e4b-b886-e37c0b4e18f7.dc1 area=wan +34:13.144 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24347 network=udp +34:13.144 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24347 network=tcp +34:13.145 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24348 network=tcp +34:13.145 [INFO] TestAgent: started state syncer +34:13.145 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24353 network=tcp +34:13.194 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:13.207 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:13.207 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24366 [Candidate]" term=2 +34:13.207 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:13.207 [DEBUG] TestAgent.server.raft: vote granted: from=a16fe06f-4110-6e4b-b886-e37c0b4e18f7 term=2 tally=1 +34:13.207 [INFO] TestAgent.server.raft: election won: tally=1 +34:13.207 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24366 [Leader]" +34:13.207 [INFO] TestAgent.server: cluster leadership acquired +34:13.207 [INFO] TestAgent.server: initializing acls +34:13.208 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:13.208 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:13.208 [INFO] TestAgent.server: New leader elected: payload=Node-a16fe06f-4110-6e4b-b886-e37c0b4e18f7 +34:13.208 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:13.209 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:13.209 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:13.209 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:13.209 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-a16fe06f-4110-6e4b-b886-e37c0b4e18f7 +34:13.209 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-a16fe06f-4110-6e4b-b886-e37c0b4e18f7.dc1 +34:13.209 [INFO] TestAgent.server: Updating LAN server: server="Node-a16fe06f-4110-6e4b-b886-e37c0b4e18f7 (Addr: tcp/127.0.0.1:24366) (DC: dc1)" +34:13.209 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-a16fe06f-4110-6e4b-b886-e37c0b4e18f7.dc1 area=wan +34:13.210 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:13.212 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:13.212 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:13.212 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:13.212 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:13.212 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:13.213 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a16fe06f-4110-6e4b-b886-e37c0b4e18f7 +34:13.213 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a16fe06f-4110-6e4b-b886-e37c0b4e18f7 +34:13.214 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a16fe06f-4110-6e4b-b886-e37c0b4e18f7 +34:13.214 [INFO] TestAgent.server: federation state anti-entropy synced +34:13.289 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:13.290 [INFO] TestAgent: Synced node info +34:13.290 [DEBUG] TestAgent: Node info in sync +34:13.306 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-a16fe06f-4110-6e4b-b886-e37c0b4e18f7 +34:13.307 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-a16fe06f-4110-6e4b-b886-e37c0b4e18f7 +34:13.308 [DEBUG] TestAgent: Node info in sync +34:13.309 [INFO] TestAgent: Synced service: service=test-id +34:13.309 [DEBUG] TestAgent: Check in sync: check=service:test-id +34:13.313 [INFO] TestAgent: Requesting shutdown +34:13.313 [INFO] TestAgent.server: shutting down server +34:13.313 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:13.313 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:13.313 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:13.313 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:13.313 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:13.313 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:13.313 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:13.313 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:13.313 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:13.313 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:13.313 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:13.313 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:13.313 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:13.313 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:13.313 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:13.313Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:13.313Z [INFO] agent.router.manager: shutting down +34:13.314 [INFO] TestAgent: consul server down +34:13.314 [INFO] TestAgent: shutdown complete +34:13.314 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24347 network=tcp +34:13.314 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24347 network=udp +34:13.314 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24348 network=tcp +34:13.814 [INFO] TestAgent: Waiting for endpoints to shut down +34:13.814 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentConnectCALeafCert_aclServiceWrite (0.71s) === CONT TestAgentConnectCARoots_list -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 4 out of 4 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:52.921Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:52.921Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:52.945Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:52.945Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:52.956 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:25b1ebbb-7db1-d09e-cdb7-1b6dab1a9ca3 Address:127.0.0.1:13071}]" -42:52.957 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-25b1ebbb-7db1-d09e-cdb7-1b6dab1a9ca3.dc1 127.0.0.1 -42:52.958 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13071 [Follower]" leader= -42:52.958 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-25b1ebbb-7db1-d09e-cdb7-1b6dab1a9ca3 127.0.0.1 -2022-09-15T21:42:52.958Z [INFO] agent.router: Initializing LAN area manager -42:52.958 [INFO] TestAgent.server: Adding LAN server: server="Node-25b1ebbb-7db1-d09e-cdb7-1b6dab1a9ca3 (Addr: tcp/127.0.0.1:13071) (DC: dc1)" -42:52.958 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-25b1ebbb-7db1-d09e-cdb7-1b6dab1a9ca3.dc1 area=wan -42:52.958 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13066 network=tcp -42:52.958 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13066 network=udp -42:52.959 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13067 network=tcp -42:52.959 [INFO] TestAgent: started state syncer -42:52.960 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13072 network=tcp -42:53.022 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:53.022 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13071 [Candidate]" term=2 -42:53.023 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:53.023 [DEBUG] TestAgent.server.raft: vote granted: from=25b1ebbb-7db1-d09e-cdb7-1b6dab1a9ca3 term=2 tally=1 -42:53.023 [INFO] TestAgent.server.raft: election won: tally=1 -42:53.023 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13071 [Leader]" -42:53.023 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=2 mode=2 found=true leader=127.0.0.1:13071 -42:53.023 [INFO] TestAgent.server: cluster leadership acquired -42:53.023 [INFO] TestAgent.server: initializing acls -42:53.024 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:53.024 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:53.025 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:53.027 [INFO] TestAgent.server: New leader elected: payload=Node-25b1ebbb-7db1-d09e-cdb7-1b6dab1a9ca3 -42:53.027 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:53.027 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:53.027 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:53.027 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-25b1ebbb-7db1-d09e-cdb7-1b6dab1a9ca3 -42:53.027 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-25b1ebbb-7db1-d09e-cdb7-1b6dab1a9ca3.dc1 -42:53.029 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:53.032 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:53.032 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:53.032 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:53.032 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:53.032 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:53.032 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-25b1ebbb-7db1-d09e-cdb7-1b6dab1a9ca3 -42:53.032 [INFO] TestAgent.server: member joined, marking health alive: member=Node-25b1ebbb-7db1-d09e-cdb7-1b6dab1a9ca3 -42:53.034 [INFO] TestAgent.server: Updating LAN server: server="Node-25b1ebbb-7db1-d09e-cdb7-1b6dab1a9ca3 (Addr: tcp/127.0.0.1:13071) (DC: dc1)" -42:53.034 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-25b1ebbb-7db1-d09e-cdb7-1b6dab1a9ca3.dc1 area=wan -42:53.054 [INFO] TestAgent.server: federation state anti-entropy synced -42:53.054 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-25b1ebbb-7db1-d09e-cdb7-1b6dab1a9ca3 -42:53.124 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-25b1ebbb-7db1-d09e-cdb7-1b6dab1a9ca3 -42:53.125 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-25b1ebbb-7db1-d09e-cdb7-1b6dab1a9ca3 -42:53.128 [INFO] TestAgent: Synced node info -42:53.128 [INFO] TestAgent: Synced service: service=test-id -42:53.128 [DEBUG] TestAgent: Check in sync: check=service:test-id -42:53.134 [INFO] TestAgent: Requesting shutdown -42:53.135 [INFO] TestAgent.server: shutting down server -42:53.135 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:53.135 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:53.135 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:53.135 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:53.135 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:53.135 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:53.135 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:53.135 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:53.135 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:53.135 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:53.135 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:53.135 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:53.135 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:53.135 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -42:53.135 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:53.135 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2022-09-15T21:42:53.135Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:53.135Z [INFO] agent.router.manager: shutting down -42:53.135 [INFO] TestAgent: consul server down -42:53.135 [INFO] TestAgent: shutdown complete -42:53.135 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13066 network=tcp -42:53.135 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13066 network=udp -42:53.136 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13067 network=tcp -42:53.636 [INFO] TestAgent: Waiting for endpoints to shut down -42:53.636 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentConnectCALeafCert_aclServiceReadDeny (0.73s) +2021-08-13T15:34:13.078Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:13.078Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:13.095Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:13.095Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:13.098 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:db5d0a21-a896-ec6d-deba-1470650cc7f7 Address:127.0.0.1:24338}]" +34:13.098 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24338 [Follower]" leader= +34:13.099 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-db5d0a21-a896-ec6d-deba-1470650cc7f7.dc1 127.0.0.1 +34:13.100 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-db5d0a21-a896-ec6d-deba-1470650cc7f7 127.0.0.1 +2021-08-13T15:34:13.100Z [INFO] agent.router: Initializing LAN area manager +34:13.100 [INFO] TestAgent.server: Adding LAN server: server="Node-db5d0a21-a896-ec6d-deba-1470650cc7f7 (Addr: tcp/127.0.0.1:24338) (DC: dc1)" +34:13.100 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-db5d0a21-a896-ec6d-deba-1470650cc7f7.dc1 area=wan +34:13.100 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24333 network=tcp +34:13.100 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24333 network=udp +34:13.101 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24334 network=tcp +34:13.101 [INFO] TestAgent: started state syncer +34:13.101 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24346 network=tcp +34:13.150 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:13.150 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24338 [Candidate]" term=2 +34:13.150 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:13.150 [DEBUG] TestAgent.server.raft: vote granted: from=db5d0a21-a896-ec6d-deba-1470650cc7f7 term=2 tally=1 +34:13.150 [INFO] TestAgent.server.raft: election won: tally=1 +34:13.150 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24338 [Leader]" +34:13.150 [INFO] TestAgent.server: cluster leadership acquired +34:13.150 [INFO] TestAgent.server: initializing acls +34:13.150 [INFO] TestAgent.server: New leader elected: payload=Node-db5d0a21-a896-ec6d-deba-1470650cc7f7 +34:13.151 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:13.151 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:13.151 [INFO] TestAgent.server: initializing acls +34:13.151 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:13.151 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:13.152 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:13.152 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:13.152 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:13.152 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-db5d0a21-a896-ec6d-deba-1470650cc7f7 +34:13.152 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-db5d0a21-a896-ec6d-deba-1470650cc7f7.dc1 +34:13.153 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:13.154 [INFO] TestAgent.server: Updating LAN server: server="Node-db5d0a21-a896-ec6d-deba-1470650cc7f7 (Addr: tcp/127.0.0.1:24338) (DC: dc1)" +34:13.154 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:13.154 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode +34:13.154 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-db5d0a21-a896-ec6d-deba-1470650cc7f7 +34:13.154 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-db5d0a21-a896-ec6d-deba-1470650cc7f7.dc1 +34:13.154 [INFO] TestAgent.server: Updating LAN server: server="Node-db5d0a21-a896-ec6d-deba-1470650cc7f7 (Addr: tcp/127.0.0.1:24338) (DC: dc1)" +34:13.155 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-db5d0a21-a896-ec6d-deba-1470650cc7f7.dc1 area=wan +34:13.155 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-db5d0a21-a896-ec6d-deba-1470650cc7f7.dc1 area=wan +34:13.156 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:13.156 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:13.156 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:13.156 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:13.156 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:13.156 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-db5d0a21-a896-ec6d-deba-1470650cc7f7 +34:13.156 [INFO] TestAgent.server: member joined, marking health alive: member=Node-db5d0a21-a896-ec6d-deba-1470650cc7f7 +34:13.157 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-db5d0a21-a896-ec6d-deba-1470650cc7f7 +34:13.157 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-db5d0a21-a896-ec6d-deba-1470650cc7f7 +34:13.157 [INFO] TestAgent.server: federation state anti-entropy synced +34:13.391 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-db5d0a21-a896-ec6d-deba-1470650cc7f7 +34:13.392 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-db5d0a21-a896-ec6d-deba-1470650cc7f7 +34:13.393 [ERROR] TestAgent: failed to get remote exec job: error="Permission denied" +34:13.394 [INFO] TestAgent: Requesting shutdown +34:13.394 [INFO] TestAgent.server: shutting down server +34:13.394 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:13.394 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:13.394 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:13.394 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:13.394 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:13.394 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:13.394 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:13.394 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:13.394 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:13.394 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:13.394 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:13.394 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +34:13.394 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:13.394 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:13.394 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:13.394 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +2021-08-13T15:34:13.394Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:13.394Z [INFO] agent.router.manager: shutting down +34:13.394 [INFO] TestAgent: consul server down +34:13.394 [INFO] TestAgent: shutdown complete +34:13.394 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24333 network=tcp +34:13.394 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24333 network=udp +34:13.394 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24334 network=tcp +34:13.894 [INFO] TestAgent: Waiting for endpoints to shut down +34:13.894 [INFO] TestAgent: Endpoints down +--- PASS: TestRemoteExecGetSpec_ACLDeny (0.84s) +=== CONT TestRemoteExecGetSpec +2021-08-13T15:34:13.334Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:13.334Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:13.350Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:13.350Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:13.353 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9b24efb9-1dab-45b5-8384-99c0697db88a Address:127.0.0.1:24345}]" +34:13.354 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24345 [Follower]" leader= +34:13.354 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9b24efb9-1dab-45b5-8384-99c0697db88a.dc1 127.0.0.1 +34:13.355 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9b24efb9-1dab-45b5-8384-99c0697db88a 127.0.0.1 +2021-08-13T15:34:13.355Z [INFO] agent.router: Initializing LAN area manager +34:13.355 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24354 network=udp +34:13.355 [INFO] TestAgent.server: Adding LAN server: server="Node-9b24efb9-1dab-45b5-8384-99c0697db88a (Addr: tcp/127.0.0.1:24345) (DC: dc1)" +34:13.355 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9b24efb9-1dab-45b5-8384-99c0697db88a.dc1 area=wan +34:13.356 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24354 network=tcp +34:13.356 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24355 network=tcp +34:13.356 [INFO] TestAgent: started state syncer +34:13.357 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24311 network=tcp +34:13.406 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:13.412 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:13.412 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24345 [Candidate]" term=2 +34:13.412 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:13.412 [DEBUG] TestAgent.server.raft: vote granted: from=9b24efb9-1dab-45b5-8384-99c0697db88a term=2 tally=1 +34:13.412 [INFO] TestAgent.server.raft: election won: tally=1 +34:13.412 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24345 [Leader]" +34:13.412 [INFO] TestAgent.server: cluster leadership acquired +34:13.412 [INFO] TestAgent.server: New leader elected: payload=Node-9b24efb9-1dab-45b5-8384-99c0697db88a +34:13.412 [INFO] TestAgent.server: initializing acls +34:13.413 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:13.413 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:13.413 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:13.414 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:13.414 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:13.414 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:13.414 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-9b24efb9-1dab-45b5-8384-99c0697db88a +34:13.414 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-9b24efb9-1dab-45b5-8384-99c0697db88a.dc1 +34:13.415 [INFO] TestAgent.server: Updating LAN server: server="Node-9b24efb9-1dab-45b5-8384-99c0697db88a (Addr: tcp/127.0.0.1:24345) (DC: dc1)" +34:13.415 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-9b24efb9-1dab-45b5-8384-99c0697db88a.dc1 area=wan +34:13.416 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:13.418 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:13.418 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:13.418 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:13.418 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:13.418 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:13.418 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9b24efb9-1dab-45b5-8384-99c0697db88a +34:13.418 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9b24efb9-1dab-45b5-8384-99c0697db88a +34:13.419 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9b24efb9-1dab-45b5-8384-99c0697db88a +34:13.419 [INFO] TestAgent.server: federation state anti-entropy synced +34:13.489 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-9b24efb9-1dab-45b5-8384-99c0697db88a +34:13.489 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-9b24efb9-1dab-45b5-8384-99c0697db88a +34:13.492 [INFO] TestAgent: Requesting shutdown +34:13.492 [INFO] TestAgent.server: shutting down server +34:13.492 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:13.492 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:13.492 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:13.492 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:13.492 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:13.492 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:13.492 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:13.492 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:13.492 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:13.492 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:13.492 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:13.492 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:13.492 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:13.492 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:13.492 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +34:13.492 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +2021-08-13T15:34:13.492Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:13.492Z [INFO] agent.router.manager: shutting down +34:13.492 [INFO] TestAgent: consul server down +34:13.492 [INFO] TestAgent: shutdown complete +34:13.492 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24354 network=tcp +34:13.492 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24354 network=udp +34:13.492 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24355 network=tcp +34:13.993 [INFO] TestAgent: Waiting for endpoints to shut down +34:13.993 [INFO] TestAgent: Endpoints down +--- PASS: TestRemoteExecGetSpec_ACLAgentToken (0.68s) === CONT TestAgentConnectCARoots_empty -2022-09-15T21:42:53.085Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:53.086Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:53.104Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:53.104Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:53.106 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:144839da-7cfb-7c35-7808-3caf38668e3d Address:127.0.0.1:13113}]" -42:53.107 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13113 [Follower]" leader= -42:53.107 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-144839da-7cfb-7c35-7808-3caf38668e3d.dc1 127.0.0.1 -42:53.108 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-144839da-7cfb-7c35-7808-3caf38668e3d 127.0.0.1 -2022-09-15T21:42:53.108Z [INFO] agent.router: Initializing LAN area manager -42:53.109 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13080 network=udp -42:53.109 [INFO] TestAgent.server: Adding LAN server: server="Node-144839da-7cfb-7c35-7808-3caf38668e3d (Addr: tcp/127.0.0.1:13113) (DC: dc1)" -42:53.109 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-144839da-7cfb-7c35-7808-3caf38668e3d.dc1 area=wan -42:53.109 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13080 network=tcp -42:53.110 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13081 network=tcp -42:53.110 [INFO] TestAgent: started state syncer -42:53.110 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13114 network=tcp -42:53.162 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -42:53.166 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:53.166 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13113 [Candidate]" term=2 -42:53.166 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:53.166 [DEBUG] TestAgent.server.raft: vote granted: from=144839da-7cfb-7c35-7808-3caf38668e3d term=2 tally=1 -42:53.166 [INFO] TestAgent.server.raft: election won: tally=1 -42:53.166 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13113 [Leader]" -42:53.166 [INFO] TestAgent.server: cluster leadership acquired -42:53.166 [INFO] TestAgent.server: New leader elected: payload=Node-144839da-7cfb-7c35-7808-3caf38668e3d -42:53.166 [INFO] TestAgent.server: initializing acls -42:53.167 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:53.167 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:53.167 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:53.168 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:53.168 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:53.168 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:53.168 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-144839da-7cfb-7c35-7808-3caf38668e3d -42:53.168 [INFO] TestAgent.server: Updating LAN server: server="Node-144839da-7cfb-7c35-7808-3caf38668e3d (Addr: tcp/127.0.0.1:13113) (DC: dc1)" -42:53.169 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-144839da-7cfb-7c35-7808-3caf38668e3d.dc1 -42:53.170 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-144839da-7cfb-7c35-7808-3caf38668e3d.dc1 area=wan -42:53.170 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:53.173 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:53.173 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:53.173 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:53.173 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:53.173 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:53.173 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-144839da-7cfb-7c35-7808-3caf38668e3d -42:53.173 [INFO] TestAgent.server: member joined, marking health alive: member=Node-144839da-7cfb-7c35-7808-3caf38668e3d -42:53.174 [INFO] TestAgent.server: federation state anti-entropy synced -42:53.175 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-144839da-7cfb-7c35-7808-3caf38668e3d -42:53.203 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:53.203 [INFO] TestAgent: Synced node info -42:53.255 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-144839da-7cfb-7c35-7808-3caf38668e3d -42:53.255 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-144839da-7cfb-7c35-7808-3caf38668e3d -42:53.256 [DEBUG] TestAgent: Node info in sync -42:53.257 [INFO] TestAgent: Synced service: service=test-id -42:53.257 [DEBUG] TestAgent: Check in sync: check=service:test-id -42:53.259 [INFO] TestAgent: Requesting shutdown -42:53.259 [INFO] TestAgent.server: shutting down server -42:53.259 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:53.259 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:53.259 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:53.259 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:53.259 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:53.259 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:53.259 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:53.259 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:53.259 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:53.259 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:53.259 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -42:53.259 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:53.259 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:53.259 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:53.259 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -2022-09-15T21:42:53.260Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:53.260Z [INFO] agent.router.manager: shutting down -42:53.260 [INFO] TestAgent: consul server down -42:53.260 [INFO] TestAgent: shutdown complete -42:53.260 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13080 network=tcp -42:53.260 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13080 network=udp -42:53.260 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13081 network=tcp -42:53.760 [INFO] TestAgent: Waiting for endpoints to shut down -42:53.761 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentConnectCALeafCert_aclDefaultDeny (0.78s) +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:13.569Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:13.569Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:13.586Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:13.586Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:13.589 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:54df12be-733e-07ab-d333-d0f8da2878ad Address:127.0.0.1:24317}]" +34:13.589 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24317 [Follower]" leader= +34:13.590 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-54df12be-733e-07ab-d333-d0f8da2878ad.dc1 127.0.0.1 +34:13.591 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-54df12be-733e-07ab-d333-d0f8da2878ad 127.0.0.1 +2021-08-13T15:34:13.591Z [INFO] agent.router: Initializing LAN area manager +34:13.592 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24312 network=udp +34:13.592 [INFO] TestAgent.server: Adding LAN server: server="Node-54df12be-733e-07ab-d333-d0f8da2878ad (Addr: tcp/127.0.0.1:24317) (DC: dc1)" +34:13.592 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-54df12be-733e-07ab-d333-d0f8da2878ad.dc1 area=wan +34:13.592 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24312 network=tcp +34:13.593 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24313 network=tcp +34:13.593 [INFO] TestAgent: started state syncer +34:13.593 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24318 network=tcp +34:13.634 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:13.634 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24317 [Candidate]" term=2 +34:13.634 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:13.634 [DEBUG] TestAgent.server.raft: vote granted: from=54df12be-733e-07ab-d333-d0f8da2878ad term=2 tally=1 +34:13.634 [INFO] TestAgent.server.raft: election won: tally=1 +34:13.634 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24317 [Leader]" +34:13.634 [INFO] TestAgent.server: cluster leadership acquired +34:13.634 [INFO] TestAgent.server: New leader elected: payload=Node-54df12be-733e-07ab-d333-d0f8da2878ad +34:13.634 [INFO] TestAgent.server: initializing acls +34:13.635 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:13.635 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:13.636 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:13.636 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:13.636 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:13.636 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:13.636 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-54df12be-733e-07ab-d333-d0f8da2878ad +34:13.636 [INFO] TestAgent.server: Updating LAN server: server="Node-54df12be-733e-07ab-d333-d0f8da2878ad (Addr: tcp/127.0.0.1:24317) (DC: dc1)" +34:13.637 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-54df12be-733e-07ab-d333-d0f8da2878ad.dc1 +34:13.637 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-54df12be-733e-07ab-d333-d0f8da2878ad.dc1 area=wan +34:13.638 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:13.640 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:13.640 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:13.640 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:13.640 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:13.640 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:13.640 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-54df12be-733e-07ab-d333-d0f8da2878ad +34:13.640 [INFO] TestAgent.server: member joined, marking health alive: member=Node-54df12be-733e-07ab-d333-d0f8da2878ad +34:13.641 [INFO] TestAgent.server: federation state anti-entropy synced +34:13.641 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-54df12be-733e-07ab-d333-d0f8da2878ad +34:13.715 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-54df12be-733e-07ab-d333-d0f8da2878ad +34:13.716 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-54df12be-733e-07ab-d333-d0f8da2878ad +34:13.736 [INFO] TestAgent: Synced node info +34:13.737 [INFO] TestAgent: Synced service: service=test-id +34:13.737 [DEBUG] TestAgent: Check in sync: check=service:test-id +34:13.739 [INFO] TestAgent: Requesting shutdown +34:13.739 [INFO] TestAgent.server: shutting down server +34:13.739 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:13.739 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:13.739 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:13.739 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:13.739 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:13.739 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:13.739 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:13.739 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:13.739 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:13.739 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:13.740 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:13.740 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:13.740 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:13.740 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:13.740 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:13.740 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:13.740Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:13.740Z [INFO] agent.router.manager: shutting down +34:13.740 [INFO] TestAgent: consul server down +34:13.740 [INFO] TestAgent: shutdown complete +34:13.740 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24312 network=tcp +34:13.740 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24312 network=udp +34:13.740 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24313 network=tcp +34:14.240 [INFO] TestAgent: Waiting for endpoints to shut down +34:14.240 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentConnectCALeafCert_aclDefaultDeny (0.69s) +=== CONT TestPreparedQuery_Integration +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:13.830Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:13.846Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:13.848 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:75dbcba8-db74-edf8-33d7-62181feb0970 Address:127.0.0.1:24359}]" +34:13.848 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24359 [Follower]" leader= +34:13.849 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-75dbcba8-db74-edf8-33d7-62181feb0970.dc1 127.0.0.1 +34:13.850 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-75dbcba8-db74-edf8-33d7-62181feb0970 127.0.0.1 +2021-08-13T15:34:13.850Z [INFO] agent.router: Initializing LAN area manager +34:13.850 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24368 network=udp +34:13.851 [INFO] TestAgent.server: Adding LAN server: server="Node-75dbcba8-db74-edf8-33d7-62181feb0970 (Addr: tcp/127.0.0.1:24359) (DC: dc1)" +34:13.851 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-75dbcba8-db74-edf8-33d7-62181feb0970.dc1 area=wan +34:13.851 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24368 network=tcp +34:13.852 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24369 network=tcp +34:13.852 [INFO] TestAgent: started state syncer +34:13.852 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24360 network=tcp +34:13.917 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:13.917 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24359 [Candidate]" term=2 +34:13.917 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:13.917 [DEBUG] TestAgent.server.raft: vote granted: from=75dbcba8-db74-edf8-33d7-62181feb0970 term=2 tally=1 +34:13.917 [INFO] TestAgent.server.raft: election won: tally=1 +34:13.917 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24359 [Leader]" +34:13.918 [INFO] TestAgent.server: cluster leadership acquired +34:13.918 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24359 +34:13.919 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:13.921 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:13.921 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:13.921 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:13.921 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:13.921 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:13.922 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-75dbcba8-db74-edf8-33d7-62181feb0970 +34:13.922 [INFO] TestAgent.server: member joined, marking health alive: member=Node-75dbcba8-db74-edf8-33d7-62181feb0970 +34:13.922 [INFO] TestAgent.server: New leader elected: payload=Node-75dbcba8-db74-edf8-33d7-62181feb0970 +34:13.923 [INFO] TestAgent.server: federation state anti-entropy synced +34:13.939 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:13.940 [INFO] TestAgent: Synced node info +34:14.121 [DEBUG] connect.ca.consul: consul CA provider configured: id=81:07:4e:3f:89:15:19:44:9b:ed:32:73:52:de:93:91:31:c9:b5:b3:43:9f:35:de:6e:c2:04:87:65:be:07:65 is_primary=true +34:14.128 [INFO] TestAgent.server.connect: CA rotated to new root under provider: provider=consul +34:14.129 [DEBUG] connect.ca.consul: consul CA provider configured: id=ad:f7:bc:d6:d1:55:c9:3f:b7:60:9b:25:b6:36:40:8c:87:bf:c7:40:84:1b:35:92:a3:2b:95:e8:6a:2d:a1:ed is_primary=true +34:14.140 [INFO] TestAgent.server.connect: CA rotated to new root under provider: provider=consul +34:14.140 [INFO] TestAgent: Requesting shutdown +34:14.140 [INFO] TestAgent.server: shutting down server +34:14.140 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:14.140 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:14.140 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:14.140 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:14.140 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:14.141 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:14.141 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:14.141 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:14.141 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:14.141 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:14.141 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:14.141Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:14.141Z [INFO] agent.router.manager: shutting down +34:14.141 [INFO] TestAgent: consul server down +34:14.141 [INFO] TestAgent: shutdown complete +34:14.141 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24368 network=tcp +34:14.141 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24368 network=udp +34:14.141 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24369 network=tcp +34:14.641 [INFO] TestAgent: Waiting for endpoints to shut down +34:14.641 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentConnectCARoots_list (0.83s) === CONT TestAgent_Token -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:53.010Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:53.010Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:53.098Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:53.098Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:53.102 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b40874a9-eea0-e3e2-2f90-46477cf620f1 Address:127.0.0.1:13078}]" -42:53.103 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b40874a9-eea0-e3e2-2f90-46477cf620f1.dc1 127.0.0.1 -42:53.103 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b40874a9-eea0-e3e2-2f90-46477cf620f1 127.0.0.1 -2022-09-15T21:42:53.103Z [INFO] agent.router: Initializing LAN area manager -42:53.104 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13122 network=udp -42:53.104 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13078 [Follower]" leader= -42:53.104 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b40874a9-eea0-e3e2-2f90-46477cf620f1.dc1 area=wan -42:53.104 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13122 network=tcp -42:53.104 [INFO] TestAgent.server: Adding LAN server: server="Node-b40874a9-eea0-e3e2-2f90-46477cf620f1 (Addr: tcp/127.0.0.1:13078) (DC: dc1)" -42:53.105 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13123 network=tcp -42:53.105 [INFO] TestAgent: started state syncer -42:53.105 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13079 network=tcp -42:53.156 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -42:53.161 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:53.162 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13078 [Candidate]" term=2 -42:53.163 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:53.163 [DEBUG] TestAgent.server.raft: vote granted: from=b40874a9-eea0-e3e2-2f90-46477cf620f1 term=2 tally=1 -42:53.163 [INFO] TestAgent.server.raft: election won: tally=1 -42:53.163 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13078 [Leader]" -42:53.163 [INFO] TestAgent.server: cluster leadership acquired -42:53.163 [INFO] TestAgent.server: initializing acls -42:53.164 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:53.164 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:53.164 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:53.165 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:53.165 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:53.165 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:53.165 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-b40874a9-eea0-e3e2-2f90-46477cf620f1 -42:53.165 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-b40874a9-eea0-e3e2-2f90-46477cf620f1.dc1 -42:53.166 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-b40874a9-eea0-e3e2-2f90-46477cf620f1.dc1 area=wan -42:53.166 [INFO] TestAgent.server: New leader elected: payload=Node-b40874a9-eea0-e3e2-2f90-46477cf620f1 -42:53.166 [INFO] TestAgent.server: Updating LAN server: server="Node-b40874a9-eea0-e3e2-2f90-46477cf620f1 (Addr: tcp/127.0.0.1:13078) (DC: dc1)" -42:53.166 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:53.169 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:53.169 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:53.169 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:53.169 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:53.169 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:53.169 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b40874a9-eea0-e3e2-2f90-46477cf620f1 -42:53.169 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b40874a9-eea0-e3e2-2f90-46477cf620f1 -42:53.170 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b40874a9-eea0-e3e2-2f90-46477cf620f1 -42:53.170 [INFO] TestAgent.server: federation state anti-entropy synced -42:53.365 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:53.366 [INFO] TestAgent: Synced node info -42:53.392 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-b40874a9-eea0-e3e2-2f90-46477cf620f1 -42:53.393 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-b40874a9-eea0-e3e2-2f90-46477cf620f1 -42:53.394 [DEBUG] TestAgent: Node info in sync -42:53.395 [INFO] TestAgent: Synced service: service=test-id -42:53.395 [DEBUG] TestAgent: Check in sync: check=service:test-id -42:53.413 [INFO] TestAgent: Requesting shutdown -42:53.413 [INFO] TestAgent.server: shutting down server -42:53.413 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:53.413 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:53.413 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:53.413 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:53.413 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:53.413 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:53.413 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:53.413 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:53.413 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:53.413 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:53.413 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:53.413 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:53.413 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:53.413 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:53.414 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:53.419Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:53.419Z [INFO] agent.router.manager: shutting down -42:53.419 [INFO] TestAgent: consul server down -42:53.419 [INFO] TestAgent: shutdown complete -42:53.419 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13122 network=tcp -42:53.419 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13122 network=udp -42:53.419 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13123 network=tcp -42:53.925 [INFO] TestAgent: Waiting for endpoints to shut down -42:53.925 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentConnectCALeafCert_aclServiceWrite (0.96s) +2021-08-13T15:34:13.680Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:13.680Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:13.737Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:13.737Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:13.741 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d6a77d40-0015-10b0-7c8e-17613537a000 Address:127.0.0.1:24324}]" +34:13.741 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24324 [Follower]" leader= +34:13.742 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d6a77d40-0015-10b0-7c8e-17613537a000.dc1 127.0.0.1 +34:13.744 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d6a77d40-0015-10b0-7c8e-17613537a000 127.0.0.1 +2021-08-13T15:34:13.744Z [INFO] agent.router: Initializing LAN area manager +34:13.745 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24319 network=udp +34:13.745 [INFO] TestAgent.server: Adding LAN server: server="Node-d6a77d40-0015-10b0-7c8e-17613537a000 (Addr: tcp/127.0.0.1:24324) (DC: dc1)" +34:13.745 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d6a77d40-0015-10b0-7c8e-17613537a000.dc1 area=wan +34:13.745 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24319 network=tcp +34:13.746 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24320 network=tcp +34:13.746 [INFO] TestAgent: started state syncer +34:13.746 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24367 network=tcp +34:13.795 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:13.801 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:13.802 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24324 [Candidate]" term=2 +34:13.802 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:13.802 [DEBUG] TestAgent.server.raft: vote granted: from=d6a77d40-0015-10b0-7c8e-17613537a000 term=2 tally=1 +34:13.802 [INFO] TestAgent.server.raft: election won: tally=1 +34:13.802 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24324 [Leader]" +34:13.802 [INFO] TestAgent.server: cluster leadership acquired +34:13.802 [INFO] TestAgent.server: initializing acls +34:13.803 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:13.803 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:13.803 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:13.804 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:13.804 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:13.804 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:13.804 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-d6a77d40-0015-10b0-7c8e-17613537a000 +34:13.804 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-d6a77d40-0015-10b0-7c8e-17613537a000.dc1 +34:13.804 [INFO] TestAgent.server: New leader elected: payload=Node-d6a77d40-0015-10b0-7c8e-17613537a000 +34:13.805 [INFO] TestAgent.server: Updating LAN server: server="Node-d6a77d40-0015-10b0-7c8e-17613537a000 (Addr: tcp/127.0.0.1:24324) (DC: dc1)" +34:13.805 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-d6a77d40-0015-10b0-7c8e-17613537a000.dc1 area=wan +34:13.805 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:13.809 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:13.809 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:13.809 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:13.809 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:13.809 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:13.809 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d6a77d40-0015-10b0-7c8e-17613537a000 +34:13.809 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d6a77d40-0015-10b0-7c8e-17613537a000 +34:13.810 [INFO] TestAgent.server: federation state anti-entropy synced +34:13.810 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d6a77d40-0015-10b0-7c8e-17613537a000 +34:13.817 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:13.818 [INFO] TestAgent: Synced node info +34:14.171 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-d6a77d40-0015-10b0-7c8e-17613537a000 +34:14.171 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-d6a77d40-0015-10b0-7c8e-17613537a000 +34:14.173 [INFO] TestAgent: Requesting shutdown +34:14.173 [INFO] TestAgent.server: shutting down server +34:14.173 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:14.173 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:14.173 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:14.173 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:14.173 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:14.173 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:14.173 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:14.173 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:14.173 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:14.173 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:14.173 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:14.173 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:14.173 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:14.173 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:14.173 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:14.173Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:14.174Z [INFO] agent.router.manager: shutting down +34:14.174 [INFO] TestAgent: consul server down +34:14.174 [INFO] TestAgent: shutdown complete +34:14.174 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24319 network=tcp +34:14.174 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24319 network=udp +34:14.174 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24320 network=tcp +34:14.675 [INFO] TestAgent: Waiting for endpoints to shut down +34:14.675 [INFO] TestAgent: Endpoints down +--- PASS: TestRemoteExecGetSpec_ACLToken (1.01s) +=== CONT TestPreparedQuery_parseLimit +--- PASS: TestPreparedQuery_parseLimit (0.00s) === CONT TestAgent_TokenTriggersFullSync === RUN TestAgent_TokenTriggersFullSync/acl_agent_token -2022-09-15T21:42:53.127Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:53.156Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:53.159 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1c9886c3-4c2a-fc16-9cdf-22c8c8b8cf6f Address:127.0.0.1:13134}]" -42:53.160 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1c9886c3-4c2a-fc16-9cdf-22c8c8b8cf6f.dc1 127.0.0.1 -42:53.161 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1c9886c3-4c2a-fc16-9cdf-22c8c8b8cf6f 127.0.0.1 -2022-09-15T21:42:53.161Z [INFO] agent.router: Initializing LAN area manager -42:53.161 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13134 [Follower]" leader= -42:53.161 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13094 network=udp -42:53.161 [INFO] TestAgent.server: Adding LAN server: server="Node-1c9886c3-4c2a-fc16-9cdf-22c8c8b8cf6f (Addr: tcp/127.0.0.1:13134) (DC: dc1)" -42:53.161 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1c9886c3-4c2a-fc16-9cdf-22c8c8b8cf6f.dc1 area=wan -42:53.162 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13094 network=tcp -42:53.162 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13095 network=tcp -42:53.163 [INFO] TestAgent: started state syncer -42:53.163 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13135 network=tcp -42:53.210 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:53.210 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13134 [Candidate]" term=2 -42:53.211 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:53.211 [DEBUG] TestAgent.server.raft: vote granted: from=1c9886c3-4c2a-fc16-9cdf-22c8c8b8cf6f term=2 tally=1 -42:53.211 [INFO] TestAgent.server.raft: election won: tally=1 -42:53.211 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13134 [Leader]" -42:53.211 [INFO] TestAgent.server: cluster leadership acquired -42:53.211 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13134 -42:53.211 [INFO] TestAgent.server: New leader elected: payload=Node-1c9886c3-4c2a-fc16-9cdf-22c8c8b8cf6f -42:53.213 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:53.216 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:53.216 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:53.216 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:53.216 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:53.216 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:53.216 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1c9886c3-4c2a-fc16-9cdf-22c8c8b8cf6f -42:53.216 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1c9886c3-4c2a-fc16-9cdf-22c8c8b8cf6f -42:53.217 [INFO] TestAgent.server: federation state anti-entropy synced -42:53.254 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:53.255 [INFO] TestAgent: Synced node info -42:53.399 [DEBUG] connect.ca.consul: consul CA provider configured: id=46:87:3c:70:fe:18:90:7f:14:5b:65:26:c6:30:d8:e7:34:cd:29:a2:72:b7:50:c8:55:22:6c:9c:21:e7:27:14 is_primary=true -42:53.407 [INFO] TestAgent.server.connect: CA rotated to new root under provider: provider=consul -42:53.409 [DEBUG] connect.ca.consul: consul CA provider configured: id=b9:71:05:61:99:2f:3d:db:25:49:dc:23:56:c4:f5:bb:dc:36:51:57:a0:b3:a9:40:56:0e:62:ae:3c:3d:c5:77 is_primary=true -42:53.422 [INFO] TestAgent.server.connect: CA rotated to new root under provider: provider=consul -42:53.422 [INFO] TestAgent: Requesting shutdown -42:53.430 [INFO] TestAgent.server: shutting down server -42:53.430 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:53.430 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:53.430 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:53.430 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:53.430 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:53.430 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:53.430 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:53.430 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:53.430 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:53.430 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:53.430 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:53.430Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:53.430Z [INFO] agent.router.manager: shutting down -42:53.430 [INFO] TestAgent: consul server down -42:53.430 [INFO] TestAgent: shutdown complete -42:53.430 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13094 network=tcp -42:53.430 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13094 network=udp -42:53.430 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13095 network=tcp -42:53.936 [INFO] TestAgent: Waiting for endpoints to shut down -42:53.936 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentConnectCARoots_list (0.85s) +2021-08-13T15:34:14.011Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:14.027Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:14.030 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f857e293-4bb9-0001-9abc-34a1c24a5a0b Address:127.0.0.1:24331}]" +34:14.030 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24331 [Follower]" leader= +34:14.031 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f857e293-4bb9-0001-9abc-34a1c24a5a0b.dc1 127.0.0.1 +34:14.032 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f857e293-4bb9-0001-9abc-34a1c24a5a0b 127.0.0.1 +2021-08-13T15:34:14.032Z [INFO] agent.router: Initializing LAN area manager +34:14.032 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24326 network=udp +34:14.033 [INFO] TestAgent.server: Adding LAN server: server="Node-f857e293-4bb9-0001-9abc-34a1c24a5a0b (Addr: tcp/127.0.0.1:24331) (DC: dc1)" +34:14.033 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f857e293-4bb9-0001-9abc-34a1c24a5a0b.dc1 area=wan +34:14.033 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24326 network=tcp +34:14.034 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24327 network=tcp +34:14.034 [INFO] TestAgent: started state syncer +34:14.034 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24381 network=tcp +34:14.082 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:14.082 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24331 [Candidate]" term=2 +34:14.082 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:14.082 [DEBUG] TestAgent.server.raft: vote granted: from=f857e293-4bb9-0001-9abc-34a1c24a5a0b term=2 tally=1 +34:14.082 [INFO] TestAgent.server.raft: election won: tally=1 +34:14.082 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24331 [Leader]" +34:14.082 [INFO] TestAgent.server: cluster leadership acquired +34:14.083 [INFO] TestAgent.server: New leader elected: payload=Node-f857e293-4bb9-0001-9abc-34a1c24a5a0b +34:14.083 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24331 +34:14.083 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:14.083 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:14.083 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f857e293-4bb9-0001-9abc-34a1c24a5a0b +34:14.083 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f857e293-4bb9-0001-9abc-34a1c24a5a0b +34:14.084 [INFO] TestAgent.server: federation state anti-entropy synced +34:14.179 [INFO] TestAgent: Requesting shutdown +34:14.180 [INFO] TestAgent.server: shutting down server +34:14.180 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:14.180 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:14.180 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:14.180 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:14.180 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:14.180 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:14.180 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:14.180 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:14.180Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:14.180Z [INFO] agent.router.manager: shutting down +34:14.180 [INFO] TestAgent: consul server down +34:14.180 [INFO] TestAgent: shutdown complete +34:14.180 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24326 network=tcp +34:14.180 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24326 network=udp +34:14.180 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24327 network=tcp +34:14.682 [INFO] TestAgent: Waiting for endpoints to shut down +34:14.682 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentConnectCARoots_empty (0.69s) === CONT TestAgent_Monitor_ACLDeny -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:13.911Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:13.926Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:13.929 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4cc9386f-b746-cced-b6ff-ae6723807d94 Address:127.0.0.1:24380}]" +34:13.929 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24380 [Follower]" leader= +34:13.929 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4cc9386f-b746-cced-b6ff-ae6723807d94.dc1 127.0.0.1 +34:13.930 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4cc9386f-b746-cced-b6ff-ae6723807d94 127.0.0.1 +2021-08-13T15:34:13.930Z [INFO] agent.router: Initializing LAN area manager +34:13.931 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24361 network=udp +34:13.931 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4cc9386f-b746-cced-b6ff-ae6723807d94.dc1 area=wan +34:13.931 [INFO] TestAgent.server: Adding LAN server: server="Node-4cc9386f-b746-cced-b6ff-ae6723807d94 (Addr: tcp/127.0.0.1:24380) (DC: dc1)" +34:13.931 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24361 network=tcp +34:13.932 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24362 network=tcp +34:13.932 [INFO] TestAgent: started state syncer +34:13.932 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24325 network=tcp +34:13.968 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:13.968 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24380 [Candidate]" term=2 +34:13.968 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:13.968 [DEBUG] TestAgent.server.raft: vote granted: from=4cc9386f-b746-cced-b6ff-ae6723807d94 term=2 tally=1 +34:13.968 [INFO] TestAgent.server.raft: election won: tally=1 +34:13.968 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24380 [Leader]" +34:13.968 [INFO] TestAgent.server: cluster leadership acquired +34:13.968 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24380 +34:13.969 [INFO] TestAgent.server: New leader elected: payload=Node-4cc9386f-b746-cced-b6ff-ae6723807d94 +34:13.970 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:13.972 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:13.972 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:13.972 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:13.972 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:13.972 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:13.972 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4cc9386f-b746-cced-b6ff-ae6723807d94 +34:13.972 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4cc9386f-b746-cced-b6ff-ae6723807d94 +34:13.973 [INFO] TestAgent.server: federation state anti-entropy synced +34:14.235 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:14.236 [INFO] TestAgent: Synced node info +34:14.292 [INFO] TestAgent: Requesting shutdown +34:14.292 [INFO] TestAgent.server: shutting down server +34:14.292 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:14.292 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:14.292 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:14.292 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:14.292 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:14.292 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:14.292 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:14.292 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:14.292 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:14.292 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:14.292 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:14.292Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:14.292Z [INFO] agent.router.manager: shutting down +34:14.292 [INFO] TestAgent: consul server down +34:14.292 [INFO] TestAgent: shutdown complete +34:14.292 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24361 network=tcp +34:14.292 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24361 network=udp +34:14.292 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24362 network=tcp +34:14.793 [INFO] TestAgent: Waiting for endpoints to shut down +34:14.793 [INFO] TestAgent: Endpoints down +--- PASS: TestRemoteExecGetSpec (0.90s) +=== CONT TestAgent_Monitor +2021-08-13T15:34:14.257Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:14.272Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:14.275 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:881e5ea3-20dd-2ec0-42fc-d8d9a69ffbee Address:127.0.0.1:24373}]" +34:14.275 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24373 [Follower]" leader= +34:14.276 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-881e5ea3-20dd-2ec0-42fc-d8d9a69ffbee.dc1 127.0.0.1 +34:14.277 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-881e5ea3-20dd-2ec0-42fc-d8d9a69ffbee 127.0.0.1 +2021-08-13T15:34:14.277Z [INFO] agent.router: Initializing LAN area manager +34:14.277 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24382 network=udp +34:14.277 [INFO] TestAgent.server: Adding LAN server: server="Node-881e5ea3-20dd-2ec0-42fc-d8d9a69ffbee (Addr: tcp/127.0.0.1:24373) (DC: dc1)" +34:14.277 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-881e5ea3-20dd-2ec0-42fc-d8d9a69ffbee.dc1 area=wan +34:14.278 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24382 network=tcp +34:14.278 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24383 network=tcp +34:14.278 [INFO] TestAgent: started state syncer +34:14.279 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24374 network=tcp +34:14.321 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:14.322 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24373 [Candidate]" term=2 +34:14.322 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:14.322 [DEBUG] TestAgent.server.raft: vote granted: from=881e5ea3-20dd-2ec0-42fc-d8d9a69ffbee term=2 tally=1 +34:14.322 [INFO] TestAgent.server.raft: election won: tally=1 +34:14.322 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24373 [Leader]" +34:14.322 [INFO] TestAgent.server: cluster leadership acquired +34:14.322 [INFO] TestAgent.server: New leader elected: payload=Node-881e5ea3-20dd-2ec0-42fc-d8d9a69ffbee +34:14.322 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24373 +34:14.323 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:14.326 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:14.326 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:14.326 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:14.326 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:14.326 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:14.326 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-881e5ea3-20dd-2ec0-42fc-d8d9a69ffbee +34:14.326 [INFO] TestAgent.server: member joined, marking health alive: member=Node-881e5ea3-20dd-2ec0-42fc-d8d9a69ffbee +34:14.327 [INFO] TestAgent.server: federation state anti-entropy synced +34:14.405 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:14.406 [INFO] TestAgent: Synced node info +34:14.406 [DEBUG] TestAgent: Node info in sync +34:14.412 [INFO] TestAgent: Requesting shutdown +34:14.412 [INFO] TestAgent.server: shutting down server +34:14.412 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:14.412 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:14.412 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:14.412 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:14.412 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:14.412 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:14.412 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:14.412 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:14.412 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:14.412 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:14.412 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:14.413Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:14.413Z [INFO] agent.router.manager: shutting down +34:14.413 [INFO] TestAgent: consul server down +34:14.413 [INFO] TestAgent: shutdown complete +34:14.413 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24382 network=tcp +34:14.413 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24382 network=udp +34:14.413 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24383 network=tcp +34:14.913 [INFO] TestAgent: Waiting for endpoints to shut down +34:14.913 [INFO] TestAgent: Endpoints down +--- PASS: TestPreparedQuery_Integration (0.67s) +=== CONT TestAgent_RegisterCheck_Service +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! === RUN TestAgent_Token/bad_token_name === RUN TestAgent_Token/bad_JSON === RUN TestAgent_Token/set_user_legacy @@ -40394,283 +32041,106 @@ === RUN TestAgent_Token/clear_repl_legacy === RUN TestAgent_Token/clear_repl === RUN TestAgent_Token/permission_denied -[WARN] freeport: 4 out of 4 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:53.667Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:53.687Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:53.690 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f1fc4d6b-dd03-b0f2-3bcf-f603d97cc58b Address:127.0.0.1:13099}]" -42:53.691 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f1fc4d6b-dd03-b0f2-3bcf-f603d97cc58b.dc1 127.0.0.1 -42:53.692 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f1fc4d6b-dd03-b0f2-3bcf-f603d97cc58b 127.0.0.1 -2022-09-15T21:42:53.692Z [INFO] agent.router: Initializing LAN area manager -42:53.692 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13108 network=udp -42:53.692 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13099 [Follower]" leader= -42:53.693 [INFO] TestAgent.server: Adding LAN server: server="Node-f1fc4d6b-dd03-b0f2-3bcf-f603d97cc58b (Addr: tcp/127.0.0.1:13099) (DC: dc1)" -42:53.693 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f1fc4d6b-dd03-b0f2-3bcf-f603d97cc58b.dc1 area=wan -42:53.693 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13108 network=tcp -42:53.695 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13109 network=tcp -42:53.695 [INFO] TestAgent: started state syncer -42:53.695 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13100 network=tcp -42:53.752 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:53.752 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13099 [Candidate]" term=2 -42:53.752 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:53.752 [DEBUG] TestAgent.server.raft: vote granted: from=f1fc4d6b-dd03-b0f2-3bcf-f603d97cc58b term=2 tally=1 -42:53.752 [INFO] TestAgent.server.raft: election won: tally=1 -42:53.752 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13099 [Leader]" -42:53.752 [INFO] TestAgent.server: cluster leadership acquired -42:53.752 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13099 -42:53.753 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:53.753 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:53.753 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f1fc4d6b-dd03-b0f2-3bcf-f603d97cc58b -42:53.753 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f1fc4d6b-dd03-b0f2-3bcf-f603d97cc58b -42:53.753 [INFO] TestAgent.server: New leader elected: payload=Node-f1fc4d6b-dd03-b0f2-3bcf-f603d97cc58b -42:53.754 [INFO] TestAgent.server: federation state anti-entropy synced -42:53.882 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:53.884 [INFO] TestAgent: Synced node info -42:53.884 [DEBUG] TestAgent: Node info in sync -42:53.971 [INFO] TestAgent: Requesting shutdown -42:53.990 [INFO] TestAgent.server: shutting down server -42:53.990 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:53.990 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:53.990 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:53.990 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:53.990 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:53.990 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:53.990 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:53.991Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:53.991Z [INFO] agent.router.manager: shutting down -42:53.991 [INFO] TestAgent: consul server down -42:53.991 [INFO] TestAgent: shutdown complete -42:53.991 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13108 network=tcp -42:53.991 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13108 network=udp -42:53.991 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13109 network=tcp -42:54.492 [INFO] TestAgent: Waiting for endpoints to shut down -42:54.492 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentConnectCARoots_empty (0.86s) -=== CONT TestAgent_Monitor -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:52.979Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:53.036Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:53.085 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2dd25615-f800-6249-11db-0464bb488c63 Address:127.0.0.1:13127}]" -42:53.086 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2dd25615-f800-6249-11db-0464bb488c63.dc1 127.0.0.1 -42:53.086 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2dd25615-f800-6249-11db-0464bb488c63 127.0.0.1 -2022-09-15T21:42:53.086Z [INFO] agent.router: Initializing LAN area manager -42:53.087 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13115 network=udp -42:53.087 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13127 [Follower]" leader= -42:53.087 [INFO] TestAgent.server: Adding LAN server: server="Node-2dd25615-f800-6249-11db-0464bb488c63 (Addr: tcp/127.0.0.1:13127) (DC: dc1)" -42:53.087 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2dd25615-f800-6249-11db-0464bb488c63.dc1 area=wan -42:53.088 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13115 network=tcp -42:53.089 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13116 network=tcp -42:53.089 [INFO] TestAgent: started state syncer -42:53.089 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13128 network=tcp -42:53.149 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:53.149 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13127 [Candidate]" term=2 -42:53.149 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:53.149 [DEBUG] TestAgent.server.raft: vote granted: from=2dd25615-f800-6249-11db-0464bb488c63 term=2 tally=1 -42:53.149 [INFO] TestAgent.server.raft: election won: tally=1 -42:53.149 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13127 [Leader]" -42:53.149 [INFO] TestAgent.server: cluster leadership acquired -42:53.149 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13127 -42:53.150 [INFO] TestAgent.server: New leader elected: payload=Node-2dd25615-f800-6249-11db-0464bb488c63 -42:53.154 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:53.166 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:53.166 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:53.166 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:53.166 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:53.166 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:53.166 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2dd25615-f800-6249-11db-0464bb488c63 -42:53.166 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2dd25615-f800-6249-11db-0464bb488c63 -42:53.168 [INFO] TestAgent.server: federation state anti-entropy synced -42:53.205 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:53.206 [INFO] TestAgent: Synced node info -42:53.512 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:35334 -42:53.513 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-97c4a4ec-c2a4-c101-d8d9-f17726f41d84 127.0.0.1 -42:53.513 [INFO] TestAgent: Requesting shutdown -42:53.513 [INFO] TestAgent.server: shutting down server -42:53.513 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:53.513 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:53.513 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:53.513 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:53.513 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:53.514 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:53.514Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:53.514Z [INFO] agent.router.manager: shutting down -42:53.514 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:53.514 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:53.514 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:53.514 [INFO] TestAgent: consul server down -42:53.514 [INFO] TestAgent: shutdown complete -42:53.514 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:53.514 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13115 network=tcp -42:53.514 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:53.514 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13115 network=udp -42:53.514 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13116 network=tcp -42:54.022 [INFO] TestAgent: Waiting for endpoints to shut down -42:54.022 [INFO] TestAgent: Endpoints down -2022-09-15T21:42:52.646Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:52.686Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:52.689 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:97c4a4ec-c2a4-c101-d8d9-f17726f41d84 Address:127.0.0.1:14063}]" -42:52.689 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14063 [Follower]" leader= -42:52.690 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-97c4a4ec-c2a4-c101-d8d9-f17726f41d84.dc1 127.0.0.1 -42:52.691 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-97c4a4ec-c2a4-c101-d8d9-f17726f41d84 127.0.0.1 -2022-09-15T21:42:52.691Z [INFO] agent.router: Initializing LAN area manager -42:52.691 [INFO] TestAgent.server: Adding LAN server: server="Node-97c4a4ec-c2a4-c101-d8d9-f17726f41d84 (Addr: tcp/127.0.0.1:14063) (DC: dc1)" -42:52.691 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-97c4a4ec-c2a4-c101-d8d9-f17726f41d84.dc1 area=wan -42:52.691 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14072 network=udp -42:52.691 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14072 network=tcp -42:52.692 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14073 network=tcp -42:52.692 [INFO] TestAgent: started state syncer -42:52.692 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14064 network=tcp -42:52.760 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:52.760 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14063 [Candidate]" term=2 -42:52.760 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:52.760 [DEBUG] TestAgent.server.raft: vote granted: from=97c4a4ec-c2a4-c101-d8d9-f17726f41d84 term=2 tally=1 -42:52.760 [INFO] TestAgent.server.raft: election won: tally=1 -42:52.760 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14063 [Leader]" -42:52.761 [INFO] TestAgent.server: cluster leadership acquired -42:52.761 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14063 -42:52.762 [INFO] TestAgent.server: New leader elected: payload=Node-97c4a4ec-c2a4-c101-d8d9-f17726f41d84 -42:52.763 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:52.765 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:52.765 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:52.765 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:52.765 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:52.765 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:52.765 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-97c4a4ec-c2a4-c101-d8d9-f17726f41d84 -42:52.766 [INFO] TestAgent.server: federation state anti-entropy synced -42:52.782 [INFO] TestAgent.server: member joined, marking health alive: member=Node-97c4a4ec-c2a4-c101-d8d9-f17726f41d84 -42:53.003 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:53.005 [INFO] TestAgent: Synced node info -42:53.511 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:13118] -42:53.512 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:13118 -42:53.513 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2dd25615-f800-6249-11db-0464bb488c63 127.0.0.1 -42:53.513 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 -42:53.513 [DEBUG] TestAgent: systemd notify failed: error="No socket" -42:53.514 [INFO] TestAgent.server: Adding LAN server: server="Node-2dd25615-f800-6249-11db-0464bb488c63 (Addr: tcp/127.0.0.1:13127) (DC: dc1)" -42:53.514 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-2dd25615-f800-6249-11db-0464bb488c63 other=Node-97c4a4ec-c2a4-c101-d8d9-f17726f41d84 -42:53.514 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2dd25615-f800-6249-11db-0464bb488c63 -42:53.515 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:13119: connect: connection refused -42:53.515 [DEBUG] TestAgent.server: Failed to flood-join server at address: server=Node-2dd25615-f800-6249-11db-0464bb488c63.dc1 address=127.0.0.1:13119 error="1 error occurred: - * Failed to join 127.0.0.1: dial tcp 127.0.0.1:13119: connect: connection refused - -" -42:54.022 [INFO] TestAgent: Requesting shutdown -42:54.022 [INFO] TestAgent.server: shutting down server -42:54.022 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:54.022 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:54.023 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:54.023 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:54.023 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:54.023 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:54.023 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:54.023 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:54.023 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:54.023 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:54.023 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:54.023Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:54.023Z [INFO] agent.router.manager: shutting down -42:54.024 [INFO] TestAgent: consul server down -42:54.024 [INFO] TestAgent: shutdown complete -42:54.024 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14072 network=tcp -42:54.024 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14072 network=udp -42:54.024 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14073 network=tcp -42:54.194 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Failed ping: Node-2dd25615-f800-6249-11db-0464bb488c63 (timeout reached) -42:54.526 [INFO] TestAgent: Waiting for endpoints to shut down -42:54.526 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Join (1.94s) -=== CONT TestAgent_RegisterCheck_Service === RUN TestAgent_Monitor/unknown_log_level === RUN TestAgent_Monitor/stream_unstructured_logs -2022-09-15T21:42:53.783Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:53.783Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:53.805Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:53.805Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:53.808 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7e884bb5-509f-490c-dfbb-609678a45806 Address:127.0.0.1:13120}]" -42:53.809 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7e884bb5-509f-490c-dfbb-609678a45806.dc1 127.0.0.1 -42:53.810 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13120 [Follower]" leader= -42:53.811 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7e884bb5-509f-490c-dfbb-609678a45806 127.0.0.1 -2022-09-15T21:42:53.811Z [INFO] agent.router: Initializing LAN area manager -42:53.811 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13129 network=udp -42:53.811 [INFO] TestAgent.server: Adding LAN server: server="Node-7e884bb5-509f-490c-dfbb-609678a45806 (Addr: tcp/127.0.0.1:13120) (DC: dc1)" -42:53.812 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7e884bb5-509f-490c-dfbb-609678a45806.dc1 area=wan -42:53.812 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13129 network=tcp -42:53.813 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13130 network=tcp -42:53.813 [INFO] TestAgent: started state syncer -42:53.813 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13121 network=tcp -42:53.851 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:53.851 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13120 [Candidate]" term=2 -42:53.851 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:53.851 [DEBUG] TestAgent.server.raft: vote granted: from=7e884bb5-509f-490c-dfbb-609678a45806 term=2 tally=1 -42:53.851 [INFO] TestAgent.server.raft: election won: tally=1 -42:53.851 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13120 [Leader]" -42:53.851 [INFO] TestAgent.server: cluster leadership acquired -42:53.851 [INFO] TestAgent.server: initializing acls -42:53.852 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:53.852 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:53.853 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:53.853 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:53.853 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:53.853 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:53.854 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-7e884bb5-509f-490c-dfbb-609678a45806 -42:53.854 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-7e884bb5-509f-490c-dfbb-609678a45806.dc1 -42:53.855 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:53.856 [INFO] TestAgent.server: New leader elected: payload=Node-7e884bb5-509f-490c-dfbb-609678a45806 -42:53.856 [INFO] TestAgent.server: Updating LAN server: server="Node-7e884bb5-509f-490c-dfbb-609678a45806 (Addr: tcp/127.0.0.1:13120) (DC: dc1)" -42:53.856 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-7e884bb5-509f-490c-dfbb-609678a45806.dc1 area=wan -42:53.859 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:53.859 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:53.859 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:53.859 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:53.859 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:53.859 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7e884bb5-509f-490c-dfbb-609678a45806 -42:53.859 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7e884bb5-509f-490c-dfbb-609678a45806 -42:53.860 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7e884bb5-509f-490c-dfbb-609678a45806 -42:53.860 [INFO] TestAgent.server: federation state anti-entropy synced -42:53.950 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth -42:53.951 [WARN] TestAgent: Node info update blocked by ACLs: node=7e884bb5-509f-490c-dfbb-609678a45806 accessorID=00000000-0000-0000-0000-000000000002 -42:54.185 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-7e884bb5-509f-490c-dfbb-609678a45806 -42:54.185 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-7e884bb5-509f-490c-dfbb-609678a45806 -42:54.187 [INFO] TestAgent: Updated agent's ACL token: token=acl_token -42:54.187 [INFO] TestAgent: Updated agent's ACL token: token=default -42:54.187 [INFO] TestAgent: Updated agent's ACL token: token=acl_agent_token -42:54.187 [INFO] TestAgent: Updated agent's ACL token: token=agent -42:54.187 [INFO] TestAgent: Updated agent's ACL token: token=acl_agent_master_token -42:54.188 [INFO] TestAgent: Updated agent's ACL token: token=agent_master -42:54.188 [INFO] TestAgent: Updated agent's ACL token: token=acl_replication_token -42:54.188 [INFO] TestAgent: Updated agent's ACL token: token=replication -42:54.188 [INFO] TestAgent: Updated agent's ACL token: token=acl_token -42:54.189 [INFO] TestAgent: Updated agent's ACL token: token=default -42:54.189 [INFO] TestAgent: Updated agent's ACL token: token=acl_agent_token -42:54.189 [INFO] TestAgent: Updated agent's ACL token: token=agent -42:54.189 [INFO] TestAgent: Updated agent's ACL token: token=acl_agent_master_token -42:54.189 [INFO] TestAgent: Updated agent's ACL token: token=agent_master -42:54.190 [INFO] TestAgent: Updated agent's ACL token: token=acl_replication_token -42:54.190 [INFO] TestAgent: Updated agent's ACL token: token=replication -42:54.190 [INFO] TestAgent: Requesting shutdown -42:54.190 [INFO] TestAgent.server: shutting down server -42:54.190 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:54.190 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:54.190 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:54.190 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:54.191 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:54.191 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:54.191 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:54.191 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:54.191 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:54.191 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:54.191 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:54.191 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:54.191 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:54.191 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:54.191 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:54.191Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:54.191Z [INFO] agent.router.manager: shutting down -42:54.191 [INFO] TestAgent: consul server down -42:54.191 [INFO] TestAgent: shutdown complete -42:54.191 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13129 network=tcp -42:54.191 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13129 network=udp -42:54.191 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13130 network=tcp -42:54.692 [INFO] TestAgent: Waiting for endpoints to shut down -42:54.692 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Token (0.93s) +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestAgent_Monitor/stream_JSON_logs +=== RUN TestAgent_Monitor/serf_shutdown_logging +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:14.664Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:14.664Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:14.736Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:14.736Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:14.748 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9d5fd328-713e-cf08-db3b-f169b1fdc413 Address:127.0.0.1:24387}]" +34:14.750 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9d5fd328-713e-cf08-db3b-f169b1fdc413.dc1 127.0.0.1 +34:14.751 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24387 [Follower]" leader= +34:14.755 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9d5fd328-713e-cf08-db3b-f169b1fdc413 127.0.0.1 +2021-08-13T15:34:14.755Z [INFO] agent.router: Initializing LAN area manager +34:14.755 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9d5fd328-713e-cf08-db3b-f169b1fdc413.dc1 area=wan +34:14.755 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24375 network=udp +34:14.756 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24375 network=tcp +34:14.756 [INFO] TestAgent.server: Adding LAN server: server="Node-9d5fd328-713e-cf08-db3b-f169b1fdc413 (Addr: tcp/127.0.0.1:24387) (DC: dc1)" +34:14.757 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24376 network=tcp +34:14.757 [INFO] TestAgent: started state syncer +34:14.766 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24388 network=tcp +34:14.806 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:14.817 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:14.817 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24387 [Candidate]" term=2 +34:14.818 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:14.818 [DEBUG] TestAgent.server.raft: vote granted: from=9d5fd328-713e-cf08-db3b-f169b1fdc413 term=2 tally=1 +34:14.818 [INFO] TestAgent.server.raft: election won: tally=1 +34:14.818 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24387 [Leader]" +34:14.818 [INFO] TestAgent.server: cluster leadership acquired +34:14.818 [INFO] TestAgent.server: initializing acls +34:14.818 [INFO] TestAgent.server: New leader elected: payload=Node-9d5fd328-713e-cf08-db3b-f169b1fdc413 +34:14.819 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:14.819 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:14.819 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:14.820 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:14.820 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:14.820 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:14.820 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-9d5fd328-713e-cf08-db3b-f169b1fdc413 +34:14.820 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-9d5fd328-713e-cf08-db3b-f169b1fdc413.dc1 +34:14.821 [INFO] TestAgent.server: Updating LAN server: server="Node-9d5fd328-713e-cf08-db3b-f169b1fdc413 (Addr: tcp/127.0.0.1:24387) (DC: dc1)" +34:14.821 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-9d5fd328-713e-cf08-db3b-f169b1fdc413.dc1 area=wan +34:14.821 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:14.824 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:14.824 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:14.824 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:14.824 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:14.824 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:14.824 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9d5fd328-713e-cf08-db3b-f169b1fdc413 +34:14.824 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9d5fd328-713e-cf08-db3b-f169b1fdc413 +34:14.825 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9d5fd328-713e-cf08-db3b-f169b1fdc413 +34:14.825 [INFO] TestAgent.server: federation state anti-entropy synced +34:14.922 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth +34:14.922 [WARN] TestAgent: Node info update blocked by ACLs: node=9d5fd328-713e-cf08-db3b-f169b1fdc413 accessorID=00000000-0000-0000-0000-000000000002 +34:14.922 [DEBUG] TestAgent: Node info in sync +34:15.087 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-9d5fd328-713e-cf08-db3b-f169b1fdc413 +34:15.087 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-9d5fd328-713e-cf08-db3b-f169b1fdc413 +34:15.089 [INFO] TestAgent: Updated agent's ACL token: token=acl_token +34:15.089 [INFO] TestAgent: Updated agent's ACL token: token=default +34:15.089 [INFO] TestAgent: Updated agent's ACL token: token=acl_agent_token +34:15.090 [INFO] TestAgent: Updated agent's ACL token: token=agent +34:15.090 [INFO] TestAgent: Updated agent's ACL token: token=acl_agent_master_token +34:15.090 [INFO] TestAgent: Updated agent's ACL token: token=agent_master +34:15.090 [INFO] TestAgent: Updated agent's ACL token: token=acl_replication_token +34:15.091 [INFO] TestAgent: Updated agent's ACL token: token=replication +34:15.091 [INFO] TestAgent: Updated agent's ACL token: token=acl_token +34:15.091 [INFO] TestAgent: Updated agent's ACL token: token=default +34:15.092 [INFO] TestAgent: Updated agent's ACL token: token=acl_agent_token +34:15.092 [INFO] TestAgent: Updated agent's ACL token: token=agent +34:15.092 [INFO] TestAgent: Updated agent's ACL token: token=acl_agent_master_token +34:15.092 [INFO] TestAgent: Updated agent's ACL token: token=agent_master +34:15.093 [INFO] TestAgent: Updated agent's ACL token: token=acl_replication_token +34:15.093 [INFO] TestAgent: Updated agent's ACL token: token=replication +34:15.093 [INFO] TestAgent: Requesting shutdown +34:15.093 [INFO] TestAgent.server: shutting down server +34:15.093 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:15.093 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:15.093 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:15.093 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:15.093 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:15.093 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:15.093 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:15.093 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:15.094 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:15.094 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:15.094 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:15.094 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:15.094 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:15.094 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:15.094 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:15.094Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:15.094Z [INFO] agent.router.manager: shutting down +34:15.094 [INFO] TestAgent: consul server down +34:15.094 [INFO] TestAgent: shutdown complete +34:15.094 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24375 network=tcp +34:15.094 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24375 network=udp +34:15.094 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24376 network=tcp +34:15.594 [INFO] TestAgent: Waiting for endpoints to shut down +34:15.594 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Token (0.95s) --- PASS: TestAgent_Token/bad_token_name (0.00s) --- PASS: TestAgent_Token/bad_JSON (0.00s) --- PASS: TestAgent_Token/set_user_legacy (0.00s) @@ -40690,2337 +32160,4194 @@ --- PASS: TestAgent_Token/clear_repl_legacy (0.00s) --- PASS: TestAgent_Token/clear_repl (0.00s) --- PASS: TestAgent_Token/permission_denied (0.00s) +=== CONT TestPreparedQuery_Update +2021-08-13T15:34:14.752Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:14.752Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:14.774Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:14.774Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:14.777 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:36ab6165-0473-7fd1-8ba4-d625d35b2ba3 Address:127.0.0.1:24394}]" +34:14.777 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24394 [Follower]" leader= +34:14.778 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-36ab6165-0473-7fd1-8ba4-d625d35b2ba3.dc1 127.0.0.1 +34:14.778 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-36ab6165-0473-7fd1-8ba4-d625d35b2ba3 127.0.0.1 +2021-08-13T15:34:14.778Z [INFO] agent.router: Initializing LAN area manager +34:14.779 [INFO] TestAgent.server: Adding LAN server: server="Node-36ab6165-0473-7fd1-8ba4-d625d35b2ba3 (Addr: tcp/127.0.0.1:24394) (DC: dc1)" +34:14.779 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-36ab6165-0473-7fd1-8ba4-d625d35b2ba3.dc1 area=wan +34:14.779 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24389 network=udp +34:14.779 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24389 network=tcp +34:14.780 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24390 network=tcp +34:14.780 [INFO] TestAgent: started state syncer +34:14.780 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24395 network=tcp +34:14.829 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:14.835 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:14.835 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24394 [Candidate]" term=2 +34:14.835 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:14.835 [DEBUG] TestAgent.server.raft: vote granted: from=36ab6165-0473-7fd1-8ba4-d625d35b2ba3 term=2 tally=1 +34:14.835 [INFO] TestAgent.server.raft: election won: tally=1 +34:14.835 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24394 [Leader]" +34:14.836 [INFO] TestAgent.server: cluster leadership acquired +34:14.836 [INFO] TestAgent.server: initializing acls +34:14.836 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:14.836 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:14.836 [INFO] TestAgent.server: New leader elected: payload=Node-36ab6165-0473-7fd1-8ba4-d625d35b2ba3 +34:14.837 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:14.837 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:14.837 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:14.837 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:14.838 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-36ab6165-0473-7fd1-8ba4-d625d35b2ba3 +34:14.838 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-36ab6165-0473-7fd1-8ba4-d625d35b2ba3.dc1 +34:14.838 [INFO] TestAgent.server: Updating LAN server: server="Node-36ab6165-0473-7fd1-8ba4-d625d35b2ba3 (Addr: tcp/127.0.0.1:24394) (DC: dc1)" +34:14.838 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-36ab6165-0473-7fd1-8ba4-d625d35b2ba3.dc1 area=wan +34:14.839 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:14.841 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:14.841 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:14.841 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:14.841 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:14.841 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:14.841 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-36ab6165-0473-7fd1-8ba4-d625d35b2ba3 +34:14.841 [INFO] TestAgent.server: member joined, marking health alive: member=Node-36ab6165-0473-7fd1-8ba4-d625d35b2ba3 +34:14.842 [INFO] TestAgent.server: federation state anti-entropy synced +34:14.843 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-36ab6165-0473-7fd1-8ba4-d625d35b2ba3 +34:14.863 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-36ab6165-0473-7fd1-8ba4-d625d35b2ba3 +34:14.863 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-36ab6165-0473-7fd1-8ba4-d625d35b2ba3 +34:14.866 [INFO] TestAgent: Updated agent's ACL token: token=acl_agent_token +34:15.112 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth +34:15.113 [INFO] TestAgent: Synced node info +34:15.113 [DEBUG] TestAgent: Node info in sync +34:15.125 [INFO] TestAgent: Requesting shutdown +34:15.125 [INFO] TestAgent.server: shutting down server +34:15.125 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:15.125 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:15.125 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:15.125 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:15.125 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:15.125 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:15.125 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:15.125 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:15.125 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:15.125 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:15.125 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:15.125 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:15.125 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:15.125 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:15.125 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:15.125Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:15.125Z [INFO] agent.router.manager: shutting down +34:15.126 [INFO] TestAgent: consul server down +34:15.126 [INFO] TestAgent: shutdown complete +34:15.126 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24389 network=tcp +34:15.126 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24389 network=udp +34:15.126 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24390 network=tcp +34:15.626 [INFO] TestAgent: Waiting for endpoints to shut down +34:15.626 [INFO] TestAgent: Endpoints down +=== RUN TestAgent_TokenTriggersFullSync/agent +2021-08-13T15:34:14.700Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:14.700Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:14.766Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:14.766Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:14.769 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0a06918d-88a9-4483-f0ea-c618866eed54 Address:127.0.0.1:24401}]" +34:14.769 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24401 [Follower]" leader= +34:14.770 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0a06918d-88a9-4483-f0ea-c618866eed54.dc1 127.0.0.1 +34:14.771 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0a06918d-88a9-4483-f0ea-c618866eed54 127.0.0.1 +2021-08-13T15:34:14.771Z [INFO] agent.router: Initializing LAN area manager +34:14.771 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0a06918d-88a9-4483-f0ea-c618866eed54.dc1 area=wan +34:14.771 [INFO] TestAgent.server: Adding LAN server: server="Node-0a06918d-88a9-4483-f0ea-c618866eed54 (Addr: tcp/127.0.0.1:24401) (DC: dc1)" +34:14.771 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24396 network=udp +34:14.771 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24396 network=tcp +34:14.772 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24397 network=tcp +34:14.773 [INFO] TestAgent: started state syncer +34:14.773 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24409 network=tcp +34:14.822 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:14.836 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:14.836 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24401 [Candidate]" term=2 +34:14.837 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:14.837 [DEBUG] TestAgent.server.raft: vote granted: from=0a06918d-88a9-4483-f0ea-c618866eed54 term=2 tally=1 +34:14.837 [INFO] TestAgent.server.raft: election won: tally=1 +34:14.837 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24401 [Leader]" +34:14.837 [INFO] TestAgent.server: cluster leadership acquired +34:14.837 [INFO] TestAgent.server: initializing acls +34:14.837 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:14.837 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:14.838 [INFO] TestAgent.server: New leader elected: payload=Node-0a06918d-88a9-4483-f0ea-c618866eed54 +34:14.838 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:14.838 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:14.838 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:14.838 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:14.839 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-0a06918d-88a9-4483-f0ea-c618866eed54 +34:14.839 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-0a06918d-88a9-4483-f0ea-c618866eed54.dc1 +34:14.839 [INFO] TestAgent.server: Updating LAN server: server="Node-0a06918d-88a9-4483-f0ea-c618866eed54 (Addr: tcp/127.0.0.1:24401) (DC: dc1)" +34:14.839 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-0a06918d-88a9-4483-f0ea-c618866eed54.dc1 area=wan +34:14.840 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:14.842 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:14.842 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:14.842 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:14.842 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:14.842 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:14.842 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0a06918d-88a9-4483-f0ea-c618866eed54 +34:14.842 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0a06918d-88a9-4483-f0ea-c618866eed54 +34:14.843 [INFO] TestAgent.server: federation state anti-entropy synced +34:14.843 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0a06918d-88a9-4483-f0ea-c618866eed54 +34:14.844 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:14.844 [INFO] TestAgent: Synced node info +34:15.177 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-0a06918d-88a9-4483-f0ea-c618866eed54 +34:15.177 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-0a06918d-88a9-4483-f0ea-c618866eed54 +34:15.178 [INFO] TestAgent: Requesting shutdown +34:15.178 [INFO] TestAgent.server: shutting down server +34:15.178 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:15.178 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:15.178 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:15.178 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:15.178 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:15.178 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:15.178 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:15.178 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:15.178 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:15.178 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:15.178 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:15.178 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:15.178 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:15.178 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:15.178 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:15.178Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:15.178Z [INFO] agent.router.manager: shutting down +34:15.178 [INFO] TestAgent: consul server down +34:15.178 [INFO] TestAgent: shutdown complete +34:15.178 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24396 network=tcp +34:15.178 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24396 network=udp +34:15.178 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24397 network=tcp +34:15.679 [INFO] TestAgent: Waiting for endpoints to shut down +34:15.679 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Monitor_ACLDeny (1.00s) === CONT TestAgent_NodeMaintenance_ACLDeny -=== RUN TestAgent_Monitor/stream_JSON_logs -2022-09-15T21:42:54.037Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:54.037Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:54.070Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:54.070Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:54.073 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5a5109e3-d124-012f-ff3e-c59b97268233 Address:127.0.0.1:13106}]" -42:54.073 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13106 [Follower]" leader= -42:54.074 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5a5109e3-d124-012f-ff3e-c59b97268233.dc1 127.0.0.1 -42:54.075 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5a5109e3-d124-012f-ff3e-c59b97268233 127.0.0.1 -2022-09-15T21:42:54.075Z [INFO] agent.router: Initializing LAN area manager -42:54.075 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13101 network=udp -42:54.076 [INFO] TestAgent.server: Adding LAN server: server="Node-5a5109e3-d124-012f-ff3e-c59b97268233 (Addr: tcp/127.0.0.1:13106) (DC: dc1)" -42:54.076 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5a5109e3-d124-012f-ff3e-c59b97268233.dc1 area=wan -42:54.076 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13101 network=tcp -42:54.077 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13102 network=tcp -42:54.077 [INFO] TestAgent: started state syncer -42:54.077 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13107 network=tcp -42:54.127 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -42:54.140 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:54.140 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13106 [Candidate]" term=2 -42:54.141 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:54.141 [DEBUG] TestAgent.server.raft: vote granted: from=5a5109e3-d124-012f-ff3e-c59b97268233 term=2 tally=1 -42:54.141 [INFO] TestAgent.server.raft: election won: tally=1 -42:54.141 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13106 [Leader]" -42:54.141 [INFO] TestAgent.server: cluster leadership acquired -42:54.141 [INFO] TestAgent.server: initializing acls -42:54.141 [INFO] TestAgent.server: New leader elected: payload=Node-5a5109e3-d124-012f-ff3e-c59b97268233 -42:54.142 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:54.142 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:54.143 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:54.143 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:54.143 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:54.143 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:54.144 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-5a5109e3-d124-012f-ff3e-c59b97268233 -42:54.144 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-5a5109e3-d124-012f-ff3e-c59b97268233.dc1 -42:54.144 [INFO] TestAgent.server: Updating LAN server: server="Node-5a5109e3-d124-012f-ff3e-c59b97268233 (Addr: tcp/127.0.0.1:13106) (DC: dc1)" -42:54.144 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-5a5109e3-d124-012f-ff3e-c59b97268233.dc1 area=wan -42:54.146 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:54.149 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:54.149 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:54.149 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:54.149 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:54.149 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:54.149 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5a5109e3-d124-012f-ff3e-c59b97268233 -42:54.149 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5a5109e3-d124-012f-ff3e-c59b97268233 -42:54.150 [INFO] TestAgent.server: federation state anti-entropy synced -42:54.150 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5a5109e3-d124-012f-ff3e-c59b97268233 -42:54.249 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:54.250 [INFO] TestAgent: Synced node info -42:54.262 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-5a5109e3-d124-012f-ff3e-c59b97268233 -42:54.263 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-5a5109e3-d124-012f-ff3e-c59b97268233 -42:54.263 [INFO] TestAgent: Requesting shutdown -42:54.263 [INFO] TestAgent.server: shutting down server -42:54.263 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:54.263 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:54.263 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:54.263 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:54.263 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:54.263 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:54.263 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:54.263 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:54.263 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:54.263 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:54.263 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:54.263 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:54.263 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:54.263 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:54.263 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:54.264Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:54.264Z [INFO] agent.router.manager: shutting down -42:54.264 [INFO] TestAgent: consul server down -42:54.264 [INFO] TestAgent: shutdown complete -42:54.264 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13101 network=tcp -42:54.264 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13101 network=udp -42:54.264 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13102 network=tcp -42:54.764 [INFO] TestAgent: Waiting for endpoints to shut down -42:54.764 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Monitor_ACLDeny (0.83s) +34:15.261 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-c81d6580-cbf4-12ea-ac61-714644b0d1fc 127.0.0.1 +2021-08-13T15:34:15.261Z [INFO] agent.router: Initializing LAN area manager +34:15.261 [INFO] TestAgent.client.serf.lan: serf: Attempting re-join to previously known node: Node-6d0fd094-9d14-e467-c7d2-cf61e7b6c9f9: 127.0.0.1:25230 +34:15.261 [DEBUG] TestAgent.client.memberlist.lan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:25230: connect: connection refused +34:15.261 [INFO] TestAgent.client.serf.lan: serf: Attempting re-join to previously known node: Node-ab9514a4-5c15-81ef-db74-0769716689ad: 127.0.0.1:25258 +34:15.262 [DEBUG] TestAgent.client.memberlist.lan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:25258: connect: connection refused +34:15.262 [DEBUG] TestAgent: added local registration for service: service=web-sidecar-proxy +34:15.262 [WARN] TestAgent.client.serf.lan: serf: Failed to re-join any previously known node +2021-08-13T15:34:15.262Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:15.262Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:15.262Z [WARN] agent.router.manager: No servers available +34:15.262 [ERROR] TestAgent: error handling service update: error="error watching service config: No known Consul servers" +2021-08-13T15:34:15.262Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:15.262Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:15.262Z [WARN] agent.router.manager: No servers available +34:15.262 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24424 network=udp +34:15.262 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24424 network=tcp +2021-08-13T15:34:15.262Z [WARN] agent.router.manager: No servers available +34:15.262 [ERROR] TestAgent: error handling service update: error="error watching service config: No known Consul servers" +2021-08-13T15:34:15.262Z [WARN] agent.router.manager: No servers available +34:15.262 [ERROR] TestAgent: error handling service update: error="error watching service config: No known Consul servers" +2021-08-13T15:34:15.262Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:15.262Z [WARN] agent.router.manager: No servers available +34:15.262 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" +34:15.262 [ERROR] TestAgent: error handling service update: error="error watching service config: No known Consul servers" +2021-08-13T15:34:15.262Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:15.262Z [WARN] agent.router.manager: No servers available +34:15.262 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=discovery-chain:redis error="error filling agent cache: No known Consul servers" +2021-08-13T15:34:15.262Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:15.262Z [WARN] agent.router.manager: No servers available +34:15.262 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=discovery-chain:redis error="error filling agent cache: No known Consul servers" +2021-08-13T15:34:15.262Z [WARN] agent.router.manager: No servers available +34:15.262 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" +2021-08-13T15:34:15.262Z [WARN] agent.router.manager: No servers available +34:15.262 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" +2021-08-13T15:34:15.262Z [WARN] agent.router.manager: No servers available +34:15.263 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=leaf error="error filling agent cache: No known Consul servers" +34:15.263 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=intentions error="error filling agent cache: No known Consul servers" +2021-08-13T15:34:15.263Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:15.263Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:15.263Z [WARN] agent.router.manager: No servers available +34:15.263 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=discovery-chain:redis error="error filling agent cache: No known Consul servers" +34:15.263 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=intentions error="error filling agent cache: No known Consul servers" +2021-08-13T15:34:15.262Z [WARN] agent.router.manager: No servers available +34:15.263 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=discovery-chain:redis error="error filling agent cache: No known Consul servers" +2021-08-13T15:34:15.263Z [WARN] agent.router.manager: No servers available +34:15.263 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" +34:15.263 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=intentions error="error filling agent cache: No known Consul servers" +2021-08-13T15:34:15.263Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:15.263Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:15.263Z [WARN] agent.router.manager: No servers available +34:15.263 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" +34:15.263 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=leaf error="error filling agent cache: No known Consul servers" +34:15.263 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=intentions error="error filling agent cache: No known Consul servers" +2021-08-13T15:34:15.263Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:15.263Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:15.263Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:15.263Z [WARN] agent.router.manager: No servers available +34:15.263 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24425 network=tcp +2021-08-13T15:34:15.263Z [WARN] agent.router.manager: No servers available +34:15.263 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" +2021-08-13T15:34:15.263Z [WARN] agent.router.manager: No servers available +34:15.263 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" +34:15.263 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" +34:15.263 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=leaf error="error filling agent cache: No known Consul servers" +2021-08-13T15:34:15.263Z [WARN] agent.router.manager: No servers available +34:15.263 [INFO] TestAgent: started state syncer +34:15.263 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=leaf error="error filling agent cache: No known Consul servers" +34:15.263 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24430 network=tcp +2021-08-13T15:34:15.263Z [WARN] agent.router.manager: No servers available +34:15.263 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" +34:15.264 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy +34:15.264 [INFO] TestAgent: Requesting shutdown +34:15.264 [INFO] TestAgent.client: shutting down client +34:15.264 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave +34:15.264 [INFO] TestAgent: consul client down +34:15.264 [INFO] TestAgent: shutdown complete +34:15.264 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24424 network=tcp +34:15.264 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24424 network=udp +34:15.265 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24425 network=tcp +34:15.765 [INFO] TestAgent: Waiting for endpoints to shut down +34:15.765 [INFO] TestAgent: Endpoints down +34:11.544 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-ab9514a4-5c15-81ef-db74-0769716689ad 127.0.0.1 +2021-08-13T15:34:11.544Z [INFO] agent.router: Initializing LAN area manager +34:11.545 [DEBUG] TestAgent: added local registration for service: service=web-sidecar-proxy +2021-08-13T15:34:11.545Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:11.545Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:11.545Z [WARN] agent.router.manager: No servers available +34:11.545 [ERROR] TestAgent: error handling service update: error="error watching service config: No known Consul servers" +34:11.545 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25255 network=udp +2021-08-13T15:34:11.545Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:11.545Z [WARN] agent.router.manager: No servers available +34:11.545 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25255 network=tcp +2021-08-13T15:34:11.545Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:11.545Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:11.545Z [WARN] agent.router.manager: No servers available +34:11.545 [ERROR] TestAgent: error handling service update: error="error watching service config: No known Consul servers" +2021-08-13T15:34:11.545Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:11.545Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:11.545Z [WARN] agent.router.manager: No servers available +34:11.545 [ERROR] TestAgent: error handling service update: error="error watching service config: No known Consul servers" +34:11.545 [ERROR] TestAgent: error handling service update: error="error watching service config: No known Consul servers" +2021-08-13T15:34:11.545Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:11.545Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:11.545Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:11.545Z [WARN] agent.router.manager: No servers available +34:11.545 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" +2021-08-13T15:34:11.545Z [WARN] agent.router.manager: No servers available +34:11.545 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=discovery-chain:redis error="error filling agent cache: No known Consul servers" +2021-08-13T15:34:11.545Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:11.545Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:11.546Z [WARN] agent.router.manager: No servers available +34:11.546 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" +2021-08-13T15:34:11.546Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:11.546Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:11.546Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:11.546Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:11.546Z [WARN] agent.router.manager: No servers available +34:11.546 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" +34:11.546 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=intentions error="error filling agent cache: No known Consul servers" +2021-08-13T15:34:11.546Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:11.546Z [WARN] agent.router.manager: No servers available +34:11.546 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" +34:11.546 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25256 network=tcp +34:11.546 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=discovery-chain:redis error="error filling agent cache: No known Consul servers" +2021-08-13T15:34:11.546Z [WARN] agent.router.manager: No servers available +34:11.546 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=intentions error="error filling agent cache: No known Consul servers" +2021-08-13T15:34:11.546Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:11.546Z [WARN] agent.router.manager: No servers available +34:11.546 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" +34:11.546 [INFO] TestAgent: started state syncer +2021-08-13T15:34:11.546Z [WARN] agent.router.manager: No servers available +34:11.546 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=discovery-chain:redis error="error filling agent cache: No known Consul servers" +34:11.546 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25261 network=tcp +2021-08-13T15:34:11.546Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:11.546Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:11.546Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:11.546Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:11.546Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:11.546Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:11.546Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:11.546Z [WARN] agent.router.manager: No servers available +34:11.546 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=intentions error="error filling agent cache: No known Consul servers" +2021-08-13T15:34:11.546Z [WARN] agent.router.manager: No servers available +34:11.546 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=leaf error="error filling agent cache: No known Consul servers" +34:11.546 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=intentions error="error filling agent cache: No known Consul servers" +2021-08-13T15:34:11.546Z [WARN] agent.router.manager: No servers available +34:11.546 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=leaf error="error filling agent cache: No known Consul servers" +2021-08-13T15:34:11.546Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:11.546Z [WARN] agent.router.manager: No servers available +34:11.546 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=discovery-chain:redis error="error filling agent cache: No known Consul servers" +2021-08-13T15:34:11.546Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:11.546Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:11.552Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:11.546Z [WARN] agent.router.manager: No servers available +34:11.552 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" +2021-08-13T15:34:11.547Z [WARN] agent.router.manager: No servers available +34:11.552 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" +34:11.552 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" +2021-08-13T15:34:11.552Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:11.552Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:11.552Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:11.552Z [WARN] agent.router.manager: No servers available +34:11.552 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=leaf error="error filling agent cache: No known Consul servers" +34:11.552 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" +2021-08-13T15:34:11.552Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:11.553Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:11.553Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:11.553Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:11.553Z [WARN] agent.router.manager: No servers available +2021-08-13T15:34:11.553Z [WARN] agent.router.manager: No servers available +34:11.553 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=leaf error="error filling agent cache: No known Consul servers" +34:11.553 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:25230] +34:11.553 [DEBUG] TestAgent.client.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:25230 +34:11.559 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-6d0fd094-9d14-e467-c7d2-cf61e7b6c9f9 127.0.0.1 +34:11.560 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 +34:11.560 [DEBUG] TestAgent: systemd notify failed: error="No socket" +2021-08-13T15:34:11.560Z [WARN] agent.router.manager: No servers available +34:11.560 [INFO] TestAgent.client: adding server: server="Node-6d0fd094-9d14-e467-c7d2-cf61e7b6c9f9 (Addr: tcp/127.0.0.1:25232) (DC: dc1)" +34:11.732 [DEBUG] TestAgent.client.serf.lan: serf: messageUserEventType: consul:new-leader +34:11.932 [DEBUG] TestAgent.client.serf.lan: serf: messageUserEventType: consul:new-leader +34:11.932 [DEBUG] TestAgent.client.serf.lan: serf: messageJoinType: Node-ab9514a4-5c15-81ef-db74-0769716689ad +34:12.131 [DEBUG] TestAgent.client.serf.lan: serf: messageUserEventType: consul:new-leader +34:12.132 [DEBUG] TestAgent.client.serf.lan: serf: messageJoinType: Node-ab9514a4-5c15-81ef-db74-0769716689ad +34:12.334 [DEBUG] TestAgent.client.serf.lan: serf: messageJoinType: Node-ab9514a4-5c15-81ef-db74-0769716689ad +34:12.334 [DEBUG] TestAgent.client.serf.lan: serf: messageUserEventType: consul:new-leader +34:12.334 [DEBUG] TestAgent.client.serf.lan: serf: messageJoinType: Node-ab9514a4-5c15-81ef-db74-0769716689ad +34:13.442 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:13.443 [INFO] TestAgent: Synced node info +34:13.444 [INFO] TestAgent: Synced service: service=web-sidecar-proxy +34:13.444 [DEBUG] TestAgent: Node info in sync +34:13.444 [DEBUG] TestAgent: Service in sync: service=web-sidecar-proxy +34:14.189 [DEBUG] TestAgent: Node info in sync +34:14.191 [INFO] TestAgent: Synced service: service=web-sidecar-proxy +34:14.191 [DEBUG] TestAgent: Node info in sync +34:14.191 [DEBUG] TestAgent: Service in sync: service=web-sidecar-proxy +34:14.193 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" +34:14.193 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" +34:14.193 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" +34:14.193 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" +34:14.204 [INFO] TestAgent: Requesting shutdown +34:14.204 [INFO] TestAgent.client: shutting down client +34:14.204 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave +34:14.204 [INFO] TestAgent: consul client down +34:14.204 [INFO] TestAgent: shutdown complete +34:14.204 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25255 network=tcp +34:14.204 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25255 network=udp +34:14.204 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25256 network=tcp +34:14.204 [ERROR] TestAgent.client: RPC failed to server: method=ConnectCA.Roots server=127.0.0.1:25232 error="rpc error making call: EOF" +34:14.204 [ERROR] TestAgent.client: RPC failed to server: method=ConfigEntry.ResolveServiceConfig server=127.0.0.1:25232 error="rpc error making call: EOF" +34:14.204 [ERROR] TestAgent.client: RPC failed to server: method=Intention.Match server=127.0.0.1:25232 error="rpc error making call: EOF" +34:14.204 [ERROR] TestAgent.client: RPC failed to server: method=Health.ServiceNodes server=127.0.0.1:25232 error="rpc error making call: EOF" +34:14.204 [ERROR] TestAgent.client: RPC failed to server: method=DiscoveryChain.Get server=127.0.0.1:25232 error="rpc error making call: EOF" +34:14.205 [ERROR] TestAgent.client: RPC failed to server: method=ConnectCA.Roots server=127.0.0.1:25232 error="rpc error making call: EOF" +34:14.715 [INFO] TestAgent: Waiting for endpoints to shut down +34:14.715 [INFO] TestAgent: Endpoints down +2021-08-13T15:34:11.310Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:11.326Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:11.329 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6d0fd094-9d14-e467-c7d2-cf61e7b6c9f9 Address:127.0.0.1:25232}]" +34:11.329 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25232 [Follower]" leader= +34:11.330 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6d0fd094-9d14-e467-c7d2-cf61e7b6c9f9.dc1 127.0.0.1 +34:11.330 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6d0fd094-9d14-e467-c7d2-cf61e7b6c9f9 127.0.0.1 +2021-08-13T15:34:11.330Z [INFO] agent.router: Initializing LAN area manager +34:11.331 [INFO] TestAgent.server: Adding LAN server: server="Node-6d0fd094-9d14-e467-c7d2-cf61e7b6c9f9 (Addr: tcp/127.0.0.1:25232) (DC: dc1)" +34:11.331 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6d0fd094-9d14-e467-c7d2-cf61e7b6c9f9.dc1 area=wan +34:11.331 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25227 network=udp +34:11.331 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25227 network=tcp +34:11.332 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25228 network=tcp +34:11.332 [INFO] TestAgent: started state syncer +34:11.332 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25233 network=tcp +34:11.393 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:11.393 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25232 [Candidate]" term=2 +34:11.393 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:11.393 [DEBUG] TestAgent.server.raft: vote granted: from=6d0fd094-9d14-e467-c7d2-cf61e7b6c9f9 term=2 tally=1 +34:11.393 [INFO] TestAgent.server.raft: election won: tally=1 +34:11.393 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25232 [Leader]" +34:11.393 [INFO] TestAgent.server: cluster leadership acquired +34:11.393 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25232 +34:11.395 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:11.396 [INFO] TestAgent.server: New leader elected: payload=Node-6d0fd094-9d14-e467-c7d2-cf61e7b6c9f9 +34:11.397 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:11.397 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:11.397 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:11.397 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:11.397 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:11.397 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6d0fd094-9d14-e467-c7d2-cf61e7b6c9f9 +34:11.397 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6d0fd094-9d14-e467-c7d2-cf61e7b6c9f9 +34:11.399 [INFO] TestAgent.server: federation state anti-entropy synced +34:11.554 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:47888 +34:11.559 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ab9514a4-5c15-81ef-db74-0769716689ad 127.0.0.1 +34:11.563 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ab9514a4-5c15-81ef-db74-0769716689ad +34:11.719 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:11.720 [INFO] TestAgent: Synced node info +34:11.746 [DEBUG] TestAgent.server.serf.lan: serf: messageJoinType: Node-ab9514a4-5c15-81ef-db74-0769716689ad +34:11.946 [DEBUG] TestAgent.server.serf.lan: serf: messageJoinType: Node-ab9514a4-5c15-81ef-db74-0769716689ad +34:12.145 [DEBUG] TestAgent.server.serf.lan: serf: messageJoinType: Node-ab9514a4-5c15-81ef-db74-0769716689ad +34:12.334 [DEBUG] TestAgent.server.serf.lan: serf: messageJoinType: Node-ab9514a4-5c15-81ef-db74-0769716689ad +34:13.415 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:13.415 [DEBUG] TestAgent: Node info in sync +34:13.415 [DEBUG] TestAgent: Node info in sync +34:14.715 [INFO] TestAgent: Requesting shutdown +34:14.716 [INFO] TestAgent.server: shutting down server +34:14.716 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:14.716 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:14.716 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:14.716 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:14.716 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:14.717 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:14.717 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:14.717 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:14.717 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:14.717 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:14.717 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:14.723Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:14.723Z [INFO] agent.router.manager: shutting down +34:14.724 [INFO] TestAgent: consul server down +34:14.724 [INFO] TestAgent: shutdown complete +34:14.724 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25227 network=tcp +34:14.724 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25227 network=udp +34:14.724 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25228 network=tcp +34:14.831 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Failed ping: Node-ab9514a4-5c15-81ef-db74-0769716689ad (timeout reached) +34:15.224 [INFO] TestAgent: Waiting for endpoints to shut down +34:15.224 [INFO] TestAgent: Endpoints down +34:15.331 [INFO] TestAgent.server.memberlist.lan: memberlist: Suspect Node-ab9514a4-5c15-81ef-db74-0769716689ad has failed, no acks received +--- PASS: TestServiceManager_PersistService_ConfigFiles (4.47s) +=== CONT TestPreparedQuery_Get +=== RUN TestPreparedQuery_Get/#00 +[WARN] freeport: 10 out of 52 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:14.931Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:14.947Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:14.950 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:90ef4b6c-0017-5e41-f449-9829c352a2c8 Address:127.0.0.1:24408}]" +34:14.950 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24408 [Follower]" leader= +34:14.951 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-90ef4b6c-0017-5e41-f449-9829c352a2c8.dc1 127.0.0.1 +34:14.952 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-90ef4b6c-0017-5e41-f449-9829c352a2c8 127.0.0.1 +2021-08-13T15:34:14.952Z [INFO] agent.router: Initializing LAN area manager +34:14.952 [INFO] TestAgent.server: Adding LAN server: server="Node-90ef4b6c-0017-5e41-f449-9829c352a2c8 (Addr: tcp/127.0.0.1:24408) (DC: dc1)" +34:14.952 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-90ef4b6c-0017-5e41-f449-9829c352a2c8.dc1 area=wan +34:14.953 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24403 network=udp +34:14.953 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24403 network=tcp +34:14.953 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24404 network=tcp +34:14.954 [INFO] TestAgent: started state syncer +34:14.954 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24423 network=tcp +34:15.020 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:15.020 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24408 [Candidate]" term=2 +34:15.020 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:15.020 [DEBUG] TestAgent.server.raft: vote granted: from=90ef4b6c-0017-5e41-f449-9829c352a2c8 term=2 tally=1 +34:15.020 [INFO] TestAgent.server.raft: election won: tally=1 +34:15.020 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24408 [Leader]" +34:15.020 [INFO] TestAgent.server: cluster leadership acquired +34:15.020 [INFO] TestAgent.server: New leader elected: payload=Node-90ef4b6c-0017-5e41-f449-9829c352a2c8 +34:15.020 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24408 +34:15.022 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:15.024 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:15.024 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:15.024 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:15.024 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:15.024 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:15.024 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-90ef4b6c-0017-5e41-f449-9829c352a2c8 +34:15.024 [INFO] TestAgent.server: member joined, marking health alive: member=Node-90ef4b6c-0017-5e41-f449-9829c352a2c8 +34:15.025 [INFO] TestAgent.server: federation state anti-entropy synced +34:15.113 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:15.114 [INFO] TestAgent: Synced node info +34:15.356 [DEBUG] TestAgent: Node info in sync +34:15.358 [INFO] TestAgent: Synced service: service=memcache +34:15.358 [DEBUG] TestAgent: Check in sync: check=service:memcache +34:15.358 [DEBUG] TestAgent: Node info in sync +34:15.358 [DEBUG] TestAgent: Service in sync: service=memcache +34:15.358 [DEBUG] TestAgent: Check in sync: check=service:memcache +34:15.359 [INFO] TestAgent: Synced check: check=memcache_check2 +34:15.359 [INFO] TestAgent: Requesting shutdown +34:15.359 [INFO] TestAgent.server: shutting down server +34:15.359 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:15.359 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:15.359 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:15.359 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:15.359 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:15.359 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:15.359 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:15.359 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:15.359 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:15.359 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:15.359 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:15.359Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:15.360Z [INFO] agent.router.manager: shutting down +34:15.360 [INFO] TestAgent: consul server down +34:15.360 [INFO] TestAgent: shutdown complete +34:15.360 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24403 network=tcp +34:15.360 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24403 network=udp +34:15.360 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24404 network=tcp +34:15.860 [INFO] TestAgent: Waiting for endpoints to shut down +34:15.860 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_RegisterCheck_Service (0.95s) === CONT TestAgent_NodeMaintenance_Disable -[WARN] freeport: 4 out of 32 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:53.945Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:53.945Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:54.041Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:54.041Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:54.047 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b06d512d-a582-784a-6fdd-ddbc99174846 Address:127.0.0.1:13092}]" -42:54.047 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13092 [Follower]" leader= -42:54.056 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b06d512d-a582-784a-6fdd-ddbc99174846.dc1 127.0.0.1 -42:54.057 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b06d512d-a582-784a-6fdd-ddbc99174846 127.0.0.1 -2022-09-15T21:42:54.057Z [INFO] agent.router: Initializing LAN area manager -42:54.057 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b06d512d-a582-784a-6fdd-ddbc99174846.dc1 area=wan -42:54.057 [INFO] TestAgent.server: Adding LAN server: server="Node-b06d512d-a582-784a-6fdd-ddbc99174846 (Addr: tcp/127.0.0.1:13092) (DC: dc1)" -42:54.058 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13087 network=udp -42:54.062 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13087 network=tcp -42:54.065 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13088 network=tcp -42:54.065 [INFO] TestAgent: started state syncer -42:54.065 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13093 network=tcp -42:54.090 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:54.090 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13092 [Candidate]" term=2 -42:54.090 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:54.090 [DEBUG] TestAgent.server.raft: vote granted: from=b06d512d-a582-784a-6fdd-ddbc99174846 term=2 tally=1 -42:54.090 [INFO] TestAgent.server.raft: election won: tally=1 -42:54.090 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13092 [Leader]" -42:54.090 [INFO] TestAgent.server: cluster leadership acquired -42:54.091 [INFO] TestAgent.server: initializing acls -42:54.091 [INFO] TestAgent.server: New leader elected: payload=Node-b06d512d-a582-784a-6fdd-ddbc99174846 -42:54.091 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:54.091 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:54.092 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:54.092 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:54.093 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:54.093 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:54.093 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-b06d512d-a582-784a-6fdd-ddbc99174846 -42:54.093 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-b06d512d-a582-784a-6fdd-ddbc99174846.dc1 -42:54.094 [INFO] TestAgent.server: Updating LAN server: server="Node-b06d512d-a582-784a-6fdd-ddbc99174846 (Addr: tcp/127.0.0.1:13092) (DC: dc1)" -42:54.094 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-b06d512d-a582-784a-6fdd-ddbc99174846.dc1 area=wan -42:54.095 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:54.099 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:54.099 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:54.099 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:54.099 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:54.099 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:54.099 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b06d512d-a582-784a-6fdd-ddbc99174846 -42:54.099 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b06d512d-a582-784a-6fdd-ddbc99174846 -42:54.100 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b06d512d-a582-784a-6fdd-ddbc99174846 -42:54.101 [INFO] TestAgent.server: federation state anti-entropy synced -42:54.149 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-b06d512d-a582-784a-6fdd-ddbc99174846 -42:54.149 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-b06d512d-a582-784a-6fdd-ddbc99174846 -42:54.152 [INFO] TestAgent: Updated agent's ACL token: token=acl_agent_token -42:54.289 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth -42:54.290 [INFO] TestAgent: Synced node info -42:54.308 [INFO] TestAgent: Requesting shutdown -42:54.308 [INFO] TestAgent.server: shutting down server -42:54.308 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:54.308 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:54.308 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:54.308 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:54.308 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:54.308 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:54.308 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:54.308 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:54.308 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:54.308 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:54.308 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:54.309 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:54.309 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:54.309 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:54.309 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:54.309Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:54.309Z [INFO] agent.router.manager: shutting down -42:54.309 [INFO] TestAgent: consul server down -42:54.309 [INFO] TestAgent: shutdown complete -42:54.309 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13087 network=tcp -42:54.309 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13087 network=udp -42:54.309 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13088 network=tcp -42:54.809 [INFO] TestAgent: Waiting for endpoints to shut down -42:54.809 [INFO] TestAgent: Endpoints down -=== RUN TestAgent_TokenTriggersFullSync/agent -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:14.810Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:14.828Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:14.831 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f996d1c5-b789-b750-a5bd-383dd7e2272f Address:127.0.0.1:24422}]" +34:14.831 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24422 [Follower]" leader= +34:14.832 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f996d1c5-b789-b750-a5bd-383dd7e2272f.dc1 127.0.0.1 +34:14.833 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f996d1c5-b789-b750-a5bd-383dd7e2272f 127.0.0.1 +2021-08-13T15:34:14.833Z [INFO] agent.router: Initializing LAN area manager +34:14.833 [INFO] TestAgent.server: Adding LAN server: server="Node-f996d1c5-b789-b750-a5bd-383dd7e2272f (Addr: tcp/127.0.0.1:24422) (DC: dc1)" +34:14.833 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f996d1c5-b789-b750-a5bd-383dd7e2272f.dc1 area=wan +34:14.833 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24410 network=tcp +34:14.833 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24410 network=udp +34:14.834 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24411 network=tcp +34:14.834 [INFO] TestAgent: started state syncer +34:14.834 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24402 network=tcp +34:14.899 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:14.899 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24422 [Candidate]" term=2 +34:14.899 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:14.899 [DEBUG] TestAgent.server.raft: vote granted: from=f996d1c5-b789-b750-a5bd-383dd7e2272f term=2 tally=1 +34:14.899 [INFO] TestAgent.server.raft: election won: tally=1 +34:14.899 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24422 [Leader]" +34:14.899 [INFO] TestAgent.server: cluster leadership acquired +34:14.899 [INFO] TestAgent.server: New leader elected: payload=Node-f996d1c5-b789-b750-a5bd-383dd7e2272f +34:14.899 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24422 +34:14.901 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:14.903 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:14.903 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:14.903 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:14.903 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:14.903 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:14.903 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f996d1c5-b789-b750-a5bd-383dd7e2272f +34:14.903 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f996d1c5-b789-b750-a5bd-383dd7e2272f +34:14.904 [INFO] TestAgent.server: federation state anti-entropy synced +34:15.162 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:15.163 [INFO] TestAgent: Synced node info +34:15.196 [DEBUG] TestAgent: Node info in sync +34:15.197 [INFO] TestAgent: Synced service: service=monitor +34:15.197 [DEBUG] TestAgent: Check in sync: check=service:monitor +34:15.298 [DEBUG] TestAgent: Node info in sync +34:15.300 [INFO] TestAgent: Synced service: service=monitor +34:15.300 [DEBUG] TestAgent: Check in sync: check=service:monitor +34:15.400 [INFO] TestAgent: Requesting shutdown +34:15.401 [INFO] TestAgent.server: shutting down server +34:15.401 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:15.401 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:15.401 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:15.401 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:15.401 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:15.401 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:15.401 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:15.401 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:15.401 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:15.401 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:15.401 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:15.402Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:15.402Z [INFO] agent.router.manager: shutting down +34:15.402 [INFO] TestAgent: consul server down +34:15.402 [INFO] TestAgent: shutdown complete +34:15.402 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24410 network=tcp +34:15.402 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24410 network=udp +34:15.402 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24411 network=tcp +34:15.902 [INFO] TestAgent: Waiting for endpoints to shut down +34:15.902 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Monitor (1.21s) + --- PASS: TestAgent_Monitor/unknown_log_level (0.00s) + --- PASS: TestAgent_Monitor/stream_unstructured_logs (0.10s) + --- PASS: TestAgent_Monitor/stream_JSON_logs (0.10s) + --- PASS: TestAgent_Monitor/serf_shutdown_logging (0.60s) +=== CONT TestPreparedQuery_Explain +=== RUN TestPreparedQuery_Explain/#00 === RUN TestAgent_NodeMaintenance_ACLDeny/no_token === RUN TestAgent_NodeMaintenance_ACLDeny/root_token -[WARN] freeport: 4 out of 4 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:54.784Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:54.803Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:54.806 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5edab77d-bde1-1b3c-d4c8-af2120b5f38f Address:127.0.0.1:13190}]" -42:54.807 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5edab77d-bde1-1b3c-d4c8-af2120b5f38f.dc1 127.0.0.1 -42:54.808 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5edab77d-bde1-1b3c-d4c8-af2120b5f38f 127.0.0.1 -2022-09-15T21:42:54.808Z [INFO] agent.router: Initializing LAN area manager -42:54.808 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13171 network=udp -42:54.808 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13190 [Follower]" leader= -42:54.809 [INFO] TestAgent.server: Adding LAN server: server="Node-5edab77d-bde1-1b3c-d4c8-af2120b5f38f (Addr: tcp/127.0.0.1:13190) (DC: dc1)" -42:54.809 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5edab77d-bde1-1b3c-d4c8-af2120b5f38f.dc1 area=wan -42:54.814 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13171 network=tcp -42:54.815 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13172 network=tcp -42:54.815 [INFO] TestAgent: started state syncer -42:54.815 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13191 network=tcp -42:54.854 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:54.854 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13190 [Candidate]" term=2 -42:54.854 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:54.854 [DEBUG] TestAgent.server.raft: vote granted: from=5edab77d-bde1-1b3c-d4c8-af2120b5f38f term=2 tally=1 -42:54.854 [INFO] TestAgent.server.raft: election won: tally=1 -42:54.854 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13190 [Leader]" -42:54.854 [INFO] TestAgent.server: cluster leadership acquired -42:54.854 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13190 -42:54.856 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:54.856 [INFO] TestAgent.server: New leader elected: payload=Node-5edab77d-bde1-1b3c-d4c8-af2120b5f38f -42:54.858 [INFO] TestAgent: Synced node info -42:54.858 [DEBUG] TestAgent: Node info in sync -42:54.858 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:54.858 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:54.858 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:54.858 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:54.858 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:54.858 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5edab77d-bde1-1b3c-d4c8-af2120b5f38f -42:54.858 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5edab77d-bde1-1b3c-d4c8-af2120b5f38f -42:54.859 [INFO] TestAgent.server: federation state anti-entropy synced -42:54.870 [INFO] TestAgent: Node entered maintenance mode -42:54.870 [DEBUG] TestAgent: removed check: check=_node_maintenance -42:54.870 [INFO] TestAgent: Node left maintenance mode -42:54.870 [DEBUG] TestAgent: Node info in sync -42:54.871 [INFO] TestAgent: Deregistered check: check=_node_maintenance -42:54.871 [INFO] TestAgent: Requesting shutdown -42:54.871 [INFO] TestAgent.server: shutting down server -42:54.871 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:54.871 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:54.871 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:54.871 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:54.871 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:54.871 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:54.871 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:54.871 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:54.871 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:54.871 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:54.871 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:54.872Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:54.872Z [INFO] agent.router.manager: shutting down -42:54.872 [INFO] TestAgent: consul server down -42:54.872 [INFO] TestAgent: shutdown complete -42:54.872 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13171 network=tcp -42:54.872 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13171 network=udp -42:54.872 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13172 network=tcp -42:55.372 [INFO] TestAgent: Waiting for endpoints to shut down -42:55.372 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_NodeMaintenance_Disable (0.61s) +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:15.611Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:15.627Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:15.630 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:361706ef-ae2b-3507-b609-bc53c4ba741a Address:127.0.0.1:24436}]" +34:15.630 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24436 [Follower]" leader= +34:15.631 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-361706ef-ae2b-3507-b609-bc53c4ba741a.dc1 127.0.0.1 +34:15.632 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-361706ef-ae2b-3507-b609-bc53c4ba741a 127.0.0.1 +2021-08-13T15:34:15.632Z [INFO] agent.router: Initializing LAN area manager +34:15.632 [INFO] TestAgent.server: Adding LAN server: server="Node-361706ef-ae2b-3507-b609-bc53c4ba741a (Addr: tcp/127.0.0.1:24436) (DC: dc1)" +34:15.632 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-361706ef-ae2b-3507-b609-bc53c4ba741a.dc1 area=wan +34:15.633 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24431 network=udp +34:15.633 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24431 network=tcp +34:15.634 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24432 network=tcp +34:15.634 [INFO] TestAgent: started state syncer +34:15.634 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24444 network=tcp +34:15.667 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:15.667 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24436 [Candidate]" term=2 +34:15.667 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:15.667 [DEBUG] TestAgent.server.raft: vote granted: from=361706ef-ae2b-3507-b609-bc53c4ba741a term=2 tally=1 +34:15.667 [INFO] TestAgent.server.raft: election won: tally=1 +34:15.667 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24436 [Leader]" +34:15.668 [INFO] TestAgent.server: cluster leadership acquired +34:15.668 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24436 +34:15.670 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:15.675 [INFO] TestAgent.server: New leader elected: payload=Node-361706ef-ae2b-3507-b609-bc53c4ba741a +34:15.682 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:15.682 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:15.682 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:15.682 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:15.682 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:15.682 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-361706ef-ae2b-3507-b609-bc53c4ba741a +34:15.682 [INFO] TestAgent.server: member joined, marking health alive: member=Node-361706ef-ae2b-3507-b609-bc53c4ba741a +34:15.686 [WARN] TestAgent.server: endpoint injected; this should only be used for testing +34:15.687 [INFO] TestAgent: Requesting shutdown +34:15.687 [INFO] TestAgent.server: shutting down server +34:15.687 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:15.687 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:15.687 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:15.687 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:15.687 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:15.687 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:15.687 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:15.688 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:15.688 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:15.688 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:15.688Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:15.688Z [INFO] agent.router.manager: shutting down +34:15.696 [INFO] TestAgent.server: federation state anti-entropy synced +34:15.696 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:15.696 [INFO] TestAgent: consul server down +34:15.697 [INFO] TestAgent: shutdown complete +34:15.697 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24431 network=tcp +34:15.697 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24431 network=udp +34:15.698 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24432 network=tcp +34:16.198 [INFO] TestAgent: Waiting for endpoints to shut down +34:16.198 [INFO] TestAgent: Endpoints down +--- PASS: TestPreparedQuery_Update (0.60s) === CONT TestAgent_NodeMaintenance_Enable -2022-09-15T21:42:54.556Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:54.577Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:54.580 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cef8343f-1da6-7a40-ced3-2410d56251a7 Address:127.0.0.1:13148}]" -42:54.580 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13148 [Follower]" leader= -42:54.581 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-cef8343f-1da6-7a40-ced3-2410d56251a7.dc1 127.0.0.1 -42:54.582 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-cef8343f-1da6-7a40-ced3-2410d56251a7 127.0.0.1 -2022-09-15T21:42:54.582Z [INFO] agent.router: Initializing LAN area manager -42:54.583 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13136 network=udp -42:54.583 [INFO] TestAgent.server: Adding LAN server: server="Node-cef8343f-1da6-7a40-ced3-2410d56251a7 (Addr: tcp/127.0.0.1:13148) (DC: dc1)" -42:54.583 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-cef8343f-1da6-7a40-ced3-2410d56251a7.dc1 area=wan -42:54.583 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13136 network=tcp -42:54.584 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13137 network=tcp -42:54.584 [INFO] TestAgent: started state syncer -42:54.584 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13149 network=tcp -42:54.626 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:54.626 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13148 [Candidate]" term=2 -42:54.626 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:54.626 [DEBUG] TestAgent.server.raft: vote granted: from=cef8343f-1da6-7a40-ced3-2410d56251a7 term=2 tally=1 -42:54.626 [INFO] TestAgent.server.raft: election won: tally=1 -42:54.626 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13148 [Leader]" -42:54.626 [INFO] TestAgent.server: cluster leadership acquired -42:54.627 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13148 -42:54.628 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:54.629 [INFO] TestAgent.server: New leader elected: payload=Node-cef8343f-1da6-7a40-ced3-2410d56251a7 -42:54.631 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:54.631 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:54.631 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:54.631 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:54.631 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:54.631 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cef8343f-1da6-7a40-ced3-2410d56251a7 -42:54.631 [INFO] TestAgent.server: member joined, marking health alive: member=Node-cef8343f-1da6-7a40-ced3-2410d56251a7 -42:54.632 [INFO] TestAgent.server: federation state anti-entropy synced -42:54.653 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:54.653 [INFO] TestAgent: Synced node info -42:54.673 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:54.673 [DEBUG] TestAgent: Node info in sync -42:54.673 [DEBUG] TestAgent: Node info in sync -42:54.925 [DEBUG] TestAgent: Node info in sync -42:54.926 [INFO] TestAgent: Synced service: service=memcache -42:54.926 [DEBUG] TestAgent: Check in sync: check=service:memcache -42:54.927 [DEBUG] TestAgent: Node info in sync -42:54.927 [DEBUG] TestAgent: Service in sync: service=memcache -42:54.927 [DEBUG] TestAgent: Check in sync: check=service:memcache -42:54.928 [INFO] TestAgent: Synced check: check=memcache_check2 -42:54.928 [INFO] TestAgent: Requesting shutdown -42:54.928 [INFO] TestAgent.server: shutting down server -42:54.928 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:54.928 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:54.928 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:54.928 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:54.928 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:54.928 [DEBUG] TestAgent: Node info in sync -42:54.928 [DEBUG] TestAgent: Service in sync: service=memcache -42:54.928 [DEBUG] TestAgent: Check in sync: check=service:memcache -42:54.928 [DEBUG] TestAgent: Check in sync: check=memcache_check2 -42:54.928 [DEBUG] TestAgent: Node info in sync -42:54.928 [DEBUG] TestAgent: Service in sync: service=memcache -42:54.928 [DEBUG] TestAgent: Check in sync: check=service:memcache -42:54.928 [DEBUG] TestAgent: Check in sync: check=memcache_check2 -42:54.928 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:54.928 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:54.928 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:54.928 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:54.928 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -42:54.928 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2022-09-15T21:42:54.929Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:54.929Z [INFO] agent.router.manager: shutting down -42:54.929 [INFO] TestAgent: consul server down -42:54.929 [INFO] TestAgent: shutdown complete -42:54.929 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13136 network=tcp -42:54.929 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13136 network=udp -42:54.929 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13137 network=tcp -42:55.432 [INFO] TestAgent: Waiting for endpoints to shut down -42:55.432 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_RegisterCheck_Service (0.91s) -=== CONT TestAgent_NodeMaintenance_BadRequest -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -42:55.095 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-bbe53cb0-90b8-8704-ba2c-012faccbade2 127.0.0.1 -2022-09-15T21:42:55.096Z [INFO] agent.router: Initializing LAN area manager -42:55.096 [DEBUG] TestAgent: added local registration for service: service=web-sidecar-proxy -42:55.096 [INFO] TestAgent.client.serf.lan: serf: Attempting re-join to previously known node: Node-63bad055-7de5-a87f-943c-7036a3647e75: 127.0.0.1:14012 -42:55.097 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13143 network=udp -2022-09-15T21:42:55.097Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.097Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.097Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.097Z [WARN] agent.router.manager: No servers available -42:55.097 [DEBUG] TestAgent.client.memberlist.lan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:14012: connect: connection refused -42:55.097 [INFO] TestAgent.client.serf.lan: serf: Attempting re-join to previously known node: Node-af92f623-df8c-6a2a-ffad-ebe3171ee740: 127.0.0.1:14040 -42:55.097 [DEBUG] TestAgent.client.memberlist.lan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:14040: connect: connection refused -42:55.097 [WARN] TestAgent.client.serf.lan: serf: Failed to re-join any previously known node -42:55.097 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13143 network=tcp -2022-09-15T21:42:55.097Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.097Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.097Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.097Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.097Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.097Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.097Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.097Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.098Z [WARN] agent.router.manager: No servers available -42:55.098 [ERROR] TestAgent: error handling service update: error="error watching service config: No known Consul servers" -2022-09-15T21:42:55.098Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.098Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.098Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.098Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.098Z [WARN] agent.router.manager: No servers available -42:55.098 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" -2022-09-15T21:42:55.098Z [WARN] agent.router.manager: No servers available -42:55.098 [ERROR] TestAgent: error handling service update: error="error watching service config: No known Consul servers" -42:55.098 [ERROR] TestAgent: error handling service update: error="error watching service config: No known Consul servers" -2022-09-15T21:42:55.098Z [WARN] agent.router.manager: No servers available -42:55.098 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=intentions error="error filling agent cache: No known Consul servers" -2022-09-15T21:42:55.098Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.098Z [WARN] agent.router.manager: No servers available -42:55.098 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" -2022-09-15T21:42:55.098Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.098Z [WARN] agent.router.manager: No servers available -42:55.098 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" -2022-09-15T21:42:55.098Z [WARN] agent.router.manager: No servers available -42:55.098 [ERROR] TestAgent: error handling service update: error="error watching service config: No known Consul servers" -42:55.098 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=discovery-chain:redis error="error filling agent cache: No known Consul servers" -42:55.098 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=intentions error="error filling agent cache: No known Consul servers" -2022-09-15T21:42:55.098Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.098Z [WARN] agent.router.manager: No servers available -42:55.098 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13144 network=tcp -2022-09-15T21:42:55.098Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.098Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.098Z [WARN] agent.router.manager: No servers available -42:55.098 [INFO] TestAgent: started state syncer -2022-09-15T21:42:55.098Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.098Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.098Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.098Z [WARN] agent.router.manager: No servers available -42:55.098 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13156 network=tcp -2022-09-15T21:42:55.098Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.098Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.098Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.099Z [WARN] agent.router.manager: No servers available -42:55.099 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" -2022-09-15T21:42:55.099Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.099Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.099Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.099Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.099Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.099Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.099Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.099Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.099Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.099Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.099Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.099Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.099Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.099Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.099Z [WARN] agent.router.manager: No servers available -42:55.098 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" -42:55.099 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" -42:55.099 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=discovery-chain:redis error="error filling agent cache: No known Consul servers" -42:55.099 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=leaf error="error filling agent cache: No known Consul servers" -42:55.099 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" -42:55.099 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=discovery-chain:redis error="error filling agent cache: No known Consul servers" -42:55.099 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=intentions error="error filling agent cache: No known Consul servers" -42:55.099 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=discovery-chain:redis error="error filling agent cache: No known Consul servers" -42:55.099 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=leaf error="error filling agent cache: No known Consul servers" -42:55.099 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" -42:55.099 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" -42:55.099 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=leaf error="error filling agent cache: No known Consul servers" -42:55.099 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=intentions error="error filling agent cache: No known Consul servers" -42:55.099 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy -2022-09-15T21:42:55.099Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.099Z [WARN] agent.router.manager: No servers available -42:55.099 [INFO] TestAgent: Requesting shutdown -2022-09-15T21:42:55.099Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:55.099Z [WARN] agent.router.manager: No servers available -42:55.099 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=leaf error="error filling agent cache: No known Consul servers" -42:55.099 [INFO] TestAgent.client: shutting down client -42:55.099 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave -42:55.100 [INFO] TestAgent: consul client down -42:55.100 [INFO] TestAgent: shutdown complete -42:55.100 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13143 network=tcp -42:55.100 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13143 network=udp -42:55.100 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13144 network=tcp -42:55.600 [INFO] TestAgent: Waiting for endpoints to shut down -42:55.600 [INFO] TestAgent: Endpoints down -42:51.676 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-af92f623-df8c-6a2a-ffad-ebe3171ee740 127.0.0.1 -2022-09-15T21:42:51.676Z [INFO] agent.router: Initializing LAN area manager -42:51.677 [DEBUG] TestAgent: added local registration for service: service=web-sidecar-proxy -42:51.677 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14037 network=udp -2022-09-15T21:42:51.678Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.678Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.678Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.678Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.678Z [WARN] agent.router.manager: No servers available -42:51.679 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" -2022-09-15T21:42:51.679Z [WARN] agent.router.manager: No servers available -42:51.682 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" -42:51.682 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" -42:51.682 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" -42:51.680 [ERROR] TestAgent: error handling service update: error="error watching service config: No known Consul servers" -42:51.682 [ERROR] TestAgent: error handling service update: error="error watching service config: No known Consul servers" -42:51.680 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14037 network=tcp -2022-09-15T21:42:51.679Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.682Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.682Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.682Z [WARN] agent.router.manager: No servers available -42:51.682 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" -2022-09-15T21:42:51.682Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.682Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.682Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.679Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.682Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.682Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.682Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.682Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.682Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.682Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.682Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.683Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.683Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.683Z [WARN] agent.router.manager: No servers available -42:51.683 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14038 network=tcp -2022-09-15T21:42:51.683Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.679Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.683Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.683Z [WARN] agent.router.manager: No servers available -42:51.683 [INFO] TestAgent: started state syncer -2022-09-15T21:42:51.683Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.683Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.683Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.683Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.683Z [WARN] agent.router.manager: No servers available -42:51.683 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=leaf error="error filling agent cache: No known Consul servers" -42:51.683 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" -42:51.683 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=intentions error="error filling agent cache: No known Consul servers" -2022-09-15T21:42:51.683Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.683Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.683Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.683Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.683Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.683Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.683Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.683Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.682Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.683Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.683Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.683Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.683Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.684Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.684Z [WARN] agent.router.manager: No servers available -42:51.684 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:14012] -2022-09-15T21:42:51.684Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.684Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.684Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.684Z [WARN] agent.router.manager: No servers available -42:51.684 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=leaf error="error filling agent cache: No known Consul servers" -42:51.684 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" -42:51.684 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=discovery-chain:redis error="error filling agent cache: No known Consul servers" -42:51.684 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=intentions error="error filling agent cache: No known Consul servers" -42:51.684 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14050 network=tcp -2022-09-15T21:42:51.684Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.684Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.684Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.684Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.684Z [WARN] agent.router.manager: No servers available -42:51.684 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" -2022-09-15T21:42:51.684Z [WARN] agent.router.manager: No servers available -42:51.684 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=leaf error="error filling agent cache: No known Consul servers" -42:51.684 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=discovery-chain:redis error="error filling agent cache: No known Consul servers" -42:51.684 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=intentions error="error filling agent cache: No known Consul servers" -2022-09-15T21:42:51.684Z [WARN] agent.router.manager: No servers available -42:51.684 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=leaf error="error filling agent cache: No known Consul servers" -2022-09-15T21:42:51.684Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.684Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.684Z [WARN] agent.router.manager: No servers available -42:51.684 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=discovery-chain:redis error="error filling agent cache: No known Consul servers" -42:51.684 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=discovery-chain:redis error="error filling agent cache: No known Consul servers" -42:51.684 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=intentions error="error filling agent cache: No known Consul servers" -2022-09-15T21:42:51.684Z [WARN] agent.router.manager: No servers available -42:51.684 [ERROR] TestAgent: error handling service update: error="error watching service config: No known Consul servers" -2022-09-15T21:42:51.684Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.684Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.684Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.684Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.684Z [WARN] agent.router.manager: No servers available -2022-09-15T21:42:51.684Z [WARN] agent.router.manager: No servers available -42:51.684 [ERROR] TestAgent: error handling service update: error="error watching service config: No known Consul servers" -42:51.685 [DEBUG] TestAgent.client.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:14012 -42:51.687 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-63bad055-7de5-a87f-943c-7036a3647e75 127.0.0.1 -42:51.688 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 -42:51.688 [DEBUG] TestAgent: systemd notify failed: error="No socket" -2022-09-15T21:42:51.688Z [WARN] agent.router.manager: No servers available -42:51.688 [INFO] TestAgent.client: adding server: server="Node-63bad055-7de5-a87f-943c-7036a3647e75 (Addr: tcp/127.0.0.1:13986) (DC: dc1)" -42:51.688 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" -42:51.752 [DEBUG] TestAgent.client.serf.lan: serf: messageUserEventType: consul:new-leader -42:51.952 [DEBUG] TestAgent.client.serf.lan: serf: messageUserEventType: consul:new-leader -42:51.952 [DEBUG] TestAgent.client.serf.lan: serf: messageJoinType: Node-af92f623-df8c-6a2a-ffad-ebe3171ee740 -42:52.153 [DEBUG] TestAgent.client.serf.lan: serf: messageUserEventType: consul:new-leader -42:52.153 [DEBUG] TestAgent.client.serf.lan: serf: messageJoinType: Node-af92f623-df8c-6a2a-ffad-ebe3171ee740 -42:52.153 [DEBUG] TestAgent.client.serf.lan: serf: messageUserEventType: consul:new-leader -42:52.153 [DEBUG] TestAgent.client.serf.lan: serf: messageJoinType: Node-af92f623-df8c-6a2a-ffad-ebe3171ee740 -42:52.355 [DEBUG] TestAgent.client.serf.lan: serf: messageJoinType: Node-af92f623-df8c-6a2a-ffad-ebe3171ee740 -42:54.004 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" -42:54.004 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" -42:54.004 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" -42:54.004 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" -42:54.010 [INFO] TestAgent: Requesting shutdown -42:54.010 [INFO] TestAgent.client: shutting down client -42:54.010 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave -2022-09-15T21:42:54.011Z [ERROR] agent: yamux: Failed to read stream data: read tcp 127.0.0.1:35883->127.0.0.1:13986: use of closed network connection -2022-09-15T21:42:54.011Z [WARN] agent: yamux: failed to send go away: session shutdown -42:54.011 [INFO] TestAgent: consul client down -42:54.011 [INFO] TestAgent: shutdown complete -42:54.011 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14037 network=tcp -42:54.011 [ERROR] TestAgent.client: RPC failed to server: method=ConnectCA.Roots server=127.0.0.1:13986 error="rpc error making call: EOF" -42:54.011 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14037 network=udp -42:54.011 [ERROR] TestAgent.client: RPC failed to server: method=DiscoveryChain.Get server=127.0.0.1:13986 error="rpc error making call: EOF" -42:54.011 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14038 network=tcp -42:54.011 [ERROR] TestAgent.client: RPC failed to server: method=Intention.Match server=127.0.0.1:13986 error="rpc error making call: EOF" -42:54.011 [ERROR] TestAgent.client: RPC failed to server: method=ConnectCA.Roots server=127.0.0.1:13986 error="rpc error making call: EOF" -42:54.011 [ERROR] TestAgent.client: RPC failed to server: method=ConfigEntry.ResolveServiceConfig server=127.0.0.1:13986 error="rpc error making call: EOF" -42:54.511 [INFO] TestAgent: Waiting for endpoints to shut down -42:54.511 [INFO] TestAgent: Endpoints down -2022-09-15T21:42:51.127Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:51.145Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:51.149 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:63bad055-7de5-a87f-943c-7036a3647e75 Address:127.0.0.1:13986}]" -42:51.150 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13986 [Follower]" leader= -42:51.150 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-63bad055-7de5-a87f-943c-7036a3647e75.dc1 127.0.0.1 -42:51.151 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-63bad055-7de5-a87f-943c-7036a3647e75 127.0.0.1 -2022-09-15T21:42:51.151Z [INFO] agent.router: Initializing LAN area manager -42:51.151 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14009 network=udp -42:51.152 [INFO] TestAgent.server: Adding LAN server: server="Node-63bad055-7de5-a87f-943c-7036a3647e75 (Addr: tcp/127.0.0.1:13986) (DC: dc1)" -42:51.152 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-63bad055-7de5-a87f-943c-7036a3647e75.dc1 area=wan -42:51.152 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14009 network=tcp -42:51.153 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14010 network=tcp -42:51.153 [INFO] TestAgent: started state syncer -42:51.153 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13987 network=tcp -42:51.202 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:51.202 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13986 [Candidate]" term=2 -42:51.202 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:51.202 [DEBUG] TestAgent.server.raft: vote granted: from=63bad055-7de5-a87f-943c-7036a3647e75 term=2 tally=1 -42:51.202 [INFO] TestAgent.server.raft: election won: tally=1 -42:51.202 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13986 [Leader]" -42:51.202 [INFO] TestAgent.server: cluster leadership acquired -42:51.202 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13986 -42:51.203 [INFO] TestAgent.server: New leader elected: payload=Node-63bad055-7de5-a87f-943c-7036a3647e75 -42:51.204 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:51.212 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:51.212 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:51.212 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:51.212 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:51.212 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:51.212 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-63bad055-7de5-a87f-943c-7036a3647e75 -42:51.212 [INFO] TestAgent.server: member joined, marking health alive: member=Node-63bad055-7de5-a87f-943c-7036a3647e75 -42:51.214 [INFO] TestAgent.server: federation state anti-entropy synced -42:51.430 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:51.431 [INFO] TestAgent: Synced node info -42:51.493 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:51.493 [DEBUG] TestAgent: Node info in sync -42:51.493 [DEBUG] TestAgent: Node info in sync -42:51.685 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:33326 -42:51.686 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-af92f623-df8c-6a2a-ffad-ebe3171ee740 127.0.0.1 -42:51.686 [INFO] TestAgent.server: member joined, marking health alive: member=Node-af92f623-df8c-6a2a-ffad-ebe3171ee740 -42:51.878 [DEBUG] TestAgent.server.serf.lan: serf: messageJoinType: Node-af92f623-df8c-6a2a-ffad-ebe3171ee740 -42:52.078 [DEBUG] TestAgent.server.serf.lan: serf: messageJoinType: Node-af92f623-df8c-6a2a-ffad-ebe3171ee740 -42:52.153 [DEBUG] TestAgent.server.serf.lan: serf: messageJoinType: Node-af92f623-df8c-6a2a-ffad-ebe3171ee740 -42:52.278 [DEBUG] TestAgent.server.serf.lan: serf: messageJoinType: Node-af92f623-df8c-6a2a-ffad-ebe3171ee740 -42:54.022 [ERROR] TestAgent.server: yamux: Failed to write body: write tcp 127.0.0.1:13986->127.0.0.1:35883: use of closed network connection -42:54.511 [INFO] TestAgent: Requesting shutdown -42:54.511 [INFO] TestAgent.server: shutting down server -42:54.511 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:54.511 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:54.511 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:54.511 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:54.511 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:54.512 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:54.512 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:54.512 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:54.512 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:54.512 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:54.512 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:54.512Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:54.512Z [INFO] agent.router.manager: shutting down -42:54.513 [INFO] TestAgent: consul server down -42:54.513 [INFO] TestAgent: shutdown complete -42:54.513 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14009 network=tcp -42:54.513 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14009 network=udp -42:54.513 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14010 network=tcp -42:54.652 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Failed ping: Node-af92f623-df8c-6a2a-ffad-ebe3171ee740 (timeout reached) -42:55.013 [INFO] TestAgent: Waiting for endpoints to shut down -42:55.013 [INFO] TestAgent: Endpoints down -42:55.152 [INFO] TestAgent.server.memberlist.lan: memberlist: Suspect Node-af92f623-df8c-6a2a-ffad-ebe3171ee740 has failed, no acks received ---- PASS: TestServiceManager_PersistService_ConfigFiles (4.51s) -=== CONT TestAgent_ServiceMaintenance_ACLDeny -2022-09-15T21:42:54.712Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:54.712Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:54.734Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:54.734Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:54.737 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3f6ceb61-9bce-dbb6-0a4d-0e8471cdf31e Address:127.0.0.1:13176}]" -42:54.738 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13176 [Follower]" leader= -42:54.738 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3f6ceb61-9bce-dbb6-0a4d-0e8471cdf31e.dc1 127.0.0.1 -42:54.739 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3f6ceb61-9bce-dbb6-0a4d-0e8471cdf31e 127.0.0.1 -2022-09-15T21:42:54.739Z [INFO] agent.router: Initializing LAN area manager -42:54.740 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13157 network=udp -42:54.740 [INFO] TestAgent.server: Adding LAN server: server="Node-3f6ceb61-9bce-dbb6-0a4d-0e8471cdf31e (Addr: tcp/127.0.0.1:13176) (DC: dc1)" -42:54.740 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3f6ceb61-9bce-dbb6-0a4d-0e8471cdf31e.dc1 area=wan -42:54.740 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13157 network=tcp -42:54.741 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13158 network=tcp -42:54.741 [INFO] TestAgent: started state syncer -42:54.742 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13177 network=tcp -42:54.782 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:54.782 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13176 [Candidate]" term=2 -42:54.783 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:54.783 [DEBUG] TestAgent.server.raft: vote granted: from=3f6ceb61-9bce-dbb6-0a4d-0e8471cdf31e term=2 tally=1 -42:54.783 [INFO] TestAgent.server.raft: election won: tally=1 -42:54.783 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13176 [Leader]" -42:54.783 [INFO] TestAgent.server: cluster leadership acquired -42:54.783 [INFO] TestAgent.server: initializing acls -42:54.784 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:54.784 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:54.784 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:54.785 [INFO] TestAgent.server: New leader elected: payload=Node-3f6ceb61-9bce-dbb6-0a4d-0e8471cdf31e -42:54.785 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:54.785 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:54.785 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:54.785 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-3f6ceb61-9bce-dbb6-0a4d-0e8471cdf31e -42:54.785 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-3f6ceb61-9bce-dbb6-0a4d-0e8471cdf31e.dc1 -42:54.785 [INFO] TestAgent.server: Updating LAN server: server="Node-3f6ceb61-9bce-dbb6-0a4d-0e8471cdf31e (Addr: tcp/127.0.0.1:13176) (DC: dc1)" -42:54.786 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-3f6ceb61-9bce-dbb6-0a4d-0e8471cdf31e.dc1 area=wan -42:54.786 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:54.789 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:54.789 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:54.789 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:54.789 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:54.789 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:54.789 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3f6ceb61-9bce-dbb6-0a4d-0e8471cdf31e -42:54.789 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3f6ceb61-9bce-dbb6-0a4d-0e8471cdf31e -42:54.790 [INFO] TestAgent.server: federation state anti-entropy synced -42:54.791 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3f6ceb61-9bce-dbb6-0a4d-0e8471cdf31e -42:54.873 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:54.874 [INFO] TestAgent: Synced node info -42:54.874 [DEBUG] TestAgent: Node info in sync -42:55.130 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-3f6ceb61-9bce-dbb6-0a4d-0e8471cdf31e -42:55.130 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-3f6ceb61-9bce-dbb6-0a4d-0e8471cdf31e -42:55.131 [INFO] TestAgent: Node entered maintenance mode -42:55.131 [DEBUG] TestAgent: Node info in sync -42:55.132 [INFO] TestAgent: Synced check: check=_node_maintenance -42:55.132 [INFO] TestAgent: Requesting shutdown -42:55.132 [INFO] TestAgent.server: shutting down server -42:55.132 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:55.132 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:55.132 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:55.132 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:55.132 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:55.132 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:55.132 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:55.132 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:55.132 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -42:55.132 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:55.133 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:55.133 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:55.133 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:55.133 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:55.133 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -2022-09-15T21:42:55.133Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:55.133Z [INFO] agent.router.manager: shutting down -42:55.133 [INFO] TestAgent: consul server down -42:55.133 [INFO] TestAgent: shutdown complete -42:55.133 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13157 network=tcp -42:55.133 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13157 network=udp -42:55.133 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13158 network=tcp -42:55.633 [INFO] TestAgent: Waiting for endpoints to shut down -42:55.633 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_NodeMaintenance_ACLDeny (0.94s) +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:15.705Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:15.705Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:15.724Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:15.724Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:15.727 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:58c7334d-8e1e-2375-e46e-6cb9cb9d5f86 Address:127.0.0.1:24464}]" +34:15.727 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24464 [Follower]" leader= +34:15.728 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-58c7334d-8e1e-2375-e46e-6cb9cb9d5f86.dc1 127.0.0.1 +34:15.729 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-58c7334d-8e1e-2375-e46e-6cb9cb9d5f86 127.0.0.1 +2021-08-13T15:34:15.729Z [INFO] agent.router: Initializing LAN area manager +34:15.729 [INFO] TestAgent.server: Adding LAN server: server="Node-58c7334d-8e1e-2375-e46e-6cb9cb9d5f86 (Addr: tcp/127.0.0.1:24464) (DC: dc1)" +34:15.729 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-58c7334d-8e1e-2375-e46e-6cb9cb9d5f86.dc1 area=wan +34:15.729 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24459 network=tcp +34:15.729 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24459 network=udp +34:15.730 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24460 network=tcp +34:15.730 [INFO] TestAgent: started state syncer +34:15.730 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24451 network=tcp +34:15.779 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:15.797 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:15.797 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24464 [Candidate]" term=2 +34:15.797 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:15.797 [DEBUG] TestAgent.server.raft: vote granted: from=58c7334d-8e1e-2375-e46e-6cb9cb9d5f86 term=2 tally=1 +34:15.797 [INFO] TestAgent.server.raft: election won: tally=1 +34:15.797 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24464 [Leader]" +34:15.797 [INFO] TestAgent.server: cluster leadership acquired +34:15.798 [INFO] TestAgent.server: New leader elected: payload=Node-58c7334d-8e1e-2375-e46e-6cb9cb9d5f86 +34:15.798 [INFO] TestAgent.server: initializing acls +34:15.798 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:15.798 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:15.799 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:15.800 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:15.800 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:15.800 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:15.800 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-58c7334d-8e1e-2375-e46e-6cb9cb9d5f86 +34:15.800 [INFO] TestAgent.server: Updating LAN server: server="Node-58c7334d-8e1e-2375-e46e-6cb9cb9d5f86 (Addr: tcp/127.0.0.1:24464) (DC: dc1)" +34:15.800 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-58c7334d-8e1e-2375-e46e-6cb9cb9d5f86.dc1 +34:15.800 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-58c7334d-8e1e-2375-e46e-6cb9cb9d5f86.dc1 area=wan +34:15.801 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:15.804 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:15.804 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:15.804 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:15.804 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:15.804 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:15.804 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-58c7334d-8e1e-2375-e46e-6cb9cb9d5f86 +34:15.804 [INFO] TestAgent.server: member joined, marking health alive: member=Node-58c7334d-8e1e-2375-e46e-6cb9cb9d5f86 +34:15.805 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-58c7334d-8e1e-2375-e46e-6cb9cb9d5f86 +34:15.805 [INFO] TestAgent.server: federation state anti-entropy synced +34:16.006 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-58c7334d-8e1e-2375-e46e-6cb9cb9d5f86 +34:16.007 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-58c7334d-8e1e-2375-e46e-6cb9cb9d5f86 +34:16.008 [INFO] TestAgent: Node entered maintenance mode +34:16.009 [INFO] TestAgent: Synced node info +34:16.010 [INFO] TestAgent: Synced check: check=_node_maintenance +34:16.010 [INFO] TestAgent: Requesting shutdown +34:16.010 [INFO] TestAgent.server: shutting down server +34:16.010 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:16.010 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:16.010 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:16.010 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:16.010 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:16.010 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:16.010 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:16.010 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:16.010 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:16.010 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:16.010 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:16.010 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:16.010 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:16.010 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:16.010 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:16.010 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:16.010Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:16.010Z [INFO] agent.router.manager: shutting down +34:16.010 [INFO] TestAgent: consul server down +34:16.010 [INFO] TestAgent: shutdown complete +34:16.010 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24459 network=tcp +34:16.010 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24459 network=udp +34:16.010 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24460 network=tcp +34:16.511 [INFO] TestAgent: Waiting for endpoints to shut down +34:16.511 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_NodeMaintenance_ACLDeny (0.83s) --- PASS: TestAgent_NodeMaintenance_ACLDeny/no_token (0.00s) --- PASS: TestAgent_NodeMaintenance_ACLDeny/root_token (0.00s) -=== CONT TestAgent_ServiceMaintenance_Disable -2022-09-15T21:42:54.829Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:54.829Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:54.851Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:54.851Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:54.860 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:34f7c173-30cf-0687-3511-7366dac3b8fc Address:127.0.0.1:13169}]" -42:54.861 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-34f7c173-30cf-0687-3511-7366dac3b8fc.dc1 127.0.0.1 -42:54.862 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-34f7c173-30cf-0687-3511-7366dac3b8fc 127.0.0.1 -2022-09-15T21:42:54.862Z [INFO] agent.router: Initializing LAN area manager -42:54.862 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13169 [Follower]" leader= -42:54.862 [INFO] TestAgent.server: Adding LAN server: server="Node-34f7c173-30cf-0687-3511-7366dac3b8fc (Addr: tcp/127.0.0.1:13169) (DC: dc1)" -42:54.862 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-34f7c173-30cf-0687-3511-7366dac3b8fc.dc1 area=wan -42:54.863 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13178 network=tcp -42:54.863 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13178 network=udp -42:54.864 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13179 network=tcp -42:54.864 [INFO] TestAgent: started state syncer -42:54.864 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13170 network=tcp -42:54.902 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:54.902 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13169 [Candidate]" term=2 -42:54.902 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:54.902 [DEBUG] TestAgent.server.raft: vote granted: from=34f7c173-30cf-0687-3511-7366dac3b8fc term=2 tally=1 -42:54.902 [INFO] TestAgent.server.raft: election won: tally=1 -42:54.902 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13169 [Leader]" -42:54.902 [INFO] TestAgent.server: cluster leadership acquired -42:54.903 [INFO] TestAgent.server: initializing acls -42:54.903 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:54.903 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:54.904 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:54.904 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:54.904 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:54.904 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:54.905 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-34f7c173-30cf-0687-3511-7366dac3b8fc -42:54.905 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-34f7c173-30cf-0687-3511-7366dac3b8fc.dc1 -42:54.906 [INFO] TestAgent.server: New leader elected: payload=Node-34f7c173-30cf-0687-3511-7366dac3b8fc -42:54.906 [INFO] TestAgent.server: Updating LAN server: server="Node-34f7c173-30cf-0687-3511-7366dac3b8fc (Addr: tcp/127.0.0.1:13169) (DC: dc1)" -42:54.906 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-34f7c173-30cf-0687-3511-7366dac3b8fc.dc1 area=wan -42:54.907 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:54.909 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:54.909 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:54.909 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:54.909 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:54.909 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:54.910 [INFO] TestAgent.server: federation state anti-entropy synced -42:54.911 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-34f7c173-30cf-0687-3511-7366dac3b8fc -42:54.911 [INFO] TestAgent.server: member joined, marking health alive: member=Node-34f7c173-30cf-0687-3511-7366dac3b8fc -42:54.912 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-34f7c173-30cf-0687-3511-7366dac3b8fc -42:55.049 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-34f7c173-30cf-0687-3511-7366dac3b8fc -42:55.049 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-34f7c173-30cf-0687-3511-7366dac3b8fc -42:55.057 [INFO] TestAgent: Updated agent's ACL token: token=agent -42:55.246 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth -42:55.247 [INFO] TestAgent: Synced node info -42:55.247 [DEBUG] TestAgent: Node info in sync -42:55.254 [INFO] TestAgent: Requesting shutdown -42:55.254 [INFO] TestAgent.server: shutting down server -42:55.254 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:55.254 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:55.254 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:55.254 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:55.254 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:55.254 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:55.254 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:55.254 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:55.254 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:55.254 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:55.254 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:55.254 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:55.254 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:55.255 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:55.255 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:55.255Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:55.255Z [INFO] agent.router.manager: shutting down -42:55.255 [INFO] TestAgent: consul server down -42:55.255 [INFO] TestAgent: shutdown complete -42:55.255 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13178 network=tcp -42:55.255 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13178 network=udp -42:55.255 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13179 network=tcp -42:55.755 [INFO] TestAgent: Waiting for endpoints to shut down -42:55.755 [INFO] TestAgent: Endpoints down -=== RUN TestAgent_TokenTriggersFullSync/acl_token +=== CONT TestPreparedQuery_ExecuteCached +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:15.877Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:15.893Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:15.896 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e3558692-946e-0a40-ce4a-a419aaa208b4 Address:127.0.0.1:24429}]" +34:15.896 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24429 [Follower]" leader= +34:15.897 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e3558692-946e-0a40-ce4a-a419aaa208b4.dc1 127.0.0.1 +34:15.898 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e3558692-946e-0a40-ce4a-a419aaa208b4 127.0.0.1 +2021-08-13T15:34:15.898Z [INFO] agent.router: Initializing LAN area manager +34:15.898 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e3558692-946e-0a40-ce4a-a419aaa208b4.dc1 area=wan +34:15.898 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24417 network=tcp +34:15.898 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24417 network=udp +34:15.899 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24418 network=tcp +34:15.899 [INFO] TestAgent: started state syncer +34:15.899 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24479 network=tcp +34:15.899 [INFO] TestAgent.server: Adding LAN server: server="Node-e3558692-946e-0a40-ce4a-a419aaa208b4 (Addr: tcp/127.0.0.1:24429) (DC: dc1)" +34:15.955 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:15.955 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24429 [Candidate]" term=2 +34:15.955 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:15.955 [DEBUG] TestAgent.server.raft: vote granted: from=e3558692-946e-0a40-ce4a-a419aaa208b4 term=2 tally=1 +34:15.955 [INFO] TestAgent.server.raft: election won: tally=1 +34:15.955 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24429 [Leader]" +34:15.955 [INFO] TestAgent.server: cluster leadership acquired +34:15.955 [INFO] TestAgent.server: New leader elected: payload=Node-e3558692-946e-0a40-ce4a-a419aaa208b4 +34:15.956 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24429 +34:15.957 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:15.960 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:15.960 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:15.960 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:15.960 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:15.960 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:15.960 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e3558692-946e-0a40-ce4a-a419aaa208b4 +34:15.960 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e3558692-946e-0a40-ce4a-a419aaa208b4 +34:15.961 [INFO] TestAgent.server: federation state anti-entropy synced +34:16.072 [INFO] TestAgent: Node entered maintenance mode +34:16.072 [DEBUG] TestAgent: removed check: check=_node_maintenance +34:16.072 [INFO] TestAgent: Node left maintenance mode +34:16.073 [INFO] TestAgent: Synced node info +34:16.073 [INFO] TestAgent: Deregistered check: check=_node_maintenance +34:16.073 [INFO] TestAgent: Requesting shutdown +34:16.073 [INFO] TestAgent.server: shutting down server +34:16.073 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:16.073 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:16.073 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:16.073 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:16.073 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:16.073 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:16.073 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:16.073 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:16.073 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:16.073 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:16.073 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:16.073 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:16.073Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:16.073Z [INFO] agent.router.manager: shutting down +34:16.074 [INFO] TestAgent: consul server down +34:16.074 [INFO] TestAgent: shutdown complete +34:16.074 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24417 network=tcp +34:16.074 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24417 network=udp +34:16.074 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24418 network=tcp +34:16.575 [INFO] TestAgent: Waiting for endpoints to shut down +34:16.575 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_NodeMaintenance_Disable (0.71s) +=== CONT TestAgent_NodeMaintenance_BadRequest +2021-08-13T15:34:15.782Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:15.797Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:15.800 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:52d26183-1c39-f98c-e0a3-e2c32a772f75 Address:127.0.0.1:24457}]" +34:15.800 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24457 [Follower]" leader= +34:15.801 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-52d26183-1c39-f98c-e0a3-e2c32a772f75.dc1 127.0.0.1 +34:15.802 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-52d26183-1c39-f98c-e0a3-e2c32a772f75 127.0.0.1 +2021-08-13T15:34:15.802Z [INFO] agent.router: Initializing LAN area manager +34:15.802 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24452 network=udp +34:15.802 [INFO] TestAgent.server: Adding LAN server: server="Node-52d26183-1c39-f98c-e0a3-e2c32a772f75 (Addr: tcp/127.0.0.1:24457) (DC: dc1)" +34:15.802 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-52d26183-1c39-f98c-e0a3-e2c32a772f75.dc1 area=wan +34:15.803 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24452 network=tcp +34:15.804 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24453 network=tcp +34:15.804 [INFO] TestAgent: started state syncer +34:15.804 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24416 network=tcp +34:15.846 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:15.846 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24457 [Candidate]" term=2 +34:15.846 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:15.846 [DEBUG] TestAgent.server.raft: vote granted: from=52d26183-1c39-f98c-e0a3-e2c32a772f75 term=2 tally=1 +34:15.846 [INFO] TestAgent.server.raft: election won: tally=1 +34:15.846 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24457 [Leader]" +34:15.846 [INFO] TestAgent.server: cluster leadership acquired +34:15.847 [INFO] TestAgent.server: New leader elected: payload=Node-52d26183-1c39-f98c-e0a3-e2c32a772f75 +34:15.847 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24457 +34:15.848 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:15.851 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:15.851 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:15.851 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:15.851 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:15.851 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:15.851 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-52d26183-1c39-f98c-e0a3-e2c32a772f75 +34:15.851 [INFO] TestAgent.server: member joined, marking health alive: member=Node-52d26183-1c39-f98c-e0a3-e2c32a772f75 +34:15.851 [INFO] TestAgent.server: federation state anti-entropy synced +34:16.084 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:16.085 [INFO] TestAgent: Synced node info +34:16.085 [DEBUG] TestAgent: Node info in sync +34:16.141 [WARN] TestAgent.server: endpoint injected; this should only be used for testing +34:16.141 [INFO] TestAgent: Requesting shutdown +34:16.141 [INFO] TestAgent.server: shutting down server +34:16.141 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:16.141 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:16.141 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:16.141 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:16.141 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:16.141 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:16.141 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:16.141 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:16.141 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:16.141 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:16.141 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:16.141Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:16.142Z [INFO] agent.router.manager: shutting down +34:16.142 [INFO] TestAgent: consul server down +34:16.142 [INFO] TestAgent: shutdown complete +34:16.142 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24452 network=tcp +34:16.142 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24452 network=udp +34:16.142 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24453 network=tcp +34:16.643 [INFO] TestAgent: Waiting for endpoints to shut down +34:16.643 [INFO] TestAgent: Endpoints down +=== RUN TestPreparedQuery_Get/#01 +2021-08-13T15:34:16.022Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:16.048Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:16.060 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ab249734-9ad1-77ba-33c6-553fc2d36a0e Address:127.0.0.1:24492}]" +34:16.060 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24492 [Follower]" leader= +34:16.061 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ab249734-9ad1-77ba-33c6-553fc2d36a0e.dc1 127.0.0.1 +34:16.062 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ab249734-9ad1-77ba-33c6-553fc2d36a0e 127.0.0.1 +2021-08-13T15:34:16.062Z [INFO] agent.router: Initializing LAN area manager +34:16.062 [INFO] TestAgent.server: Adding LAN server: server="Node-ab249734-9ad1-77ba-33c6-553fc2d36a0e (Addr: tcp/127.0.0.1:24492) (DC: dc1)" +34:16.062 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ab249734-9ad1-77ba-33c6-553fc2d36a0e.dc1 area=wan +34:16.062 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24480 network=udp +34:16.062 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24480 network=tcp +34:16.063 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24481 network=tcp +34:16.063 [INFO] TestAgent: started state syncer +34:16.063 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24472 network=tcp +34:16.101 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:16.101 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24492 [Candidate]" term=2 +34:16.101 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:16.101 [DEBUG] TestAgent.server.raft: vote granted: from=ab249734-9ad1-77ba-33c6-553fc2d36a0e term=2 tally=1 +34:16.101 [INFO] TestAgent.server.raft: election won: tally=1 +34:16.101 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24492 [Leader]" +34:16.101 [INFO] TestAgent.server: cluster leadership acquired +34:16.102 [INFO] TestAgent.server: New leader elected: payload=Node-ab249734-9ad1-77ba-33c6-553fc2d36a0e +34:16.102 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24492 +34:16.103 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:16.106 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:16.106 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:16.106 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:16.106 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:16.106 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:16.106 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ab249734-9ad1-77ba-33c6-553fc2d36a0e +34:16.106 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ab249734-9ad1-77ba-33c6-553fc2d36a0e +34:16.107 [INFO] TestAgent.server: federation state anti-entropy synced +34:16.131 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:16.132 [INFO] TestAgent: Synced node info +34:16.132 [DEBUG] TestAgent: Node info in sync +34:16.158 [WARN] TestAgent.server: endpoint injected; this should only be used for testing +34:16.158 [INFO] TestAgent: Requesting shutdown +34:16.158 [INFO] TestAgent.server: shutting down server +34:16.158 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:16.158 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:16.158 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:16.158 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:16.158 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:16.158 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:16.158 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:16.158 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:16.158 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:16.158 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:16.158 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:16.158Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:16.158Z [INFO] agent.router.manager: shutting down +34:16.159 [INFO] TestAgent: consul server down +34:16.159 [INFO] TestAgent: shutdown complete +34:16.159 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24480 network=tcp +34:16.159 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24480 network=udp +34:16.159 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24481 network=tcp +34:16.659 [INFO] TestAgent: Waiting for endpoints to shut down +34:16.659 [INFO] TestAgent: Endpoints down +=== RUN TestPreparedQuery_Explain/#01 +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:16.214Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:16.230Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:16.233 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:527b48be-9d7a-7b64-6c34-a5e5f0f063d2 Address:127.0.0.1:24478}]" +34:16.233 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24478 [Follower]" leader= +34:16.234 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-527b48be-9d7a-7b64-6c34-a5e5f0f063d2.dc1 127.0.0.1 +34:16.235 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-527b48be-9d7a-7b64-6c34-a5e5f0f063d2 127.0.0.1 +2021-08-13T15:34:16.235Z [INFO] agent.router: Initializing LAN area manager +34:16.235 [INFO] TestAgent.server: Adding LAN server: server="Node-527b48be-9d7a-7b64-6c34-a5e5f0f063d2 (Addr: tcp/127.0.0.1:24478) (DC: dc1)" +34:16.235 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-527b48be-9d7a-7b64-6c34-a5e5f0f063d2.dc1 area=wan +34:16.235 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24473 network=udp +34:16.235 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24473 network=tcp +34:16.236 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24474 network=tcp +34:16.236 [INFO] TestAgent: started state syncer +34:16.236 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24493 network=tcp +34:16.293 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:16.293 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24478 [Candidate]" term=2 +34:16.294 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:16.294 [DEBUG] TestAgent.server.raft: vote granted: from=527b48be-9d7a-7b64-6c34-a5e5f0f063d2 term=2 tally=1 +34:16.294 [INFO] TestAgent.server.raft: election won: tally=1 +34:16.294 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24478 [Leader]" +34:16.294 [INFO] TestAgent.server: cluster leadership acquired +34:16.294 [INFO] TestAgent.server: New leader elected: payload=Node-527b48be-9d7a-7b64-6c34-a5e5f0f063d2 +34:16.294 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24478 +34:16.295 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:16.298 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:16.298 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:16.298 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:16.298 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:16.298 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:16.298 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-527b48be-9d7a-7b64-6c34-a5e5f0f063d2 +34:16.298 [INFO] TestAgent.server: member joined, marking health alive: member=Node-527b48be-9d7a-7b64-6c34-a5e5f0f063d2 +34:16.298 [INFO] TestAgent.server: federation state anti-entropy synced +34:16.325 [INFO] TestAgent: Node entered maintenance mode +34:16.325 [INFO] TestAgent: Synced node info +34:16.326 [INFO] TestAgent: Synced check: check=_node_maintenance +34:16.326 [INFO] TestAgent: Requesting shutdown +34:16.326 [INFO] TestAgent.server: shutting down server +34:16.326 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:16.326 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:16.326 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:16.326 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:16.326 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:16.326 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:16.326 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:16.326 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:16.326 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:16.326 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:16.326 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:16.326 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:16.326Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:16.326Z [INFO] agent.router.manager: shutting down +34:16.327 [INFO] TestAgent: consul server down +34:16.327 [INFO] TestAgent: shutdown complete +34:16.327 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24473 network=tcp +34:16.327 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24473 network=udp +34:16.327 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24474 network=tcp +34:16.827 [INFO] TestAgent: Waiting for endpoints to shut down +34:16.827 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_NodeMaintenance_Enable (0.63s) +=== CONT TestAgent_ServiceMaintenance_ACLDeny +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! === RUN TestAgent_ServiceMaintenance_ACLDeny/no_token === RUN TestAgent_ServiceMaintenance_ACLDeny/root_token -[WARN] freeport: 4 out of 39 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:55.392Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:55.418Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:55.421 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0e2557e3-b904-ead0-ef37-11f1fb09916c Address:127.0.0.1:13162}]" -42:55.421 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13162 [Follower]" leader= -42:55.422 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0e2557e3-b904-ead0-ef37-11f1fb09916c.dc1 127.0.0.1 -42:55.423 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0e2557e3-b904-ead0-ef37-11f1fb09916c 127.0.0.1 -2022-09-15T21:42:55.423Z [INFO] agent.router: Initializing LAN area manager -42:55.424 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13150 network=udp -42:55.424 [INFO] TestAgent.server: Adding LAN server: server="Node-0e2557e3-b904-ead0-ef37-11f1fb09916c (Addr: tcp/127.0.0.1:13162) (DC: dc1)" -42:55.424 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0e2557e3-b904-ead0-ef37-11f1fb09916c.dc1 area=wan -42:55.424 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13150 network=tcp -42:55.425 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13151 network=tcp -42:55.425 [INFO] TestAgent: started state syncer -42:55.425 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13163 network=tcp -42:55.478 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:55.478 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13162 [Candidate]" term=2 -42:55.478 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:55.478 [DEBUG] TestAgent.server.raft: vote granted: from=0e2557e3-b904-ead0-ef37-11f1fb09916c term=2 tally=1 -42:55.478 [INFO] TestAgent.server.raft: election won: tally=1 -42:55.478 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13162 [Leader]" -42:55.478 [INFO] TestAgent.server: cluster leadership acquired -42:55.479 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13162 -42:55.479 [INFO] TestAgent.server: New leader elected: payload=Node-0e2557e3-b904-ead0-ef37-11f1fb09916c -42:55.480 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:55.483 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:55.483 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:55.483 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:55.483 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:55.483 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:55.483 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0e2557e3-b904-ead0-ef37-11f1fb09916c -42:55.483 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0e2557e3-b904-ead0-ef37-11f1fb09916c -42:55.484 [INFO] TestAgent.server: federation state anti-entropy synced -42:55.505 [INFO] TestAgent: Node entered maintenance mode -42:55.506 [INFO] TestAgent: Synced node info -42:55.506 [INFO] TestAgent: Synced check: check=_node_maintenance -42:55.507 [INFO] TestAgent: Requesting shutdown -42:55.507 [INFO] TestAgent.server: shutting down server -42:55.507 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:55.507 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:55.507 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:55.507 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:55.507 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:55.507 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:55.507 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:55.507 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:55.507 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:55.507 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:55.507 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:55.507 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:55.507Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:55.507Z [INFO] agent.router.manager: shutting down -42:55.508 [INFO] TestAgent: consul server down -42:55.508 [INFO] TestAgent: shutdown complete -42:55.508 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13150 network=tcp -42:55.508 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13150 network=udp -42:55.508 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13151 network=tcp -42:56.008 [INFO] TestAgent: Waiting for endpoints to shut down -42:56.008 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_NodeMaintenance_Enable (0.64s) -=== CONT TestAgent_ServiceMaintenance_Enable -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:55.449Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:55.467Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:55.470 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0a1d9581-b1f5-695e-e880-6ff4d10fd6d9 Address:127.0.0.1:13141}]" -42:55.470 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13141 [Follower]" leader= -42:55.471 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0a1d9581-b1f5-695e-e880-6ff4d10fd6d9.dc1 127.0.0.1 -42:55.472 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0a1d9581-b1f5-695e-e880-6ff4d10fd6d9 127.0.0.1 -2022-09-15T21:42:55.472Z [INFO] agent.router: Initializing LAN area manager -42:55.472 [INFO] TestAgent.server: Adding LAN server: server="Node-0a1d9581-b1f5-695e-e880-6ff4d10fd6d9 (Addr: tcp/127.0.0.1:13141) (DC: dc1)" -42:55.472 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0a1d9581-b1f5-695e-e880-6ff4d10fd6d9.dc1 area=wan -42:55.472 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13164 network=tcp -42:55.472 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13164 network=udp -42:55.473 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13165 network=tcp -42:55.473 [INFO] TestAgent: started state syncer -42:55.473 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13142 network=tcp -42:55.526 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:55.526 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13141 [Candidate]" term=2 -42:55.526 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:55.526 [DEBUG] TestAgent.server.raft: vote granted: from=0a1d9581-b1f5-695e-e880-6ff4d10fd6d9 term=2 tally=1 -42:55.526 [INFO] TestAgent.server.raft: election won: tally=1 -42:55.526 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13141 [Leader]" -42:55.526 [INFO] TestAgent.server: cluster leadership acquired -42:55.527 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13141 -42:55.528 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:55.530 [INFO] TestAgent.server: New leader elected: payload=Node-0a1d9581-b1f5-695e-e880-6ff4d10fd6d9 -42:55.539 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:55.539 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:55.539 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:55.539 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:55.539 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:55.539 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0a1d9581-b1f5-695e-e880-6ff4d10fd6d9 -42:55.539 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0a1d9581-b1f5-695e-e880-6ff4d10fd6d9 -42:55.540 [INFO] TestAgent.server: federation state anti-entropy synced -42:55.606 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:55.607 [INFO] TestAgent: Synced node info -42:55.608 [INFO] TestAgent: Requesting shutdown -42:55.608 [INFO] TestAgent.server: shutting down server -42:55.608 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:55.608 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:55.608 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:55.608 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:55.608 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:55.608 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:55.608 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:55.608 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:55.608 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:55.608 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:55.608 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:55.608Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:55.608Z [INFO] agent.router.manager: shutting down -42:55.608 [INFO] TestAgent: consul server down -42:55.608 [INFO] TestAgent: shutdown complete -42:55.608 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13164 network=tcp -42:55.609 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13164 network=udp -42:55.609 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13165 network=tcp -42:56.110 [INFO] TestAgent: Waiting for endpoints to shut down -42:56.110 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_NodeMaintenance_BadRequest (0.68s) -=== CONT TestAgent_ServiceMaintenance_BadRequest -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:55.626Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:55.626Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:55.645Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:55.645Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:55.649 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7b0029cb-adcc-0a28-9c44-a59939ff70ed Address:127.0.0.1:13232}]" -42:55.649 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13232 [Follower]" leader= -42:55.651 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7b0029cb-adcc-0a28-9c44-a59939ff70ed.dc1 127.0.0.1 -42:55.652 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7b0029cb-adcc-0a28-9c44-a59939ff70ed 127.0.0.1 -2022-09-15T21:42:55.652Z [INFO] agent.router: Initializing LAN area manager -42:55.652 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13227 network=udp -42:55.653 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7b0029cb-adcc-0a28-9c44-a59939ff70ed.dc1 area=wan -42:55.653 [INFO] TestAgent.server: Adding LAN server: server="Node-7b0029cb-adcc-0a28-9c44-a59939ff70ed (Addr: tcp/127.0.0.1:13232) (DC: dc1)" -42:55.653 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13227 network=tcp -42:55.654 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13228 network=tcp -42:55.654 [INFO] TestAgent: started state syncer -42:55.654 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13233 network=tcp -42:55.687 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:55.687 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13232 [Candidate]" term=2 -42:55.687 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:55.687 [DEBUG] TestAgent.server.raft: vote granted: from=7b0029cb-adcc-0a28-9c44-a59939ff70ed term=2 tally=1 -42:55.687 [INFO] TestAgent.server.raft: election won: tally=1 -42:55.687 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13232 [Leader]" -42:55.687 [INFO] TestAgent.server: cluster leadership acquired -42:55.687 [INFO] TestAgent.server: New leader elected: payload=Node-7b0029cb-adcc-0a28-9c44-a59939ff70ed -42:55.688 [INFO] TestAgent.server: initializing acls -42:55.688 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:55.688 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:55.689 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:55.689 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:55.689 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:55.689 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:55.689 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-7b0029cb-adcc-0a28-9c44-a59939ff70ed -42:55.690 [INFO] TestAgent.server: Updating LAN server: server="Node-7b0029cb-adcc-0a28-9c44-a59939ff70ed (Addr: tcp/127.0.0.1:13232) (DC: dc1)" -42:55.690 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-7b0029cb-adcc-0a28-9c44-a59939ff70ed.dc1 -42:55.690 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-7b0029cb-adcc-0a28-9c44-a59939ff70ed.dc1 area=wan -42:55.691 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:55.693 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:55.693 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:55.693 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:55.693 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:55.693 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:55.693 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7b0029cb-adcc-0a28-9c44-a59939ff70ed -42:55.693 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7b0029cb-adcc-0a28-9c44-a59939ff70ed -42:55.694 [INFO] TestAgent.server: federation state anti-entropy synced -42:55.695 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7b0029cb-adcc-0a28-9c44-a59939ff70ed -42:55.719 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:55.720 [INFO] TestAgent: Synced node info -42:55.720 [DEBUG] TestAgent: Node info in sync -42:55.768 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-7b0029cb-adcc-0a28-9c44-a59939ff70ed -42:55.768 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-7b0029cb-adcc-0a28-9c44-a59939ff70ed -42:55.769 [INFO] TestAgent: Service entered maintenance mode: service=test -42:55.769 [DEBUG] TestAgent: Node info in sync -42:55.770 [WARN] TestAgent: Service registration blocked by ACLs: service=test accessorID=00000000-0000-0000-0000-000000000002 -42:55.771 [INFO] TestAgent: Synced check: check=_service_maintenance:test -42:55.771 [INFO] TestAgent: Requesting shutdown -42:55.771 [INFO] TestAgent.server: shutting down server -42:55.771 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:55.771 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:55.771 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:55.771 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:55.771 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:55.771 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:55.771 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:55.771 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:55.771 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:55.771 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:55.771 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:55.771 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:55.772 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:55.772 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:55.772 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:55.772Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:55.772Z [INFO] agent.router.manager: shutting down -42:55.772 [INFO] TestAgent: consul server down -42:55.772 [INFO] TestAgent: shutdown complete -42:55.772 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13227 network=tcp -42:55.772 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13227 network=udp -42:55.772 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13228 network=tcp -42:56.274 [INFO] TestAgent: Waiting for endpoints to shut down -42:56.274 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_ServiceMaintenance_ACLDeny (0.67s) +2021-08-13T15:34:16.528Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:16.543Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:16.546 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:80d6ba1d-ef2b-56c4-1dd3-6ab058abf16b Address:127.0.0.1:24485}]" +34:16.546 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24485 [Follower]" leader= +34:16.547 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-80d6ba1d-ef2b-56c4-1dd3-6ab058abf16b.dc1 127.0.0.1 +34:16.547 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-80d6ba1d-ef2b-56c4-1dd3-6ab058abf16b 127.0.0.1 +2021-08-13T15:34:16.547Z [INFO] agent.router: Initializing LAN area manager +34:16.548 [INFO] TestAgent.server: Adding LAN server: server="Node-80d6ba1d-ef2b-56c4-1dd3-6ab058abf16b (Addr: tcp/127.0.0.1:24485) (DC: dc1)" +34:16.548 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-80d6ba1d-ef2b-56c4-1dd3-6ab058abf16b.dc1 area=wan +34:16.548 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24494 network=tcp +34:16.548 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24494 network=udp +34:16.549 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24495 network=tcp +34:16.549 [INFO] TestAgent: started state syncer +34:16.549 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24465 network=tcp +34:16.617 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:16.617 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24485 [Candidate]" term=2 +34:16.618 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:16.618 [DEBUG] TestAgent.server.raft: vote granted: from=80d6ba1d-ef2b-56c4-1dd3-6ab058abf16b term=2 tally=1 +34:16.618 [INFO] TestAgent.server.raft: election won: tally=1 +34:16.618 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24485 [Leader]" +34:16.618 [INFO] TestAgent.server: cluster leadership acquired +34:16.618 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24485 +34:16.619 [INFO] TestAgent.server: New leader elected: payload=Node-80d6ba1d-ef2b-56c4-1dd3-6ab058abf16b +34:16.620 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:16.624 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:16.624 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:16.624 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:16.624 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:16.624 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:16.624 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-80d6ba1d-ef2b-56c4-1dd3-6ab058abf16b +34:16.624 [INFO] TestAgent.server: member joined, marking health alive: member=Node-80d6ba1d-ef2b-56c4-1dd3-6ab058abf16b +34:16.626 [INFO] TestAgent.server: federation state anti-entropy synced +34:16.723 [WARN] TestAgent.server: endpoint injected; this should only be used for testing +34:16.724 [INFO] TestAgent: Requesting shutdown +34:16.724 [INFO] TestAgent.server: shutting down server +34:16.724 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:16.724 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:16.724 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:16.724 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:16.724 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:16.724 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:16.724 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:16.724 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:16.724 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:16.724 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:16.724 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:16.725 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:16.725Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:16.725Z [INFO] agent.router.manager: shutting down +34:16.725 [INFO] TestAgent: consul server down +34:16.725 [INFO] TestAgent: shutdown complete +34:16.725 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24494 network=tcp +34:16.725 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24494 network=udp +34:16.725 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24495 network=tcp +34:17.225 [INFO] TestAgent: Waiting for endpoints to shut down +34:17.225 [INFO] TestAgent: Endpoints down +--- PASS: TestPreparedQuery_ExecuteCached (0.71s) +=== CONT TestPreparedQuery_Execute +=== RUN TestPreparedQuery_Execute/#00 +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:16.617Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:16.652Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:16.655 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4a3dfa1d-c528-eccb-bfc6-0e0f8cf3c07c Address:127.0.0.1:24471}]" +34:16.655 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24471 [Follower]" leader= +34:16.656 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4a3dfa1d-c528-eccb-bfc6-0e0f8cf3c07c.dc1 127.0.0.1 +34:16.657 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4a3dfa1d-c528-eccb-bfc6-0e0f8cf3c07c 127.0.0.1 +2021-08-13T15:34:16.657Z [INFO] agent.router: Initializing LAN area manager +34:16.658 [INFO] TestAgent.server: Adding LAN server: server="Node-4a3dfa1d-c528-eccb-bfc6-0e0f8cf3c07c (Addr: tcp/127.0.0.1:24471) (DC: dc1)" +34:16.658 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4a3dfa1d-c528-eccb-bfc6-0e0f8cf3c07c.dc1 area=wan +34:16.658 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24466 network=tcp +34:16.658 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24466 network=udp +34:16.659 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24467 network=tcp +34:16.659 [INFO] TestAgent: started state syncer +34:16.659 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24437 network=tcp +34:16.706 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:16.706 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24471 [Candidate]" term=2 +34:16.706 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:16.706 [DEBUG] TestAgent.server.raft: vote granted: from=4a3dfa1d-c528-eccb-bfc6-0e0f8cf3c07c term=2 tally=1 +34:16.706 [INFO] TestAgent.server.raft: election won: tally=1 +34:16.706 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24471 [Leader]" +34:16.706 [INFO] TestAgent.server: cluster leadership acquired +34:16.706 [INFO] TestAgent.server: New leader elected: payload=Node-4a3dfa1d-c528-eccb-bfc6-0e0f8cf3c07c +34:16.706 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24471 +34:16.707 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:16.710 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:16.710 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:16.710 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:16.710 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:16.710 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:16.710 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4a3dfa1d-c528-eccb-bfc6-0e0f8cf3c07c +34:16.710 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4a3dfa1d-c528-eccb-bfc6-0e0f8cf3c07c +34:16.716 [INFO] TestAgent.server: federation state anti-entropy synced +34:16.788 [INFO] TestAgent: Requesting shutdown +34:16.788 [INFO] TestAgent.server: shutting down server +34:16.788 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:16.788 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:16.788 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:16.788 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:16.788 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:16.788 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:16.788 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:16.788 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:16.788 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:16.788 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:16.788 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:16.788 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:16.788Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:16.788Z [INFO] agent.router.manager: shutting down +34:16.789 [INFO] TestAgent: consul server down +34:16.789 [INFO] TestAgent: shutdown complete +34:16.789 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24466 network=tcp +34:16.789 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24466 network=udp +34:16.789 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24467 network=tcp +34:17.289 [INFO] TestAgent: Waiting for endpoints to shut down +34:17.289 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_NodeMaintenance_BadRequest (0.71s) +=== CONT TestPreparedQuery_List +=== RUN TestPreparedQuery_List/#00 +2021-08-13T15:34:16.666Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:16.697Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:16.700 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0b745c68-2cfe-14bb-2829-fbe53c53e571 Address:127.0.0.1:24443}]" +34:16.700 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24443 [Follower]" leader= +34:16.701 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0b745c68-2cfe-14bb-2829-fbe53c53e571.dc1 127.0.0.1 +34:16.702 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0b745c68-2cfe-14bb-2829-fbe53c53e571 127.0.0.1 +2021-08-13T15:34:16.702Z [INFO] agent.router: Initializing LAN area manager +34:16.702 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24438 network=udp +34:16.702 [INFO] TestAgent.server: Adding LAN server: server="Node-0b745c68-2cfe-14bb-2829-fbe53c53e571 (Addr: tcp/127.0.0.1:24443) (DC: dc1)" +34:16.702 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0b745c68-2cfe-14bb-2829-fbe53c53e571.dc1 area=wan +34:16.703 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24438 network=tcp +34:16.704 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24439 network=tcp +34:16.704 [INFO] TestAgent: started state syncer +34:16.704 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24510 network=tcp +34:16.754 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:16.754 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24443 [Candidate]" term=2 +34:16.754 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:16.754 [DEBUG] TestAgent.server.raft: vote granted: from=0b745c68-2cfe-14bb-2829-fbe53c53e571 term=2 tally=1 +34:16.754 [INFO] TestAgent.server.raft: election won: tally=1 +34:16.754 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24443 [Leader]" +34:16.755 [INFO] TestAgent.server: cluster leadership acquired +34:16.755 [INFO] TestAgent.server: New leader elected: payload=Node-0b745c68-2cfe-14bb-2829-fbe53c53e571 +34:16.755 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24443 +34:16.756 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:16.759 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:16.759 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:16.759 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:16.759 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:16.759 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:16.759 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0b745c68-2cfe-14bb-2829-fbe53c53e571 +34:16.759 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0b745c68-2cfe-14bb-2829-fbe53c53e571 +34:16.760 [INFO] TestAgent.server: federation state anti-entropy synced +34:16.851 [INFO] TestAgent: Requesting shutdown +34:16.851 [INFO] TestAgent.server: shutting down server +34:16.851 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:16.851 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:16.851 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:16.851 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:16.851 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:16.852 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:16.852 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:16.852 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:16.852 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:16.852 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:16.852 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:16.852 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:16.852Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:16.852Z [INFO] agent.router.manager: shutting down +34:16.852 [INFO] TestAgent: consul server down +34:16.852 [INFO] TestAgent: shutdown complete +34:16.852 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24438 network=tcp +34:16.852 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24438 network=udp +34:16.852 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24439 network=tcp +34:17.353 [INFO] TestAgent: Waiting for endpoints to shut down +34:17.353 [INFO] TestAgent: Endpoints down +--- PASS: TestPreparedQuery_Get (1.59s) + --- PASS: TestPreparedQuery_Get/#00 (0.88s) + --- PASS: TestPreparedQuery_Get/#01 (0.71s) +=== CONT TestAgent_ServiceMaintenance_Disable +2021-08-13T15:34:16.685Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:16.709Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:16.711 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:783e88ea-b18d-2264-1302-5bf6d5a17a8e Address:127.0.0.1:24502}]" +34:16.712 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24502 [Follower]" leader= +34:16.712 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-783e88ea-b18d-2264-1302-5bf6d5a17a8e.dc1 127.0.0.1 +34:16.713 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-783e88ea-b18d-2264-1302-5bf6d5a17a8e 127.0.0.1 +2021-08-13T15:34:16.713Z [INFO] agent.router: Initializing LAN area manager +34:16.714 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24511 network=udp +34:16.714 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-783e88ea-b18d-2264-1302-5bf6d5a17a8e.dc1 area=wan +34:16.714 [INFO] TestAgent.server: Adding LAN server: server="Node-783e88ea-b18d-2264-1302-5bf6d5a17a8e (Addr: tcp/127.0.0.1:24502) (DC: dc1)" +34:16.714 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24511 network=tcp +34:16.716 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24498 network=tcp +34:16.716 [INFO] TestAgent: started state syncer +34:16.716 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24517 network=tcp +34:16.774 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:16.774 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24502 [Candidate]" term=2 +34:16.774 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:16.774 [DEBUG] TestAgent.server.raft: vote granted: from=783e88ea-b18d-2264-1302-5bf6d5a17a8e term=2 tally=1 +34:16.774 [INFO] TestAgent.server.raft: election won: tally=1 +34:16.774 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24502 [Leader]" +34:16.775 [INFO] TestAgent.server: cluster leadership acquired +34:16.775 [INFO] TestAgent.server: New leader elected: payload=Node-783e88ea-b18d-2264-1302-5bf6d5a17a8e +34:16.775 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24502 +34:16.776 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:16.779 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:16.779 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:16.779 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:16.779 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:16.779 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:16.780 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-783e88ea-b18d-2264-1302-5bf6d5a17a8e +34:16.780 [INFO] TestAgent.server: member joined, marking health alive: member=Node-783e88ea-b18d-2264-1302-5bf6d5a17a8e +34:16.780 [INFO] TestAgent.server: federation state anti-entropy synced +34:16.886 [INFO] TestAgent: Requesting shutdown +34:16.886 [INFO] TestAgent.server: shutting down server +34:16.886 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:16.886 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:16.886 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:16.886 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:16.886 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:16.886 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:16.886 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:16.886 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:16.886 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:16.886 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:16.886 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:16.886 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:16.887Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:16.887Z [INFO] agent.router.manager: shutting down +34:16.887 [INFO] TestAgent: consul server down +34:16.887 [INFO] TestAgent: shutdown complete +34:16.887 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24511 network=tcp +34:16.887 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24511 network=udp +34:16.887 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24498 network=tcp +34:17.387 [INFO] TestAgent: Waiting for endpoints to shut down +34:17.387 [INFO] TestAgent: Endpoints down +=== RUN TestPreparedQuery_Explain/#02 +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:16.854Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:16.854Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:16.880Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:16.880Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:16.883 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4a34ef61-908a-8559-2c1d-33555cd28bfd Address:127.0.0.1:24523}]" +34:16.884 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24523 [Follower]" leader= +34:16.884 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4a34ef61-908a-8559-2c1d-33555cd28bfd.dc1 127.0.0.1 +34:16.885 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4a34ef61-908a-8559-2c1d-33555cd28bfd 127.0.0.1 +2021-08-13T15:34:16.885Z [INFO] agent.router: Initializing LAN area manager +34:16.885 [INFO] TestAgent.server: Adding LAN server: server="Node-4a34ef61-908a-8559-2c1d-33555cd28bfd (Addr: tcp/127.0.0.1:24523) (DC: dc1)" +34:16.886 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4a34ef61-908a-8559-2c1d-33555cd28bfd.dc1 area=wan +34:16.886 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24518 network=tcp +34:16.886 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24518 network=udp +34:16.887 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24519 network=tcp +34:16.887 [INFO] TestAgent: started state syncer +34:16.887 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24503 network=tcp +34:16.928 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:16.928 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24523 [Candidate]" term=2 +34:16.928 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:16.928 [DEBUG] TestAgent.server.raft: vote granted: from=4a34ef61-908a-8559-2c1d-33555cd28bfd term=2 tally=1 +34:16.928 [INFO] TestAgent.server.raft: election won: tally=1 +34:16.928 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24523 [Leader]" +34:16.928 [INFO] TestAgent.server: cluster leadership acquired +34:16.928 [INFO] TestAgent.server: New leader elected: payload=Node-4a34ef61-908a-8559-2c1d-33555cd28bfd +34:16.929 [INFO] TestAgent.server: initializing acls +34:16.929 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:16.929 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:16.930 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:16.931 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:16.931 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:16.931 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:16.931 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-4a34ef61-908a-8559-2c1d-33555cd28bfd +34:16.931 [INFO] TestAgent.server: Updating LAN server: server="Node-4a34ef61-908a-8559-2c1d-33555cd28bfd (Addr: tcp/127.0.0.1:24523) (DC: dc1)" +34:16.931 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-4a34ef61-908a-8559-2c1d-33555cd28bfd.dc1 +34:16.931 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-4a34ef61-908a-8559-2c1d-33555cd28bfd.dc1 area=wan +34:16.932 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:16.935 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:16.935 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:16.935 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:16.935 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:16.935 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:16.935 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4a34ef61-908a-8559-2c1d-33555cd28bfd +34:16.935 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4a34ef61-908a-8559-2c1d-33555cd28bfd +34:16.936 [INFO] TestAgent.server: federation state anti-entropy synced +34:16.937 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4a34ef61-908a-8559-2c1d-33555cd28bfd +34:16.960 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:16.960 [INFO] TestAgent: Synced node info +34:16.960 [DEBUG] TestAgent: Node info in sync +34:17.097 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-4a34ef61-908a-8559-2c1d-33555cd28bfd +34:17.097 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-4a34ef61-908a-8559-2c1d-33555cd28bfd +34:17.098 [INFO] TestAgent: Service entered maintenance mode: service=test +34:17.098 [DEBUG] TestAgent: Node info in sync +34:17.098 [WARN] TestAgent: Service registration blocked by ACLs: service=test accessorID=00000000-0000-0000-0000-000000000002 +34:17.100 [INFO] TestAgent: Synced check: check=_service_maintenance:test +34:17.100 [INFO] TestAgent: Requesting shutdown +34:17.100 [INFO] TestAgent.server: shutting down server +34:17.100 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:17.100 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:17.100 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:17.100 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:17.100 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:17.100 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:17.100 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:17.100 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:17.100 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:17.100 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:17.100 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:17.100 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:17.100 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:17.100 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:17.100 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:17.101Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:17.101Z [INFO] agent.router.manager: shutting down +34:17.101 [INFO] TestAgent: consul server down +34:17.101 [INFO] TestAgent: shutdown complete +34:17.101 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24518 network=tcp +34:17.101 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24518 network=udp +34:17.101 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24519 network=tcp +34:17.601 [INFO] TestAgent: Waiting for endpoints to shut down +34:17.601 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_ServiceMaintenance_ACLDeny (0.77s) --- PASS: TestAgent_ServiceMaintenance_ACLDeny/no_token (0.00s) --- PASS: TestAgent_ServiceMaintenance_ACLDeny/root_token (0.00s) -=== CONT TestAgent_DeregisterService_ACLDeny -=== RUN TestAgent_DeregisterService_ACLDeny/no_token -=== RUN TestAgent_DeregisterService_ACLDeny/root_token +=== CONT TestPreparedQuery_Create +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:17.315Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:17.347Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:17.350 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5f316449-9c03-4ced-82ad-0099164000fc Address:127.0.0.1:24530}]" +34:17.350 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24530 [Follower]" leader= +34:17.358 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5f316449-9c03-4ced-82ad-0099164000fc.dc1 127.0.0.1 +34:17.359 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5f316449-9c03-4ced-82ad-0099164000fc 127.0.0.1 +2021-08-13T15:34:17.359Z [INFO] agent.router: Initializing LAN area manager +34:17.359 [INFO] TestAgent.server: Adding LAN server: server="Node-5f316449-9c03-4ced-82ad-0099164000fc (Addr: tcp/127.0.0.1:24530) (DC: dc1)" +34:17.359 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5f316449-9c03-4ced-82ad-0099164000fc.dc1 area=wan +34:17.359 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24525 network=tcp +34:17.359 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24525 network=udp +34:17.360 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24526 network=tcp +34:17.360 [INFO] TestAgent: started state syncer +34:17.360 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24531 network=tcp +34:17.423 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:17.423 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24530 [Candidate]" term=2 +34:17.423 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:17.423 [DEBUG] TestAgent.server.raft: vote granted: from=5f316449-9c03-4ced-82ad-0099164000fc term=2 tally=1 +34:17.423 [INFO] TestAgent.server.raft: election won: tally=1 +34:17.423 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24530 [Leader]" +34:17.424 [INFO] TestAgent.server: cluster leadership acquired +34:17.424 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24530 +34:17.424 [INFO] TestAgent.server: New leader elected: payload=Node-5f316449-9c03-4ced-82ad-0099164000fc +34:17.426 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:17.429 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:17.429 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:17.429 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:17.429 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:17.429 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:17.429 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5f316449-9c03-4ced-82ad-0099164000fc +34:17.429 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5f316449-9c03-4ced-82ad-0099164000fc +34:17.430 [INFO] TestAgent.server: federation state anti-entropy synced +34:17.470 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:17.471 [INFO] TestAgent: Synced node info +34:17.591 [WARN] TestAgent.server: endpoint injected; this should only be used for testing +34:17.591 [INFO] TestAgent: Requesting shutdown +34:17.591 [INFO] TestAgent.server: shutting down server +34:17.592 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:17.592 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:17.592 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:17.592 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:17.592 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:17.592 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:17.592 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:17.592 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:17.592 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:17.592 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:17.592 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:17.592Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:17.592Z [INFO] agent.router.manager: shutting down +34:17.592 [INFO] TestAgent: consul server down +34:17.592 [INFO] TestAgent: shutdown complete +34:17.592 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24525 network=tcp +34:17.592 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24525 network=udp +34:17.592 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24526 network=tcp +34:18.092 [INFO] TestAgent: Waiting for endpoints to shut down +34:18.092 [INFO] TestAgent: Endpoints down +=== RUN TestPreparedQuery_List/#01 +2021-08-13T15:34:17.243Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:17.261Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:17.264 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0340cb07-290e-7ca6-5c30-4e4d07d422be Address:127.0.0.1:24516}]" +34:17.264 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24516 [Follower]" leader= +34:17.265 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0340cb07-290e-7ca6-5c30-4e4d07d422be.dc1 127.0.0.1 +34:17.266 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0340cb07-290e-7ca6-5c30-4e4d07d422be 127.0.0.1 +2021-08-13T15:34:17.266Z [INFO] agent.router: Initializing LAN area manager +34:17.266 [INFO] TestAgent.server: Adding LAN server: server="Node-0340cb07-290e-7ca6-5c30-4e4d07d422be (Addr: tcp/127.0.0.1:24516) (DC: dc1)" +34:17.266 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0340cb07-290e-7ca6-5c30-4e4d07d422be.dc1 area=wan +34:17.266 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24504 network=tcp +34:17.266 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24504 network=udp +34:17.267 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24512 network=tcp +34:17.267 [INFO] TestAgent: started state syncer +34:17.267 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24524 network=tcp +34:17.322 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:17.322 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24516 [Candidate]" term=2 +34:17.322 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:17.322 [DEBUG] TestAgent.server.raft: vote granted: from=0340cb07-290e-7ca6-5c30-4e4d07d422be term=2 tally=1 +34:17.322 [INFO] TestAgent.server.raft: election won: tally=1 +34:17.322 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24516 [Leader]" +34:17.322 [INFO] TestAgent.server: cluster leadership acquired +34:17.322 [INFO] TestAgent.server: New leader elected: payload=Node-0340cb07-290e-7ca6-5c30-4e4d07d422be +34:17.323 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24516 +34:17.324 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:17.326 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:17.326 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:17.326 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:17.326 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:17.326 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:17.326 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0340cb07-290e-7ca6-5c30-4e4d07d422be +34:17.326 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0340cb07-290e-7ca6-5c30-4e4d07d422be +34:17.327 [INFO] TestAgent.server: federation state anti-entropy synced +34:17.537 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:17.537 [INFO] TestAgent: Synced node info +34:17.592 [WARN] TestAgent.server: endpoint injected; this should only be used for testing +34:17.592 [INFO] TestAgent: Requesting shutdown +34:17.592 [INFO] TestAgent.server: shutting down server +34:17.592 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:17.592 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:17.592 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:17.592 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:17.592 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:17.592 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:17.592 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:17.592 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:17.592 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:17.592 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +34:17.592 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +2021-08-13T15:34:17.593Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:17.593Z [INFO] agent.router.manager: shutting down +34:17.593 [INFO] TestAgent: consul server down +34:17.593 [INFO] TestAgent: shutdown complete +34:17.593 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24504 network=tcp +34:17.593 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24504 network=udp +34:17.593 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24512 network=tcp +34:18.093 [INFO] TestAgent: Waiting for endpoints to shut down +34:18.093 [INFO] TestAgent: Endpoints down +=== RUN TestPreparedQuery_Execute/#01 +2021-08-13T15:34:17.430Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:17.448Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:17.451 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8c6236dd-7bfa-1111-c8e9-88b8737c7c95 Address:127.0.0.1:24551}]" +34:17.451 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24551 [Follower]" leader= +34:17.452 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8c6236dd-7bfa-1111-c8e9-88b8737c7c95.dc1 127.0.0.1 +34:17.452 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8c6236dd-7bfa-1111-c8e9-88b8737c7c95 127.0.0.1 +2021-08-13T15:34:17.452Z [INFO] agent.router: Initializing LAN area manager +34:17.453 [INFO] TestAgent.server: Adding LAN server: server="Node-8c6236dd-7bfa-1111-c8e9-88b8737c7c95 (Addr: tcp/127.0.0.1:24551) (DC: dc1)" +34:17.453 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8c6236dd-7bfa-1111-c8e9-88b8737c7c95.dc1 area=wan +34:17.453 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24546 network=tcp +34:17.453 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24546 network=udp +34:17.454 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24547 network=tcp +34:17.454 [INFO] TestAgent: started state syncer +34:17.454 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24538 network=tcp +34:17.517 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:17.517 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24551 [Candidate]" term=2 +34:17.517 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:17.517 [DEBUG] TestAgent.server.raft: vote granted: from=8c6236dd-7bfa-1111-c8e9-88b8737c7c95 term=2 tally=1 +34:17.517 [INFO] TestAgent.server.raft: election won: tally=1 +34:17.517 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24551 [Leader]" +34:17.517 [INFO] TestAgent.server: cluster leadership acquired +34:17.517 [INFO] TestAgent.server: New leader elected: payload=Node-8c6236dd-7bfa-1111-c8e9-88b8737c7c95 +34:17.517 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24551 +34:17.519 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:17.521 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:17.521 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:17.521 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:17.521 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:17.521 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:17.521 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8c6236dd-7bfa-1111-c8e9-88b8737c7c95 +34:17.521 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8c6236dd-7bfa-1111-c8e9-88b8737c7c95 +34:17.521 [INFO] TestAgent.server: federation state anti-entropy synced +34:17.644 [WARN] TestAgent.server: endpoint injected; this should only be used for testing +34:17.644 [INFO] TestAgent: Requesting shutdown +34:17.644 [INFO] TestAgent.server: shutting down server +34:17.644 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:17.644 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:17.644 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:17.644 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:17.644 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:17.644 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:17.644 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:17.644 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:17.644 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:17.644 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:17.644 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:17.644 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:17.645Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:17.645Z [INFO] agent.router.manager: shutting down +34:17.645 [INFO] TestAgent: consul server down +34:17.645 [INFO] TestAgent: shutdown complete +34:17.645 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24546 network=tcp +34:17.645 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24546 network=udp +34:17.645 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24547 network=tcp +34:18.145 [INFO] TestAgent: Waiting for endpoints to shut down +34:18.145 [INFO] TestAgent: Endpoints down +--- PASS: TestPreparedQuery_Explain (2.14s) + --- PASS: TestPreparedQuery_Explain/#00 (0.66s) + --- PASS: TestPreparedQuery_Explain/#01 (0.73s) + --- PASS: TestPreparedQuery_Explain/#02 (0.76s) +=== CONT TestAgent_ServiceMaintenance_Enable +2021-08-13T15:34:15.644Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:15.644Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:15.690Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:15.690Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:15.694 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:05179876-67aa-f63c-7b5f-87caac9b1e80 Address:127.0.0.1:24450}]" +34:15.694 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24450 [Follower]" leader= +34:15.695 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-05179876-67aa-f63c-7b5f-87caac9b1e80.dc1 127.0.0.1 +34:15.698 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-05179876-67aa-f63c-7b5f-87caac9b1e80 127.0.0.1 +2021-08-13T15:34:15.698Z [INFO] agent.router: Initializing LAN area manager +34:15.698 [INFO] TestAgent.server: Adding LAN server: server="Node-05179876-67aa-f63c-7b5f-87caac9b1e80 (Addr: tcp/127.0.0.1:24450) (DC: dc1)" +34:15.698 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-05179876-67aa-f63c-7b5f-87caac9b1e80.dc1 area=wan +34:15.698 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24445 network=udp +34:15.698 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24445 network=tcp +34:15.699 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24446 network=tcp +34:15.699 [INFO] TestAgent: started state syncer +34:15.700 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24458 network=tcp +34:15.748 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:15.754 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:15.754 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24450 [Candidate]" term=2 +34:15.755 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:15.755 [DEBUG] TestAgent.server.raft: vote granted: from=05179876-67aa-f63c-7b5f-87caac9b1e80 term=2 tally=1 +34:15.755 [INFO] TestAgent.server.raft: election won: tally=1 +34:15.755 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24450 [Leader]" +34:15.755 [INFO] TestAgent.server: cluster leadership acquired +34:15.755 [INFO] TestAgent.server: New leader elected: payload=Node-05179876-67aa-f63c-7b5f-87caac9b1e80 +34:15.755 [INFO] TestAgent.server: initializing acls +34:15.756 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:15.756 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:15.756 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:15.757 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:15.757 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:15.757 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:15.757 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-05179876-67aa-f63c-7b5f-87caac9b1e80 +34:15.757 [INFO] TestAgent.server: Updating LAN server: server="Node-05179876-67aa-f63c-7b5f-87caac9b1e80 (Addr: tcp/127.0.0.1:24450) (DC: dc1)" +34:15.757 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-05179876-67aa-f63c-7b5f-87caac9b1e80.dc1 +34:15.757 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-05179876-67aa-f63c-7b5f-87caac9b1e80.dc1 area=wan +34:15.759 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:15.761 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:15.761 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:15.761 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:15.761 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:15.761 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:15.761 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-05179876-67aa-f63c-7b5f-87caac9b1e80 +34:15.761 [INFO] TestAgent.server: member joined, marking health alive: member=Node-05179876-67aa-f63c-7b5f-87caac9b1e80 +34:15.762 [INFO] TestAgent.server: federation state anti-entropy synced +34:15.762 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-05179876-67aa-f63c-7b5f-87caac9b1e80 +34:15.902 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth +34:15.902 [WARN] TestAgent: Node info update blocked by ACLs: node=05179876-67aa-f63c-7b5f-87caac9b1e80 accessorID=00000000-0000-0000-0000-000000000002 +34:15.902 [DEBUG] TestAgent: Node info in sync +34:16.058 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-05179876-67aa-f63c-7b5f-87caac9b1e80 +34:16.059 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-05179876-67aa-f63c-7b5f-87caac9b1e80 +34:16.061 [INFO] TestAgent: Updated agent's ACL token: token=agent +34:17.656 [DEBUG] TestAgent.acl: dropping service from result due to ACLs: service="{consul {}}" +34:17.656 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:17.657 [INFO] TestAgent: Synced node info +34:17.664 [INFO] TestAgent: Requesting shutdown +34:17.664 [INFO] TestAgent.server: shutting down server +34:17.664 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:17.664 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:17.664 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:17.664 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:17.664 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:17.664 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:17.664 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:17.664 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:17.664 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:17.664 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:17.664 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:17.664 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:17.664 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:17.664 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:17.664 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:17.665Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:17.665Z [INFO] agent.router.manager: shutting down +34:17.665 [INFO] TestAgent: consul server down +34:17.665 [INFO] TestAgent: shutdown complete +34:17.665 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24445 network=tcp +34:17.665 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24445 network=udp +34:17.665 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24446 network=tcp +34:18.169 [INFO] TestAgent: Waiting for endpoints to shut down +34:18.169 [INFO] TestAgent: Endpoints down +=== RUN TestAgent_TokenTriggersFullSync/acl_token +[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:17.371Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:17.387Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:17.400 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:87d904cd-5526-2ab1-4f48-31584f851247 Address:127.0.0.1:24537}]" +34:17.402 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-87d904cd-5526-2ab1-4f48-31584f851247.dc1 127.0.0.1 +34:17.403 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24537 [Follower]" leader= +34:17.405 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-87d904cd-5526-2ab1-4f48-31584f851247 127.0.0.1 +2021-08-13T15:34:17.405Z [INFO] agent.router: Initializing LAN area manager +34:17.406 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24532 network=udp +34:17.407 [INFO] TestAgent.server: Adding LAN server: server="Node-87d904cd-5526-2ab1-4f48-31584f851247 (Addr: tcp/127.0.0.1:24537) (DC: dc1)" +34:17.407 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-87d904cd-5526-2ab1-4f48-31584f851247.dc1 area=wan +34:17.408 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24532 network=tcp +34:17.410 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24533 network=tcp +34:17.410 [INFO] TestAgent: started state syncer +34:17.410 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24545 network=tcp +34:17.455 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:17.455 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24537 [Candidate]" term=2 +34:17.455 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:17.455 [DEBUG] TestAgent.server.raft: vote granted: from=87d904cd-5526-2ab1-4f48-31584f851247 term=2 tally=1 +34:17.455 [INFO] TestAgent.server.raft: election won: tally=1 +34:17.455 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24537 [Leader]" +34:17.455 [INFO] TestAgent.server: cluster leadership acquired +34:17.456 [INFO] TestAgent.server: New leader elected: payload=Node-87d904cd-5526-2ab1-4f48-31584f851247 +34:17.456 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24537 +34:17.457 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:17.459 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:17.459 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:17.459 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:17.459 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:17.459 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:17.460 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-87d904cd-5526-2ab1-4f48-31584f851247 +34:17.460 [INFO] TestAgent.server: member joined, marking health alive: member=Node-87d904cd-5526-2ab1-4f48-31584f851247 +34:17.460 [INFO] TestAgent.server: federation state anti-entropy synced +34:17.495 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:17.495 [INFO] TestAgent: Synced node info +34:17.495 [DEBUG] TestAgent: Node info in sync +34:17.811 [INFO] TestAgent: Service entered maintenance mode: service=test +34:17.811 [DEBUG] TestAgent: removed check: check=_service_maintenance:test +34:17.811 [INFO] TestAgent: Service left maintenance mode: service=test +34:17.811 [DEBUG] TestAgent: Node info in sync +34:17.812 [INFO] TestAgent: Synced service: service=test +34:17.812 [INFO] TestAgent: Deregistered check: check=_service_maintenance:test +34:17.812 [INFO] TestAgent: Requesting shutdown +34:17.813 [INFO] TestAgent.server: shutting down server +34:17.813 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:17.813 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:17.813 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:17.813 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:17.813 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:17.813 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:17.813 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:17.813 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:17.813 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:17.813 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:17.813 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:17.813Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:17.813Z [INFO] agent.router.manager: shutting down +34:17.813 [INFO] TestAgent: consul server down +34:17.813 [INFO] TestAgent: shutdown complete +34:17.813 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24532 network=tcp +34:17.813 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24532 network=udp +34:17.813 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24533 network=tcp +34:18.313 [INFO] TestAgent: Waiting for endpoints to shut down +34:18.314 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_ServiceMaintenance_Disable (0.96s) +=== CONT TestOperator_ServerHealth_Unhealthy +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:17.617Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:17.633Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:17.635 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e0b0e243-bbdb-44a3-cd4a-3409f6c867b7 Address:127.0.0.1:24544}]" +34:17.635 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24544 [Follower]" leader= +34:17.636 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e0b0e243-bbdb-44a3-cd4a-3409f6c867b7.dc1 127.0.0.1 +34:17.637 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e0b0e243-bbdb-44a3-cd4a-3409f6c867b7 127.0.0.1 +2021-08-13T15:34:17.637Z [INFO] agent.router: Initializing LAN area manager +34:17.637 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e0b0e243-bbdb-44a3-cd4a-3409f6c867b7.dc1 area=wan +34:17.637 [INFO] TestAgent.server: Adding LAN server: server="Node-e0b0e243-bbdb-44a3-cd4a-3409f6c867b7 (Addr: tcp/127.0.0.1:24544) (DC: dc1)" +34:17.638 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24539 network=udp +34:17.638 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24539 network=tcp +34:17.638 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24540 network=tcp +34:17.639 [INFO] TestAgent: started state syncer +34:17.639 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24559 network=tcp +34:17.695 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:17.695 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24544 [Candidate]" term=2 +34:17.695 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:17.695 [DEBUG] TestAgent.server.raft: vote granted: from=e0b0e243-bbdb-44a3-cd4a-3409f6c867b7 term=2 tally=1 +34:17.695 [INFO] TestAgent.server.raft: election won: tally=1 +34:17.695 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24544 [Leader]" +34:17.696 [INFO] TestAgent.server: cluster leadership acquired +34:17.696 [INFO] TestAgent.server: New leader elected: payload=Node-e0b0e243-bbdb-44a3-cd4a-3409f6c867b7 +34:17.696 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24544 +34:17.697 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:17.700 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:17.700 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:17.700 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:17.700 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:17.700 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:17.700 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e0b0e243-bbdb-44a3-cd4a-3409f6c867b7 +34:17.700 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e0b0e243-bbdb-44a3-cd4a-3409f6c867b7 +34:17.701 [INFO] TestAgent.server: federation state anti-entropy synced +34:18.044 [WARN] TestAgent.server: endpoint injected; this should only be used for testing +34:18.045 [INFO] TestAgent: Requesting shutdown +34:18.045 [INFO] TestAgent.server: shutting down server +34:18.045 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:18.045 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:18.045 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:18.045 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:18.045 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:18.045 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:18.045 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:18.045 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:18.045 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:18.045 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:18.045 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:18.045 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:18.045Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:18.045Z [INFO] agent.router.manager: shutting down +34:18.045 [INFO] TestAgent: consul server down +34:18.045 [INFO] TestAgent: shutdown complete +34:18.045 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24539 network=tcp +34:18.046 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24539 network=udp +34:18.046 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24540 network=tcp +34:18.546 [INFO] TestAgent: Waiting for endpoints to shut down +34:18.546 [INFO] TestAgent: Endpoints down +--- PASS: TestPreparedQuery_Create (0.94s) +=== CONT TestAgent_ServiceMaintenance_BadRequest === RUN TestAgent_ServiceMaintenance_BadRequest/not_enabled === RUN TestAgent_ServiceMaintenance_BadRequest/no_service_id === RUN TestAgent_ServiceMaintenance_BadRequest/bad_service_id -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:55.667Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:55.685Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:55.688 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ce58e383-cb39-f738-76b6-e618b3e1b248 Address:127.0.0.1:13197}]" -42:55.688 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13197 [Follower]" leader= -42:55.689 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ce58e383-cb39-f738-76b6-e618b3e1b248.dc1 127.0.0.1 -42:55.690 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ce58e383-cb39-f738-76b6-e618b3e1b248 127.0.0.1 -2022-09-15T21:42:55.690Z [INFO] agent.router: Initializing LAN area manager -42:55.690 [INFO] TestAgent.server: Adding LAN server: server="Node-ce58e383-cb39-f738-76b6-e618b3e1b248 (Addr: tcp/127.0.0.1:13197) (DC: dc1)" -42:55.690 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ce58e383-cb39-f738-76b6-e618b3e1b248.dc1 area=wan -42:55.690 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13206 network=udp -42:55.690 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13206 network=tcp -42:55.691 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13207 network=tcp -42:55.691 [INFO] TestAgent: started state syncer -42:55.691 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13198 network=tcp -42:55.726 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:55.726 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13197 [Candidate]" term=2 -42:55.726 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:55.726 [DEBUG] TestAgent.server.raft: vote granted: from=ce58e383-cb39-f738-76b6-e618b3e1b248 term=2 tally=1 -42:55.726 [INFO] TestAgent.server.raft: election won: tally=1 -42:55.726 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13197 [Leader]" -42:55.727 [INFO] TestAgent.server: cluster leadership acquired -42:55.727 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13197 -42:55.728 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:55.728 [INFO] TestAgent.server: New leader elected: payload=Node-ce58e383-cb39-f738-76b6-e618b3e1b248 -42:55.731 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:55.731 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:55.731 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:55.731 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:55.731 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:55.731 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ce58e383-cb39-f738-76b6-e618b3e1b248 -42:55.731 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ce58e383-cb39-f738-76b6-e618b3e1b248 -42:55.732 [INFO] TestAgent.server: federation state anti-entropy synced -42:55.850 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:55.850 [INFO] TestAgent: Synced node info -42:56.118 [INFO] TestAgent: Service entered maintenance mode: service=test -42:56.119 [DEBUG] TestAgent: removed check: check=_service_maintenance:test -42:56.119 [INFO] TestAgent: Service left maintenance mode: service=test -42:56.119 [DEBUG] TestAgent: Node info in sync -42:56.119 [INFO] TestAgent: Synced service: service=test -42:56.120 [INFO] TestAgent: Deregistered check: check=_service_maintenance:test -42:56.120 [INFO] TestAgent: Requesting shutdown -42:56.120 [INFO] TestAgent.server: shutting down server -42:56.120 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:56.120 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:56.120 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:56.120 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:56.120 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:56.120 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:56.120 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:56.120 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:56.120 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:56.120Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:56.121Z [INFO] agent.router.manager: shutting down -42:56.121 [INFO] TestAgent: consul server down -42:56.121 [INFO] TestAgent: shutdown complete -42:56.121 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13206 network=tcp -42:56.121 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13206 network=udp -42:56.121 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13207 network=tcp -42:56.125 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="shutdown waiting for leader" -42:56.125 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:56.621 [INFO] TestAgent: Waiting for endpoints to shut down -42:56.621 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_ServiceMaintenance_Disable (0.99s) +2021-08-13T15:34:18.109Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:18.124Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:18.127 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:406b6035-0392-79f5-9fd8-ce9f26c6d67d Address:127.0.0.1:24565}]" +34:18.127 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24565 [Follower]" leader= +34:18.128 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-406b6035-0392-79f5-9fd8-ce9f26c6d67d.dc1 127.0.0.1 +34:18.128 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-406b6035-0392-79f5-9fd8-ce9f26c6d67d 127.0.0.1 +2021-08-13T15:34:18.129Z [INFO] agent.router: Initializing LAN area manager +34:18.129 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-406b6035-0392-79f5-9fd8-ce9f26c6d67d.dc1 area=wan +34:18.129 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24560 network=udp +34:18.129 [INFO] TestAgent.server: Adding LAN server: server="Node-406b6035-0392-79f5-9fd8-ce9f26c6d67d (Addr: tcp/127.0.0.1:24565) (DC: dc1)" +34:18.129 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24560 network=tcp +34:18.130 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24561 network=tcp +34:18.130 [INFO] TestAgent: started state syncer +34:18.130 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24552 network=tcp +34:18.164 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:18.164 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24565 [Candidate]" term=2 +34:18.164 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:18.164 [DEBUG] TestAgent.server.raft: vote granted: from=406b6035-0392-79f5-9fd8-ce9f26c6d67d term=2 tally=1 +34:18.164 [INFO] TestAgent.server.raft: election won: tally=1 +34:18.164 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24565 [Leader]" +34:18.164 [INFO] TestAgent.server: cluster leadership acquired +34:18.164 [INFO] TestAgent.server: New leader elected: payload=Node-406b6035-0392-79f5-9fd8-ce9f26c6d67d +34:18.164 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24565 +34:18.165 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:18.168 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:18.168 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:18.168 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:18.168 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:18.168 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:18.168 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-406b6035-0392-79f5-9fd8-ce9f26c6d67d +34:18.168 [INFO] TestAgent.server: member joined, marking health alive: member=Node-406b6035-0392-79f5-9fd8-ce9f26c6d67d +34:18.169 [INFO] TestAgent.server: federation state anti-entropy synced +34:18.214 [WARN] TestAgent.server: endpoint injected; this should only be used for testing +34:18.215 [INFO] TestAgent: Requesting shutdown +34:18.215 [INFO] TestAgent.server: shutting down server +34:18.215 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:18.215 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:18.215 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:18.215 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:18.215 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:18.215 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:18.215 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:18.215 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:18.215 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:18.215 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:18.215 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:18.215 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:18.215Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:18.215Z [INFO] agent.router.manager: shutting down +34:18.215 [INFO] TestAgent: consul server down +34:18.215 [INFO] TestAgent: shutdown complete +34:18.215 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24560 network=tcp +34:18.215 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24560 network=udp +34:18.216 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24561 network=tcp +34:18.716 [INFO] TestAgent: Waiting for endpoints to shut down +34:18.716 [INFO] TestAgent: Endpoints down +--- PASS: TestPreparedQuery_List (1.43s) + --- PASS: TestPreparedQuery_List/#00 (0.80s) + --- PASS: TestPreparedQuery_List/#01 (0.62s) +=== CONT TestOperator_ServerHealth +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:18.109Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:18.124Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:18.127 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3e5c43ee-396b-af4b-b823-6898ccc9510d Address:127.0.0.1:24558}]" +34:18.127 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24558 [Follower]" leader= +34:18.128 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3e5c43ee-396b-af4b-b823-6898ccc9510d.dc1 127.0.0.1 +34:18.128 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3e5c43ee-396b-af4b-b823-6898ccc9510d 127.0.0.1 +2021-08-13T15:34:18.129Z [INFO] agent.router: Initializing LAN area manager +34:18.129 [INFO] TestAgent.server: Adding LAN server: server="Node-3e5c43ee-396b-af4b-b823-6898ccc9510d (Addr: tcp/127.0.0.1:24558) (DC: dc1)" +34:18.129 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24553 network=udp +34:18.129 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3e5c43ee-396b-af4b-b823-6898ccc9510d.dc1 area=wan +34:18.129 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24553 network=tcp +34:18.130 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24554 network=tcp +34:18.130 [INFO] TestAgent: started state syncer +34:18.130 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24566 network=tcp +34:18.164 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:18.164 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24558 [Candidate]" term=2 +34:18.165 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:18.165 [DEBUG] TestAgent.server.raft: vote granted: from=3e5c43ee-396b-af4b-b823-6898ccc9510d term=2 tally=1 +34:18.165 [INFO] TestAgent.server.raft: election won: tally=1 +34:18.165 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24558 [Leader]" +34:18.165 [INFO] TestAgent.server: cluster leadership acquired +34:18.165 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24558 +34:18.166 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:18.168 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:18.168 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:18.168 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:18.168 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:18.168 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:18.168 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3e5c43ee-396b-af4b-b823-6898ccc9510d +34:18.168 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3e5c43ee-396b-af4b-b823-6898ccc9510d +34:18.169 [INFO] TestAgent.server: New leader elected: payload=Node-3e5c43ee-396b-af4b-b823-6898ccc9510d +34:18.169 [INFO] TestAgent.server: federation state anti-entropy synced +34:18.355 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:18.356 [INFO] TestAgent: Synced node info +34:18.356 [DEBUG] TestAgent: Node info in sync +34:18.469 [WARN] TestAgent.server: endpoint injected; this should only be used for testing +34:18.470 [INFO] TestAgent: Requesting shutdown +34:18.470 [INFO] TestAgent.server: shutting down server +34:18.470 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:18.470 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:18.470 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:18.470 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:18.470 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:18.470 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:18.470 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:18.470 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:18.470 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:18.470 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:18.470 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:18.470Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:18.470Z [INFO] agent.router.manager: shutting down +34:18.470 [INFO] TestAgent: consul server down +34:18.470 [INFO] TestAgent: shutdown complete +34:18.470 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24553 network=tcp +34:18.470 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24553 network=udp +34:18.470 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24554 network=tcp +34:18.971 [INFO] TestAgent: Waiting for endpoints to shut down +34:18.971 [INFO] TestAgent: Endpoints down +=== RUN TestPreparedQuery_Execute/#02 +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:18.162Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:18.177Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:18.180 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e88ed5e4-c2d1-1756-996f-4d2f1769f678 Address:127.0.0.1:24579}]" +34:18.180 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24579 [Follower]" leader= +34:18.181 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e88ed5e4-c2d1-1756-996f-4d2f1769f678.dc1 127.0.0.1 +34:18.181 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e88ed5e4-c2d1-1756-996f-4d2f1769f678 127.0.0.1 +2021-08-13T15:34:18.182Z [INFO] agent.router: Initializing LAN area manager +34:18.182 [INFO] TestAgent.server: Adding LAN server: server="Node-e88ed5e4-c2d1-1756-996f-4d2f1769f678 (Addr: tcp/127.0.0.1:24579) (DC: dc1)" +34:18.182 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e88ed5e4-c2d1-1756-996f-4d2f1769f678.dc1 area=wan +34:18.182 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24567 network=tcp +34:18.182 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24567 network=udp +34:18.183 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24575 network=tcp +34:18.183 [INFO] TestAgent: started state syncer +34:18.183 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24580 network=tcp +34:18.244 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:18.244 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24579 [Candidate]" term=2 +34:18.244 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:18.244 [DEBUG] TestAgent.server.raft: vote granted: from=e88ed5e4-c2d1-1756-996f-4d2f1769f678 term=2 tally=1 +34:18.244 [INFO] TestAgent.server.raft: election won: tally=1 +34:18.244 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24579 [Leader]" +34:18.244 [INFO] TestAgent.server: cluster leadership acquired +34:18.245 [INFO] TestAgent.server: New leader elected: payload=Node-e88ed5e4-c2d1-1756-996f-4d2f1769f678 +34:18.245 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24579 +34:18.246 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:18.248 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:18.248 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:18.248 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:18.248 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:18.248 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:18.248 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e88ed5e4-c2d1-1756-996f-4d2f1769f678 +34:18.249 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e88ed5e4-c2d1-1756-996f-4d2f1769f678 +34:18.249 [INFO] TestAgent.server: federation state anti-entropy synced +34:18.508 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:18.508 [INFO] TestAgent: Synced node info +34:18.568 [INFO] TestAgent: Service entered maintenance mode: service=test +34:18.568 [DEBUG] TestAgent: Node info in sync +34:18.569 [INFO] TestAgent: Synced service: service=test +34:18.570 [INFO] TestAgent: Synced check: check=_service_maintenance:test +34:18.570 [INFO] TestAgent: Requesting shutdown +34:18.570 [INFO] TestAgent.server: shutting down server +34:18.570 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:18.570 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:18.570 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:18.570 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:18.570 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:18.570 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:18.570 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:18.570 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:18.570 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:18.570 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:18.570 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:18.570Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:18.571Z [INFO] agent.router.manager: shutting down +34:18.571 [INFO] TestAgent: consul server down +34:18.571 [INFO] TestAgent: shutdown complete +34:18.571 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24567 network=tcp +34:18.571 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24567 network=udp +34:18.571 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24575 network=tcp +34:19.071 [INFO] TestAgent: Waiting for endpoints to shut down +34:19.071 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_ServiceMaintenance_Enable (0.93s) +=== CONT TestAgent_DeregisterService_ACLDeny +2021-08-13T15:34:18.562Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:18.578Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:18.580 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ee3afe91-ac38-6fc9-33e1-7b048ef4f636 Address:127.0.0.1:24593}]" +34:18.580 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24593 [Follower]" leader= +34:18.581 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ee3afe91-ac38-6fc9-33e1-7b048ef4f636.dc1 127.0.0.1 +34:18.582 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ee3afe91-ac38-6fc9-33e1-7b048ef4f636 127.0.0.1 +2021-08-13T15:34:18.582Z [INFO] agent.router: Initializing LAN area manager +34:18.582 [INFO] TestAgent.server: Adding LAN server: server="Node-ee3afe91-ac38-6fc9-33e1-7b048ef4f636 (Addr: tcp/127.0.0.1:24593) (DC: dc1)" +34:18.582 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ee3afe91-ac38-6fc9-33e1-7b048ef4f636.dc1 area=wan +34:18.583 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24588 network=udp +34:18.583 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24588 network=tcp +34:18.583 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24589 network=tcp +34:18.584 [INFO] TestAgent: started state syncer +34:18.584 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24601 network=tcp +34:18.628 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:18.628 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24593 [Candidate]" term=2 +34:18.628 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:18.628 [DEBUG] TestAgent.server.raft: vote granted: from=ee3afe91-ac38-6fc9-33e1-7b048ef4f636 term=2 tally=1 +34:18.628 [INFO] TestAgent.server.raft: election won: tally=1 +34:18.628 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24593 [Leader]" +34:18.628 [INFO] TestAgent.server: cluster leadership acquired +34:18.629 [INFO] TestAgent.server: New leader elected: payload=Node-ee3afe91-ac38-6fc9-33e1-7b048ef4f636 +34:18.629 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24593 +34:18.630 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:18.632 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:18.632 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:18.632 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:18.632 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:18.632 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:18.632 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ee3afe91-ac38-6fc9-33e1-7b048ef4f636 +34:18.632 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ee3afe91-ac38-6fc9-33e1-7b048ef4f636 +34:18.633 [INFO] TestAgent.server: federation state anti-entropy synced +34:18.654 [INFO] TestAgent: Requesting shutdown +34:18.655 [INFO] TestAgent.server: shutting down server +34:18.655 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:18.655 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:18.655 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:18.655 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:18.655 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:18.655 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:18.655 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:18.655 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:18.655 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:18.655 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:18.655 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:18.655 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:18.655Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:18.655Z [INFO] agent.router.manager: shutting down +34:18.655 [INFO] TestAgent: consul server down +34:18.655 [INFO] TestAgent: shutdown complete +34:18.655 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24588 network=tcp +34:18.655 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24588 network=udp +34:18.655 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24589 network=tcp +34:19.155 [INFO] TestAgent: Waiting for endpoints to shut down +34:19.155 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_ServiceMaintenance_BadRequest (0.61s) + --- PASS: TestAgent_ServiceMaintenance_BadRequest/not_enabled (0.00s) + --- PASS: TestAgent_ServiceMaintenance_BadRequest/no_service_id (0.00s) + --- PASS: TestAgent_ServiceMaintenance_BadRequest/bad_service_id (0.00s) +=== CONT TestOperator_AutopilotCASConfiguration +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestAgent_DeregisterService_ACLDeny/no_token +=== RUN TestAgent_DeregisterService_ACLDeny/root_token +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:18.989Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:19.007Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:19.010 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:77d18176-6995-a741-95d1-cc2f85386c5f Address:127.0.0.1:24614}]" +34:19.010 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24614 [Follower]" leader= +34:19.011 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-77d18176-6995-a741-95d1-cc2f85386c5f.dc1 127.0.0.1 +34:19.012 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-77d18176-6995-a741-95d1-cc2f85386c5f 127.0.0.1 +2021-08-13T15:34:19.012Z [INFO] agent.router: Initializing LAN area manager +34:19.012 [INFO] TestAgent.server: Adding LAN server: server="Node-77d18176-6995-a741-95d1-cc2f85386c5f (Addr: tcp/127.0.0.1:24614) (DC: dc1)" +34:19.012 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-77d18176-6995-a741-95d1-cc2f85386c5f.dc1 area=wan +34:19.012 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24609 network=tcp +34:19.012 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24609 network=udp +34:19.013 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24610 network=tcp +34:19.013 [INFO] TestAgent: started state syncer +34:19.013 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24636 network=tcp +34:19.052 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:19.052 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24614 [Candidate]" term=2 +34:19.053 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:19.053 [DEBUG] TestAgent.server.raft: vote granted: from=77d18176-6995-a741-95d1-cc2f85386c5f term=2 tally=1 +34:19.053 [INFO] TestAgent.server.raft: election won: tally=1 +34:19.053 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24614 [Leader]" +34:19.053 [INFO] TestAgent.server: cluster leadership acquired +34:19.053 [INFO] TestAgent.server: New leader elected: payload=Node-77d18176-6995-a741-95d1-cc2f85386c5f +34:19.053 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24614 +34:19.055 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:19.057 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:19.057 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:19.057 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:19.057 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:19.057 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:19.057 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-77d18176-6995-a741-95d1-cc2f85386c5f +34:19.057 [INFO] TestAgent.server: member joined, marking health alive: member=Node-77d18176-6995-a741-95d1-cc2f85386c5f +34:19.058 [INFO] TestAgent.server: federation state anti-entropy synced +34:19.265 [WARN] TestAgent.server: endpoint injected; this should only be used for testing +34:19.265 [INFO] TestAgent: Requesting shutdown +34:19.265 [INFO] TestAgent.server: shutting down server +34:19.265 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:19.265 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:19.265 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:19.265 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:19.265 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:19.265 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:19.265 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:19.265 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:19.265 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:19.265 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:19.265 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:19.265 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:19.265Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:19.265Z [INFO] agent.router.manager: shutting down +34:19.266 [INFO] TestAgent: consul server down +34:19.266 [INFO] TestAgent: shutdown complete +34:19.266 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24609 network=tcp +34:19.266 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24609 network=udp +34:19.266 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24610 network=tcp +34:19.769 [INFO] TestAgent: Waiting for endpoints to shut down +34:19.769 [INFO] TestAgent: Endpoints down +2021-08-13T15:34:19.173Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:19.189Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:19.192 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:35c5dc10-be1b-9109-9a6f-68b98f1d94af Address:127.0.0.1:24621}]" +34:19.192 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24621 [Follower]" leader= +34:19.193 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-35c5dc10-be1b-9109-9a6f-68b98f1d94af.dc1 127.0.0.1 +34:19.194 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-35c5dc10-be1b-9109-9a6f-68b98f1d94af 127.0.0.1 +2021-08-13T15:34:19.194Z [INFO] agent.router: Initializing LAN area manager +34:19.194 [INFO] TestAgent.server: Adding LAN server: server="Node-35c5dc10-be1b-9109-9a6f-68b98f1d94af (Addr: tcp/127.0.0.1:24621) (DC: dc1)" +34:19.194 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-35c5dc10-be1b-9109-9a6f-68b98f1d94af.dc1 area=wan +34:19.194 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24616 network=tcp +34:19.194 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24616 network=udp +34:19.195 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24617 network=tcp +34:19.195 [INFO] TestAgent: started state syncer +34:19.195 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24622 network=tcp +34:19.233 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:19.233 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24621 [Candidate]" term=2 +34:19.233 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:19.233 [DEBUG] TestAgent.server.raft: vote granted: from=35c5dc10-be1b-9109-9a6f-68b98f1d94af term=2 tally=1 +34:19.233 [INFO] TestAgent.server.raft: election won: tally=1 +34:19.233 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24621 [Leader]" +34:19.233 [INFO] TestAgent.server: cluster leadership acquired +34:19.234 [INFO] TestAgent.server: New leader elected: payload=Node-35c5dc10-be1b-9109-9a6f-68b98f1d94af +34:19.234 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24621 +34:19.235 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:19.237 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:19.237 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:19.238 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:19.238 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:19.238 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:19.238 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-35c5dc10-be1b-9109-9a6f-68b98f1d94af +34:19.238 [INFO] TestAgent.server: member joined, marking health alive: member=Node-35c5dc10-be1b-9109-9a6f-68b98f1d94af +34:19.238 [INFO] TestAgent.server: federation state anti-entropy synced +34:19.266 [INFO] TestAgent: Requesting shutdown +34:19.266 [INFO] TestAgent.server: shutting down server +34:19.266 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:19.266 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:19.266 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:19.266 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:19.266 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:19.266 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:19.266 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:19.266 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:19.266 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:19.266 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:19.266 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:19.266 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:19.266Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:19.266Z [INFO] agent.router.manager: shutting down +34:19.267 [INFO] TestAgent: consul server down +34:19.267 [INFO] TestAgent: shutdown complete +34:19.267 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24616 network=tcp +34:19.267 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24616 network=udp +34:19.267 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24617 network=tcp +34:19.769 [INFO] TestAgent: Waiting for endpoints to shut down +34:19.769 [INFO] TestAgent: Endpoints down +--- PASS: TestOperator_AutopilotCASConfiguration (0.61s) === CONT TestAgent_DeregisterService -=== RUN TestAgent_Monitor/serf_shutdown_logging -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:56.295Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:56.295Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:56.316Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:56.316Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:56.319 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:48a77c52-f018-c499-44b5-853228d63f33 Address:127.0.0.1:13211}]" -42:56.320 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13211 [Follower]" leader= -42:56.320 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-48a77c52-f018-c499-44b5-853228d63f33.dc1 127.0.0.1 -42:56.321 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-48a77c52-f018-c499-44b5-853228d63f33 127.0.0.1 -2022-09-15T21:42:56.321Z [INFO] agent.router: Initializing LAN area manager -42:56.322 [INFO] TestAgent.server: Adding LAN server: server="Node-48a77c52-f018-c499-44b5-853228d63f33 (Addr: tcp/127.0.0.1:13211) (DC: dc1)" -42:56.322 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-48a77c52-f018-c499-44b5-853228d63f33.dc1 area=wan -42:56.322 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13192 network=udp -42:56.322 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13192 network=tcp -42:56.323 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13193 network=tcp -42:56.323 [INFO] TestAgent: started state syncer -42:56.323 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13212 network=tcp -42:56.373 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -42:56.376 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:56.376 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13211 [Candidate]" term=2 -42:56.376 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:56.376 [DEBUG] TestAgent.server.raft: vote granted: from=48a77c52-f018-c499-44b5-853228d63f33 term=2 tally=1 -42:56.376 [INFO] TestAgent.server.raft: election won: tally=1 -42:56.376 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13211 [Leader]" -42:56.376 [INFO] TestAgent.server: cluster leadership acquired -42:56.377 [INFO] TestAgent.server: initializing acls -42:56.377 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:56.377 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:56.378 [INFO] TestAgent.server: New leader elected: payload=Node-48a77c52-f018-c499-44b5-853228d63f33 -42:56.378 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:56.379 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:56.379 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:56.379 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:56.379 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-48a77c52-f018-c499-44b5-853228d63f33 -42:56.379 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-48a77c52-f018-c499-44b5-853228d63f33.dc1 -42:56.380 [INFO] TestAgent.server: Updating LAN server: server="Node-48a77c52-f018-c499-44b5-853228d63f33 (Addr: tcp/127.0.0.1:13211) (DC: dc1)" -42:56.381 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-48a77c52-f018-c499-44b5-853228d63f33.dc1 area=wan -42:56.381 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:56.384 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:56.384 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:56.384 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:56.384 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:56.384 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:56.384 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-48a77c52-f018-c499-44b5-853228d63f33 -42:56.384 [INFO] TestAgent.server: member joined, marking health alive: member=Node-48a77c52-f018-c499-44b5-853228d63f33 -42:56.385 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-48a77c52-f018-c499-44b5-853228d63f33 -42:56.386 [INFO] TestAgent.server: federation state anti-entropy synced -42:56.408 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-48a77c52-f018-c499-44b5-853228d63f33 -42:56.409 [DEBUG] TestAgent: removed service: service=test -42:56.410 [INFO] TestAgent: Synced node info -42:56.410 [INFO] TestAgent: Deregistered service: service=test -42:56.410 [INFO] TestAgent: Requesting shutdown -42:56.410 [INFO] TestAgent.server: shutting down server -42:56.410 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:56.410 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:56.410 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:56.410 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:56.410 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:56.410 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:56.410 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:56.410 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:56.410 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:56.410 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:56.410 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:56.410 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:56.410 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:56.410 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:56.410 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:56.411 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:56.411Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:56.411Z [INFO] agent.router.manager: shutting down -42:56.411 [INFO] TestAgent: consul server down -42:56.411 [INFO] TestAgent: shutdown complete -42:56.411 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13192 network=tcp -42:56.411 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13192 network=udp -42:56.411 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13193 network=tcp -42:56.912 [INFO] TestAgent: Waiting for endpoints to shut down -42:56.912 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_DeregisterService_ACLDeny (0.64s) +=== RUN TestPreparedQuery_Execute/#03 +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:19.089Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:19.089Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:19.106Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:19.106Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:19.109 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ef59552e-3c55-4483-1cfe-ba7f0a1a6409 Address:127.0.0.1:24649}]" +34:19.109 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24649 [Follower]" leader= +34:19.109 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ef59552e-3c55-4483-1cfe-ba7f0a1a6409.dc1 127.0.0.1 +34:19.110 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ef59552e-3c55-4483-1cfe-ba7f0a1a6409 127.0.0.1 +2021-08-13T15:34:19.110Z [INFO] agent.router: Initializing LAN area manager +34:19.111 [INFO] TestAgent.server: Adding LAN server: server="Node-ef59552e-3c55-4483-1cfe-ba7f0a1a6409 (Addr: tcp/127.0.0.1:24649) (DC: dc1)" +34:19.111 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ef59552e-3c55-4483-1cfe-ba7f0a1a6409.dc1 area=wan +34:19.111 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24637 network=tcp +34:19.111 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24637 network=udp +34:19.112 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24645 network=tcp +34:19.112 [INFO] TestAgent: started state syncer +34:19.112 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24615 network=tcp +34:19.153 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:19.153 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24649 [Candidate]" term=2 +34:19.154 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:19.154 [DEBUG] TestAgent.server.raft: vote granted: from=ef59552e-3c55-4483-1cfe-ba7f0a1a6409 term=2 tally=1 +34:19.154 [INFO] TestAgent.server.raft: election won: tally=1 +34:19.154 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24649 [Leader]" +34:19.154 [INFO] TestAgent.server: cluster leadership acquired +34:19.154 [INFO] TestAgent.server: New leader elected: payload=Node-ef59552e-3c55-4483-1cfe-ba7f0a1a6409 +34:19.154 [INFO] TestAgent.server: initializing acls +34:19.155 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:19.155 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:19.155 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:19.156 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:19.156 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:19.156 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:19.156 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-ef59552e-3c55-4483-1cfe-ba7f0a1a6409 +34:19.156 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-ef59552e-3c55-4483-1cfe-ba7f0a1a6409.dc1 +34:19.156 [INFO] TestAgent.server: Updating LAN server: server="Node-ef59552e-3c55-4483-1cfe-ba7f0a1a6409 (Addr: tcp/127.0.0.1:24649) (DC: dc1)" +34:19.157 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-ef59552e-3c55-4483-1cfe-ba7f0a1a6409.dc1 area=wan +34:19.157 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:19.159 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:19.160 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:19.160 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:19.160 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:19.160 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:19.160 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ef59552e-3c55-4483-1cfe-ba7f0a1a6409 +34:19.160 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ef59552e-3c55-4483-1cfe-ba7f0a1a6409 +34:19.160 [INFO] TestAgent.server: federation state anti-entropy synced +34:19.161 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ef59552e-3c55-4483-1cfe-ba7f0a1a6409 +34:19.286 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-ef59552e-3c55-4483-1cfe-ba7f0a1a6409 +34:19.286 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-ef59552e-3c55-4483-1cfe-ba7f0a1a6409 +34:19.287 [DEBUG] TestAgent: removed service: service=test +34:19.288 [INFO] TestAgent: Synced node info +34:19.288 [INFO] TestAgent: Deregistered service: service=test +34:19.288 [INFO] TestAgent: Requesting shutdown +34:19.288 [INFO] TestAgent.server: shutting down server +34:19.288 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:19.288 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:19.288 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:19.288 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:19.288 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:19.288 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:19.288 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:19.288 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:19.289 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:19.289 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:19.288 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:19.288 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:19.289 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:19.288 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:19.289 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:19.289 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:19.289Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:19.289Z [INFO] agent.router.manager: shutting down +34:19.289 [INFO] TestAgent: consul server down +34:19.289 [INFO] TestAgent: shutdown complete +34:19.289 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24637 network=tcp +34:19.289 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24637 network=udp +34:19.289 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24645 network=tcp +34:19.789 [INFO] TestAgent: Waiting for endpoints to shut down +34:19.789 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_DeregisterService_ACLDeny (0.72s) --- PASS: TestAgent_DeregisterService_ACLDeny/no_token (0.00s) --- PASS: TestAgent_DeregisterService_ACLDeny/root_token (0.00s) +=== CONT TestOperator_AutopilotSetConfiguration +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:18.186Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:18.186Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:18.203Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:18.203Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:18.206 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:33f8a2e5-7921-c0b4-b474-887953abb3b8 Address:127.0.0.1:24572}]" +34:18.206 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24572 [Follower]" leader= +34:18.207 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-33f8a2e5-7921-c0b4-b474-887953abb3b8.dc1 127.0.0.1 +34:18.208 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-33f8a2e5-7921-c0b4-b474-887953abb3b8 127.0.0.1 +2021-08-13T15:34:18.208Z [INFO] agent.router: Initializing LAN area manager +34:18.208 [INFO] TestAgent.server: Adding LAN server: server="Node-33f8a2e5-7921-c0b4-b474-887953abb3b8 (Addr: tcp/127.0.0.1:24572) (DC: dc1)" +34:18.208 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-33f8a2e5-7921-c0b4-b474-887953abb3b8.dc1 area=wan +34:18.208 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24581 network=udp +34:18.208 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24581 network=tcp +34:18.209 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24568 network=tcp +34:18.209 [INFO] TestAgent: started state syncer +34:18.209 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24594 network=tcp +34:18.250 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:18.251 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24572 [Candidate]" term=2 +34:18.251 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:18.251 [DEBUG] TestAgent.server.raft: vote granted: from=33f8a2e5-7921-c0b4-b474-887953abb3b8 term=2 tally=1 +34:18.251 [INFO] TestAgent.server.raft: election won: tally=1 +34:18.251 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24572 [Leader]" +34:18.251 [INFO] TestAgent.server: cluster leadership acquired +34:18.251 [INFO] TestAgent.server: New leader elected: payload=Node-33f8a2e5-7921-c0b4-b474-887953abb3b8 +34:18.251 [INFO] TestAgent.server: initializing acls +34:18.251 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:18.252 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:18.252 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:18.253 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:18.253 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:18.253 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:18.253 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-33f8a2e5-7921-c0b4-b474-887953abb3b8 +34:18.253 [INFO] TestAgent.server: Updating LAN server: server="Node-33f8a2e5-7921-c0b4-b474-887953abb3b8 (Addr: tcp/127.0.0.1:24572) (DC: dc1)" +34:18.253 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-33f8a2e5-7921-c0b4-b474-887953abb3b8.dc1 +34:18.253 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-33f8a2e5-7921-c0b4-b474-887953abb3b8.dc1 area=wan +34:18.254 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:18.256 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:18.256 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:18.256 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:18.256 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:18.256 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:18.256 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-33f8a2e5-7921-c0b4-b474-887953abb3b8 +34:18.257 [INFO] TestAgent.server: member joined, marking health alive: member=Node-33f8a2e5-7921-c0b4-b474-887953abb3b8 +34:18.257 [INFO] TestAgent.server: federation state anti-entropy synced +34:18.258 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-33f8a2e5-7921-c0b4-b474-887953abb3b8 +34:18.328 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth +34:18.328 [WARN] TestAgent: Node info update blocked by ACLs: node=33f8a2e5-7921-c0b4-b474-887953abb3b8 accessorID=00000000-0000-0000-0000-000000000002 +34:18.328 [DEBUG] TestAgent: Node info in sync +34:18.572 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-33f8a2e5-7921-c0b4-b474-887953abb3b8 +34:18.572 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-33f8a2e5-7921-c0b4-b474-887953abb3b8 +34:18.575 [INFO] TestAgent: Updated agent's ACL token: token=acl_token +34:19.936 [DEBUG] TestAgent.acl: dropping service from result due to ACLs: service="{consul {}}" +34:19.937 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:19.937 [INFO] TestAgent: Synced node info +34:19.955 [INFO] TestAgent: Requesting shutdown +34:19.955 [INFO] TestAgent.server: shutting down server +34:19.955 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:19.955 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:19.955 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:19.955 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:19.955 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:19.955 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:19.955 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:19.955 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:19.955 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:19.955 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:19.955 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:19.955 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:19.955 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +34:19.955 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:19.955 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2021-08-13T15:34:19.955Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:19.955Z [INFO] agent.router.manager: shutting down +34:19.956 [INFO] TestAgent: consul server down +34:19.956 [INFO] TestAgent: shutdown complete +34:19.956 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24581 network=tcp +34:19.956 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24581 network=udp +34:19.956 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24568 network=tcp +34:20.456 [INFO] TestAgent: Waiting for endpoints to shut down +34:20.456 [INFO] TestAgent: Endpoints down +=== RUN TestAgent_TokenTriggersFullSync/default +2021-08-13T15:34:19.787Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:19.828Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:19.832 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7e135dd9-bc5d-9df9-1bc4-03e097f71798 Address:127.0.0.1:24628}]" +34:19.837 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24628 [Follower]" leader= +34:19.838 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7e135dd9-bc5d-9df9-1bc4-03e097f71798.dc1 127.0.0.1 +34:19.838 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7e135dd9-bc5d-9df9-1bc4-03e097f71798 127.0.0.1 +2021-08-13T15:34:19.838Z [INFO] agent.router: Initializing LAN area manager +34:19.839 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24623 network=udp +34:19.841 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7e135dd9-bc5d-9df9-1bc4-03e097f71798.dc1 area=wan +34:19.841 [INFO] TestAgent.server: Adding LAN server: server="Node-7e135dd9-bc5d-9df9-1bc4-03e097f71798 (Addr: tcp/127.0.0.1:24628) (DC: dc1)" +34:19.841 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24623 network=tcp +34:19.842 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24624 network=tcp +34:19.842 [INFO] TestAgent: started state syncer +34:19.842 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24573 network=tcp +34:19.883 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:19.883 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24628 [Candidate]" term=2 +34:19.883 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:19.883 [DEBUG] TestAgent.server.raft: vote granted: from=7e135dd9-bc5d-9df9-1bc4-03e097f71798 term=2 tally=1 +34:19.883 [INFO] TestAgent.server.raft: election won: tally=1 +34:19.883 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24628 [Leader]" +34:19.883 [INFO] TestAgent.server: cluster leadership acquired +34:19.884 [INFO] TestAgent.server: New leader elected: payload=Node-7e135dd9-bc5d-9df9-1bc4-03e097f71798 +34:19.884 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24628 +34:19.885 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:19.887 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:19.887 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:19.887 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:19.887 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:19.887 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:19.887 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7e135dd9-bc5d-9df9-1bc4-03e097f71798 +34:19.887 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7e135dd9-bc5d-9df9-1bc4-03e097f71798 +34:19.888 [INFO] TestAgent.server: federation state anti-entropy synced +34:19.961 [DEBUG] TestAgent: removed service: service=test +34:19.961 [INFO] TestAgent: Synced node info +34:19.962 [INFO] TestAgent: Deregistered service: service=test +34:19.962 [INFO] TestAgent: Requesting shutdown +34:19.962 [INFO] TestAgent.server: shutting down server +34:19.962 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:19.962 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:19.962 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:19.962 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:19.962 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:19.962 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:19.962 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:19.962 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:19.962 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +34:19.962 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:19.962 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:19.962 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2021-08-13T15:34:19.962Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:19.962Z [INFO] agent.router.manager: shutting down +34:19.962 [INFO] TestAgent: consul server down +34:19.962 [INFO] TestAgent: shutdown complete +34:19.962 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24623 network=tcp +34:19.962 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24623 network=udp +34:19.962 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24624 network=tcp +34:20.463 [INFO] TestAgent: Waiting for endpoints to shut down +34:20.463 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_DeregisterService (0.69s) === CONT TestAgent_UpdateCheck_ACLDeny -2022-09-15T21:42:56.030Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:56.049Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:56.057 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d7ae38ae-9476-ba77-da36-04f838db4d24 Address:127.0.0.1:13239}]" -42:56.058 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13239 [Follower]" leader= -42:56.062 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d7ae38ae-9476-ba77-da36-04f838db4d24.dc1 127.0.0.1 -42:56.063 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d7ae38ae-9476-ba77-da36-04f838db4d24 127.0.0.1 -2022-09-15T21:42:56.063Z [INFO] agent.router: Initializing LAN area manager -42:56.064 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13234 network=udp -42:56.064 [INFO] TestAgent.server: Adding LAN server: server="Node-d7ae38ae-9476-ba77-da36-04f838db4d24 (Addr: tcp/127.0.0.1:13239) (DC: dc1)" -42:56.064 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d7ae38ae-9476-ba77-da36-04f838db4d24.dc1 area=wan -42:56.065 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13234 network=tcp -42:56.065 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13235 network=tcp -42:56.066 [INFO] TestAgent: started state syncer -42:56.066 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13240 network=tcp -42:56.108 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:56.109 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13239 [Candidate]" term=2 -42:56.109 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:56.109 [DEBUG] TestAgent.server.raft: vote granted: from=d7ae38ae-9476-ba77-da36-04f838db4d24 term=2 tally=1 -42:56.109 [INFO] TestAgent.server.raft: election won: tally=1 -42:56.109 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13239 [Leader]" -42:56.109 [INFO] TestAgent.server: cluster leadership acquired -42:56.109 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13239 -42:56.110 [INFO] TestAgent.server: New leader elected: payload=Node-d7ae38ae-9476-ba77-da36-04f838db4d24 -42:56.111 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:56.114 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:56.114 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:56.114 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:56.114 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:56.114 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:56.114 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d7ae38ae-9476-ba77-da36-04f838db4d24 -42:56.114 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d7ae38ae-9476-ba77-da36-04f838db4d24 -42:56.116 [INFO] TestAgent.server: federation state anti-entropy synced -42:56.262 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:56.263 [INFO] TestAgent: Synced node info -42:56.467 [INFO] TestAgent: Service entered maintenance mode: service=test -42:56.467 [DEBUG] TestAgent: Node info in sync -42:56.468 [INFO] TestAgent: Synced service: service=test -42:56.469 [INFO] TestAgent: Synced check: check=_service_maintenance:test -42:56.469 [INFO] TestAgent: Requesting shutdown -42:56.469 [INFO] TestAgent.server: shutting down server -42:56.469 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:56.469 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:56.469 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:56.469 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:56.469 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:56.469 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:56.469 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:56.469 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:56.469 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:56.469 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:56.469 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:56.469Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:56.469Z [INFO] agent.router.manager: shutting down -42:56.469 [INFO] TestAgent: consul server down -42:56.469 [INFO] TestAgent: shutdown complete -42:56.469 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13234 network=tcp -42:56.469 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13234 network=udp -42:56.469 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13235 network=tcp -42:56.973 [INFO] TestAgent: Waiting for endpoints to shut down -42:56.973 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_ServiceMaintenance_Enable (0.96s) -=== CONT TestAgent_FailCheck_ACLDeny -2022-09-15T21:42:56.203Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:56.243Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:56.247 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:275507e1-baeb-bbf9-ca2f-fa5a49fbd306 Address:127.0.0.1:13204}]" -42:56.248 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-275507e1-baeb-bbf9-ca2f-fa5a49fbd306.dc1 127.0.0.1 -42:56.249 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-275507e1-baeb-bbf9-ca2f-fa5a49fbd306 127.0.0.1 -2022-09-15T21:42:56.249Z [INFO] agent.router: Initializing LAN area manager -42:56.249 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13213 network=udp -42:56.249 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13204 [Follower]" leader= -42:56.250 [INFO] TestAgent.server: Adding LAN server: server="Node-275507e1-baeb-bbf9-ca2f-fa5a49fbd306 (Addr: tcp/127.0.0.1:13204) (DC: dc1)" -42:56.250 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-275507e1-baeb-bbf9-ca2f-fa5a49fbd306.dc1 area=wan -42:56.251 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13213 network=tcp -42:56.251 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13214 network=tcp -42:56.252 [INFO] TestAgent: started state syncer -42:56.252 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13205 network=tcp -42:56.290 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:56.290 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13204 [Candidate]" term=2 -42:56.290 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:56.290 [DEBUG] TestAgent.server.raft: vote granted: from=275507e1-baeb-bbf9-ca2f-fa5a49fbd306 term=2 tally=1 -42:56.290 [INFO] TestAgent.server.raft: election won: tally=1 -42:56.290 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13204 [Leader]" -42:56.290 [INFO] TestAgent.server: cluster leadership acquired -42:56.290 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13204 -42:56.292 [INFO] TestAgent.server: New leader elected: payload=Node-275507e1-baeb-bbf9-ca2f-fa5a49fbd306 -42:56.292 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:56.296 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:56.296 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:56.296 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:56.296 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:56.296 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:56.297 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-275507e1-baeb-bbf9-ca2f-fa5a49fbd306 -42:56.297 [INFO] TestAgent.server: member joined, marking health alive: member=Node-275507e1-baeb-bbf9-ca2f-fa5a49fbd306 -42:56.298 [INFO] TestAgent.server: federation state anti-entropy synced -42:56.387 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:56.387 [INFO] TestAgent: Synced node info -42:56.387 [DEBUG] TestAgent: Node info in sync -42:56.497 [INFO] TestAgent: Requesting shutdown -42:56.497 [INFO] TestAgent.server: shutting down server -42:56.497 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:56.497 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:56.497 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:56.497 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:56.497 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:56.497 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:56.497 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:56.497 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:56.498 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:56.498 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:56.498 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:56.498Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:56.498Z [INFO] agent.router.manager: shutting down -42:56.498 [INFO] TestAgent: consul server down -42:56.498 [INFO] TestAgent: shutdown complete -42:56.498 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13213 network=tcp -42:56.498 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13213 network=udp -42:56.498 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13214 network=tcp -42:56.999 [INFO] TestAgent: Waiting for endpoints to shut down -42:56.999 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_ServiceMaintenance_BadRequest (0.89s) - --- PASS: TestAgent_ServiceMaintenance_BadRequest/not_enabled (0.00s) - --- PASS: TestAgent_ServiceMaintenance_BadRequest/no_service_id (0.00s) - --- PASS: TestAgent_ServiceMaintenance_BadRequest/bad_service_id (0.00s) -=== CONT TestAgent_FailCheck -[WARN] freeport: 4 out of 25 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:19.786Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:19.828Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:19.833 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b3947cd1-5d21-3d54-7b7e-6053c9529a52 Address:127.0.0.1:24586}]" +34:19.836 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b3947cd1-5d21-3d54-7b7e-6053c9529a52.dc1 127.0.0.1 +34:19.837 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b3947cd1-5d21-3d54-7b7e-6053c9529a52 127.0.0.1 +2021-08-13T15:34:19.837Z [INFO] agent.router: Initializing LAN area manager +34:19.837 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24574 network=udp +34:19.837 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24586 [Follower]" leader= +34:19.839 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24574 network=tcp +34:19.840 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24582 network=tcp +34:19.840 [INFO] TestAgent: started state syncer +34:19.841 [INFO] TestAgent.server: Adding LAN server: server="Node-b3947cd1-5d21-3d54-7b7e-6053c9529a52 (Addr: tcp/127.0.0.1:24586) (DC: dc1)" +34:19.841 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24643 network=tcp +34:19.841 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b3947cd1-5d21-3d54-7b7e-6053c9529a52.dc1 area=wan +34:19.884 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:19.884 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24586 [Candidate]" term=2 +34:19.885 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:19.885 [DEBUG] TestAgent.server.raft: vote granted: from=b3947cd1-5d21-3d54-7b7e-6053c9529a52 term=2 tally=1 +34:19.885 [INFO] TestAgent.server.raft: election won: tally=1 +34:19.885 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24586 [Leader]" +34:19.885 [INFO] TestAgent.server: cluster leadership acquired +34:19.885 [INFO] TestAgent.server: New leader elected: payload=Node-b3947cd1-5d21-3d54-7b7e-6053c9529a52 +34:19.885 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24586 +34:19.886 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:19.888 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:19.888 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:19.888 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:19.888 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:19.888 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:19.888 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b3947cd1-5d21-3d54-7b7e-6053c9529a52 +34:19.888 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b3947cd1-5d21-3d54-7b7e-6053c9529a52 +34:19.889 [INFO] TestAgent.server: federation state anti-entropy synced +34:20.066 [WARN] TestAgent.server: endpoint injected; this should only be used for testing +34:20.066 [INFO] TestAgent: Requesting shutdown +34:20.066 [INFO] TestAgent.server: shutting down server +34:20.066 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:20.066 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:20.066 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:20.066 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:20.066 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:20.066 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:20.067 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:20.066 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:20.067 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:20.067 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:20.067 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:20.067 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:20.067Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:20.067Z [INFO] agent.router.manager: shutting down +34:20.067 [INFO] TestAgent: consul server down +34:20.067 [INFO] TestAgent: shutdown complete +34:20.067 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24574 network=tcp +34:20.067 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24574 network=udp +34:20.067 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24582 network=tcp +34:20.567 [INFO] TestAgent: Waiting for endpoints to shut down +34:20.567 [INFO] TestAgent: Endpoints down +=== RUN TestPreparedQuery_Execute/#04 === RUN TestAgent_UpdateCheck_ACLDeny/no_token === RUN TestAgent_UpdateCheck_ACLDeny/root_token -[WARN] freeport: 4 out of 4 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:54.511Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:54.537Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:54.540 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7a609d1f-8fc8-5751-bb0e-358ef07f111b Address:127.0.0.1:13085}]" -42:54.541 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7a609d1f-8fc8-5751-bb0e-358ef07f111b.dc1 127.0.0.1 -42:54.542 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13085 [Follower]" leader= -42:54.547 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7a609d1f-8fc8-5751-bb0e-358ef07f111b 127.0.0.1 -2022-09-15T21:42:54.547Z [INFO] agent.router: Initializing LAN area manager -42:54.547 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13073 network=udp -42:54.547 [INFO] TestAgent.server: Adding LAN server: server="Node-7a609d1f-8fc8-5751-bb0e-358ef07f111b (Addr: tcp/127.0.0.1:13085) (DC: dc1)" -42:54.547 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7a609d1f-8fc8-5751-bb0e-358ef07f111b.dc1 area=wan -42:54.548 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13073 network=tcp -42:54.549 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13074 network=tcp -42:54.549 [INFO] TestAgent: started state syncer -42:54.549 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13086 network=tcp -42:54.588 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:54.588 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13085 [Candidate]" term=2 -42:54.588 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:54.588 [DEBUG] TestAgent.server.raft: vote granted: from=7a609d1f-8fc8-5751-bb0e-358ef07f111b term=2 tally=1 -42:54.588 [INFO] TestAgent.server.raft: election won: tally=1 -42:54.588 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13085 [Leader]" -42:54.588 [INFO] TestAgent.server: cluster leadership acquired -42:54.588 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13085 -42:54.590 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:54.590 [INFO] TestAgent.server: New leader elected: payload=Node-7a609d1f-8fc8-5751-bb0e-358ef07f111b -42:54.592 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:54.592 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:54.592 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:54.592 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:54.592 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:54.592 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7a609d1f-8fc8-5751-bb0e-358ef07f111b -42:54.592 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7a609d1f-8fc8-5751-bb0e-358ef07f111b -42:54.594 [INFO] TestAgent.server: federation state anti-entropy synced -42:54.596 [INFO] TestAgent: Synced node info -42:54.596 [INFO] TestAgent: Synced service: service=monitor -42:54.596 [DEBUG] TestAgent: Check in sync: check=service:monitor -42:54.647 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:54.647 [DEBUG] TestAgent: Node info in sync -42:54.647 [DEBUG] TestAgent: Service in sync: service=monitor -42:54.647 [DEBUG] TestAgent: Check in sync: check=service:monitor -42:54.698 [DEBUG] TestAgent: Node info in sync -42:54.699 [INFO] TestAgent: Synced service: service=monitor -42:54.699 [DEBUG] TestAgent: Check in sync: check=service:monitor -42:56.673 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:56.673 [DEBUG] TestAgent: Node info in sync -42:56.673 [DEBUG] TestAgent: Service in sync: service=monitor -42:56.673 [DEBUG] TestAgent: Check in sync: check=service:monitor -42:56.673 [DEBUG] TestAgent: Node info in sync -42:56.673 [DEBUG] TestAgent: Service in sync: service=monitor -42:56.673 [DEBUG] TestAgent: Check in sync: check=service:monitor -42:56.702 [INFO] TestAgent: Requesting shutdown -42:56.702 [INFO] TestAgent.server: shutting down server -42:56.702 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:56.702 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:56.702 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:56.702 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:56.702 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:56.703 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:56.703 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:56.703 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:56.703 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:56.703 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:56.703 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:56.703Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:56.703Z [INFO] agent.router.manager: shutting down -42:56.703 [INFO] TestAgent: consul server down -42:56.703 [INFO] TestAgent: shutdown complete -42:56.703 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13073 network=tcp -42:56.704 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13073 network=udp -42:56.704 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13074 network=tcp -42:57.204 [INFO] TestAgent: Waiting for endpoints to shut down -42:57.204 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Monitor (2.81s) - --- PASS: TestAgent_Monitor/unknown_log_level (0.00s) - --- PASS: TestAgent_Monitor/stream_unstructured_logs (0.10s) - --- PASS: TestAgent_Monitor/stream_JSON_logs (2.01s) - --- PASS: TestAgent_Monitor/serf_shutdown_logging (0.60s) +2021-08-13T15:34:19.842Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:19.859Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:19.862 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a345a019-3c85-294e-7502-42401376d600 Address:127.0.0.1:24656}]" +34:19.862 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24656 [Follower]" leader= +34:19.863 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a345a019-3c85-294e-7502-42401376d600.dc1 127.0.0.1 +34:19.864 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a345a019-3c85-294e-7502-42401376d600 127.0.0.1 +2021-08-13T15:34:19.864Z [INFO] agent.router: Initializing LAN area manager +34:19.864 [INFO] TestAgent.server: Adding LAN server: server="Node-a345a019-3c85-294e-7502-42401376d600 (Addr: tcp/127.0.0.1:24656) (DC: dc1)" +34:19.864 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a345a019-3c85-294e-7502-42401376d600.dc1 area=wan +34:19.865 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24644 network=udp +34:19.865 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24644 network=tcp +34:19.865 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24652 network=tcp +34:19.866 [INFO] TestAgent: started state syncer +34:19.866 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24664 network=tcp +34:19.924 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:19.924 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24656 [Candidate]" term=2 +34:19.925 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:19.925 [DEBUG] TestAgent.server.raft: vote granted: from=a345a019-3c85-294e-7502-42401376d600 term=2 tally=1 +34:19.925 [INFO] TestAgent.server.raft: election won: tally=1 +34:19.925 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24656 [Leader]" +34:19.925 [INFO] TestAgent.server: cluster leadership acquired +34:19.925 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24656 +34:19.926 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:19.927 [INFO] TestAgent.server: New leader elected: payload=Node-a345a019-3c85-294e-7502-42401376d600 +34:19.929 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:19.929 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:19.929 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:19.929 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:19.929 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:19.929 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a345a019-3c85-294e-7502-42401376d600 +34:19.929 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a345a019-3c85-294e-7502-42401376d600 +34:19.930 [INFO] TestAgent.server: federation state anti-entropy synced +34:20.202 [INFO] TestAgent: Requesting shutdown +34:20.202 [INFO] TestAgent.server: shutting down server +34:20.202 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:20.202 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:20.202 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:20.202 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:20.202 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:20.203 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:20.203 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:20.203 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:20.203 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:20.203 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:20.203 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:20.203 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:20.203Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:20.203Z [INFO] agent.router.manager: shutting down +34:20.203 [INFO] TestAgent: consul server down +34:20.203 [INFO] TestAgent: shutdown complete +34:20.203 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24644 network=tcp +34:20.203 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24644 network=udp +34:20.203 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24652 network=tcp +34:20.703 [INFO] TestAgent: Waiting for endpoints to shut down +34:20.703 [INFO] TestAgent: Endpoints down +--- PASS: TestOperator_AutopilotSetConfiguration (0.91s) +=== CONT TestOperator_AutopilotGetConfiguration +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:18.331Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:18.353Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:18.357 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:163e7d71-a4a9-e5e5-756d-07028193b425 Address:127.0.0.1:24600}]" +34:18.358 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-163e7d71-a4a9-e5e5-756d-07028193b425.dc1 127.0.0.1 +34:18.359 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24600 [Follower]" leader= +34:18.359 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-163e7d71-a4a9-e5e5-756d-07028193b425 127.0.0.1 +2021-08-13T15:34:18.359Z [INFO] agent.router: Initializing LAN area manager +34:18.360 [INFO] TestAgent.server: Adding LAN server: server="Node-163e7d71-a4a9-e5e5-756d-07028193b425 (Addr: tcp/127.0.0.1:24600) (DC: dc1)" +34:18.362 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24595 network=udp +34:18.362 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24595 network=tcp +34:18.362 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-163e7d71-a4a9-e5e5-756d-07028193b425.dc1 area=wan +34:18.364 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24596 network=tcp +34:18.364 [INFO] TestAgent: started state syncer +34:18.364 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24587 network=tcp +34:18.398 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:18.398 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24600 [Candidate]" term=2 +34:18.398 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:18.398 [DEBUG] TestAgent.server.raft: vote granted: from=163e7d71-a4a9-e5e5-756d-07028193b425 term=2 tally=1 +34:18.398 [INFO] TestAgent.server.raft: election won: tally=1 +34:18.398 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24600 [Leader]" +34:18.398 [INFO] TestAgent.server: cluster leadership acquired +34:18.398 [INFO] TestAgent.server: New leader elected: payload=Node-163e7d71-a4a9-e5e5-756d-07028193b425 +34:18.398 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24600 +34:18.400 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:18.402 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:18.402 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:18.402 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:18.402 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:18.402 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:18.402 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-163e7d71-a4a9-e5e5-756d-07028193b425 +34:18.402 [INFO] TestAgent.server: member joined, marking health alive: member=Node-163e7d71-a4a9-e5e5-756d-07028193b425 +34:18.403 [INFO] TestAgent.server: federation state anti-entropy synced +34:18.544 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:18.545 [INFO] TestAgent: Synced node info +34:20.407 [INFO] TestAgent: Requesting shutdown +34:20.407 [INFO] TestAgent.server: shutting down server +34:20.407 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:20.407 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:20.407 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:20.407 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:20.407 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:20.407 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:20.407 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:20.407 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:20.407 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:20.407 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:20.408 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:20.408Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:20.408Z [INFO] agent.router.manager: shutting down +34:20.408 [INFO] TestAgent: consul server down +34:20.408 [INFO] TestAgent: shutdown complete +34:20.408 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24595 network=tcp +34:20.408 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24595 network=udp +34:20.408 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24596 network=tcp +34:20.908 [INFO] TestAgent: Waiting for endpoints to shut down +34:20.908 [INFO] TestAgent: Endpoints down +--- PASS: TestOperator_ServerHealth_Unhealthy (2.59s) +=== CONT TestAgent_FailCheck_ACLDeny +2021-08-13T15:34:09.042Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:09.059Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:09.062 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:dacb70d2-c301-0fbf-2a68-7955a45a9682 Address:127.0.0.1:25074}]" +34:09.062 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25074 [Follower]" leader= +34:09.063 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-dacb70d2-c301-0fbf-2a68-7955a45a9682.dc1 127.0.0.1 +34:09.064 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-dacb70d2-c301-0fbf-2a68-7955a45a9682 127.0.0.1 +2021-08-13T15:34:09.064Z [INFO] agent.router: Initializing LAN area manager +34:09.064 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25083 network=udp +34:09.064 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-dacb70d2-c301-0fbf-2a68-7955a45a9682.dc1 area=wan +34:09.064 [INFO] TestAgent.server: Adding LAN server: server="Node-dacb70d2-c301-0fbf-2a68-7955a45a9682 (Addr: tcp/127.0.0.1:25074) (DC: dc1)" +34:09.065 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25083 network=tcp +34:09.066 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25084 network=tcp +34:09.066 [INFO] TestAgent: started state syncer +34:09.066 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25075 network=tcp +34:09.119 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:09.119 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25074 [Candidate]" term=2 +34:09.119 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:09.119 [DEBUG] TestAgent.server.raft: vote granted: from=dacb70d2-c301-0fbf-2a68-7955a45a9682 term=2 tally=1 +34:09.119 [INFO] TestAgent.server.raft: election won: tally=1 +34:09.119 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25074 [Leader]" +34:09.119 [INFO] TestAgent.server: cluster leadership acquired +34:09.120 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25074 +34:09.121 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:09.123 [INFO] TestAgent.server: New leader elected: payload=Node-dacb70d2-c301-0fbf-2a68-7955a45a9682 +34:09.124 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:09.124 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:09.124 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:09.124 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:09.124 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:09.124 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-dacb70d2-c301-0fbf-2a68-7955a45a9682 +34:09.124 [INFO] TestAgent.server: member joined, marking health alive: member=Node-dacb70d2-c301-0fbf-2a68-7955a45a9682 +34:09.124 [INFO] TestAgent.server: federation state anti-entropy synced +34:09.133 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:09.133 [INFO] TestAgent: Synced node info +34:09.133 [DEBUG] TestAgent: Node info in sync +34:09.465 [INFO] TestAgent.server: server starting leave +34:09.465 [INFO] TestAgent.server.serf.wan: serf: EventMemberLeave: Node-dacb70d2-c301-0fbf-2a68-7955a45a9682.dc1 127.0.0.1 +34:09.465 [INFO] TestAgent.server: Handled event for server in area: event=member-leave server=Node-dacb70d2-c301-0fbf-2a68-7955a45a9682.dc1 area=wan +2021-08-13T15:34:09.465Z [INFO] agent.router.manager: shutting down +34:10.789 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:10.790 [DEBUG] TestAgent: Node info in sync +34:12.465 [INFO] TestAgent.server.serf.lan: serf: EventMemberLeave: Node-dacb70d2-c301-0fbf-2a68-7955a45a9682 127.0.0.1 +34:12.465 [INFO] TestAgent.server: Removing LAN server: server="Node-dacb70d2-c301-0fbf-2a68-7955a45a9682 (Addr: tcp/127.0.0.1:25074) (DC: dc1)" +34:12.465 [WARN] TestAgent.server: deregistering self should be done by follower: name=Node-dacb70d2-c301-0fbf-2a68-7955a45a9682 +34:13.123 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" +34:15.123 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" +34:15.465 [INFO] TestAgent.server: Waiting to drain RPC traffic: drain_time=5s +34:17.123 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" +34:19.123 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" +34:19.123 [ERROR] TestAgent.server.autopilot: Error promoting servers: error="error getting server raft protocol versions: No servers found" +34:20.465 [INFO] TestAgent: Requesting shutdown +34:20.465 [INFO] TestAgent.server: shutting down server +34:20.465 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:20.465 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:20.465 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:20.465 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:20.466 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:20.466 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:20.466 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:20.466 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:20.466 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2021-08-13T15:34:20.466Z [INFO] agent.router.manager: shutting down +34:20.466 [INFO] TestAgent: consul server down +34:20.466 [INFO] TestAgent: shutdown complete +34:20.466 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25083 network=tcp +34:20.466 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25083 network=udp +34:20.466 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25084 network=tcp +34:20.971 [INFO] TestAgent: Waiting for endpoints to shut down +34:20.971 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_StartStop (11.95s) +=== CONT TestAgent_FailCheck +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:12.939Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:12.966Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:12.969 [INFO] dc2.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:079a64ed-ffed-79c5-0a75-fe99309f35bc Address:127.0.0.1:24352}]" +34:12.969 [INFO] dc2.server.raft: entering follower state: follower="Node at 127.0.0.1:24352 [Follower]" leader= +34:12.970 [INFO] dc2.server.serf.wan: serf: EventMemberJoin: Node-079a64ed-ffed-79c5-0a75-fe99309f35bc.dc2 127.0.0.1 +34:12.971 [INFO] dc2.server.serf.lan: serf: EventMemberJoin: Node-079a64ed-ffed-79c5-0a75-fe99309f35bc 127.0.0.1 +2021-08-13T15:34:12.971Z [INFO] agent.router: Initializing LAN area manager +34:12.971 [INFO] dc2.server: Handled event for server in area: event=member-join server=Node-079a64ed-ffed-79c5-0a75-fe99309f35bc.dc2 area=wan +34:12.971 [INFO] dc2.server: Adding LAN server: server="Node-079a64ed-ffed-79c5-0a75-fe99309f35bc (Addr: tcp/127.0.0.1:24352) (DC: dc2)" +34:12.971 [INFO] dc2: Started DNS server: address=127.0.0.1:24340 network=udp +34:12.971 [INFO] dc2: Started DNS server: address=127.0.0.1:24340 network=tcp +34:12.972 [INFO] dc2: Started HTTP server: address=127.0.0.1:24341 network=tcp +34:12.972 [INFO] dc2: started state syncer +34:12.972 [INFO] dc2: Started gRPC server: address=127.0.0.1:24332 network=tcp +34:13.033 [WARN] dc2.server.raft: heartbeat timeout reached, starting election: last-leader= +34:13.033 [INFO] dc2.server.raft: entering candidate state: node="Node at 127.0.0.1:24352 [Candidate]" term=2 +34:13.033 [DEBUG] dc2.server.raft: votes: needed=1 +34:13.033 [DEBUG] dc2.server.raft: vote granted: from=079a64ed-ffed-79c5-0a75-fe99309f35bc term=2 tally=1 +34:13.033 [INFO] dc2.server.raft: election won: tally=1 +34:13.033 [INFO] dc2.server.raft: entering leader state: leader="Node at 127.0.0.1:24352 [Leader]" +34:13.033 [INFO] dc2.server: cluster leadership acquired +34:13.033 [INFO] dc2.server: New leader elected: payload=Node-079a64ed-ffed-79c5-0a75-fe99309f35bc +34:13.033 [DEBUG] dc2.server: Cannot upgrade to new ACLs, servers in acl datacenter are not yet upgraded: ACLDatacenter=dc1 mode=1 found=false +34:13.034 [WARN] dc2.server.connect: primary datacenter is configured but unreachable - deferring initialization of the secondary datacenter CA +34:13.034 [INFO] dc2.leader: started routine: routine="config entry replication" +34:13.034 [INFO] dc2.leader: started routine: routine="federation state replication" +34:13.034 [INFO] dc2.leader: started routine: routine="federation state anti-entropy" +34:13.034 [INFO] dc2.leader: started routine: routine="secondary CA roots watch" +34:13.034 [INFO] dc2.leader: started routine: routine="intention replication" +34:13.034 [INFO] dc2.leader: started routine: routine="intermediate cert renew watch" +34:13.034 [INFO] dc2.leader: started routine: routine="CA root pruning" +34:13.034 [DEBUG] dc2.server.connect: starting Connect intention replication from primary datacenter: primary=dc1 +34:13.034 [DEBUG] dc2.server: Skipping self join check for node since the cluster is too small: node=Node-079a64ed-ffed-79c5-0a75-fe99309f35bc +34:13.034 [INFO] dc2.server: member joined, marking health alive: member=Node-079a64ed-ffed-79c5-0a75-fe99309f35bc +34:13.034 [DEBUG] dc2.server: federation states are not enabled in the primary dc +34:13.035 [WARN] dc2.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ConfigEntry.ListAll +34:13.034 [WARN] dc2.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=Intention.List +34:13.035 [ERROR] dc2.server.connect: error replicating intentions: routine="intention replication" error="No path to datacenter" +34:13.034 [DEBUG] dc2.server: federation states are not enabled in the primary dc +34:13.035 [DEBUG] dc2.server: federation states are not enabled in the primary dc +34:13.035 [DEBUG] dc2.server: federation states are not enabled in the primary dc +34:13.035 [DEBUG] dc2.server: federation states are not enabled in the primary dc +34:13.035 [DEBUG] dc2.server: federation states are not enabled in the primary dc +34:13.034 [DEBUG] dc2.server.connect: starting Connect CA root replication from primary datacenter: primary=dc1 +34:13.035 [WARN] dc2.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ConnectCA.Roots +34:13.035 [ERROR] dc2.server.connect: CA root replication failed, will retry: routine="secondary CA roots watch" error="Error retrieving the primary datacenter's roots: No path to datacenter" +34:13.341 [DEBUG] dc2: Skipping remote check since it is managed automatically: check=serfHealth +34:13.341 [INFO] dc2: Synced node info +34:13.384 [INFO] dc2: (WAN) joining: wan_addresses=[127.0.0.1:24295] +34:13.385 [DEBUG] dc2.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:24295 +34:13.386 [INFO] dc2.server.serf.wan: serf: EventMemberJoin: Node-1e324b1a-b9d0-f591-283c-590f188e1a48.dc1 127.0.0.1 +34:13.386 [INFO] dc2: (WAN) joined: number_of_nodes=1 +34:13.386 [INFO] dc2.server: Handled event for server in area: event=member-join server=Node-1e324b1a-b9d0-f591-283c-590f188e1a48.dc1 area=wan +34:13.609 [DEBUG] dc2.server.serf.wan: serf: messageJoinType: Node-079a64ed-ffed-79c5-0a75-fe99309f35bc.dc2 +34:14.088 [DEBUG] dc2.server.replication.config_entry: finished fetching config entries: amount=0 +34:14.088 [DEBUG] dc2.server.replication.config_entry: Config Entry replication: local=0 remote=0 +34:14.088 [DEBUG] dc2.server.replication.config_entry: Config Entry replication: deletions=0 updates=0 +34:14.088 [DEBUG] dc2.server.replication.config_entry: replication completed through remote index: index=1 +34:14.120 [DEBUG] dc2.server: finished fetching remote objects: amount=1 +34:14.120 [DEBUG] dc2.server: diffing replication state: local_amount=0 remote_amount=1 +34:14.120 [DEBUG] dc2.server: diffed replication state: deletions=0 updates=1 +34:14.120 [DEBUG] dc2.server: performing updates: updates=1 +34:14.120 [DEBUG] dc2.server: finished updates +34:14.120 [DEBUG] dc2.replication.federation_state: replication completed through remote index: index=10 +34:14.328 [DEBUG] dc2: Skipping remote check since it is managed automatically: check=serfHealth +34:14.328 [DEBUG] dc2: Node info in sync +34:14.328 [DEBUG] dc2: Node info in sync +34:14.615 [DEBUG] dc2.server.serf.wan: serf: messageJoinType: Node-079a64ed-ffed-79c5-0a75-fe99309f35bc.dc2 +34:15.049 [DEBUG] connect.ca.consul: consul CA provider configured: id=ad:4a:c6:ab:ef:63:c9:60:1a:51:7f:19:62:e3:e9:d9:0e:76:55:10:6e:74:24:69:28:a1:6c:b8:b9:8f:fd:89 is_primary=false +34:15.062 [INFO] dc2.server.connect: received new intermediate certificate from primary datacenter +34:15.062 [INFO] dc2.server.connect: updated root certificates from primary datacenter +34:15.068 [DEBUG] dc2: Node info in sync +34:15.069 [INFO] dc2: Synced service: service=foo +34:15.069 [DEBUG] dc2: Check in sync: check=service:foo +34:15.071 [DEBUG] dc2: Node info in sync +34:15.071 [DEBUG] dc2: Service in sync: service=foo +34:15.071 [DEBUG] dc2: Check in sync: check=service:foo +34:15.071 [DEBUG] dc2: Node info in sync +34:15.071 [DEBUG] dc2: Service in sync: service=foo +34:15.071 [DEBUG] dc2: Check in sync: check=service:foo +34:15.113 [DEBUG] dc2.server.serf.wan: serf: messageJoinType: Node-079a64ed-ffed-79c5-0a75-fe99309f35bc.dc2 +34:15.610 [DEBUG] dc2.server.serf.wan: serf: messageJoinType: Node-079a64ed-ffed-79c5-0a75-fe99309f35bc.dc2 +34:18.036 [INFO] dc2.server: federation state anti-entropy synced +34:18.036 [DEBUG] dc2.server: finished fetching remote objects: amount=2 +34:18.036 [DEBUG] dc2.server: diffing replication state: local_amount=1 remote_amount=2 +34:18.036 [DEBUG] dc2.server: diffed replication state: deletions=0 updates=1 +34:18.036 [DEBUG] dc2.server: performing updates: updates=1 +34:18.036 [DEBUG] dc2.server: finished updates +34:18.036 [DEBUG] dc2.replication.federation_state: replication completed through remote index: index=19 +34:20.094 [INFO] dc2.server.connect: received new intermediate certificate from primary datacenter +34:20.095 [INFO] dc2.server.connect: updated root certificates from primary datacenter +34:20.096 [INFO] dc2.server.connect: updated root certificates from primary datacenter +34:20.118 [INFO] dc2: Requesting shutdown +34:20.118 [INFO] dc2.server: shutting down server +34:20.118 [DEBUG] dc2.leader: stopping routine: routine="intermediate cert renew watch" +34:20.118 [DEBUG] dc2.leader: stopping routine: routine="CA root pruning" +34:20.118 [DEBUG] dc2.leader: stopping routine: routine="config entry replication" +34:20.118 [DEBUG] dc2.leader: stopping routine: routine="federation state replication" +34:20.118 [DEBUG] dc2.leader: stopping routine: routine="federation state anti-entropy" +34:20.118 [DEBUG] dc2.leader: stopping routine: routine="secondary CA roots watch" +34:20.118 [DEBUG] dc2.leader: stopping routine: routine="intention replication" +34:20.118 [WARN] dc2.server.serf.lan: serf: Shutdown without a Leave +34:20.118 [DEBUG] dc2.leader: stopped routine: routine="CA root pruning" +34:20.118 [DEBUG] dc2.leader: stopped routine: routine="federation state anti-entropy" +34:20.118 [DEBUG] dc2.leader: stopped routine: routine="intermediate cert renew watch" +34:20.118 [WARN] dc2.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:20.119Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:20.119Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:20.119Z [INFO] agent.router.manager: shutting down +34:20.119 [INFO] dc2: consul server down +34:20.119 [INFO] dc2: shutdown complete +34:20.119 [INFO] dc2: Stopping server: protocol=DNS address=127.0.0.1:24340 network=tcp +34:20.119 [INFO] dc2: Stopping server: protocol=DNS address=127.0.0.1:24340 network=udp +34:20.119 [INFO] dc2: Stopping server: protocol=HTTP address=127.0.0.1:24341 network=tcp +34:20.119 [ERROR] dc2.server.rpc: RPC failed to server in DC: server=127.0.0.1:24296 datacenter=dc1 method=Intention.List error="rpc error making call: EOF" +34:20.119 [ERROR] dc2.server.connect: error replicating intentions: routine="intention replication" error="rpc error making call: EOF" +34:20.119 [DEBUG] dc2.leader: stopped routine: routine="intention replication" +34:20.119 [ERROR] dc2.server.rpc: RPC failed to server in DC: server=127.0.0.1:24296 datacenter=dc1 method=ConnectCA.Roots error="rpc error making call: EOF" +34:20.119 [ERROR] dc2.server.connect: CA root replication failed, will retry: routine="secondary CA roots watch" error="Error retrieving the primary datacenter's roots: rpc error making call: EOF" +34:20.119 [DEBUG] dc2.leader: stopped routine: routine="secondary CA roots watch" +34:20.119 [ERROR] dc2.server.rpc: RPC failed to server in DC: server=127.0.0.1:24296 datacenter=dc1 method=FederationState.List error="rpc error making call: EOF" +34:20.119 [WARN] dc2.replication.federation_state: replication error (will retry if still leader): error="failed to retrieve federation states: rpc error making call: EOF" +34:20.119 [INFO] dc2.replication.federation_state: stopped replication +34:20.119 [DEBUG] dc2.leader: stopped routine: routine="federation state replication" +34:20.119 [ERROR] dc2.server.rpc: RPC failed to server in DC: server=127.0.0.1:24296 datacenter=dc1 method=ConfigEntry.ListAll error="rpc error making call: EOF" +34:20.119 [INFO] dc2.server.replication.config_entry: stopped replication +34:20.119 [DEBUG] dc2.leader: stopped routine: routine="config entry replication" +34:20.619 [INFO] dc2: Waiting for endpoints to shut down +34:20.619 [INFO] dc2: Endpoints down +2021-08-13T15:34:12.587Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:12.604Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:12.607 [INFO] dc1.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1e324b1a-b9d0-f591-283c-590f188e1a48 Address:127.0.0.1:24296}]" +34:12.608 [INFO] dc1.server.serf.wan: serf: EventMemberJoin: Node-1e324b1a-b9d0-f591-283c-590f188e1a48.dc1 127.0.0.1 +34:12.609 [INFO] dc1.server.serf.lan: serf: EventMemberJoin: Node-1e324b1a-b9d0-f591-283c-590f188e1a48 127.0.0.1 +2021-08-13T15:34:12.609Z [INFO] agent.router: Initializing LAN area manager +34:12.609 [INFO] dc1: Started DNS server: address=127.0.0.1:25311 network=udp +34:12.609 [INFO] dc1.server.raft: entering follower state: follower="Node at 127.0.0.1:24296 [Follower]" leader= +34:12.609 [INFO] dc1.server: Adding LAN server: server="Node-1e324b1a-b9d0-f591-283c-590f188e1a48 (Addr: tcp/127.0.0.1:24296) (DC: dc1)" +34:12.610 [INFO] dc1.server: Handled event for server in area: event=member-join server=Node-1e324b1a-b9d0-f591-283c-590f188e1a48.dc1 area=wan +34:12.610 [INFO] dc1: Started DNS server: address=127.0.0.1:25311 network=tcp +34:12.611 [INFO] dc1: Started HTTP server: address=127.0.0.1:25312 network=tcp +34:12.611 [INFO] dc1: started state syncer +34:12.611 [INFO] dc1: Started gRPC server: address=127.0.0.1:24297 network=tcp +34:12.651 [WARN] dc1.server.raft: heartbeat timeout reached, starting election: last-leader= +34:12.651 [INFO] dc1.server.raft: entering candidate state: node="Node at 127.0.0.1:24296 [Candidate]" term=2 +34:12.651 [DEBUG] dc1.server.raft: votes: needed=1 +34:12.651 [DEBUG] dc1.server.raft: vote granted: from=1e324b1a-b9d0-f591-283c-590f188e1a48 term=2 tally=1 +34:12.651 [INFO] dc1.server.raft: election won: tally=1 +34:12.651 [INFO] dc1.server.raft: entering leader state: leader="Node at 127.0.0.1:24296 [Leader]" +34:12.651 [INFO] dc1.server: cluster leadership acquired +34:12.651 [INFO] dc1.server: New leader elected: payload=Node-1e324b1a-b9d0-f591-283c-590f188e1a48 +34:12.651 [DEBUG] dc1.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24296 +34:12.653 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:12.655 [INFO] dc1.server.connect: initialized primary datacenter CA with provider: provider=consul +34:12.655 [INFO] dc1.leader: started routine: routine="federation state anti-entropy" +34:12.655 [INFO] dc1.leader: started routine: routine="federation state pruning" +34:12.655 [INFO] dc1.leader: started routine: routine="intermediate cert renew watch" +34:12.655 [INFO] dc1.leader: started routine: routine="CA root pruning" +34:12.655 [DEBUG] dc1.server: Skipping self join check for node since the cluster is too small: node=Node-1e324b1a-b9d0-f591-283c-590f188e1a48 +34:12.655 [INFO] dc1.server: member joined, marking health alive: member=Node-1e324b1a-b9d0-f591-283c-590f188e1a48 +34:12.656 [INFO] dc1.server: federation state anti-entropy synced +34:12.780 [DEBUG] dc1: Skipping remote check since it is managed automatically: check=serfHealth +34:12.781 [INFO] dc1: Synced node info +34:12.781 [DEBUG] dc1: Node info in sync +34:13.385 [DEBUG] dc1.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:53064 +34:13.386 [INFO] dc1.server.serf.wan: serf: EventMemberJoin: Node-079a64ed-ffed-79c5-0a75-fe99309f35bc.dc2 127.0.0.1 +34:13.386 [INFO] dc1.server: Handled event for server in area: event=member-join server=Node-079a64ed-ffed-79c5-0a75-fe99309f35bc.dc2 area=wan +34:13.388 [DEBUG] connect.ca.consul: consul CA provider configured: id=6f:a5:ca:02:21:a0:7e:a2:0d:c7:5e:15:f8:c0:e8:4f:53:35:4e:9b:56:de:f0:c5:2f:c4:2d:ba:48:04:7c:40 is_primary=true +34:13.391 [INFO] dc1.server.connect: CA rotated to new root under provider: provider=consul +34:13.471 [DEBUG] dc1.server.serf.wan: serf: messageJoinType: Node-079a64ed-ffed-79c5-0a75-fe99309f35bc.dc2 +34:13.974 [DEBUG] dc1.server.serf.wan: serf: messageJoinType: Node-079a64ed-ffed-79c5-0a75-fe99309f35bc.dc2 +34:14.135 [DEBUG] dc1: Skipping remote check since it is managed automatically: check=serfHealth +34:14.135 [DEBUG] dc1: Node info in sync +34:14.471 [DEBUG] dc1.server.serf.wan: serf: messageJoinType: Node-079a64ed-ffed-79c5-0a75-fe99309f35bc.dc2 +34:14.971 [DEBUG] dc1.server.serf.wan: serf: messageJoinType: Node-079a64ed-ffed-79c5-0a75-fe99309f35bc.dc2 +34:20.087 [DEBUG] connect.ca.consul: consul CA provider configured: id=26:18:bb:9c:12:56:3b:70:94:e3:a4:97:e6:34:c2:7d:ce:c2:ca:a9:b4:c3:d5:61:43:6e:e6:1e:31:6b:81:6d is_primary=true +34:20.089 [INFO] dc1.server.connect: CA rotated to new root under provider: provider=consul +34:20.619 [INFO] dc1: Requesting shutdown +34:20.619 [INFO] dc1.server: shutting down server +34:20.619 [DEBUG] dc1.leader: stopping routine: routine="federation state anti-entropy" +34:20.619 [DEBUG] dc1.leader: stopping routine: routine="federation state pruning" +34:20.619 [DEBUG] dc1.leader: stopping routine: routine="intermediate cert renew watch" +34:20.619 [DEBUG] dc1.leader: stopping routine: routine="CA root pruning" +34:20.619 [WARN] dc1.server.serf.lan: serf: Shutdown without a Leave +34:20.620 [ERROR] dc1.server: error performing anti-entropy sync of federation state: error="context canceled" +34:20.620 [DEBUG] dc1.leader: stopped routine: routine="federation state anti-entropy" +34:20.620 [DEBUG] dc1.leader: stopped routine: routine="federation state pruning" +34:20.620 [DEBUG] dc1.leader: stopped routine: routine="intermediate cert renew watch" +34:20.620 [WARN] dc1.server.serf.wan: serf: Shutdown without a Leave +34:20.620 [DEBUG] dc1.leader: stopped routine: routine="CA root pruning" +2021-08-13T15:34:20.620Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:20.620Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:20.620Z [INFO] agent.router.manager: shutting down +34:20.620 [INFO] dc1: consul server down +34:20.620 [INFO] dc1: shutdown complete +34:20.620 [INFO] dc1: Stopping server: protocol=DNS address=127.0.0.1:25311 network=tcp +34:20.620 [INFO] dc1: Stopping server: protocol=DNS address=127.0.0.1:25311 network=udp +34:20.620 [INFO] dc1: Stopping server: protocol=HTTP address=127.0.0.1:25312 network=tcp +34:21.121 [INFO] dc1: Waiting for endpoints to shut down +34:21.121 [INFO] dc1: Endpoints down +--- PASS: TestAgentConnectCALeafCert_secondaryDC_good (8.55s) === CONT TestAgent_WarnCheck_ACLDeny +2021-08-13T15:34:20.481Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:20.481Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:20.499Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:20.499Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:20.502 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cbe3a7fc-e1ae-0fc2-dd38-dd91d37ed0ae Address:127.0.0.1:24663}]" +34:20.502 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24663 [Follower]" leader= +34:20.503 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-cbe3a7fc-e1ae-0fc2-dd38-dd91d37ed0ae.dc1 127.0.0.1 +34:20.504 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-cbe3a7fc-e1ae-0fc2-dd38-dd91d37ed0ae 127.0.0.1 +2021-08-13T15:34:20.504Z [INFO] agent.router: Initializing LAN area manager +34:20.504 [INFO] TestAgent.server: Adding LAN server: server="Node-cbe3a7fc-e1ae-0fc2-dd38-dd91d37ed0ae (Addr: tcp/127.0.0.1:24663) (DC: dc1)" +34:20.504 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-cbe3a7fc-e1ae-0fc2-dd38-dd91d37ed0ae.dc1 area=wan +34:20.504 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24658 network=udp +34:20.504 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24658 network=tcp +34:20.505 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24659 network=tcp +34:20.505 [INFO] TestAgent: started state syncer +34:20.505 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24692 network=tcp +34:20.554 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:20.554 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:20.554 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24663 [Candidate]" term=2 +34:20.554 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:20.554 [DEBUG] TestAgent.server.raft: vote granted: from=cbe3a7fc-e1ae-0fc2-dd38-dd91d37ed0ae term=2 tally=1 +34:20.554 [INFO] TestAgent.server.raft: election won: tally=1 +34:20.554 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24663 [Leader]" +34:20.554 [INFO] TestAgent.server: cluster leadership acquired +34:20.555 [INFO] TestAgent.server: initializing acls +34:20.555 [INFO] TestAgent.server: New leader elected: payload=Node-cbe3a7fc-e1ae-0fc2-dd38-dd91d37ed0ae +34:20.555 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:20.555 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:20.556 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:20.556 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:20.556 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:20.556 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:20.556 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-cbe3a7fc-e1ae-0fc2-dd38-dd91d37ed0ae +34:20.557 [INFO] TestAgent.server: Updating LAN server: server="Node-cbe3a7fc-e1ae-0fc2-dd38-dd91d37ed0ae (Addr: tcp/127.0.0.1:24663) (DC: dc1)" +34:20.557 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-cbe3a7fc-e1ae-0fc2-dd38-dd91d37ed0ae.dc1 +34:20.557 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-cbe3a7fc-e1ae-0fc2-dd38-dd91d37ed0ae.dc1 area=wan +34:20.558 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:20.560 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:20.560 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:20.560 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:20.560 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:20.560 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:20.560 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cbe3a7fc-e1ae-0fc2-dd38-dd91d37ed0ae +34:20.560 [INFO] TestAgent.server: member joined, marking health alive: member=Node-cbe3a7fc-e1ae-0fc2-dd38-dd91d37ed0ae +34:20.561 [INFO] TestAgent.server: federation state anti-entropy synced +34:20.561 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cbe3a7fc-e1ae-0fc2-dd38-dd91d37ed0ae +34:20.652 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-cbe3a7fc-e1ae-0fc2-dd38-dd91d37ed0ae +34:20.652 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-cbe3a7fc-e1ae-0fc2-dd38-dd91d37ed0ae +34:20.653 [DEBUG] TestAgent: Check status updated: check=test status=passing +34:20.654 [INFO] TestAgent: Synced node info +34:20.655 [WARN] TestAgent: Check registration blocked by ACLs: check=test accessorID=00000000-0000-0000-0000-000000000002 +34:20.655 [INFO] TestAgent: Requesting shutdown +34:20.655 [INFO] TestAgent.server: shutting down server +34:20.655 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:20.655 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:20.655 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:20.655 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:20.655 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:20.655 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:20.655 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:20.655 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:20.655 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:20.655 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:20.655 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:20.655 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:20.655 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:20.655 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:20.655 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:20.655 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:20.655Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:20.655Z [INFO] agent.router.manager: shutting down +34:20.655 [INFO] TestAgent: consul server down +34:20.655 [INFO] TestAgent: shutdown complete +34:20.656 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24658 network=tcp +34:20.656 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24658 network=udp +34:20.656 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24659 network=tcp +34:21.158 [INFO] TestAgent: Waiting for endpoints to shut down +34:21.158 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_UpdateCheck_ACLDeny (0.70s) + --- PASS: TestAgent_UpdateCheck_ACLDeny/no_token (0.00s) + --- PASS: TestAgent_UpdateCheck_ACLDeny/root_token (0.00s) +=== CONT TestAgent_WarnCheck === RUN TestAgent_FailCheck_ACLDeny/no_token === RUN TestAgent_FailCheck_ACLDeny/root_token -2022-09-15T21:42:56.642Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:56.671Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:56.675 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e323fcd0-a62e-3709-0021-499852e6a983 Address:127.0.0.1:13225}]" -42:56.675 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13225 [Follower]" leader= -42:56.676 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e323fcd0-a62e-3709-0021-499852e6a983.dc1 127.0.0.1 -42:56.677 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e323fcd0-a62e-3709-0021-499852e6a983 127.0.0.1 -2022-09-15T21:42:56.677Z [INFO] agent.router: Initializing LAN area manager -42:56.677 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13241 network=udp -42:56.678 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e323fcd0-a62e-3709-0021-499852e6a983.dc1 area=wan -42:56.678 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13241 network=tcp -42:56.678 [INFO] TestAgent.server: Adding LAN server: server="Node-e323fcd0-a62e-3709-0021-499852e6a983 (Addr: tcp/127.0.0.1:13225) (DC: dc1)" -42:56.679 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13242 network=tcp -42:56.679 [INFO] TestAgent: started state syncer -42:56.679 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13226 network=tcp -42:56.726 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:56.726 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13225 [Candidate]" term=2 -42:56.726 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:56.726 [DEBUG] TestAgent.server.raft: vote granted: from=e323fcd0-a62e-3709-0021-499852e6a983 term=2 tally=1 -42:56.726 [INFO] TestAgent.server.raft: election won: tally=1 -42:56.726 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13225 [Leader]" -42:56.726 [INFO] TestAgent.server: cluster leadership acquired -42:56.727 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13225 -42:56.728 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:56.730 [INFO] TestAgent.server: New leader elected: payload=Node-e323fcd0-a62e-3709-0021-499852e6a983 -42:56.731 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:56.731 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:56.731 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:56.731 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:56.731 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:56.731 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e323fcd0-a62e-3709-0021-499852e6a983 -42:56.731 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e323fcd0-a62e-3709-0021-499852e6a983 -42:56.732 [INFO] TestAgent.server: federation state anti-entropy synced -42:56.826 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:56.827 [INFO] TestAgent: Synced node info -42:56.827 [DEBUG] TestAgent: Node info in sync -42:57.007 [DEBUG] TestAgent: removed service: service=test -42:57.007 [DEBUG] TestAgent: Node info in sync -42:57.008 [INFO] TestAgent: Deregistered service: service=test -42:57.008 [INFO] TestAgent: Requesting shutdown -42:57.008 [INFO] TestAgent.server: shutting down server -42:57.008 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:57.008 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:57.008 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:57.008 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:57.008 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:57.008 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:57.008 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:57.008 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:57.008 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:57.008 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:57.008 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:57.008Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:57.008Z [INFO] agent.router.manager: shutting down -42:57.008 [INFO] TestAgent: consul server down -42:57.008 [INFO] TestAgent: shutdown complete -42:57.008 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13241 network=tcp -42:57.008 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13241 network=udp -42:57.009 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13242 network=tcp -42:57.510 [INFO] TestAgent: Waiting for endpoints to shut down -42:57.510 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_DeregisterService (0.89s) -=== CONT TestAgent_WarnCheck -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:18.733Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:18.749Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:18.752 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1fcdb351-ce9f-7466-f43b-db8ab609d11a Address:127.0.0.1:24607}]" +34:18.752 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24607 [Follower]" leader= +34:18.753 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1fcdb351-ce9f-7466-f43b-db8ab609d11a.dc1 127.0.0.1 +34:18.754 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1fcdb351-ce9f-7466-f43b-db8ab609d11a 127.0.0.1 +2021-08-13T15:34:18.754Z [INFO] agent.router: Initializing LAN area manager +34:18.754 [INFO] TestAgent.server: Adding LAN server: server="Node-1fcdb351-ce9f-7466-f43b-db8ab609d11a (Addr: tcp/127.0.0.1:24607) (DC: dc1)" +34:18.754 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1fcdb351-ce9f-7466-f43b-db8ab609d11a.dc1 area=wan +34:18.754 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24602 network=tcp +34:18.754 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24602 network=udp +34:18.755 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24603 network=tcp +34:18.755 [INFO] TestAgent: started state syncer +34:18.756 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24608 network=tcp +34:18.808 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:18.808 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24607 [Candidate]" term=2 +34:18.808 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:18.808 [DEBUG] TestAgent.server.raft: vote granted: from=1fcdb351-ce9f-7466-f43b-db8ab609d11a term=2 tally=1 +34:18.808 [INFO] TestAgent.server.raft: election won: tally=1 +34:18.808 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24607 [Leader]" +34:18.809 [INFO] TestAgent.server: cluster leadership acquired +34:18.809 [INFO] TestAgent.server: New leader elected: payload=Node-1fcdb351-ce9f-7466-f43b-db8ab609d11a +34:18.809 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24607 +34:18.810 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:18.813 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:18.813 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:18.813 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:18.813 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:18.813 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:18.813 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1fcdb351-ce9f-7466-f43b-db8ab609d11a +34:18.813 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1fcdb351-ce9f-7466-f43b-db8ab609d11a +34:18.814 [INFO] TestAgent.server: federation state anti-entropy synced +34:18.835 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:18.836 [INFO] TestAgent: Synced node info +34:19.123 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:19.123 [DEBUG] TestAgent: Node info in sync +34:19.123 [DEBUG] TestAgent: Node info in sync +34:20.814 [INFO] TestAgent: Requesting shutdown +34:20.814 [INFO] TestAgent.server: shutting down server +34:20.814 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:20.814 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:20.814 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:20.814 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:20.814 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:20.814 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:20.814 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:20.815 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:20.815 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:20.815 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:20.815 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:20.815Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:20.815Z [INFO] agent.router.manager: shutting down +34:20.815 [INFO] TestAgent: consul server down +34:20.815 [INFO] TestAgent: shutdown complete +34:20.815 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24602 network=tcp +34:20.815 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24602 network=udp +34:20.815 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24603 network=tcp +34:21.315 [INFO] TestAgent: Waiting for endpoints to shut down +34:21.316 [INFO] TestAgent: Endpoints down +--- PASS: TestOperator_ServerHealth (2.60s) +=== CONT TestAgent_PassCheck_ACLDeny === RUN TestAgent_WarnCheck_ACLDeny/no_token === RUN TestAgent_WarnCheck_ACLDeny/root_token -2022-09-15T21:42:55.783Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:55.783Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:55.801Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:55.801Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:55.804 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:df5ef5c9-f3ca-a5a1-ae1c-7dc64a4078b3 Address:127.0.0.1:13218}]" -42:55.805 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13218 [Follower]" leader= -42:55.805 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-df5ef5c9-f3ca-a5a1-ae1c-7dc64a4078b3.dc1 127.0.0.1 -42:55.808 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-df5ef5c9-f3ca-a5a1-ae1c-7dc64a4078b3 127.0.0.1 -2022-09-15T21:42:55.808Z [INFO] agent.router: Initializing LAN area manager -42:55.808 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13199 network=udp -42:55.808 [INFO] TestAgent.server: Adding LAN server: server="Node-df5ef5c9-f3ca-a5a1-ae1c-7dc64a4078b3 (Addr: tcp/127.0.0.1:13218) (DC: dc1)" -42:55.808 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-df5ef5c9-f3ca-a5a1-ae1c-7dc64a4078b3.dc1 area=wan -42:55.808 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13199 network=tcp -42:55.809 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13200 network=tcp -42:55.809 [INFO] TestAgent: started state syncer -42:55.810 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13219 network=tcp -42:55.858 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -42:55.868 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:55.868 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13218 [Candidate]" term=2 -42:55.868 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:55.868 [DEBUG] TestAgent.server.raft: vote granted: from=df5ef5c9-f3ca-a5a1-ae1c-7dc64a4078b3 term=2 tally=1 -42:55.868 [INFO] TestAgent.server.raft: election won: tally=1 -42:55.868 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13218 [Leader]" -42:55.868 [INFO] TestAgent.server: cluster leadership acquired -42:55.868 [INFO] TestAgent.server: New leader elected: payload=Node-df5ef5c9-f3ca-a5a1-ae1c-7dc64a4078b3 -42:55.869 [INFO] TestAgent.server: initializing acls -42:55.869 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:55.869 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:55.870 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:55.871 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:55.871 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:55.871 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:55.871 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-df5ef5c9-f3ca-a5a1-ae1c-7dc64a4078b3 -42:55.871 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-df5ef5c9-f3ca-a5a1-ae1c-7dc64a4078b3.dc1 -42:55.871 [INFO] TestAgent.server: Updating LAN server: server="Node-df5ef5c9-f3ca-a5a1-ae1c-7dc64a4078b3 (Addr: tcp/127.0.0.1:13218) (DC: dc1)" -42:55.872 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-df5ef5c9-f3ca-a5a1-ae1c-7dc64a4078b3.dc1 area=wan -42:55.872 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:55.875 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:55.875 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:55.875 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:55.875 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:55.875 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:55.876 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-df5ef5c9-f3ca-a5a1-ae1c-7dc64a4078b3 -42:55.876 [INFO] TestAgent.server: member joined, marking health alive: member=Node-df5ef5c9-f3ca-a5a1-ae1c-7dc64a4078b3 -42:55.877 [INFO] TestAgent.server: federation state anti-entropy synced -42:55.877 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-df5ef5c9-f3ca-a5a1-ae1c-7dc64a4078b3 -42:56.250 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth -42:56.250 [WARN] TestAgent: Node info update blocked by ACLs: node=df5ef5c9-f3ca-a5a1-ae1c-7dc64a4078b3 accessorID=00000000-0000-0000-0000-000000000002 -42:56.290 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-df5ef5c9-f3ca-a5a1-ae1c-7dc64a4078b3 -42:56.290 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-df5ef5c9-f3ca-a5a1-ae1c-7dc64a4078b3 -42:56.294 [INFO] TestAgent: Updated agent's ACL token: token=acl_token -42:57.138 [DEBUG] TestAgent.acl: dropping service from result due to ACLs: service="{consul {}}" -42:57.138 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:57.139 [INFO] TestAgent: Synced node info -42:57.154 [INFO] TestAgent: Requesting shutdown -42:57.154 [INFO] TestAgent.server: shutting down server -42:57.154 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:57.154 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:57.154 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:57.154 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:57.154 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:57.154 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:57.154 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:57.154 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:57.154 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:57.154 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:57.154 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:57.154 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:57.154 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:57.154 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:57.154 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:57.154Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:57.154Z [INFO] agent.router.manager: shutting down -42:57.154 [INFO] TestAgent: consul server down -42:57.154 [INFO] TestAgent: shutdown complete -42:57.154 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13199 network=tcp -42:57.155 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13199 network=udp -42:57.155 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13200 network=tcp -42:57.655 [INFO] TestAgent: Waiting for endpoints to shut down -42:57.655 [INFO] TestAgent: Endpoints down -=== RUN TestAgent_TokenTriggersFullSync/default -2022-09-15T21:42:56.930Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:56.930Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:56.948Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:56.948Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:56.951 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d05712df-801c-8f36-8618-b73e52a818fe Address:127.0.0.1:13246}]" -42:56.951 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13246 [Follower]" leader= -42:56.952 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d05712df-801c-8f36-8618-b73e52a818fe.dc1 127.0.0.1 -42:56.953 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d05712df-801c-8f36-8618-b73e52a818fe 127.0.0.1 -2022-09-15T21:42:56.953Z [INFO] agent.router: Initializing LAN area manager -42:56.954 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13220 network=udp -42:56.954 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d05712df-801c-8f36-8618-b73e52a818fe.dc1 area=wan -42:56.954 [INFO] TestAgent.server: Adding LAN server: server="Node-d05712df-801c-8f36-8618-b73e52a818fe (Addr: tcp/127.0.0.1:13246) (DC: dc1)" -42:56.954 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13220 network=tcp -42:56.955 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13221 network=tcp -42:56.955 [INFO] TestAgent: started state syncer -42:56.955 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13247 network=tcp -42:56.987 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:56.987 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13246 [Candidate]" term=2 -42:56.987 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:56.987 [DEBUG] TestAgent.server.raft: vote granted: from=d05712df-801c-8f36-8618-b73e52a818fe term=2 tally=1 -42:56.987 [INFO] TestAgent.server.raft: election won: tally=1 -42:56.987 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13246 [Leader]" -42:56.987 [INFO] TestAgent.server: cluster leadership acquired -42:56.988 [INFO] TestAgent.server: initializing acls -42:56.988 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:56.988 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:56.988 [INFO] TestAgent.server: New leader elected: payload=Node-d05712df-801c-8f36-8618-b73e52a818fe -42:56.989 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:56.989 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:56.989 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:56.989 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:56.989 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-d05712df-801c-8f36-8618-b73e52a818fe -42:56.990 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-d05712df-801c-8f36-8618-b73e52a818fe.dc1 -42:56.990 [INFO] TestAgent.server: Updating LAN server: server="Node-d05712df-801c-8f36-8618-b73e52a818fe (Addr: tcp/127.0.0.1:13246) (DC: dc1)" -42:56.990 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-d05712df-801c-8f36-8618-b73e52a818fe.dc1 area=wan -42:56.991 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:56.993 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:56.993 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:56.993 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:56.993 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:56.993 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:56.993 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d05712df-801c-8f36-8618-b73e52a818fe -42:56.993 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d05712df-801c-8f36-8618-b73e52a818fe -42:56.994 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d05712df-801c-8f36-8618-b73e52a818fe -42:56.996 [INFO] TestAgent.server: federation state anti-entropy synced -42:57.269 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-d05712df-801c-8f36-8618-b73e52a818fe -42:57.270 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-d05712df-801c-8f36-8618-b73e52a818fe -42:57.271 [DEBUG] TestAgent: Check status updated: check=test status=passing -42:57.272 [INFO] TestAgent: Synced node info -42:57.272 [WARN] TestAgent: Check registration blocked by ACLs: check=test accessorID=00000000-0000-0000-0000-000000000002 -42:57.272 [INFO] TestAgent: Requesting shutdown -42:57.272 [INFO] TestAgent.server: shutting down server -42:57.272 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:57.272 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:57.272 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:57.272 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:57.272 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:57.272 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:57.272 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:57.272 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:57.272 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:57.272 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:57.272 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:57.272 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:57.272 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:57.272 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:57.272 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:57.273 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:57.273Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:57.273Z [INFO] agent.router.manager: shutting down -42:57.273 [INFO] TestAgent: consul server down -42:57.273 [INFO] TestAgent: shutdown complete -42:57.273 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13220 network=tcp -42:57.273 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13220 network=udp -42:57.273 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13221 network=tcp -42:57.773 [INFO] TestAgent: Waiting for endpoints to shut down -42:57.773 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_UpdateCheck_ACLDeny (0.86s) - --- PASS: TestAgent_UpdateCheck_ACLDeny/no_token (0.00s) - --- PASS: TestAgent_UpdateCheck_ACLDeny/root_token (0.00s) -=== CONT TestAgent_PassCheck_ACLDeny -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:57.017Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:57.033Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:57.036 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:53a1185b-9ffa-1775-8c9b-0e26e933db41 Address:127.0.0.1:13253}]" -42:57.037 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-53a1185b-9ffa-1775-8c9b-0e26e933db41.dc1 127.0.0.1 -42:57.038 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13253 [Follower]" leader= -42:57.038 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-53a1185b-9ffa-1775-8c9b-0e26e933db41 127.0.0.1 -2022-09-15T21:42:57.038Z [INFO] agent.router: Initializing LAN area manager -42:57.039 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13248 network=udp -42:57.039 [INFO] TestAgent.server: Adding LAN server: server="Node-53a1185b-9ffa-1775-8c9b-0e26e933db41 (Addr: tcp/127.0.0.1:13253) (DC: dc1)" -42:57.039 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-53a1185b-9ffa-1775-8c9b-0e26e933db41.dc1 area=wan -42:57.039 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13248 network=tcp -42:57.040 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13249 network=tcp -42:57.040 [INFO] TestAgent: started state syncer -42:57.040 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13254 network=tcp -42:57.080 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:57.080 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13253 [Candidate]" term=2 -42:57.080 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:57.080 [DEBUG] TestAgent.server.raft: vote granted: from=53a1185b-9ffa-1775-8c9b-0e26e933db41 term=2 tally=1 -42:57.080 [INFO] TestAgent.server.raft: election won: tally=1 -42:57.080 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13253 [Leader]" -42:57.080 [INFO] TestAgent.server: cluster leadership acquired -42:57.080 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13253 -42:57.082 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:57.084 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:57.084 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:57.084 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:57.084 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:57.084 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:57.085 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-53a1185b-9ffa-1775-8c9b-0e26e933db41 -42:57.085 [INFO] TestAgent.server: member joined, marking health alive: member=Node-53a1185b-9ffa-1775-8c9b-0e26e933db41 -42:57.085 [INFO] TestAgent.server: New leader elected: payload=Node-53a1185b-9ffa-1775-8c9b-0e26e933db41 -42:57.085 [INFO] TestAgent.server: federation state anti-entropy synced -42:57.394 [DEBUG] TestAgent: Check status updated: check=test status=critical -42:57.395 [INFO] TestAgent: Synced node info -42:57.396 [INFO] TestAgent: Synced check: check=test -42:57.396 [INFO] TestAgent: Requesting shutdown -42:57.396 [INFO] TestAgent.server: shutting down server -42:57.396 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:57.396 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:57.396 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:57.396 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:57.396 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:57.396 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:57.396 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:57.396 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:57.396 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:57.396 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:57.396 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:57.397 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:57.397Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:57.397Z [INFO] agent.router.manager: shutting down -42:57.397 [INFO] TestAgent: consul server down -42:57.397 [INFO] TestAgent: shutdown complete -42:57.397 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13248 network=tcp -42:57.397 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13248 network=udp -42:57.397 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13249 network=tcp -42:57.897 [INFO] TestAgent: Waiting for endpoints to shut down -42:57.897 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_FailCheck (0.90s) +2021-08-13T15:34:20.584Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:20.599Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:20.602 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2b8b9def-b77b-82e4-7460-56354bdf8d2c Address:127.0.0.1:24670}]" +34:20.602 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24670 [Follower]" leader= +34:20.603 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2b8b9def-b77b-82e4-7460-56354bdf8d2c.dc1 127.0.0.1 +34:20.604 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2b8b9def-b77b-82e4-7460-56354bdf8d2c 127.0.0.1 +2021-08-13T15:34:20.604Z [INFO] agent.router: Initializing LAN area manager +34:20.604 [INFO] TestAgent.server: Adding LAN server: server="Node-2b8b9def-b77b-82e4-7460-56354bdf8d2c (Addr: tcp/127.0.0.1:24670) (DC: dc1)" +34:20.604 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2b8b9def-b77b-82e4-7460-56354bdf8d2c.dc1 area=wan +34:20.604 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24693 network=udp +34:20.604 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24693 network=tcp +34:20.605 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24666 network=tcp +34:20.605 [INFO] TestAgent: started state syncer +34:20.606 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24699 network=tcp +34:20.672 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:20.672 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24670 [Candidate]" term=2 +34:20.672 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:20.672 [DEBUG] TestAgent.server.raft: vote granted: from=2b8b9def-b77b-82e4-7460-56354bdf8d2c term=2 tally=1 +34:20.672 [INFO] TestAgent.server.raft: election won: tally=1 +34:20.672 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24670 [Leader]" +34:20.672 [INFO] TestAgent.server: cluster leadership acquired +34:20.673 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24670 +34:20.673 [INFO] TestAgent.server: New leader elected: payload=Node-2b8b9def-b77b-82e4-7460-56354bdf8d2c +34:20.674 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:20.677 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:20.677 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:20.677 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:20.677 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:20.677 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:20.677 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2b8b9def-b77b-82e4-7460-56354bdf8d2c +34:20.677 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2b8b9def-b77b-82e4-7460-56354bdf8d2c +34:20.678 [INFO] TestAgent.server: federation state anti-entropy synced +34:20.749 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:20.750 [INFO] TestAgent: Synced node info +34:20.750 [DEBUG] TestAgent: Node info in sync +34:20.971 [WARN] TestAgent.server: endpoint injected; this should only be used for testing +34:20.972 [INFO] TestAgent: Requesting shutdown +34:20.972 [INFO] TestAgent.server: shutting down server +34:20.972 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:20.972 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:20.972 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:20.972 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:20.972 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:20.972 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:20.972 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:20.972 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:20.972 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:20.972 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:20.973 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:20.973Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:20.973Z [INFO] agent.router.manager: shutting down +34:20.973 [INFO] TestAgent: consul server down +34:20.973 [INFO] TestAgent: shutdown complete +34:20.973 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24693 network=tcp +34:20.973 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24693 network=udp +34:20.973 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24666 network=tcp +34:21.479 [INFO] TestAgent: Waiting for endpoints to shut down +34:21.479 [INFO] TestAgent: Endpoints down +=== RUN TestPreparedQuery_Execute/#05 +2021-08-13T15:34:20.721Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:20.737Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:20.740 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:af3f39da-9439-28d9-a0c1-94f523e5779b Address:127.0.0.1:24684}]" +34:20.740 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24684 [Follower]" leader= +34:20.741 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-af3f39da-9439-28d9-a0c1-94f523e5779b.dc1 127.0.0.1 +34:20.742 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-af3f39da-9439-28d9-a0c1-94f523e5779b 127.0.0.1 +2021-08-13T15:34:20.742Z [INFO] agent.router: Initializing LAN area manager +34:20.742 [INFO] TestAgent.server: Adding LAN server: server="Node-af3f39da-9439-28d9-a0c1-94f523e5779b (Addr: tcp/127.0.0.1:24684) (DC: dc1)" +34:20.742 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-af3f39da-9439-28d9-a0c1-94f523e5779b.dc1 area=wan +34:20.743 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24700 network=tcp +34:20.743 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24700 network=udp +34:20.743 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24680 network=tcp +34:20.744 [INFO] TestAgent: started state syncer +34:20.744 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24678 network=tcp +34:20.790 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:20.790 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24684 [Candidate]" term=2 +34:20.790 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:20.790 [DEBUG] TestAgent.server.raft: vote granted: from=af3f39da-9439-28d9-a0c1-94f523e5779b term=2 tally=1 +34:20.790 [INFO] TestAgent.server.raft: election won: tally=1 +34:20.790 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24684 [Leader]" +34:20.790 [INFO] TestAgent.server: cluster leadership acquired +34:20.790 [INFO] TestAgent.server: New leader elected: payload=Node-af3f39da-9439-28d9-a0c1-94f523e5779b +34:20.790 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24684 +34:20.792 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:20.794 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:20.794 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:20.794 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:20.794 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:20.794 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:20.794 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-af3f39da-9439-28d9-a0c1-94f523e5779b +34:20.794 [INFO] TestAgent.server: member joined, marking health alive: member=Node-af3f39da-9439-28d9-a0c1-94f523e5779b +34:20.795 [INFO] TestAgent.server: federation state anti-entropy synced +34:20.827 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:20.828 [INFO] TestAgent: Synced node info +34:20.979 [INFO] TestAgent: Requesting shutdown +34:20.979 [INFO] TestAgent.server: shutting down server +34:20.979 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:20.979 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:20.979 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:20.979 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:20.979 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:20.979 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:20.980 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:20.980 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:20.980 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:20.980 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:20.980 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:20.980Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:20.980Z [INFO] agent.router.manager: shutting down +34:20.980 [INFO] TestAgent: consul server down +34:20.980 [INFO] TestAgent: shutdown complete +34:20.980 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24700 network=tcp +34:20.980 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24700 network=udp +34:20.980 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24680 network=tcp +34:21.482 [INFO] TestAgent: Waiting for endpoints to shut down +34:21.482 [INFO] TestAgent: Endpoints down +--- PASS: TestOperator_AutopilotGetConfiguration (0.78s) === CONT TestAgent_PassCheck -2022-09-15T21:42:56.991Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:56.991Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:57.008Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:57.008Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:57.011 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0101eee3-3b90-619a-43d9-94f115287978 Address:127.0.0.1:13183}]" -42:57.011 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13183 [Follower]" leader= -42:57.012 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0101eee3-3b90-619a-43d9-94f115287978.dc1 127.0.0.1 -42:57.013 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0101eee3-3b90-619a-43d9-94f115287978 127.0.0.1 -2022-09-15T21:42:57.013Z [INFO] agent.router: Initializing LAN area manager -42:57.013 [INFO] TestAgent.server: Adding LAN server: server="Node-0101eee3-3b90-619a-43d9-94f115287978 (Addr: tcp/127.0.0.1:13183) (DC: dc1)" -42:57.013 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0101eee3-3b90-619a-43d9-94f115287978.dc1 area=wan -42:57.013 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13185 network=udp -42:57.013 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13185 network=tcp -42:57.014 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13186 network=tcp -42:57.014 [INFO] TestAgent: started state syncer -42:57.014 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13184 network=tcp -42:57.056 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:57.056 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13183 [Candidate]" term=2 -42:57.056 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:57.056 [DEBUG] TestAgent.server.raft: vote granted: from=0101eee3-3b90-619a-43d9-94f115287978 term=2 tally=1 -42:57.057 [INFO] TestAgent.server.raft: election won: tally=1 -42:57.057 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13183 [Leader]" -42:57.057 [INFO] TestAgent.server: cluster leadership acquired -42:57.057 [INFO] TestAgent.server: initializing acls -42:57.057 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:57.057 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:57.057 [INFO] TestAgent.server: New leader elected: payload=Node-0101eee3-3b90-619a-43d9-94f115287978 -42:57.058 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:57.058 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:57.058 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:57.059 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:57.059 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-0101eee3-3b90-619a-43d9-94f115287978 -42:57.059 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-0101eee3-3b90-619a-43d9-94f115287978.dc1 -42:57.059 [INFO] TestAgent.server: Updating LAN server: server="Node-0101eee3-3b90-619a-43d9-94f115287978 (Addr: tcp/127.0.0.1:13183) (DC: dc1)" -42:57.059 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-0101eee3-3b90-619a-43d9-94f115287978.dc1 area=wan -42:57.060 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:57.062 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:57.062 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:57.062 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:57.062 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:57.062 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:57.062 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0101eee3-3b90-619a-43d9-94f115287978 -42:57.062 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0101eee3-3b90-619a-43d9-94f115287978 -42:57.063 [INFO] TestAgent.server: federation state anti-entropy synced -42:57.064 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0101eee3-3b90-619a-43d9-94f115287978 -42:57.389 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:57.407 [INFO] TestAgent: Synced node info -42:57.416 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-0101eee3-3b90-619a-43d9-94f115287978 -42:57.417 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-0101eee3-3b90-619a-43d9-94f115287978 -42:57.417 [DEBUG] TestAgent: Check status updated: check=test status=critical -42:57.417 [DEBUG] TestAgent: Node info in sync -42:57.417 [WARN] TestAgent: Check registration blocked by ACLs: check=test accessorID=00000000-0000-0000-0000-000000000002 -42:57.417 [INFO] TestAgent: Requesting shutdown -42:57.418 [INFO] TestAgent.server: shutting down server -42:57.418 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:57.418 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:57.418 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:57.418 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:57.418 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:57.418 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:57.418 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:57.418 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:57.418 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:57.418 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:57.418 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:57.418 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:57.418 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -42:57.418 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:57.418 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -2022-09-15T21:42:57.418Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:57.418Z [INFO] agent.router.manager: shutting down -42:57.418 [INFO] TestAgent: consul server down -42:57.418 [INFO] TestAgent: shutdown complete -42:57.419 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13185 network=tcp -42:57.419 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13185 network=udp -42:57.419 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13186 network=tcp -42:57.920 [INFO] TestAgent: Waiting for endpoints to shut down -42:57.920 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_FailCheck_ACLDeny (0.95s) - --- PASS: TestAgent_FailCheck_ACLDeny/no_token (0.00s) - --- PASS: TestAgent_FailCheck_ACLDeny/root_token (0.00s) +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:20.473Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:20.473Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:20.489Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:20.489Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:20.492 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:595b1ab1-a8ad-1425-2801-a3ec5327fd8a Address:127.0.0.1:24691}]" +34:20.492 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24691 [Follower]" leader= +34:20.493 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-595b1ab1-a8ad-1425-2801-a3ec5327fd8a.dc1 127.0.0.1 +34:20.494 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-595b1ab1-a8ad-1425-2801-a3ec5327fd8a 127.0.0.1 +2021-08-13T15:34:20.494Z [INFO] agent.router: Initializing LAN area manager +34:20.494 [INFO] TestAgent.server: Adding LAN server: server="Node-595b1ab1-a8ad-1425-2801-a3ec5327fd8a (Addr: tcp/127.0.0.1:24691) (DC: dc1)" +34:20.494 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-595b1ab1-a8ad-1425-2801-a3ec5327fd8a.dc1 area=wan +34:20.494 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24665 network=udp +34:20.494 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24665 network=tcp +34:20.495 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24687 network=tcp +34:20.495 [INFO] TestAgent: started state syncer +34:20.495 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24657 network=tcp +34:20.533 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:20.533 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24691 [Candidate]" term=2 +34:20.533 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:20.533 [DEBUG] TestAgent.server.raft: vote granted: from=595b1ab1-a8ad-1425-2801-a3ec5327fd8a term=2 tally=1 +34:20.533 [INFO] TestAgent.server.raft: election won: tally=1 +34:20.533 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24691 [Leader]" +34:20.533 [INFO] TestAgent.server: cluster leadership acquired +34:20.533 [INFO] TestAgent.server: New leader elected: payload=Node-595b1ab1-a8ad-1425-2801-a3ec5327fd8a +34:20.534 [INFO] TestAgent.server: initializing acls +34:20.534 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:20.534 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:20.535 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:20.535 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:20.535 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:20.535 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:20.536 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-595b1ab1-a8ad-1425-2801-a3ec5327fd8a +34:20.536 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-595b1ab1-a8ad-1425-2801-a3ec5327fd8a.dc1 +34:20.537 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:20.539 [INFO] TestAgent.server: Updating LAN server: server="Node-595b1ab1-a8ad-1425-2801-a3ec5327fd8a (Addr: tcp/127.0.0.1:24691) (DC: dc1)" +34:20.539 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-595b1ab1-a8ad-1425-2801-a3ec5327fd8a.dc1 area=wan +34:20.539 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:20.539 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:20.539 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:20.539 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:20.539 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:20.540 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-595b1ab1-a8ad-1425-2801-a3ec5327fd8a +34:20.540 [INFO] TestAgent.server: member joined, marking health alive: member=Node-595b1ab1-a8ad-1425-2801-a3ec5327fd8a +34:20.540 [INFO] TestAgent.server: federation state anti-entropy synced +34:20.541 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-595b1ab1-a8ad-1425-2801-a3ec5327fd8a +34:20.771 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth +34:20.771 [WARN] TestAgent: Node info update blocked by ACLs: node=595b1ab1-a8ad-1425-2801-a3ec5327fd8a accessorID=00000000-0000-0000-0000-000000000002 +34:20.886 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-595b1ab1-a8ad-1425-2801-a3ec5327fd8a +34:20.886 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-595b1ab1-a8ad-1425-2801-a3ec5327fd8a +34:20.889 [INFO] TestAgent: Updated agent's ACL token: token=default +34:21.007 [DEBUG] TestAgent.acl: dropping service from result due to ACLs: service="{consul {}}" +34:21.007 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:21.008 [INFO] TestAgent: Synced node info +34:21.008 [DEBUG] TestAgent: Node info in sync +34:21.024 [INFO] TestAgent: Requesting shutdown +34:21.024 [INFO] TestAgent.server: shutting down server +34:21.024 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:21.024 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:21.024 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:21.024 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:21.024 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:21.024 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:21.024 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:21.025 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:21.025 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:21.025 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:21.025 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:21.025 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:21.025 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:21.025 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:21.025 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:21.025Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:21.025Z [INFO] agent.router.manager: shutting down +34:21.025 [INFO] TestAgent: consul server down +34:21.025 [INFO] TestAgent: shutdown complete +34:21.025 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24665 network=tcp +34:21.025 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24665 network=udp +34:21.025 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24687 network=tcp +34:21.526 [INFO] TestAgent: Waiting for endpoints to shut down +34:21.526 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_TokenTriggersFullSync (6.85s) + --- PASS: TestAgent_TokenTriggersFullSync/acl_agent_token (0.95s) + --- PASS: TestAgent_TokenTriggersFullSync/agent (2.54s) + --- PASS: TestAgent_TokenTriggersFullSync/acl_token (2.29s) + --- PASS: TestAgent_TokenTriggersFullSync/default (1.07s) === CONT TestAgent_DeregisterCheckACLDeny === RUN TestAgent_PassCheck_ACLDeny/no_token === RUN TestAgent_PassCheck_ACLDeny/root_token -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestAgent_DeregisterCheckACLDeny/no_token -=== RUN TestAgent_DeregisterCheckACLDeny/root_token -2022-09-15T21:42:57.323Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:57.323Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:57.357Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:57.357Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:57.369 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:717f6ab8-d8f1-eca5-b4f0-27cb48b3448d Address:127.0.0.1:13267}]" -42:57.379 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-717f6ab8-d8f1-eca5-b4f0-27cb48b3448d.dc1 127.0.0.1 -42:57.380 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-717f6ab8-d8f1-eca5-b4f0-27cb48b3448d 127.0.0.1 -2022-09-15T21:42:57.380Z [INFO] agent.router: Initializing LAN area manager -42:57.381 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13262 network=udp -42:57.381 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13267 [Follower]" leader= -42:57.381 [INFO] TestAgent.server: Adding LAN server: server="Node-717f6ab8-d8f1-eca5-b4f0-27cb48b3448d (Addr: tcp/127.0.0.1:13267) (DC: dc1)" -42:57.381 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-717f6ab8-d8f1-eca5-b4f0-27cb48b3448d.dc1 area=wan -42:57.394 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13262 network=tcp -42:57.395 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13263 network=tcp -42:57.395 [INFO] TestAgent: started state syncer -42:57.395 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13268 network=tcp -42:57.429 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:57.429 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13267 [Candidate]" term=2 -42:57.430 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:57.430 [DEBUG] TestAgent.server.raft: vote granted: from=717f6ab8-d8f1-eca5-b4f0-27cb48b3448d term=2 tally=1 -42:57.430 [INFO] TestAgent.server.raft: election won: tally=1 -42:57.430 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13267 [Leader]" -42:57.430 [INFO] TestAgent.server: cluster leadership acquired -42:57.430 [INFO] TestAgent.server: initializing acls -42:57.430 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:57.430 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:57.431 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:57.432 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:57.432 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:57.432 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:57.432 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-717f6ab8-d8f1-eca5-b4f0-27cb48b3448d -42:57.432 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-717f6ab8-d8f1-eca5-b4f0-27cb48b3448d.dc1 -42:57.433 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:57.434 [INFO] TestAgent.server: New leader elected: payload=Node-717f6ab8-d8f1-eca5-b4f0-27cb48b3448d -42:57.434 [INFO] TestAgent.server: Updating LAN server: server="Node-717f6ab8-d8f1-eca5-b4f0-27cb48b3448d (Addr: tcp/127.0.0.1:13267) (DC: dc1)" -42:57.434 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-717f6ab8-d8f1-eca5-b4f0-27cb48b3448d.dc1 area=wan -42:57.436 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:57.436 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:57.436 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:57.436 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:57.436 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:57.436 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-717f6ab8-d8f1-eca5-b4f0-27cb48b3448d -42:57.436 [INFO] TestAgent.server: member joined, marking health alive: member=Node-717f6ab8-d8f1-eca5-b4f0-27cb48b3448d -42:57.437 [INFO] TestAgent.server: federation state anti-entropy synced -42:57.437 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-717f6ab8-d8f1-eca5-b4f0-27cb48b3448d -42:57.561 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:57.562 [INFO] TestAgent: Synced node info -42:57.609 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-717f6ab8-d8f1-eca5-b4f0-27cb48b3448d -42:57.609 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-717f6ab8-d8f1-eca5-b4f0-27cb48b3448d -42:57.610 [DEBUG] TestAgent: Check status updated: check=test status=warning -42:57.611 [DEBUG] TestAgent: Node info in sync -42:57.611 [WARN] TestAgent: Check registration blocked by ACLs: check=test accessorID=00000000-0000-0000-0000-000000000002 -42:57.611 [INFO] TestAgent: Requesting shutdown -42:57.611 [INFO] TestAgent.server: shutting down server -42:57.611 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:57.611 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:57.611 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:57.611 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:57.611 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:57.611 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:57.611 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:57.611 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:57.611 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:57.611 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:57.611 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:57.611 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:57.611 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:57.611 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:57.611 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:57.611Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:57.612Z [INFO] agent.router.manager: shutting down -42:57.612 [INFO] TestAgent: consul server down -42:57.612 [INFO] TestAgent: shutdown complete -42:57.612 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13262 network=tcp -42:57.612 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13262 network=udp -42:57.612 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13263 network=tcp -42:58.112 [INFO] TestAgent: Waiting for endpoints to shut down -42:58.112 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_WarnCheck_ACLDeny (0.81s) +2021-08-13T15:34:20.927Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:20.927Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:20.945Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:20.945Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:20.947 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c8d56b5b-da5c-20a7-f833-0f4fd4f9009c Address:127.0.0.1:24698}]" +34:20.947 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24698 [Follower]" leader= +34:20.948 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c8d56b5b-da5c-20a7-f833-0f4fd4f9009c.dc1 127.0.0.1 +34:20.949 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c8d56b5b-da5c-20a7-f833-0f4fd4f9009c 127.0.0.1 +2021-08-13T15:34:20.949Z [INFO] agent.router: Initializing LAN area manager +34:20.949 [INFO] TestAgent.server: Adding LAN server: server="Node-c8d56b5b-da5c-20a7-f833-0f4fd4f9009c (Addr: tcp/127.0.0.1:24698) (DC: dc1)" +34:20.949 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c8d56b5b-da5c-20a7-f833-0f4fd4f9009c.dc1 area=wan +34:20.950 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24679 network=tcp +34:20.950 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24679 network=udp +34:20.960 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24694 network=tcp +34:20.960 [INFO] TestAgent: started state syncer +34:20.960 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24727 network=tcp +34:21.000 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:21.011 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:21.011 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24698 [Candidate]" term=2 +34:21.011 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:21.011 [DEBUG] TestAgent.server.raft: vote granted: from=c8d56b5b-da5c-20a7-f833-0f4fd4f9009c term=2 tally=1 +34:21.011 [INFO] TestAgent.server.raft: election won: tally=1 +34:21.011 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24698 [Leader]" +34:21.011 [INFO] TestAgent.server: cluster leadership acquired +34:21.012 [INFO] TestAgent.server: New leader elected: payload=Node-c8d56b5b-da5c-20a7-f833-0f4fd4f9009c +34:21.012 [INFO] TestAgent.server: initializing acls +34:21.012 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:21.012 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:21.013 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:21.013 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:21.013 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:21.013 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:21.013 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-c8d56b5b-da5c-20a7-f833-0f4fd4f9009c +34:21.014 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-c8d56b5b-da5c-20a7-f833-0f4fd4f9009c.dc1 +34:21.014 [INFO] TestAgent.server: Updating LAN server: server="Node-c8d56b5b-da5c-20a7-f833-0f4fd4f9009c (Addr: tcp/127.0.0.1:24698) (DC: dc1)" +34:21.014 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-c8d56b5b-da5c-20a7-f833-0f4fd4f9009c.dc1 area=wan +34:21.015 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:21.017 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:21.017 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:21.017 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:21.017 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:21.017 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:21.017 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c8d56b5b-da5c-20a7-f833-0f4fd4f9009c +34:21.017 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c8d56b5b-da5c-20a7-f833-0f4fd4f9009c +34:21.018 [INFO] TestAgent.server: federation state anti-entropy synced +34:21.018 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c8d56b5b-da5c-20a7-f833-0f4fd4f9009c +34:21.216 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-c8d56b5b-da5c-20a7-f833-0f4fd4f9009c +34:21.216 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-c8d56b5b-da5c-20a7-f833-0f4fd4f9009c +34:21.217 [DEBUG] TestAgent: Check status updated: check=test status=critical +34:21.218 [INFO] TestAgent: Synced node info +34:21.218 [WARN] TestAgent: Check registration blocked by ACLs: check=test accessorID=00000000-0000-0000-0000-000000000002 +34:21.218 [INFO] TestAgent: Requesting shutdown +34:21.219 [INFO] TestAgent.server: shutting down server +34:21.219 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:21.219 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:21.219 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:21.219 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:21.219 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:21.219 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:21.219 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:21.219 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:21.219 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:21.219 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:21.219 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:21.219 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:21.219 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:21.219 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:21.219 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:21.219 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:21.219Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:21.219Z [INFO] agent.router.manager: shutting down +34:21.219 [INFO] TestAgent: consul server down +34:21.219 [INFO] TestAgent: shutdown complete +34:21.219 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24679 network=tcp +34:21.219 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24679 network=udp +34:21.219 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24694 network=tcp +34:21.719 [INFO] TestAgent: Waiting for endpoints to shut down +34:21.719 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_FailCheck_ACLDeny (0.81s) + --- PASS: TestAgent_FailCheck_ACLDeny/no_token (0.00s) + --- PASS: TestAgent_FailCheck_ACLDeny/root_token (0.00s) +=== CONT TestOperator_Keyring_LocalOnly +2021-08-13T15:34:20.991Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:21.008Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:21.011 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:53023ccd-3900-bac4-3071-2fe41be9ab8b Address:127.0.0.1:24733}]" +34:21.011 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24733 [Follower]" leader= +34:21.012 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-53023ccd-3900-bac4-3071-2fe41be9ab8b.dc1 127.0.0.1 +34:21.013 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-53023ccd-3900-bac4-3071-2fe41be9ab8b 127.0.0.1 +2021-08-13T15:34:21.013Z [INFO] agent.router: Initializing LAN area manager +34:21.013 [INFO] TestAgent.server: Adding LAN server: server="Node-53023ccd-3900-bac4-3071-2fe41be9ab8b (Addr: tcp/127.0.0.1:24733) (DC: dc1)" +34:21.013 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-53023ccd-3900-bac4-3071-2fe41be9ab8b.dc1 area=wan +34:21.013 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24728 network=udp +34:21.013 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24728 network=tcp +34:21.014 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24729 network=tcp +34:21.014 [INFO] TestAgent: started state syncer +34:21.014 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24671 network=tcp +34:21.079 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:21.079 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24733 [Candidate]" term=2 +34:21.079 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:21.079 [DEBUG] TestAgent.server.raft: vote granted: from=53023ccd-3900-bac4-3071-2fe41be9ab8b term=2 tally=1 +34:21.079 [INFO] TestAgent.server.raft: election won: tally=1 +34:21.079 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24733 [Leader]" +34:21.079 [INFO] TestAgent.server: cluster leadership acquired +34:21.079 [INFO] TestAgent.server: New leader elected: payload=Node-53023ccd-3900-bac4-3071-2fe41be9ab8b +34:21.079 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24733 +34:21.081 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:21.083 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:21.083 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:21.083 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:21.083 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:21.083 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:21.084 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-53023ccd-3900-bac4-3071-2fe41be9ab8b +34:21.084 [INFO] TestAgent.server: member joined, marking health alive: member=Node-53023ccd-3900-bac4-3071-2fe41be9ab8b +34:21.084 [INFO] TestAgent.server: federation state anti-entropy synced +34:21.256 [DEBUG] TestAgent: Check status updated: check=test status=critical +34:21.257 [INFO] TestAgent: Synced node info +34:21.258 [INFO] TestAgent: Synced check: check=test +34:21.258 [INFO] TestAgent: Requesting shutdown +34:21.258 [INFO] TestAgent.server: shutting down server +34:21.258 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:21.258 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:21.258 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:21.258 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:21.258 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:21.258 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:21.258 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:21.258 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:21.258 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:21.258 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:21.258 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:21.258 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:21.258Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:21.258Z [INFO] agent.router.manager: shutting down +34:21.258 [INFO] TestAgent: consul server down +34:21.258 [INFO] TestAgent: shutdown complete +34:21.258 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24728 network=tcp +34:21.258 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24728 network=udp +34:21.259 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24729 network=tcp +34:21.759 [INFO] TestAgent: Waiting for endpoints to shut down +34:21.759 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_FailCheck (0.79s) +=== CONT TestAgent_DeregisterCheck +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:21.149Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:21.149Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:21.167Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:21.167Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:21.170 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c95ecd8f-c188-c39c-693a-67ee8f14535b Address:127.0.0.1:24712}]" +34:21.171 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c95ecd8f-c188-c39c-693a-67ee8f14535b.dc1 127.0.0.1 +34:21.171 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24712 [Follower]" leader= +34:21.172 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c95ecd8f-c188-c39c-693a-67ee8f14535b 127.0.0.1 +2021-08-13T15:34:21.172Z [INFO] agent.router: Initializing LAN area manager +34:21.172 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24672 network=udp +34:21.172 [INFO] TestAgent.server: Adding LAN server: server="Node-c95ecd8f-c188-c39c-693a-67ee8f14535b (Addr: tcp/127.0.0.1:24712) (DC: dc1)" +34:21.172 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c95ecd8f-c188-c39c-693a-67ee8f14535b.dc1 area=wan +34:21.173 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24672 network=tcp +34:21.173 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24708 network=tcp +34:21.173 [INFO] TestAgent: started state syncer +34:21.174 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24734 network=tcp +34:21.222 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:21.232 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:21.232 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24712 [Candidate]" term=2 +34:21.232 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:21.232 [DEBUG] TestAgent.server.raft: vote granted: from=c95ecd8f-c188-c39c-693a-67ee8f14535b term=2 tally=1 +34:21.232 [INFO] TestAgent.server.raft: election won: tally=1 +34:21.232 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24712 [Leader]" +34:21.232 [INFO] TestAgent.server: cluster leadership acquired +34:21.232 [INFO] TestAgent.server: New leader elected: payload=Node-c95ecd8f-c188-c39c-693a-67ee8f14535b +34:21.232 [INFO] TestAgent.server: initializing acls +34:21.233 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:21.233 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:21.233 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:21.234 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:21.234 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:21.234 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:21.234 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-c95ecd8f-c188-c39c-693a-67ee8f14535b +34:21.234 [INFO] TestAgent.server: Updating LAN server: server="Node-c95ecd8f-c188-c39c-693a-67ee8f14535b (Addr: tcp/127.0.0.1:24712) (DC: dc1)" +34:21.234 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-c95ecd8f-c188-c39c-693a-67ee8f14535b.dc1 +34:21.235 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-c95ecd8f-c188-c39c-693a-67ee8f14535b.dc1 area=wan +34:21.235 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:21.238 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:21.238 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:21.238 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:21.238 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:21.238 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:21.238 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c95ecd8f-c188-c39c-693a-67ee8f14535b +34:21.238 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c95ecd8f-c188-c39c-693a-67ee8f14535b +34:21.239 [INFO] TestAgent.server: federation state anti-entropy synced +34:21.239 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c95ecd8f-c188-c39c-693a-67ee8f14535b +34:21.322 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:21.323 [INFO] TestAgent: Synced node info +34:21.347 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-c95ecd8f-c188-c39c-693a-67ee8f14535b +34:21.347 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-c95ecd8f-c188-c39c-693a-67ee8f14535b +34:21.347 [DEBUG] TestAgent: Check status updated: check=test status=warning +34:21.348 [DEBUG] TestAgent: Node info in sync +34:21.348 [WARN] TestAgent: Check registration blocked by ACLs: check=test accessorID=00000000-0000-0000-0000-000000000002 +34:21.348 [INFO] TestAgent: Requesting shutdown +34:21.348 [INFO] TestAgent.server: shutting down server +34:21.348 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:21.348 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:21.348 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:21.348 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:21.348 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:21.348 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:21.348 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:21.348 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:21.348 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:21.348 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:21.348 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:21.348 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:21.348 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:21.348 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:21.348 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:21.348Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:21.348Z [INFO] agent.router.manager: shutting down +34:21.349 [INFO] TestAgent: consul server down +34:21.349 [INFO] TestAgent: shutdown complete +34:21.349 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24672 network=tcp +34:21.349 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24672 network=udp +34:21.349 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24708 network=tcp +34:21.849 [INFO] TestAgent: Waiting for endpoints to shut down +34:21.849 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_WarnCheck_ACLDeny (0.73s) --- PASS: TestAgent_WarnCheck_ACLDeny/no_token (0.00s) --- PASS: TestAgent_WarnCheck_ACLDeny/root_token (0.00s) -=== CONT TestAgent_DeregisterCheck -2022-09-15T21:42:57.529Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:57.548Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:57.551 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f3935ade-4e18-38fa-bff8-48f480d1afb4 Address:127.0.0.1:13281}]" -42:57.551 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13281 [Follower]" leader= -42:57.552 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f3935ade-4e18-38fa-bff8-48f480d1afb4.dc1 127.0.0.1 -42:57.552 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f3935ade-4e18-38fa-bff8-48f480d1afb4 127.0.0.1 -2022-09-15T21:42:57.552Z [INFO] agent.router: Initializing LAN area manager -42:57.553 [INFO] TestAgent.server: Adding LAN server: server="Node-f3935ade-4e18-38fa-bff8-48f480d1afb4 (Addr: tcp/127.0.0.1:13281) (DC: dc1)" -42:57.553 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f3935ade-4e18-38fa-bff8-48f480d1afb4.dc1 area=wan -42:57.553 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13290 network=udp -42:57.553 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13290 network=tcp -42:57.554 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13291 network=tcp -42:57.554 [INFO] TestAgent: started state syncer -42:57.554 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13282 network=tcp -42:57.603 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:57.603 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13281 [Candidate]" term=2 -42:57.603 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:57.603 [DEBUG] TestAgent.server.raft: vote granted: from=f3935ade-4e18-38fa-bff8-48f480d1afb4 term=2 tally=1 -42:57.603 [INFO] TestAgent.server.raft: election won: tally=1 -42:57.603 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13281 [Leader]" -42:57.604 [INFO] TestAgent.server: cluster leadership acquired -42:57.604 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13281 -42:57.605 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:57.606 [INFO] TestAgent.server: New leader elected: payload=Node-f3935ade-4e18-38fa-bff8-48f480d1afb4 -42:57.608 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:57.608 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:57.608 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:57.608 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:57.608 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:57.608 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f3935ade-4e18-38fa-bff8-48f480d1afb4 -42:57.608 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f3935ade-4e18-38fa-bff8-48f480d1afb4 -42:57.610 [INFO] TestAgent.server: federation state anti-entropy synced -42:57.635 [DEBUG] TestAgent: Check status updated: check=test status=warning -42:57.636 [INFO] TestAgent: Synced node info -42:57.636 [INFO] TestAgent: Synced check: check=test -42:57.636 [INFO] TestAgent: Requesting shutdown -42:57.637 [INFO] TestAgent.server: shutting down server -42:57.637 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:57.637 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:57.637 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:57.637 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:57.637 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:57.637 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:57.637 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:57.637 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:57.637 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:57.637 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:57.637 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:57.637 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:57.637Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:57.637Z [INFO] agent.router.manager: shutting down -42:57.637 [INFO] TestAgent: consul server down -42:57.637 [INFO] TestAgent: shutdown complete -42:57.638 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13290 network=tcp -42:57.638 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13290 network=udp -42:57.638 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13291 network=tcp -42:58.140 [INFO] TestAgent: Waiting for endpoints to shut down -42:58.140 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_WarnCheck (0.63s) +=== CONT TestOperator_Keyring_InvalidRelayFactor +=== RUN TestAgent_DeregisterCheckACLDeny/no_token +=== RUN TestAgent_DeregisterCheckACLDeny/root_token +2021-08-13T15:34:21.183Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:21.199Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:21.202 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:dee4b4f2-edbd-6bea-e6be-d3d33a6fc4bb Address:127.0.0.1:24740}]" +34:21.202 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24740 [Follower]" leader= +34:21.203 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-dee4b4f2-edbd-6bea-e6be-d3d33a6fc4bb.dc1 127.0.0.1 +34:21.203 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-dee4b4f2-edbd-6bea-e6be-d3d33a6fc4bb 127.0.0.1 +2021-08-13T15:34:21.203Z [INFO] agent.router: Initializing LAN area manager +34:21.204 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-dee4b4f2-edbd-6bea-e6be-d3d33a6fc4bb.dc1 area=wan +34:21.204 [INFO] TestAgent.server: Adding LAN server: server="Node-dee4b4f2-edbd-6bea-e6be-d3d33a6fc4bb (Addr: tcp/127.0.0.1:24740) (DC: dc1)" +34:21.204 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24735 network=udp +34:21.204 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24735 network=tcp +34:21.205 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24736 network=tcp +34:21.205 [INFO] TestAgent: started state syncer +34:21.205 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24713 network=tcp +34:21.264 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:21.264 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24740 [Candidate]" term=2 +34:21.264 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:21.264 [DEBUG] TestAgent.server.raft: vote granted: from=dee4b4f2-edbd-6bea-e6be-d3d33a6fc4bb term=2 tally=1 +34:21.264 [INFO] TestAgent.server.raft: election won: tally=1 +34:21.264 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24740 [Leader]" +34:21.264 [INFO] TestAgent.server: cluster leadership acquired +34:21.264 [INFO] TestAgent.server: New leader elected: payload=Node-dee4b4f2-edbd-6bea-e6be-d3d33a6fc4bb +34:21.265 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24740 +34:21.266 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:21.268 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:21.268 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:21.268 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:21.268 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:21.268 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:21.268 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-dee4b4f2-edbd-6bea-e6be-d3d33a6fc4bb +34:21.268 [INFO] TestAgent.server: member joined, marking health alive: member=Node-dee4b4f2-edbd-6bea-e6be-d3d33a6fc4bb +34:21.269 [INFO] TestAgent.server: federation state anti-entropy synced +34:21.422 [DEBUG] TestAgent: Check status updated: check=test status=warning +34:21.423 [INFO] TestAgent: Synced node info +34:21.424 [INFO] TestAgent: Synced check: check=test +34:21.424 [INFO] TestAgent: Requesting shutdown +34:21.424 [INFO] TestAgent.server: shutting down server +34:21.424 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:21.424 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:21.424 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:21.424 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:21.424 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:21.424 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:21.424 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:21.424 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:21.424 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:21.424 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:21.424 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:21.424 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:21.424Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:21.424Z [INFO] agent.router.manager: shutting down +34:21.424 [INFO] TestAgent: consul server down +34:21.424 [INFO] TestAgent: shutdown complete +34:21.424 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24735 network=tcp +34:21.424 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24735 network=udp +34:21.424 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24736 network=tcp +34:21.924 [INFO] TestAgent: Waiting for endpoints to shut down +34:21.924 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_WarnCheck (0.77s) === CONT TestAgent_RegisterCheck_ACLDeny -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:57.800Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:57.800Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:57.833Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:57.833Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:57.842 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3a8f0cdf-c583-35d1-9df3-8bbae35793aa Address:127.0.0.1:13302}]" -42:57.843 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3a8f0cdf-c583-35d1-9df3-8bbae35793aa.dc1 127.0.0.1 -42:57.844 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3a8f0cdf-c583-35d1-9df3-8bbae35793aa 127.0.0.1 -2022-09-15T21:42:57.844Z [INFO] agent.router: Initializing LAN area manager -42:57.844 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13302 [Follower]" leader= -42:57.845 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3a8f0cdf-c583-35d1-9df3-8bbae35793aa.dc1 area=wan -42:57.846 [INFO] TestAgent.server: Adding LAN server: server="Node-3a8f0cdf-c583-35d1-9df3-8bbae35793aa (Addr: tcp/127.0.0.1:13302) (DC: dc1)" -42:57.846 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13276 network=udp -42:57.846 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13276 network=tcp -42:57.847 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13277 network=tcp -42:57.847 [INFO] TestAgent: started state syncer -42:57.847 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13303 network=tcp -42:57.896 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -42:57.912 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:57.912 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13302 [Candidate]" term=2 -42:57.912 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:57.912 [DEBUG] TestAgent.server.raft: vote granted: from=3a8f0cdf-c583-35d1-9df3-8bbae35793aa term=2 tally=1 -42:57.912 [INFO] TestAgent.server.raft: election won: tally=1 -42:57.912 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13302 [Leader]" -42:57.913 [INFO] TestAgent.server: cluster leadership acquired -42:57.913 [INFO] TestAgent.server: initializing acls -42:57.913 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:57.913 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:57.914 [INFO] TestAgent.server: New leader elected: payload=Node-3a8f0cdf-c583-35d1-9df3-8bbae35793aa -42:57.915 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:57.915 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:57.915 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:57.915 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:57.915 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-3a8f0cdf-c583-35d1-9df3-8bbae35793aa -42:57.916 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-3a8f0cdf-c583-35d1-9df3-8bbae35793aa.dc1 -42:57.917 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:57.917 [INFO] TestAgent.server: Updating LAN server: server="Node-3a8f0cdf-c583-35d1-9df3-8bbae35793aa (Addr: tcp/127.0.0.1:13302) (DC: dc1)" -42:57.917 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-3a8f0cdf-c583-35d1-9df3-8bbae35793aa.dc1 area=wan -42:57.919 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:57.919 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:57.919 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:57.919 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:57.920 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:57.920 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3a8f0cdf-c583-35d1-9df3-8bbae35793aa -42:57.920 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3a8f0cdf-c583-35d1-9df3-8bbae35793aa -42:57.921 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3a8f0cdf-c583-35d1-9df3-8bbae35793aa -42:57.921 [INFO] TestAgent.server: federation state anti-entropy synced -42:57.939 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-3a8f0cdf-c583-35d1-9df3-8bbae35793aa -42:57.940 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-3a8f0cdf-c583-35d1-9df3-8bbae35793aa -42:57.941 [DEBUG] TestAgent: Check status updated: check=test status=passing -42:57.943 [INFO] TestAgent: Synced node info -42:57.943 [WARN] TestAgent: Check registration blocked by ACLs: check=test accessorID=00000000-0000-0000-0000-000000000002 -42:57.943 [INFO] TestAgent: Requesting shutdown -42:57.943 [INFO] TestAgent.server: shutting down server -42:57.943 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:57.943 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:57.943 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:57.943 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:57.943 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:57.943 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:57.943 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:57.943 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:57.943 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:57.943 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:57.943 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:57.943 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:57.943 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:57.943 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:57.943 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:57.943 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:57.944Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:57.944Z [INFO] agent.router.manager: shutting down -42:57.944 [INFO] TestAgent: consul server down -42:57.944 [INFO] TestAgent: shutdown complete -42:57.944 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13276 network=tcp -42:57.944 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13276 network=udp -42:57.944 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13277 network=tcp -42:58.444 [INFO] TestAgent: Waiting for endpoints to shut down -42:58.444 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_PassCheck_ACLDeny (0.67s) +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:21.335Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:21.335Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:21.353Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:21.353Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:21.356 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:56e310fb-8e4e-0892-475d-752120c58124 Address:127.0.0.1:24677}]" +34:21.356 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24677 [Follower]" leader= +34:21.357 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-56e310fb-8e4e-0892-475d-752120c58124.dc1 127.0.0.1 +34:21.358 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-56e310fb-8e4e-0892-475d-752120c58124 127.0.0.1 +2021-08-13T15:34:21.358Z [INFO] agent.router: Initializing LAN area manager +34:21.358 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-56e310fb-8e4e-0892-475d-752120c58124.dc1 area=wan +34:21.358 [INFO] TestAgent.server: Adding LAN server: server="Node-56e310fb-8e4e-0892-475d-752120c58124 (Addr: tcp/127.0.0.1:24677) (DC: dc1)" +34:21.358 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24714 network=tcp +34:21.358 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24714 network=udp +34:21.359 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24673 network=tcp +34:21.359 [INFO] TestAgent: started state syncer +34:21.359 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24706 network=tcp +34:21.406 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:21.406 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24677 [Candidate]" term=2 +34:21.407 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:21.407 [DEBUG] TestAgent.server.raft: vote granted: from=56e310fb-8e4e-0892-475d-752120c58124 term=2 tally=1 +34:21.407 [INFO] TestAgent.server.raft: election won: tally=1 +34:21.407 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24677 [Leader]" +34:21.407 [INFO] TestAgent.server: cluster leadership acquired +34:21.407 [INFO] TestAgent.server: New leader elected: payload=Node-56e310fb-8e4e-0892-475d-752120c58124 +34:21.407 [INFO] TestAgent.server: initializing acls +34:21.408 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:21.408 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:21.408 [INFO] TestAgent.server: initializing acls +34:21.408 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:21.409 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:21.409 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:21.409 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:21.409 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:21.409 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:21.410 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-56e310fb-8e4e-0892-475d-752120c58124 +34:21.410 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-56e310fb-8e4e-0892-475d-752120c58124.dc1 +34:21.411 [INFO] TestAgent.server: Updating LAN server: server="Node-56e310fb-8e4e-0892-475d-752120c58124 (Addr: tcp/127.0.0.1:24677) (DC: dc1)" +34:21.411 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-56e310fb-8e4e-0892-475d-752120c58124.dc1 area=wan +34:21.411 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:21.411 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode +34:21.411 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:21.411 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-56e310fb-8e4e-0892-475d-752120c58124 +34:21.411 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-56e310fb-8e4e-0892-475d-752120c58124.dc1 +34:21.411 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-56e310fb-8e4e-0892-475d-752120c58124.dc1 area=wan +34:21.411 [INFO] TestAgent.server: Updating LAN server: server="Node-56e310fb-8e4e-0892-475d-752120c58124 (Addr: tcp/127.0.0.1:24677) (DC: dc1)" +34:21.414 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:21.414 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:21.414 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:21.414 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:21.414 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:21.414 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-56e310fb-8e4e-0892-475d-752120c58124 +34:21.414 [INFO] TestAgent.server: member joined, marking health alive: member=Node-56e310fb-8e4e-0892-475d-752120c58124 +34:21.415 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-56e310fb-8e4e-0892-475d-752120c58124 +34:21.415 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-56e310fb-8e4e-0892-475d-752120c58124 +34:21.415 [INFO] TestAgent.server: federation state anti-entropy synced +34:21.536 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-56e310fb-8e4e-0892-475d-752120c58124 +34:21.537 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-56e310fb-8e4e-0892-475d-752120c58124 +34:21.538 [DEBUG] TestAgent: Check status updated: check=test status=passing +34:21.540 [INFO] TestAgent: Synced node info +34:21.540 [WARN] TestAgent: Check registration blocked by ACLs: check=test accessorID=00000000-0000-0000-0000-000000000002 +34:21.540 [INFO] TestAgent: Requesting shutdown +34:21.540 [INFO] TestAgent.server: shutting down server +34:21.540 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:21.540 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:21.540 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:21.540 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:21.540 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:21.540 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:21.540 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:21.540 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:21.540 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:21.540 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:21.540 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:21.540 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:21.541 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:21.541 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:21.541 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:21.541 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:21.541Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:21.541Z [INFO] agent.router.manager: shutting down +34:21.542 [INFO] TestAgent: consul server down +34:21.542 [INFO] TestAgent: shutdown complete +34:21.542 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24714 network=tcp +34:21.542 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24714 network=udp +34:21.542 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24673 network=tcp +34:22.042 [INFO] TestAgent: Waiting for endpoints to shut down +34:22.042 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_PassCheck_ACLDeny (0.73s) --- PASS: TestAgent_PassCheck_ACLDeny/no_token (0.00s) --- PASS: TestAgent_PassCheck_ACLDeny/root_token (0.00s) -=== CONT TestAgent_RegisterCheck_BadStatus -2022-09-15T21:42:57.673Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:57.673Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:57.697Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:57.697Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:57.700 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:28935b32-68b5-f782-8981-3ffe6514b3db Address:127.0.0.1:13274}]" -42:57.700 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13274 [Follower]" leader= -42:57.701 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-28935b32-68b5-f782-8981-3ffe6514b3db.dc1 127.0.0.1 -42:57.702 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-28935b32-68b5-f782-8981-3ffe6514b3db 127.0.0.1 -2022-09-15T21:42:57.702Z [INFO] agent.router: Initializing LAN area manager -42:57.702 [INFO] TestAgent.server: Adding LAN server: server="Node-28935b32-68b5-f782-8981-3ffe6514b3db (Addr: tcp/127.0.0.1:13274) (DC: dc1)" -42:57.702 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-28935b32-68b5-f782-8981-3ffe6514b3db.dc1 area=wan -42:57.702 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13297 network=tcp -42:57.702 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13297 network=udp -42:57.703 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13298 network=tcp -42:57.703 [INFO] TestAgent: started state syncer -42:57.703 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13275 network=tcp -42:57.749 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:57.749 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13274 [Candidate]" term=2 -42:57.749 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:57.749 [DEBUG] TestAgent.server.raft: vote granted: from=28935b32-68b5-f782-8981-3ffe6514b3db term=2 tally=1 -42:57.749 [INFO] TestAgent.server.raft: election won: tally=1 -42:57.749 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13274 [Leader]" -42:57.749 [INFO] TestAgent.server: cluster leadership acquired -42:57.750 [INFO] TestAgent.server: initializing acls -42:57.750 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:57.750 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:57.751 [INFO] TestAgent.server: New leader elected: payload=Node-28935b32-68b5-f782-8981-3ffe6514b3db -42:57.751 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:57.751 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:57.751 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:57.751 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:57.752 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-28935b32-68b5-f782-8981-3ffe6514b3db -42:57.752 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-28935b32-68b5-f782-8981-3ffe6514b3db.dc1 -42:57.752 [INFO] TestAgent.server: Updating LAN server: server="Node-28935b32-68b5-f782-8981-3ffe6514b3db (Addr: tcp/127.0.0.1:13274) (DC: dc1)" -42:57.752 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-28935b32-68b5-f782-8981-3ffe6514b3db.dc1 area=wan -42:57.753 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:57.755 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:57.756 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:57.756 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:57.756 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:57.756 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:57.756 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-28935b32-68b5-f782-8981-3ffe6514b3db -42:57.756 [INFO] TestAgent.server: member joined, marking health alive: member=Node-28935b32-68b5-f782-8981-3ffe6514b3db -42:57.757 [INFO] TestAgent.server: federation state anti-entropy synced -42:57.757 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-28935b32-68b5-f782-8981-3ffe6514b3db -42:57.774 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-28935b32-68b5-f782-8981-3ffe6514b3db -42:57.774 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-28935b32-68b5-f782-8981-3ffe6514b3db -42:57.777 [INFO] TestAgent: Updated agent's ACL token: token=default -42:57.928 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth -42:57.929 [INFO] TestAgent: Synced node info -42:57.929 [DEBUG] TestAgent: Node info in sync -42:57.949 [INFO] TestAgent: Requesting shutdown -42:57.949 [INFO] TestAgent.server: shutting down server -42:57.949 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:57.949 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:57.949 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:57.949 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:57.949 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:57.949 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:57.950 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:57.950 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:57.950 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:57.950 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:57.950 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:57.950 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:57.950 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:57.950 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:57.950 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:57.950Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:57.950Z [INFO] agent.router.manager: shutting down -42:57.950 [INFO] TestAgent: consul server down -42:57.950 [INFO] TestAgent: shutdown complete -42:57.950 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13297 network=tcp -42:57.950 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13297 network=udp -42:57.950 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13298 network=tcp -42:58.454 [INFO] TestAgent: Waiting for endpoints to shut down -42:58.454 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_TokenTriggersFullSync (4.53s) - --- PASS: TestAgent_TokenTriggersFullSync/acl_agent_token (0.88s) - --- PASS: TestAgent_TokenTriggersFullSync/agent (0.95s) - --- PASS: TestAgent_TokenTriggersFullSync/acl_token (1.90s) - --- PASS: TestAgent_TokenTriggersFullSync/default (0.80s) -=== CONT TestAgent_RegisterCheck_Passing +=== CONT TestOperator_KeyringUse === RUN TestAgent_RegisterCheck_ACLDeny/no_token_-_node_check === RUN TestAgent_RegisterCheck_ACLDeny/svc_token_-_node_check === RUN TestAgent_RegisterCheck_ACLDeny/node_token_-_node_check === RUN TestAgent_RegisterCheck_ACLDeny/no_token_-_svc_check === RUN TestAgent_RegisterCheck_ACLDeny/node_token_-_svc_check === RUN TestAgent_RegisterCheck_ACLDeny/svc_token_-_svc_check -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:57.943Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:57.943Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:57.962Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:57.962Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:57.965 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e73fa57b-5f67-bc86-9875-aceb89023dfb Address:127.0.0.1:13288}]" -42:57.966 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13288 [Follower]" leader= -42:57.967 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e73fa57b-5f67-bc86-9875-aceb89023dfb.dc1 127.0.0.1 -42:57.967 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e73fa57b-5f67-bc86-9875-aceb89023dfb 127.0.0.1 -2022-09-15T21:42:57.967Z [INFO] agent.router: Initializing LAN area manager -42:57.968 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13318 network=udp -42:57.968 [INFO] TestAgent.server: Adding LAN server: server="Node-e73fa57b-5f67-bc86-9875-aceb89023dfb (Addr: tcp/127.0.0.1:13288) (DC: dc1)" -42:57.968 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e73fa57b-5f67-bc86-9875-aceb89023dfb.dc1 area=wan -42:57.968 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13318 network=tcp -42:57.969 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13319 network=tcp -42:57.969 [INFO] TestAgent: started state syncer -42:57.969 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13289 network=tcp -42:58.018 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -42:58.022 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:58.022 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13288 [Candidate]" term=2 -42:58.022 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:58.022 [DEBUG] TestAgent.server.raft: vote granted: from=e73fa57b-5f67-bc86-9875-aceb89023dfb term=2 tally=1 -42:58.022 [INFO] TestAgent.server.raft: election won: tally=1 -42:58.022 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13288 [Leader]" -42:58.023 [INFO] TestAgent.server: cluster leadership acquired -42:58.023 [INFO] TestAgent.server: initializing acls -42:58.023 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:58.023 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:58.024 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:58.025 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:58.025 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:58.025 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:58.025 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-e73fa57b-5f67-bc86-9875-aceb89023dfb -42:58.025 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-e73fa57b-5f67-bc86-9875-aceb89023dfb.dc1 -42:58.026 [INFO] TestAgent.server: New leader elected: payload=Node-e73fa57b-5f67-bc86-9875-aceb89023dfb -42:58.026 [INFO] TestAgent.server: Updating LAN server: server="Node-e73fa57b-5f67-bc86-9875-aceb89023dfb (Addr: tcp/127.0.0.1:13288) (DC: dc1)" -42:58.026 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-e73fa57b-5f67-bc86-9875-aceb89023dfb.dc1 area=wan -42:58.028 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:58.030 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:58.030 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:58.030 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:58.030 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:58.030 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:58.030 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e73fa57b-5f67-bc86-9875-aceb89023dfb -42:58.031 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e73fa57b-5f67-bc86-9875-aceb89023dfb -42:58.032 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e73fa57b-5f67-bc86-9875-aceb89023dfb -42:58.032 [INFO] TestAgent.server: federation state anti-entropy synced -42:58.076 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-e73fa57b-5f67-bc86-9875-aceb89023dfb -42:58.078 [DEBUG] TestAgent: removed check: check=test -42:58.079 [INFO] TestAgent: Synced node info -42:58.079 [INFO] TestAgent: Deregistered check: check=test -42:58.079 [INFO] TestAgent: Requesting shutdown -42:58.079 [INFO] TestAgent.server: shutting down server -42:58.079 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:58.079 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:58.079 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:58.079 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:58.079 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:58.079 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:58.079 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:58.079 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:58.079 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:58.079 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:58.079 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:58.079 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:58.079 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:58.079 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:58.079 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:58.080 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:58.080Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:58.080Z [INFO] agent.router.manager: shutting down -42:58.080 [INFO] TestAgent: consul server down -42:58.080 [INFO] TestAgent: shutdown complete -42:58.080 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13318 network=tcp -42:58.080 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13318 network=udp -42:58.080 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13319 network=tcp -42:58.580 [INFO] TestAgent: Waiting for endpoints to shut down -42:58.580 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_DeregisterCheckACLDeny (0.66s) +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:21.496Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:21.512Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:21.514 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:13023108-6ab5-5eef-9d21-ca2116d4e8d9 Address:127.0.0.1:24719}]" +34:21.515 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24719 [Follower]" leader= +34:21.516 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-13023108-6ab5-5eef-9d21-ca2116d4e8d9.dc1 127.0.0.1 +34:21.516 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-13023108-6ab5-5eef-9d21-ca2116d4e8d9 127.0.0.1 +2021-08-13T15:34:21.516Z [INFO] agent.router: Initializing LAN area manager +34:21.517 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24707 network=udp +34:21.517 [INFO] TestAgent.server: Adding LAN server: server="Node-13023108-6ab5-5eef-9d21-ca2116d4e8d9 (Addr: tcp/127.0.0.1:24719) (DC: dc1)" +34:21.517 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-13023108-6ab5-5eef-9d21-ca2116d4e8d9.dc1 area=wan +34:21.517 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24707 network=tcp +34:21.518 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24715 network=tcp +34:21.518 [INFO] TestAgent: started state syncer +34:21.518 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24720 network=tcp +34:21.580 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:21.580 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24719 [Candidate]" term=2 +34:21.581 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:21.581 [DEBUG] TestAgent.server.raft: vote granted: from=13023108-6ab5-5eef-9d21-ca2116d4e8d9 term=2 tally=1 +34:21.581 [INFO] TestAgent.server.raft: election won: tally=1 +34:21.581 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24719 [Leader]" +34:21.581 [INFO] TestAgent.server: cluster leadership acquired +34:21.581 [INFO] TestAgent.server: New leader elected: payload=Node-13023108-6ab5-5eef-9d21-ca2116d4e8d9 +34:21.581 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24719 +34:21.604 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:21.606 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:21.606 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:21.606 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:21.606 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:21.606 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:21.607 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-13023108-6ab5-5eef-9d21-ca2116d4e8d9 +34:21.607 [INFO] TestAgent.server: member joined, marking health alive: member=Node-13023108-6ab5-5eef-9d21-ca2116d4e8d9 +34:21.610 [INFO] TestAgent.server: federation state anti-entropy synced +34:21.779 [WARN] TestAgent.server: endpoint injected; this should only be used for testing +34:21.779 [INFO] TestAgent: Requesting shutdown +34:21.779 [INFO] TestAgent.server: shutting down server +34:21.779 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:21.779 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:21.779 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:21.779 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:21.779 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:21.779 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:21.779 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:21.780 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:21.780 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:21.780 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:21.780 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:21.780 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:21.780Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:21.780Z [INFO] agent.router.manager: shutting down +34:21.780 [INFO] TestAgent: consul server down +34:21.780 [INFO] TestAgent: shutdown complete +34:21.780 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24707 network=tcp +34:21.780 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24707 network=udp +34:21.780 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24715 network=tcp +34:22.280 [INFO] TestAgent: Waiting for endpoints to shut down +34:22.280 [INFO] TestAgent: Endpoints down +=== RUN TestPreparedQuery_Execute/#06 +2021-08-13T15:34:21.506Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:21.523Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:21.526 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cff3e799-1f75-3771-6ce4-a029f69279b5 Address:127.0.0.1:24726}]" +34:21.526 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24726 [Follower]" leader= +34:21.527 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-cff3e799-1f75-3771-6ce4-a029f69279b5.dc1 127.0.0.1 +34:21.533 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-cff3e799-1f75-3771-6ce4-a029f69279b5 127.0.0.1 +2021-08-13T15:34:21.533Z [INFO] agent.router: Initializing LAN area manager +34:21.535 [INFO] TestAgent.server: Adding LAN server: server="Node-cff3e799-1f75-3771-6ce4-a029f69279b5 (Addr: tcp/127.0.0.1:24726) (DC: dc1)" +34:21.535 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-cff3e799-1f75-3771-6ce4-a029f69279b5.dc1 area=wan +34:21.541 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24721 network=udp +34:21.541 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24721 network=tcp +34:21.544 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24722 network=tcp +34:21.545 [INFO] TestAgent: started state syncer +34:21.545 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24741 network=tcp +34:21.564 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:21.564 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24726 [Candidate]" term=2 +34:21.564 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:21.564 [DEBUG] TestAgent.server.raft: vote granted: from=cff3e799-1f75-3771-6ce4-a029f69279b5 term=2 tally=1 +34:21.564 [INFO] TestAgent.server.raft: election won: tally=1 +34:21.564 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24726 [Leader]" +34:21.565 [INFO] TestAgent.server: cluster leadership acquired +34:21.565 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24726 +34:21.566 [INFO] TestAgent.server: New leader elected: payload=Node-cff3e799-1f75-3771-6ce4-a029f69279b5 +34:21.603 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:21.606 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:21.606 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:21.606 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:21.606 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:21.606 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:21.606 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cff3e799-1f75-3771-6ce4-a029f69279b5 +34:21.606 [INFO] TestAgent.server: member joined, marking health alive: member=Node-cff3e799-1f75-3771-6ce4-a029f69279b5 +34:21.607 [INFO] TestAgent.server: federation state anti-entropy synced +34:21.794 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:21.795 [INFO] TestAgent: Synced node info +34:21.799 [DEBUG] TestAgent: Check status updated: check=test status=passing +34:21.799 [DEBUG] TestAgent: Node info in sync +34:21.800 [INFO] TestAgent: Synced check: check=test +34:21.800 [INFO] TestAgent: Requesting shutdown +34:21.800 [INFO] TestAgent.server: shutting down server +34:21.800 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:21.800 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:21.800 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:21.800 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:21.800 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:21.800 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:21.800 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:21.800 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:21.800 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:21.800 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:21.800 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:21.800Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:21.800Z [INFO] agent.router.manager: shutting down +34:21.801 [INFO] TestAgent: consul server down +34:21.801 [INFO] TestAgent: shutdown complete +34:21.801 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24721 network=tcp +34:21.801 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24721 network=udp +34:21.801 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24722 network=tcp +34:22.301 [INFO] TestAgent: Waiting for endpoints to shut down +34:22.301 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_PassCheck (0.82s) +=== CONT TestAgent_RegisterCheck_BadStatus +2021-08-13T15:34:21.599Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:21.599Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:21.623Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:21.623Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:21.626 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:23610a1b-bcd3-cc4a-5046-21adffd07c05 Address:127.0.0.1:24747}]" +34:21.626 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24747 [Follower]" leader= +34:21.627 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-23610a1b-bcd3-cc4a-5046-21adffd07c05.dc1 127.0.0.1 +34:21.628 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-23610a1b-bcd3-cc4a-5046-21adffd07c05 127.0.0.1 +2021-08-13T15:34:21.628Z [INFO] agent.router: Initializing LAN area manager +34:21.628 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24742 network=udp +34:21.628 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-23610a1b-bcd3-cc4a-5046-21adffd07c05.dc1 area=wan +34:21.628 [INFO] TestAgent.server: Adding LAN server: server="Node-23610a1b-bcd3-cc4a-5046-21adffd07c05 (Addr: tcp/127.0.0.1:24747) (DC: dc1)" +34:21.628 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24742 network=tcp +34:21.629 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24743 network=tcp +34:21.629 [INFO] TestAgent: started state syncer +34:21.630 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24685 network=tcp +34:21.668 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:21.668 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24747 [Candidate]" term=2 +34:21.669 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:21.669 [DEBUG] TestAgent.server.raft: vote granted: from=23610a1b-bcd3-cc4a-5046-21adffd07c05 term=2 tally=1 +34:21.669 [INFO] TestAgent.server.raft: election won: tally=1 +34:21.669 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24747 [Leader]" +34:21.669 [INFO] TestAgent.server: cluster leadership acquired +34:21.669 [INFO] TestAgent.server: New leader elected: payload=Node-23610a1b-bcd3-cc4a-5046-21adffd07c05 +34:21.669 [INFO] TestAgent.server: initializing acls +34:21.670 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:21.670 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:21.670 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:21.671 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:21.671 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:21.671 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:21.671 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-23610a1b-bcd3-cc4a-5046-21adffd07c05 +34:21.671 [INFO] TestAgent.server: Updating LAN server: server="Node-23610a1b-bcd3-cc4a-5046-21adffd07c05 (Addr: tcp/127.0.0.1:24747) (DC: dc1)" +34:21.671 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-23610a1b-bcd3-cc4a-5046-21adffd07c05.dc1 +34:21.671 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-23610a1b-bcd3-cc4a-5046-21adffd07c05.dc1 area=wan +34:21.672 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:21.674 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:21.674 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:21.674 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:21.674 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:21.674 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:21.675 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-23610a1b-bcd3-cc4a-5046-21adffd07c05 +34:21.675 [INFO] TestAgent.server: member joined, marking health alive: member=Node-23610a1b-bcd3-cc4a-5046-21adffd07c05 +34:21.676 [INFO] TestAgent.server: federation state anti-entropy synced +34:21.676 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-23610a1b-bcd3-cc4a-5046-21adffd07c05 +34:21.851 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-23610a1b-bcd3-cc4a-5046-21adffd07c05 +34:21.853 [DEBUG] TestAgent: removed check: check=test +34:21.853 [INFO] TestAgent: Synced node info +34:21.853 [INFO] TestAgent: Deregistered check: check=test +34:21.854 [INFO] TestAgent: Requesting shutdown +34:21.854 [INFO] TestAgent.server: shutting down server +34:21.854 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:21.854 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:21.854 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:21.854 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:21.854 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:21.854 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:21.854 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:21.854 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:21.854 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:21.854 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:21.854 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:21.854 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:21.854 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:21.854 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:21.854 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:21.854 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:21.854Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:21.854Z [INFO] agent.router.manager: shutting down +34:21.855 [INFO] TestAgent: consul server down +34:21.855 [INFO] TestAgent: shutdown complete +34:21.855 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24742 network=tcp +34:21.855 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24742 network=udp +34:21.855 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24743 network=tcp +34:22.355 [INFO] TestAgent: Waiting for endpoints to shut down +34:22.355 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_DeregisterCheckACLDeny (0.83s) --- PASS: TestAgent_DeregisterCheckACLDeny/no_token (0.00s) --- PASS: TestAgent_DeregisterCheckACLDeny/root_token (0.00s) -=== CONT TestAgent_RegisterCheckScriptsExecRemoteDisable -2022-09-15T21:42:57.916Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:57.937Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:57.940 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f5b66b3c-a5a2-afa7-69de-0f5ab896554a Address:127.0.0.1:13316}]" -42:57.940 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13316 [Follower]" leader= -42:57.941 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f5b66b3c-a5a2-afa7-69de-0f5ab896554a.dc1 127.0.0.1 -42:57.942 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f5b66b3c-a5a2-afa7-69de-0f5ab896554a 127.0.0.1 -2022-09-15T21:42:57.942Z [INFO] agent.router: Initializing LAN area manager -42:57.942 [INFO] TestAgent.server: Adding LAN server: server="Node-f5b66b3c-a5a2-afa7-69de-0f5ab896554a (Addr: tcp/127.0.0.1:13316) (DC: dc1)" -42:57.942 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f5b66b3c-a5a2-afa7-69de-0f5ab896554a.dc1 area=wan -42:57.942 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13283 network=udp -42:57.942 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13283 network=tcp -42:57.943 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13284 network=tcp -42:57.943 [INFO] TestAgent: started state syncer -42:57.944 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13317 network=tcp -42:58.014 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:58.014 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13316 [Candidate]" term=2 -42:58.014 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:58.014 [DEBUG] TestAgent.server.raft: vote granted: from=f5b66b3c-a5a2-afa7-69de-0f5ab896554a term=2 tally=1 -42:58.014 [INFO] TestAgent.server.raft: election won: tally=1 -42:58.014 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13316 [Leader]" -42:58.014 [INFO] TestAgent.server: cluster leadership acquired -42:58.015 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13316 -42:58.016 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:58.018 [INFO] TestAgent.server: New leader elected: payload=Node-f5b66b3c-a5a2-afa7-69de-0f5ab896554a -42:58.031 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:58.031 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:58.031 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:58.031 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:58.031 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:58.031 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f5b66b3c-a5a2-afa7-69de-0f5ab896554a -42:58.031 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f5b66b3c-a5a2-afa7-69de-0f5ab896554a -42:58.032 [INFO] TestAgent.server: federation state anti-entropy synced -42:58.089 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:58.089 [INFO] TestAgent: Synced node info -42:58.103 [DEBUG] TestAgent: Check status updated: check=test status=passing -42:58.104 [DEBUG] TestAgent: Node info in sync -42:58.105 [INFO] TestAgent: Synced check: check=test -42:58.105 [INFO] TestAgent: Requesting shutdown -42:58.105 [INFO] TestAgent.server: shutting down server -42:58.105 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:58.105 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:58.105 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:58.105 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:58.105 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:58.105 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:58.105 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:58.105 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:58.105 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -42:58.105 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:58.105 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -2022-09-15T21:42:58.105Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:58.105Z [INFO] agent.router.manager: shutting down -42:58.105 [INFO] TestAgent: consul server down -42:58.105 [INFO] TestAgent: shutdown complete -42:58.105 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13283 network=tcp -42:58.105 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13283 network=udp -42:58.105 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13284 network=tcp -42:58.606 [INFO] TestAgent: Waiting for endpoints to shut down -42:58.606 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_PassCheck (0.71s) -=== CONT TestAgent_RegisterCheckScriptsExecDisable -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:58.193Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:58.220Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:58.229 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:849cd1af-0be6-00ec-27a9-befd6232bfac Address:127.0.0.1:13323}]" -42:58.229 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13323 [Follower]" leader= -42:58.230 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-849cd1af-0be6-00ec-27a9-befd6232bfac.dc1 127.0.0.1 -42:58.231 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-849cd1af-0be6-00ec-27a9-befd6232bfac 127.0.0.1 -2022-09-15T21:42:58.231Z [INFO] agent.router: Initializing LAN area manager -42:58.231 [INFO] TestAgent.server: Adding LAN server: server="Node-849cd1af-0be6-00ec-27a9-befd6232bfac (Addr: tcp/127.0.0.1:13323) (DC: dc1)" -42:58.231 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-849cd1af-0be6-00ec-27a9-befd6232bfac.dc1 area=wan -42:58.232 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13255 network=udp -42:58.232 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13255 network=tcp -42:58.233 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13256 network=tcp -42:58.233 [INFO] TestAgent: started state syncer -42:58.233 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13324 network=tcp -42:58.282 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -42:58.295 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:58.295 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13323 [Candidate]" term=2 -42:58.295 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:58.296 [DEBUG] TestAgent.server.raft: vote granted: from=849cd1af-0be6-00ec-27a9-befd6232bfac term=2 tally=1 -42:58.296 [INFO] TestAgent.server.raft: election won: tally=1 -42:58.296 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13323 [Leader]" -42:58.296 [INFO] TestAgent.server: cluster leadership acquired -42:58.296 [INFO] TestAgent.server: initializing acls -42:58.296 [INFO] TestAgent.server: New leader elected: payload=Node-849cd1af-0be6-00ec-27a9-befd6232bfac -42:58.297 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:58.297 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:58.297 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:58.298 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:58.298 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:58.298 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:58.298 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-849cd1af-0be6-00ec-27a9-befd6232bfac -42:58.298 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-849cd1af-0be6-00ec-27a9-befd6232bfac.dc1 -42:58.299 [INFO] TestAgent.server: Updating LAN server: server="Node-849cd1af-0be6-00ec-27a9-befd6232bfac (Addr: tcp/127.0.0.1:13323) (DC: dc1)" -42:58.299 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:58.299 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-849cd1af-0be6-00ec-27a9-befd6232bfac.dc1 area=wan -42:58.302 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:58.302 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:58.302 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:58.302 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:58.302 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:58.302 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-849cd1af-0be6-00ec-27a9-befd6232bfac -42:58.302 [INFO] TestAgent.server: member joined, marking health alive: member=Node-849cd1af-0be6-00ec-27a9-befd6232bfac -42:58.303 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-849cd1af-0be6-00ec-27a9-befd6232bfac -42:58.303 [INFO] TestAgent.server: federation state anti-entropy synced -42:58.359 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:58.359 [INFO] TestAgent: Synced node info -42:58.359 [DEBUG] TestAgent: Node info in sync -42:58.465 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-849cd1af-0be6-00ec-27a9-befd6232bfac -42:58.465 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-849cd1af-0be6-00ec-27a9-befd6232bfac -42:58.466 [DEBUG] TestAgent: Node info in sync -42:58.467 [INFO] TestAgent: Synced service: service=foo:1234 -42:58.472 [DEBUG] TestAgent: Node info in sync -42:58.472 [DEBUG] TestAgent: Service in sync: service=foo:1234 -42:58.473 [INFO] TestAgent: Synced check: check=test -42:58.474 [DEBUG] TestAgent: Node info in sync -42:58.474 [DEBUG] TestAgent: Service in sync: service=foo:1234 -42:58.474 [DEBUG] TestAgent: Check in sync: check=test -42:58.475 [INFO] TestAgent: Synced check: check=test2 -42:58.475 [INFO] TestAgent: Requesting shutdown -42:58.475 [INFO] TestAgent.server: shutting down server -42:58.475 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:58.475 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:58.475 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:58.475 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:58.475 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:58.475 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:58.475 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:58.476 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:58.476 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:58.476 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:58.476 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:58.476 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:58.476 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:58.476 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:58.476 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:58.476Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:58.476Z [INFO] agent.router.manager: shutting down -42:58.476 [INFO] TestAgent: consul server down -42:58.476 [INFO] TestAgent: shutdown complete -42:58.476 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13255 network=tcp -42:58.476 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13255 network=udp -42:58.476 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13256 network=tcp -42:58.977 [INFO] TestAgent: Waiting for endpoints to shut down -42:58.977 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_RegisterCheck_ACLDeny (0.84s) +=== CONT TestOperator_KeyringRemove +2021-08-13T15:34:21.739Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:21.757Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:21.760 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8b0b9935-8862-d25a-577f-dda0852bb08f Address:127.0.0.1:24705}]" +34:21.761 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8b0b9935-8862-d25a-577f-dda0852bb08f.dc1 127.0.0.1 +34:21.762 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8b0b9935-8862-d25a-577f-dda0852bb08f 127.0.0.1 +2021-08-13T15:34:21.762Z [INFO] agent.router: Initializing LAN area manager +34:21.762 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24705 [Follower]" leader= +34:21.763 [INFO] TestAgent.server: Adding LAN server: server="Node-8b0b9935-8862-d25a-577f-dda0852bb08f (Addr: tcp/127.0.0.1:24705) (DC: dc1)" +34:21.763 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8b0b9935-8862-d25a-577f-dda0852bb08f.dc1 area=wan +34:21.763 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24686 network=tcp +34:21.763 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24686 network=udp +34:21.764 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24701 network=tcp +34:21.764 [INFO] TestAgent: started state syncer +34:21.764 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24748 network=tcp +34:21.808 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:21.808 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24705 [Candidate]" term=2 +34:21.808 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:21.808 [DEBUG] TestAgent.server.raft: vote granted: from=8b0b9935-8862-d25a-577f-dda0852bb08f term=2 tally=1 +34:21.808 [INFO] TestAgent.server.raft: election won: tally=1 +34:21.808 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24705 [Leader]" +34:21.808 [INFO] TestAgent.server: cluster leadership acquired +34:21.808 [INFO] TestAgent.server: New leader elected: payload=Node-8b0b9935-8862-d25a-577f-dda0852bb08f +34:21.809 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24705 +34:21.810 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:21.813 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:21.813 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:21.813 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:21.813 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:21.813 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:21.813 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8b0b9935-8862-d25a-577f-dda0852bb08f +34:21.813 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8b0b9935-8862-d25a-577f-dda0852bb08f +34:21.814 [INFO] TestAgent.server: federation state anti-entropy synced +34:21.999 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query +34:22.000 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-8b0b9935-8862-d25a-577f-dda0852bb08f +34:22.000 [INFO] TestAgent: Requesting shutdown +34:22.000 [INFO] TestAgent.server: shutting down server +34:22.000 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:22.000 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:22.000 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:22.000 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:22.000 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:22.000 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:22.000 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:22.000 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:22.000 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:22.000 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:22.000 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +34:22.000 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2021-08-13T15:34:22.000Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:22.000Z [INFO] agent.router.manager: shutting down +34:22.001 [INFO] TestAgent: consul server down +34:22.001 [INFO] TestAgent: shutdown complete +34:22.001 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24686 network=tcp +34:22.001 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24686 network=udp +34:22.001 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24701 network=tcp +34:22.501 [INFO] TestAgent: Waiting for endpoints to shut down +34:22.501 [INFO] TestAgent: Endpoints down +--- PASS: TestOperator_Keyring_LocalOnly (0.78s) +=== CONT TestAgent_RegisterCheck_Passing +[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:21.944Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:21.961Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:21.964 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:731bb637-4d63-5e13-17dd-6a03537bc77c Address:127.0.0.1:24761}]" +34:21.964 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24761 [Follower]" leader= +34:21.965 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-731bb637-4d63-5e13-17dd-6a03537bc77c.dc1 127.0.0.1 +34:21.966 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-731bb637-4d63-5e13-17dd-6a03537bc77c 127.0.0.1 +2021-08-13T15:34:21.966Z [INFO] agent.router: Initializing LAN area manager +34:21.966 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-731bb637-4d63-5e13-17dd-6a03537bc77c.dc1 area=wan +34:21.966 [INFO] TestAgent.server: Adding LAN server: server="Node-731bb637-4d63-5e13-17dd-6a03537bc77c (Addr: tcp/127.0.0.1:24761) (DC: dc1)" +34:21.966 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24770 network=udp +34:21.966 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24770 network=tcp +34:21.967 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24757 network=tcp +34:21.967 [INFO] TestAgent: started state syncer +34:21.967 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24776 network=tcp +34:22.006 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:22.006 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24761 [Candidate]" term=2 +34:22.006 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:22.006 [DEBUG] TestAgent.server.raft: vote granted: from=731bb637-4d63-5e13-17dd-6a03537bc77c term=2 tally=1 +34:22.006 [INFO] TestAgent.server.raft: election won: tally=1 +34:22.006 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24761 [Leader]" +34:22.006 [INFO] TestAgent.server: cluster leadership acquired +34:22.006 [INFO] TestAgent.server: initializing acls +34:22.007 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:22.007 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:22.008 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:22.008 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:22.008 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:22.008 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:22.008 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-731bb637-4d63-5e13-17dd-6a03537bc77c +34:22.009 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-731bb637-4d63-5e13-17dd-6a03537bc77c.dc1 +34:22.010 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:22.010 [INFO] TestAgent.server: New leader elected: payload=Node-731bb637-4d63-5e13-17dd-6a03537bc77c +34:22.010 [INFO] TestAgent.server: Updating LAN server: server="Node-731bb637-4d63-5e13-17dd-6a03537bc77c (Addr: tcp/127.0.0.1:24761) (DC: dc1)" +34:22.010 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-731bb637-4d63-5e13-17dd-6a03537bc77c.dc1 area=wan +34:22.013 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:22.013 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:22.013 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:22.013 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:22.013 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:22.013 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-731bb637-4d63-5e13-17dd-6a03537bc77c +34:22.013 [INFO] TestAgent.server: member joined, marking health alive: member=Node-731bb637-4d63-5e13-17dd-6a03537bc77c +34:22.014 [INFO] TestAgent.server: federation state anti-entropy synced +34:22.014 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-731bb637-4d63-5e13-17dd-6a03537bc77c +34:22.069 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-731bb637-4d63-5e13-17dd-6a03537bc77c +34:22.069 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-731bb637-4d63-5e13-17dd-6a03537bc77c +34:22.071 [INFO] TestAgent: Synced node info +34:22.072 [INFO] TestAgent: Synced service: service=foo:1234 +34:22.078 [DEBUG] TestAgent: Node info in sync +34:22.078 [DEBUG] TestAgent: Service in sync: service=foo:1234 +34:22.078 [INFO] TestAgent: Synced check: check=test +34:22.083 [DEBUG] TestAgent: Node info in sync +34:22.083 [DEBUG] TestAgent: Service in sync: service=foo:1234 +34:22.083 [DEBUG] TestAgent: Check in sync: check=test +34:22.084 [INFO] TestAgent: Synced check: check=test2 +34:22.084 [INFO] TestAgent: Requesting shutdown +34:22.085 [INFO] TestAgent.server: shutting down server +34:22.085 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:22.085 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:22.085 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:22.085 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:22.085 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:22.085 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:22.085 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:22.085 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:22.085 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:22.085 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:22.085 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:22.085 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:22.085 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:22.085 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:22.085 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:22.085 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:22.085Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:22.085Z [INFO] agent.router.manager: shutting down +34:22.085 [INFO] TestAgent: consul server down +34:22.085 [INFO] TestAgent: shutdown complete +34:22.085 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24770 network=tcp +34:22.086 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24770 network=udp +34:22.086 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24757 network=tcp +34:22.586 [INFO] TestAgent: Waiting for endpoints to shut down +34:22.586 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_RegisterCheck_ACLDeny (0.66s) --- PASS: TestAgent_RegisterCheck_ACLDeny/no_token_-_node_check (0.00s) --- PASS: TestAgent_RegisterCheck_ACLDeny/svc_token_-_node_check (0.00s) --- PASS: TestAgent_RegisterCheck_ACLDeny/node_token_-_node_check (0.00s) --- PASS: TestAgent_RegisterCheck_ACLDeny/no_token_-_svc_check (0.00s) --- PASS: TestAgent_RegisterCheck_ACLDeny/node_token_-_svc_check (0.00s) --- PASS: TestAgent_RegisterCheck_ACLDeny/svc_token_-_svc_check (0.00s) +=== CONT TestAgent_RegisterCheckScriptsExecRemoteDisable +2021-08-13T15:34:21.777Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:21.795Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:21.798 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:76d0ee84-d866-a9fb-d90d-77583cbeebb1 Address:127.0.0.1:24754}]" +34:21.798 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24754 [Follower]" leader= +34:21.799 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-76d0ee84-d866-a9fb-d90d-77583cbeebb1.dc1 127.0.0.1 +34:21.800 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-76d0ee84-d866-a9fb-d90d-77583cbeebb1 127.0.0.1 +2021-08-13T15:34:21.800Z [INFO] agent.router: Initializing LAN area manager +34:21.800 [INFO] TestAgent.server: Adding LAN server: server="Node-76d0ee84-d866-a9fb-d90d-77583cbeebb1 (Addr: tcp/127.0.0.1:24754) (DC: dc1)" +34:21.800 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-76d0ee84-d866-a9fb-d90d-77583cbeebb1.dc1 area=wan +34:21.801 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24749 network=udp +34:21.801 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24749 network=tcp +34:21.802 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24750 network=tcp +34:21.802 [INFO] TestAgent: started state syncer +34:21.802 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24803 network=tcp +34:21.849 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:21.849 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24754 [Candidate]" term=2 +34:21.849 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:21.849 [DEBUG] TestAgent.server.raft: vote granted: from=76d0ee84-d866-a9fb-d90d-77583cbeebb1 term=2 tally=1 +34:21.849 [INFO] TestAgent.server.raft: election won: tally=1 +34:21.849 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24754 [Leader]" +34:21.849 [INFO] TestAgent.server: cluster leadership acquired +34:21.850 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24754 +34:21.851 [INFO] TestAgent.server: New leader elected: payload=Node-76d0ee84-d866-a9fb-d90d-77583cbeebb1 +34:21.852 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:21.855 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:21.855 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:21.855 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:21.855 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:21.855 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:21.855 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-76d0ee84-d866-a9fb-d90d-77583cbeebb1 +34:21.855 [INFO] TestAgent.server: member joined, marking health alive: member=Node-76d0ee84-d866-a9fb-d90d-77583cbeebb1 +34:21.857 [INFO] TestAgent.server: federation state anti-entropy synced +34:22.025 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:22.026 [INFO] TestAgent: Synced node info +34:22.026 [DEBUG] TestAgent: Node info in sync +34:22.034 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:22.034 [DEBUG] TestAgent: Node info in sync +34:22.087 [DEBUG] TestAgent: removed check: check=test +34:22.087 [DEBUG] TestAgent: Node info in sync +34:22.088 [INFO] TestAgent: Deregistered check: check=test +34:22.088 [INFO] TestAgent: Requesting shutdown +34:22.088 [INFO] TestAgent.server: shutting down server +34:22.088 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:22.088 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:22.088 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:22.088 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:22.088 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:22.088 [DEBUG] TestAgent: Node info in sync +34:22.088 [DEBUG] TestAgent: Node info in sync +34:22.088 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:22.088 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:22.088 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:22.088 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:22.088 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:22.088 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:22.088Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:22.088Z [INFO] agent.router.manager: shutting down +34:22.088 [INFO] TestAgent: consul server down +34:22.088 [INFO] TestAgent: shutdown complete +34:22.088 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24749 network=tcp +34:22.088 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24749 network=udp +34:22.088 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24750 network=tcp +34:22.589 [INFO] TestAgent: Waiting for endpoints to shut down +34:22.589 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_DeregisterCheck (0.83s) +=== CONT TestOperator_KeyringListLocalOnly +2021-08-13T15:34:22.061Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:22.079Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:22.082 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3afbc33c-c563-272c-f07e-b0291fd33712 Address:127.0.0.1:24795}]" +34:22.083 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24795 [Follower]" leader= +34:22.087 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3afbc33c-c563-272c-f07e-b0291fd33712.dc1 127.0.0.1 +34:22.088 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3afbc33c-c563-272c-f07e-b0291fd33712 127.0.0.1 +2021-08-13T15:34:22.088Z [INFO] agent.router: Initializing LAN area manager +34:22.089 [INFO] TestAgent.server: Adding LAN server: server="Node-3afbc33c-c563-272c-f07e-b0291fd33712 (Addr: tcp/127.0.0.1:24795) (DC: dc1)" +34:22.089 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3afbc33c-c563-272c-f07e-b0291fd33712.dc1 area=wan +34:22.089 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24777 network=udp +34:22.089 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24777 network=tcp +34:22.090 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24787 network=tcp +34:22.090 [INFO] TestAgent: started state syncer +34:22.090 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24783 network=tcp +34:22.119 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:22.119 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24795 [Candidate]" term=2 +34:22.119 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:22.119 [DEBUG] TestAgent.server.raft: vote granted: from=3afbc33c-c563-272c-f07e-b0291fd33712 term=2 tally=1 +34:22.119 [INFO] TestAgent.server.raft: election won: tally=1 +34:22.119 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24795 [Leader]" +34:22.119 [INFO] TestAgent.server: cluster leadership acquired +34:22.120 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24795 +34:22.121 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:22.123 [INFO] TestAgent.server: New leader elected: payload=Node-3afbc33c-c563-272c-f07e-b0291fd33712 +34:22.123 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:22.123 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:22.123 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:22.123 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:22.123 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:22.123 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3afbc33c-c563-272c-f07e-b0291fd33712 +34:22.123 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3afbc33c-c563-272c-f07e-b0291fd33712 +34:22.124 [INFO] TestAgent.server: federation state anti-entropy synced +34:22.137 [INFO] TestAgent.server.serf.wan: serf: Received install-key query +34:22.147 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-3afbc33c-c563-272c-f07e-b0291fd33712.dc1 +34:22.147 [INFO] TestAgent.server.serf.lan: serf: Received install-key query +34:22.148 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-3afbc33c-c563-272c-f07e-b0291fd33712 +34:22.148 [INFO] TestAgent.server.serf.wan: serf: Received use-key query +34:22.149 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-3afbc33c-c563-272c-f07e-b0291fd33712.dc1 +34:22.149 [INFO] TestAgent.server.serf.lan: serf: Received use-key query +34:22.168 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-3afbc33c-c563-272c-f07e-b0291fd33712 +34:22.168 [INFO] TestAgent.server.serf.wan: serf: Received remove-key query +34:22.168 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-3afbc33c-c563-272c-f07e-b0291fd33712.dc1 +34:22.169 [INFO] TestAgent.server.serf.lan: serf: Received remove-key query +34:22.169 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-3afbc33c-c563-272c-f07e-b0291fd33712 +34:22.169 [INFO] TestAgent.server.serf.wan: serf: Received list-keys query +34:22.170 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-3afbc33c-c563-272c-f07e-b0291fd33712.dc1 +34:22.170 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query +34:22.170 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-3afbc33c-c563-272c-f07e-b0291fd33712 +34:22.170 [INFO] TestAgent: Requesting shutdown +34:22.171 [INFO] TestAgent.server: shutting down server +34:22.171 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:22.171 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:22.171 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:22.171 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:22.171 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:22.171 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:22.171 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:22.173 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:22.173 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:22.173 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:22.173 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:22.173 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:22.174Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:22.174Z [INFO] agent.router.manager: shutting down +34:22.174 [INFO] TestAgent: consul server down +34:22.174 [INFO] TestAgent: shutdown complete +34:22.174 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24777 network=tcp +34:22.174 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24777 network=udp +34:22.175 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24787 network=tcp +34:22.675 [INFO] TestAgent: Waiting for endpoints to shut down +34:22.675 [INFO] TestAgent: Endpoints down +--- PASS: TestOperator_KeyringUse (0.63s) +=== CONT TestAgent_RegisterCheckScriptsExecDisable +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:21.870Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:21.887Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:21.890 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:201cc144-5b82-1b40-5821-ef52e79c3583 Address:127.0.0.1:24816}]" +34:21.891 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24816 [Follower]" leader= +34:21.891 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-201cc144-5b82-1b40-5821-ef52e79c3583.dc1 127.0.0.1 +34:21.892 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-201cc144-5b82-1b40-5821-ef52e79c3583 127.0.0.1 +2021-08-13T15:34:21.892Z [INFO] agent.router: Initializing LAN area manager +34:21.892 [INFO] TestAgent.server: Adding LAN server: server="Node-201cc144-5b82-1b40-5821-ef52e79c3583 (Addr: tcp/127.0.0.1:24816) (DC: dc1)" +34:21.893 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-201cc144-5b82-1b40-5821-ef52e79c3583.dc1 area=wan +34:21.893 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24804 network=udp +34:21.893 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24804 network=tcp +34:21.894 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24805 network=tcp +34:21.894 [INFO] TestAgent: started state syncer +34:21.894 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24769 network=tcp +34:21.939 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:21.939 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24816 [Candidate]" term=2 +34:21.939 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:21.939 [DEBUG] TestAgent.server.raft: vote granted: from=201cc144-5b82-1b40-5821-ef52e79c3583 term=2 tally=1 +34:21.939 [INFO] TestAgent.server.raft: election won: tally=1 +34:21.939 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24816 [Leader]" +34:21.939 [INFO] TestAgent.server: cluster leadership acquired +34:21.939 [INFO] TestAgent.server: New leader elected: payload=Node-201cc144-5b82-1b40-5821-ef52e79c3583 +34:21.939 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24816 +34:21.941 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:21.943 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:21.943 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:21.943 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:21.943 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:21.944 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:21.944 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-201cc144-5b82-1b40-5821-ef52e79c3583 +34:21.944 [INFO] TestAgent.server: member joined, marking health alive: member=Node-201cc144-5b82-1b40-5821-ef52e79c3583 +34:21.945 [INFO] TestAgent.server: federation state anti-entropy synced +34:22.262 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:22.262 [INFO] TestAgent: Synced node info +34:22.262 [DEBUG] TestAgent: Node info in sync +34:22.292 [INFO] TestAgent: Requesting shutdown +34:22.292 [INFO] TestAgent.server: shutting down server +34:22.292 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:22.292 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:22.292 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:22.292 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:22.292 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:22.293 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:22.293 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:22.293 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:22.293 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:22.293 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:22.293 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:22.293Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:22.293Z [INFO] agent.router.manager: shutting down +34:22.293 [INFO] TestAgent: consul server down +34:22.293 [INFO] TestAgent: shutdown complete +34:22.293 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24804 network=tcp +34:22.293 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24804 network=udp +34:22.293 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24805 network=tcp +34:22.793 [INFO] TestAgent: Waiting for endpoints to shut down +34:22.793 [INFO] TestAgent: Endpoints down +--- PASS: TestOperator_Keyring_InvalidRelayFactor (0.94s) +=== CONT TestOperator_KeyringList +2021-08-13T15:34:22.318Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:22.334Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:22.338 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6dcdfa6d-2533-c20a-97ec-ad1e1df938ed Address:127.0.0.1:24802}]" +34:22.338 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24802 [Follower]" leader= +34:22.343 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6dcdfa6d-2533-c20a-97ec-ad1e1df938ed.dc1 127.0.0.1 +34:22.344 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6dcdfa6d-2533-c20a-97ec-ad1e1df938ed 127.0.0.1 +2021-08-13T15:34:22.344Z [INFO] agent.router: Initializing LAN area manager +34:22.344 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24797 network=udp +34:22.345 [INFO] TestAgent.server: Adding LAN server: server="Node-6dcdfa6d-2533-c20a-97ec-ad1e1df938ed (Addr: tcp/127.0.0.1:24802) (DC: dc1)" +34:22.345 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6dcdfa6d-2533-c20a-97ec-ad1e1df938ed.dc1 area=wan +34:22.345 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24797 network=tcp +34:22.346 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24798 network=tcp +34:22.346 [INFO] TestAgent: started state syncer +34:22.346 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24762 network=tcp +34:22.391 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:22.391 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24802 [Candidate]" term=2 +34:22.391 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:22.391 [DEBUG] TestAgent.server.raft: vote granted: from=6dcdfa6d-2533-c20a-97ec-ad1e1df938ed term=2 tally=1 +34:22.391 [INFO] TestAgent.server.raft: election won: tally=1 +34:22.391 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24802 [Leader]" +34:22.391 [INFO] TestAgent.server: cluster leadership acquired +34:22.392 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24802 +34:22.393 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:22.395 [INFO] TestAgent.server: New leader elected: payload=Node-6dcdfa6d-2533-c20a-97ec-ad1e1df938ed +34:22.395 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:22.395 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:22.395 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:22.395 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:22.395 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:22.395 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6dcdfa6d-2533-c20a-97ec-ad1e1df938ed +34:22.395 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6dcdfa6d-2533-c20a-97ec-ad1e1df938ed +34:22.397 [INFO] TestAgent.server: federation state anti-entropy synced +34:22.476 [INFO] TestAgent: Requesting shutdown +34:22.476 [INFO] TestAgent.server: shutting down server +34:22.476 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:22.476 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:22.476 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:22.476 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:22.476 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:22.477 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:22.477 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:22.477 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:22.477 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:22.477 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:22.477 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:22.477 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:22.477Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:22.477Z [INFO] agent.router.manager: shutting down +34:22.477 [INFO] TestAgent: consul server down +34:22.477 [INFO] TestAgent: shutdown complete +34:22.477 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24797 network=tcp +34:22.477 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24797 network=udp +34:22.477 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24798 network=tcp +34:22.977 [INFO] TestAgent: Waiting for endpoints to shut down +34:22.977 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_RegisterCheck_BadStatus (0.68s) === CONT TestAgent_RegisterCheck_Scripts -2022-09-15T21:42:58.130Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:58.150Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:58.161 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:17893282-48a7-3cef-943a-43a8383fcad5 Address:127.0.0.1:13309}]" -42:58.162 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13309 [Follower]" leader= -42:58.168 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-17893282-48a7-3cef-943a-43a8383fcad5.dc1 127.0.0.1 -42:58.175 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-17893282-48a7-3cef-943a-43a8383fcad5 127.0.0.1 -2022-09-15T21:42:58.175Z [INFO] agent.router: Initializing LAN area manager -42:58.175 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13311 network=udp -42:58.176 [INFO] TestAgent.server: Adding LAN server: server="Node-17893282-48a7-3cef-943a-43a8383fcad5 (Addr: tcp/127.0.0.1:13309) (DC: dc1)" -42:58.176 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-17893282-48a7-3cef-943a-43a8383fcad5.dc1 area=wan -42:58.176 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13311 network=tcp -42:58.180 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13312 network=tcp -42:58.181 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13310 network=tcp -42:58.192 [INFO] TestAgent: started state syncer -42:58.219 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:58.219 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13309 [Candidate]" term=2 -42:58.219 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:58.219 [DEBUG] TestAgent.server.raft: vote granted: from=17893282-48a7-3cef-943a-43a8383fcad5 term=2 tally=1 -42:58.219 [INFO] TestAgent.server.raft: election won: tally=1 -42:58.219 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13309 [Leader]" -42:58.220 [INFO] TestAgent.server: cluster leadership acquired -42:58.220 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13309 -42:58.221 [INFO] TestAgent.server: New leader elected: payload=Node-17893282-48a7-3cef-943a-43a8383fcad5 -42:58.221 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:58.224 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:58.224 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:58.224 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:58.224 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:58.224 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:58.224 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-17893282-48a7-3cef-943a-43a8383fcad5 -42:58.224 [INFO] TestAgent.server: member joined, marking health alive: member=Node-17893282-48a7-3cef-943a-43a8383fcad5 -42:58.225 [INFO] TestAgent.server: federation state anti-entropy synced -42:58.489 [DEBUG] TestAgent: removed check: check=test -42:58.489 [INFO] TestAgent: Synced node info -42:58.490 [INFO] TestAgent: Deregistered check: check=test -42:58.490 [INFO] TestAgent: Requesting shutdown -42:58.490 [INFO] TestAgent.server: shutting down server -42:58.490 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:58.490 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:58.490 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:58.490 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:58.490 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:58.490 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:58.490 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:58.490 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:58.490 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:58.490 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:58.490 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:58.490 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:58.490Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:58.490Z [INFO] agent.router.manager: shutting down -42:58.490 [INFO] TestAgent: consul server down -42:58.490 [INFO] TestAgent: shutdown complete -42:58.490 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13311 network=tcp -42:58.490 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13311 network=udp -42:58.491 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13312 network=tcp -42:58.991 [INFO] TestAgent: Waiting for endpoints to shut down -42:58.991 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_DeregisterCheck (0.88s) -=== CONT TestAgent_ForceLeavePrune -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:22.373Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:22.389Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:22.392 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bfa49c02-dd9b-1d15-3fa9-b1c9d0258828 Address:127.0.0.1:24782}]" +34:22.393 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bfa49c02-dd9b-1d15-3fa9-b1c9d0258828.dc1 127.0.0.1 +34:22.394 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bfa49c02-dd9b-1d15-3fa9-b1c9d0258828 127.0.0.1 +2021-08-13T15:34:22.394Z [INFO] agent.router: Initializing LAN area manager +34:22.394 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24763 network=udp +34:22.394 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24782 [Follower]" leader= +34:22.395 [INFO] TestAgent.server: Adding LAN server: server="Node-bfa49c02-dd9b-1d15-3fa9-b1c9d0258828 (Addr: tcp/127.0.0.1:24782) (DC: dc1)" +34:22.395 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bfa49c02-dd9b-1d15-3fa9-b1c9d0258828.dc1 area=wan +34:22.395 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24763 network=tcp +34:22.396 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24778 network=tcp +34:22.396 [INFO] TestAgent: started state syncer +34:22.396 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24755 network=tcp +34:22.443 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:22.443 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24782 [Candidate]" term=2 +34:22.444 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:22.444 [DEBUG] TestAgent.server.raft: vote granted: from=bfa49c02-dd9b-1d15-3fa9-b1c9d0258828 term=2 tally=1 +34:22.444 [INFO] TestAgent.server.raft: election won: tally=1 +34:22.444 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24782 [Leader]" +34:22.444 [INFO] TestAgent.server: cluster leadership acquired +34:22.444 [INFO] TestAgent.server: New leader elected: payload=Node-bfa49c02-dd9b-1d15-3fa9-b1c9d0258828 +34:22.444 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24782 +34:22.445 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:22.448 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:22.448 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:22.448 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:22.448 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:22.448 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:22.448 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bfa49c02-dd9b-1d15-3fa9-b1c9d0258828 +34:22.448 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bfa49c02-dd9b-1d15-3fa9-b1c9d0258828 +34:22.449 [INFO] TestAgent.server: federation state anti-entropy synced +34:22.519 [INFO] TestAgent.server.serf.wan: serf: Received install-key query +34:22.519 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-bfa49c02-dd9b-1d15-3fa9-b1c9d0258828.dc1 +34:22.519 [INFO] TestAgent.server.serf.lan: serf: Received install-key query +34:22.520 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-bfa49c02-dd9b-1d15-3fa9-b1c9d0258828 +34:22.520 [INFO] TestAgent.server.serf.wan: serf: Received list-keys query +34:22.520 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-bfa49c02-dd9b-1d15-3fa9-b1c9d0258828.dc1 +34:22.520 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query +34:22.521 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-bfa49c02-dd9b-1d15-3fa9-b1c9d0258828 +34:22.521 [INFO] TestAgent.server.serf.wan: serf: Received remove-key query +34:22.521 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-bfa49c02-dd9b-1d15-3fa9-b1c9d0258828.dc1 +34:22.521 [INFO] TestAgent.server.serf.lan: serf: Received remove-key query +34:22.522 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-bfa49c02-dd9b-1d15-3fa9-b1c9d0258828 +34:22.522 [INFO] TestAgent.server.serf.wan: serf: Received list-keys query +34:22.522 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-bfa49c02-dd9b-1d15-3fa9-b1c9d0258828.dc1 +34:22.522 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query +34:22.523 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-bfa49c02-dd9b-1d15-3fa9-b1c9d0258828 +34:22.523 [INFO] TestAgent: Requesting shutdown +34:22.523 [INFO] TestAgent.server: shutting down server +34:22.523 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:22.523 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:22.523 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:22.523 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:22.523 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:22.524 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:22.524 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:22.524 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:22.524 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:22.524 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:22.524 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:22.525 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:22.525Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:22.525Z [INFO] agent.router.manager: shutting down +34:22.525 [INFO] TestAgent: consul server down +34:22.525 [INFO] TestAgent: shutdown complete +34:22.525 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24763 network=tcp +34:22.525 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24763 network=udp +34:22.525 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24778 network=tcp +34:23.025 [INFO] TestAgent: Waiting for endpoints to shut down +34:23.025 [INFO] TestAgent: Endpoints down +--- PASS: TestOperator_KeyringRemove (0.67s) +=== CONT TestOperator_KeyringInstall +2021-08-13T15:34:22.298Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:22.315Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:22.317 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3e815018-07da-d6a2-36cc-3a3614b1e3ce Address:127.0.0.1:24775}]" +34:22.318 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3e815018-07da-d6a2-36cc-3a3614b1e3ce.dc1 127.0.0.1 +34:22.319 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24775 [Follower]" leader= +34:22.323 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3e815018-07da-d6a2-36cc-3a3614b1e3ce 127.0.0.1 +2021-08-13T15:34:22.323Z [INFO] agent.router: Initializing LAN area manager +34:22.324 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24784 network=udp +34:22.324 [INFO] TestAgent.server: Adding LAN server: server="Node-3e815018-07da-d6a2-36cc-3a3614b1e3ce (Addr: tcp/127.0.0.1:24775) (DC: dc1)" +34:22.324 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3e815018-07da-d6a2-36cc-3a3614b1e3ce.dc1 area=wan +34:22.324 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24784 network=tcp +34:22.325 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24771 network=tcp +34:22.325 [INFO] TestAgent: started state syncer +34:22.325 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24796 network=tcp +34:22.378 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:22.378 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24775 [Candidate]" term=2 +34:22.379 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:22.379 [DEBUG] TestAgent.server.raft: vote granted: from=3e815018-07da-d6a2-36cc-3a3614b1e3ce term=2 tally=1 +34:22.379 [INFO] TestAgent.server.raft: election won: tally=1 +34:22.379 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24775 [Leader]" +34:22.379 [INFO] TestAgent.server: cluster leadership acquired +34:22.379 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24775 +34:22.380 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:22.382 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:22.383 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:22.383 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:22.383 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:22.383 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:22.383 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3e815018-07da-d6a2-36cc-3a3614b1e3ce +34:22.383 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3e815018-07da-d6a2-36cc-3a3614b1e3ce +34:22.383 [INFO] TestAgent.server: New leader elected: payload=Node-3e815018-07da-d6a2-36cc-3a3614b1e3ce +34:22.384 [INFO] TestAgent.server: federation state anti-entropy synced +34:22.471 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:22.472 [INFO] TestAgent: Synced node info +34:22.472 [DEBUG] TestAgent: Node info in sync +34:22.700 [WARN] TestAgent.server: endpoint injected; this should only be used for testing +34:22.700 [INFO] TestAgent: Requesting shutdown +34:22.700 [INFO] TestAgent.server: shutting down server +34:22.700 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:22.700 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:22.700 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:22.700 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:22.700 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:22.700 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:22.700 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:22.701 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:22.701 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:22.701 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:22.701 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:22.701Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:22.701Z [INFO] agent.router.manager: shutting down +34:22.701 [INFO] TestAgent: consul server down +34:22.701 [INFO] TestAgent: shutdown complete +34:22.701 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24784 network=tcp +34:22.701 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24784 network=udp +34:22.701 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24771 network=tcp +34:23.201 [INFO] TestAgent: Waiting for endpoints to shut down +34:23.201 [INFO] TestAgent: Endpoints down +=== RUN TestPreparedQuery_Execute/#07 === RUN TestAgent_RegisterCheck_Scripts/==_Consul_1.0.0_as_node_check === RUN TestAgent_RegisterCheck_Scripts/==_Consul_1.0.0_as_top-level_service_check === RUN TestAgent_RegisterCheck_Scripts/==_Consul_1.0.0_as_slice-based_service_check @@ -43029,1900 +36356,3505 @@ === RUN TestAgent_RegisterCheck_Scripts/>_Consul_1.0.0_(fixup)_as_slice-based_service_check === RUN TestAgent_RegisterCheck_Scripts/>_Consul_1.0.0_as_node_check === RUN TestAgent_RegisterCheck_Scripts/>_Consul_1.0.0_as_top-level_service_check -2022-09-15T21:42:58.600Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:58.617Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:58.620 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d0d1c096-9017-8ef9-2e66-974c49cd3b0d Address:127.0.0.1:13330}]" -42:58.621 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d0d1c096-9017-8ef9-2e66-974c49cd3b0d.dc1 127.0.0.1 -42:58.622 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13330 [Follower]" leader= -42:58.623 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d0d1c096-9017-8ef9-2e66-974c49cd3b0d 127.0.0.1 -2022-09-15T21:42:58.623Z [INFO] agent.router: Initializing LAN area manager -42:58.623 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13325 network=udp -42:58.623 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d0d1c096-9017-8ef9-2e66-974c49cd3b0d.dc1 area=wan -42:58.623 [INFO] TestAgent.server: Adding LAN server: server="Node-d0d1c096-9017-8ef9-2e66-974c49cd3b0d (Addr: tcp/127.0.0.1:13330) (DC: dc1)" -42:58.623 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13325 network=tcp -42:58.624 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13326 network=tcp -42:58.624 [INFO] TestAgent: started state syncer -42:58.624 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13331 network=tcp -42:58.662 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:58.662 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13330 [Candidate]" term=2 -42:58.662 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:58.662 [DEBUG] TestAgent.server.raft: vote granted: from=d0d1c096-9017-8ef9-2e66-974c49cd3b0d term=2 tally=1 -42:58.662 [INFO] TestAgent.server.raft: election won: tally=1 -42:58.662 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13330 [Leader]" -42:58.662 [INFO] TestAgent.server: cluster leadership acquired -42:58.663 [INFO] TestAgent.server: New leader elected: payload=Node-d0d1c096-9017-8ef9-2e66-974c49cd3b0d -42:58.663 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13330 -42:58.664 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:58.667 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:58.667 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:58.667 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:58.667 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:58.667 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:58.667 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d0d1c096-9017-8ef9-2e66-974c49cd3b0d -42:58.667 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d0d1c096-9017-8ef9-2e66-974c49cd3b0d -42:58.668 [INFO] TestAgent.server: federation state anti-entropy synced -42:58.744 [INFO] TestAgent: Requesting shutdown -42:58.744 [INFO] TestAgent.server: shutting down server -42:58.744 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:58.744 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:58.744 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:58.744 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:58.744 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:58.744 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:58.744 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:58.744 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:58.744 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:58.744 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:58.744 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:58.744 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:58.745Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:58.745Z [INFO] agent.router.manager: shutting down -42:58.745 [INFO] TestAgent: consul server down -42:58.745 [INFO] TestAgent: shutdown complete -42:58.745 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13325 network=tcp -42:58.745 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13325 network=udp -42:58.745 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13326 network=tcp -42:59.246 [INFO] TestAgent: Waiting for endpoints to shut down -42:59.246 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_RegisterCheckScriptsExecRemoteDisable (0.67s) -=== CONT TestAgent_ForceLeave_ACLDeny +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! === RUN TestAgent_RegisterCheck_Scripts/>_Consul_1.0.0_as_slice-based_service_check -2022-09-15T21:42:58.474Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:58.490Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:58.493 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bd437daf-529a-86ca-5409-456fa46f2150 Address:127.0.0.1:13295}]" -42:58.494 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13295 [Follower]" leader= -42:58.494 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bd437daf-529a-86ca-5409-456fa46f2150.dc1 127.0.0.1 -42:58.495 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bd437daf-529a-86ca-5409-456fa46f2150 127.0.0.1 -2022-09-15T21:42:58.495Z [INFO] agent.router: Initializing LAN area manager -42:58.496 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13304 network=udp -42:58.496 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bd437daf-529a-86ca-5409-456fa46f2150.dc1 area=wan -42:58.496 [INFO] TestAgent.server: Adding LAN server: server="Node-bd437daf-529a-86ca-5409-456fa46f2150 (Addr: tcp/127.0.0.1:13295) (DC: dc1)" -42:58.496 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13304 network=tcp -42:58.497 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13305 network=tcp -42:58.497 [INFO] TestAgent: started state syncer -42:58.497 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13296 network=tcp -42:58.548 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:58.548 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13295 [Candidate]" term=2 -42:58.548 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:58.548 [DEBUG] TestAgent.server.raft: vote granted: from=bd437daf-529a-86ca-5409-456fa46f2150 term=2 tally=1 -42:58.548 [INFO] TestAgent.server.raft: election won: tally=1 -42:58.548 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13295 [Leader]" -42:58.548 [INFO] TestAgent.server: cluster leadership acquired -42:58.548 [INFO] TestAgent.server: New leader elected: payload=Node-bd437daf-529a-86ca-5409-456fa46f2150 -42:58.549 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13295 -42:58.550 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:58.552 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:58.552 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:58.552 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:58.552 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:58.552 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:58.552 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bd437daf-529a-86ca-5409-456fa46f2150 -42:58.552 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bd437daf-529a-86ca-5409-456fa46f2150 -42:58.553 [INFO] TestAgent.server: federation state anti-entropy synced -42:58.644 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:58.645 [INFO] TestAgent: Synced node info -42:58.759 [INFO] TestAgent: Requesting shutdown -42:58.759 [INFO] TestAgent.server: shutting down server -42:58.759 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:58.759 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:58.759 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:58.759 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:58.759 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:58.759 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:58.759 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:58.759 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:58.760 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:58.760 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:58.760 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:58.760Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:58.760Z [INFO] agent.router.manager: shutting down -42:58.760 [INFO] TestAgent: consul server down -42:58.760 [INFO] TestAgent: shutdown complete -42:58.760 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13304 network=tcp -42:58.760 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13304 network=udp -42:58.760 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13305 network=tcp -42:59.261 [INFO] TestAgent: Waiting for endpoints to shut down -42:59.261 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_RegisterCheck_BadStatus (0.83s) -=== CONT TestOpenMetricsMimeTypeHeaders ---- PASS: TestOpenMetricsMimeTypeHeaders (0.00s) -=== CONT TestAgent_ForceLeave -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:58.478Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:58.494Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:58.497 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:09e7fd33-5cf3-6478-136c-f597f9a4c345 Address:127.0.0.1:13260}]" -42:58.497 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13260 [Follower]" leader= -42:58.498 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-09e7fd33-5cf3-6478-136c-f597f9a4c345.dc1 127.0.0.1 -42:58.499 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-09e7fd33-5cf3-6478-136c-f597f9a4c345 127.0.0.1 -2022-09-15T21:42:58.499Z [INFO] agent.router: Initializing LAN area manager -42:58.499 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-09e7fd33-5cf3-6478-136c-f597f9a4c345.dc1 area=wan -42:58.499 [INFO] TestAgent.server: Adding LAN server: server="Node-09e7fd33-5cf3-6478-136c-f597f9a4c345 (Addr: tcp/127.0.0.1:13260) (DC: dc1)" -42:58.499 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13269 network=udp -42:58.499 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13269 network=tcp -42:58.500 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13270 network=tcp -42:58.500 [INFO] TestAgent: started state syncer -42:58.500 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13261 network=tcp -42:58.534 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:58.534 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13260 [Candidate]" term=2 -42:58.534 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:58.534 [DEBUG] TestAgent.server.raft: vote granted: from=09e7fd33-5cf3-6478-136c-f597f9a4c345 term=2 tally=1 -42:58.534 [INFO] TestAgent.server.raft: election won: tally=1 -42:58.534 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13260 [Leader]" -42:58.535 [INFO] TestAgent.server: cluster leadership acquired -42:58.535 [INFO] TestAgent.server: New leader elected: payload=Node-09e7fd33-5cf3-6478-136c-f597f9a4c345 -42:58.535 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13260 -42:58.536 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:58.539 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:58.539 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:58.539 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:58.539 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:58.539 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:58.539 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-09e7fd33-5cf3-6478-136c-f597f9a4c345 -42:58.539 [INFO] TestAgent.server: member joined, marking health alive: member=Node-09e7fd33-5cf3-6478-136c-f597f9a4c345 -42:58.540 [INFO] TestAgent.server: federation state anti-entropy synced -42:58.827 [INFO] TestAgent: Synced node info -42:58.828 [INFO] TestAgent: Synced check: check=test -42:58.828 [INFO] TestAgent: Requesting shutdown -42:58.828 [INFO] TestAgent.server: shutting down server -42:58.828 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:58.828 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:58.828 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:58.828 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:58.828 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:58.828 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:58.828 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:58.828 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:58.828 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:58.828 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:58.828 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:58.828 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:58.829Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:58.829Z [INFO] agent.router.manager: shutting down -42:58.829 [INFO] TestAgent: consul server down -42:58.829 [INFO] TestAgent: shutdown complete -42:58.829 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13269 network=tcp -42:58.829 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13269 network=udp -42:58.829 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13270 network=tcp -42:59.330 [INFO] TestAgent: Waiting for endpoints to shut down -42:59.330 [INFO] TestAgent: Endpoints down +2021-08-13T15:34:22.517Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:22.532Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:22.535 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a8a1c880-3af7-bc30-d1ab-583efefb7c75 Address:127.0.0.1:24768}]" +34:22.535 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24768 [Follower]" leader= +34:22.536 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a8a1c880-3af7-bc30-d1ab-583efefb7c75.dc1 127.0.0.1 +34:22.537 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a8a1c880-3af7-bc30-d1ab-583efefb7c75 127.0.0.1 +2021-08-13T15:34:22.537Z [INFO] agent.router: Initializing LAN area manager +34:22.537 [INFO] TestAgent.server: Adding LAN server: server="Node-a8a1c880-3af7-bc30-d1ab-583efefb7c75 (Addr: tcp/127.0.0.1:24768) (DC: dc1)" +34:22.537 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a8a1c880-3af7-bc30-d1ab-583efefb7c75.dc1 area=wan +34:22.537 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24756 network=udp +34:22.537 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24756 network=tcp +34:22.538 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24764 network=tcp +34:22.538 [INFO] TestAgent: started state syncer +34:22.538 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24810 network=tcp +34:22.583 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:22.583 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24768 [Candidate]" term=2 +34:22.584 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:22.584 [DEBUG] TestAgent.server.raft: vote granted: from=a8a1c880-3af7-bc30-d1ab-583efefb7c75 term=2 tally=1 +34:22.584 [INFO] TestAgent.server.raft: election won: tally=1 +34:22.584 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24768 [Leader]" +34:22.584 [INFO] TestAgent.server: cluster leadership acquired +34:22.584 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24768 +34:22.585 [INFO] TestAgent.server: New leader elected: payload=Node-a8a1c880-3af7-bc30-d1ab-583efefb7c75 +34:22.585 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:22.588 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:22.588 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:22.588 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:22.588 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:22.588 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:22.588 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a8a1c880-3af7-bc30-d1ab-583efefb7c75 +34:22.588 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a8a1c880-3af7-bc30-d1ab-583efefb7c75 +34:22.589 [INFO] TestAgent.server: federation state anti-entropy synced +34:22.841 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:22.842 [INFO] TestAgent: Synced node info +34:22.882 [DEBUG] TestAgent: Node info in sync +34:22.882 [INFO] TestAgent: Synced check: check=test +34:22.882 [INFO] TestAgent: Requesting shutdown +34:22.882 [INFO] TestAgent.server: shutting down server +34:22.882 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:22.882 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:22.882 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:22.882 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:22.882 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:22.883 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:22.883 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:22.883 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:22.883 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:22.883 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:22.883 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:22.883Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:22.883Z [INFO] agent.router.manager: shutting down +34:22.883 [INFO] TestAgent: consul server down +34:22.883 [INFO] TestAgent: shutdown complete +34:22.883 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24756 network=tcp +34:22.883 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24756 network=udp +34:22.883 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24764 network=tcp +34:23.383 [INFO] TestAgent: Waiting for endpoints to shut down +34:23.383 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_RegisterCheck_Passing (0.88s) -=== CONT TestAgent_Leave_ACLDeny -2022-09-15T21:42:58.624Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:58.642Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:58.645 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5a9eca08-92ab-b4f7-1639-f803cc3058a3 Address:127.0.0.1:13337}]" -42:58.645 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13337 [Follower]" leader= -42:58.646 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5a9eca08-92ab-b4f7-1639-f803cc3058a3.dc1 127.0.0.1 -42:58.647 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5a9eca08-92ab-b4f7-1639-f803cc3058a3 127.0.0.1 -2022-09-15T21:42:58.647Z [INFO] agent.router: Initializing LAN area manager -42:58.648 [INFO] TestAgent.server: Adding LAN server: server="Node-5a9eca08-92ab-b4f7-1639-f803cc3058a3 (Addr: tcp/127.0.0.1:13337) (DC: dc1)" -42:58.648 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5a9eca08-92ab-b4f7-1639-f803cc3058a3.dc1 area=wan -42:58.648 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13332 network=udp -42:58.648 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13332 network=tcp -42:58.649 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13333 network=tcp -42:58.649 [INFO] TestAgent: started state syncer -42:58.649 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13338 network=tcp -42:58.706 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:58.706 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13337 [Candidate]" term=2 -42:58.707 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:58.707 [DEBUG] TestAgent.server.raft: vote granted: from=5a9eca08-92ab-b4f7-1639-f803cc3058a3 term=2 tally=1 -42:58.707 [INFO] TestAgent.server.raft: election won: tally=1 -42:58.707 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13337 [Leader]" -42:58.707 [INFO] TestAgent.server: cluster leadership acquired -42:58.707 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13337 -42:58.708 [INFO] TestAgent.server: New leader elected: payload=Node-5a9eca08-92ab-b4f7-1639-f803cc3058a3 -42:58.709 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:58.711 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:58.711 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:58.711 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:58.711 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:58.711 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:58.711 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5a9eca08-92ab-b4f7-1639-f803cc3058a3 -42:58.711 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5a9eca08-92ab-b4f7-1639-f803cc3058a3 -42:58.713 [INFO] TestAgent.server: federation state anti-entropy synced -42:58.855 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:58.855 [INFO] TestAgent: Synced node info -42:58.881 [INFO] TestAgent: Requesting shutdown -42:58.881 [INFO] TestAgent.server: shutting down server -42:58.881 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:58.881 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:58.881 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:58.881 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:58.881 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:58.881 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:58.881 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:58.881 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:58.881 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:58.881 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:58.882 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:58.882Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:58.882Z [INFO] agent.router.manager: shutting down -42:58.882 [INFO] TestAgent: consul server down -42:58.882 [INFO] TestAgent: shutdown complete -42:58.882 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13332 network=tcp -42:58.882 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13332 network=udp -42:58.882 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13333 network=tcp -42:59.382 [INFO] TestAgent: Waiting for endpoints to shut down -42:59.382 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_RegisterCheckScriptsExecDisable (0.78s) -=== CONT TestAgent_Leave -=== RUN TestAgent_ForceLeave_ACLDeny/no_token -=== RUN TestAgent_ForceLeave_ACLDeny/agent_master_token -=== RUN TestAgent_ForceLeave_ACLDeny/read-only_token -=== RUN TestAgent_ForceLeave_ACLDeny/operator_write_token -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestAgent_Leave_ACLDeny/no_token -=== RUN TestAgent_Leave_ACLDeny/read-only_token -=== RUN TestAgent_Leave_ACLDeny/agent_master_token -2022-09-15T21:42:58.995Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:58.995Z [WARN] agent: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ -2022-09-15T21:42:59.017Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:59.017Z [WARN] agent.auto_config: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ -42:59.020 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c4e66f42-214f-4dd6-cfbf-4b90e3584dbb Address:127.0.0.1:13358}]" -42:59.020 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13358 [Follower]" leader= -42:59.021 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c4e66f42-214f-4dd6-cfbf-4b90e3584dbb.dc1 127.0.0.1 -42:59.022 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c4e66f42-214f-4dd6-cfbf-4b90e3584dbb 127.0.0.1 -2022-09-15T21:42:59.022Z [INFO] agent.router: Initializing LAN area manager -42:59.022 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13346 network=udp -42:59.022 [INFO] TestAgent.server: Adding LAN server: server="Node-c4e66f42-214f-4dd6-cfbf-4b90e3584dbb (Addr: tcp/127.0.0.1:13358) (DC: dc1)" -42:59.022 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c4e66f42-214f-4dd6-cfbf-4b90e3584dbb.dc1 area=wan -42:59.023 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13346 network=tcp -42:59.023 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13347 network=tcp -42:59.024 [INFO] TestAgent: started state syncer -42:59.024 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13359 network=tcp -42:59.078 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:59.078 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13358 [Candidate]" term=2 -42:59.078 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:59.078 [DEBUG] TestAgent.server.raft: vote granted: from=c4e66f42-214f-4dd6-cfbf-4b90e3584dbb term=2 tally=1 -42:59.078 [INFO] TestAgent.server.raft: election won: tally=1 -42:59.078 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13358 [Leader]" -42:59.078 [INFO] TestAgent.server: cluster leadership acquired -42:59.079 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13358 -42:59.080 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:59.081 [INFO] TestAgent.server: New leader elected: payload=Node-c4e66f42-214f-4dd6-cfbf-4b90e3584dbb -42:59.083 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:59.083 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:59.083 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:59.083 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:59.083 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:59.083 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c4e66f42-214f-4dd6-cfbf-4b90e3584dbb -42:59.083 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c4e66f42-214f-4dd6-cfbf-4b90e3584dbb -42:59.084 [INFO] TestAgent.server: federation state anti-entropy synced -42:59.233 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:59.234 [INFO] TestAgent: Synced node info -42:59.235 [DEBUG] TestAgent: Node info in sync -42:59.236 [INFO] TestAgent: Synced check: check=test -42:59.237 [DEBUG] TestAgent: Node info in sync -42:59.238 [INFO] TestAgent: Synced service: service=a -42:59.238 [DEBUG] TestAgent: Check in sync: check=test -42:59.238 [DEBUG] TestAgent: Check in sync: check=service:a -42:59.239 [DEBUG] TestAgent: Node info in sync -42:59.240 [INFO] TestAgent: Synced service: service=a -42:59.240 [DEBUG] TestAgent: Check in sync: check=test -42:59.240 [DEBUG] TestAgent: Check in sync: check=service:a -42:59.240 [DEBUG] TestAgent: Node info in sync -42:59.240 [DEBUG] TestAgent: Service in sync: service=a -42:59.241 [INFO] TestAgent: Synced check: check=test -42:59.241 [DEBUG] TestAgent: Check in sync: check=service:a -42:59.241 [DEBUG] TestAgent: Node info in sync -42:59.242 [INFO] TestAgent: Synced service: service=a -42:59.242 [DEBUG] TestAgent: Check in sync: check=test -42:59.242 [DEBUG] TestAgent: Check in sync: check=service:a -42:59.243 [DEBUG] TestAgent: Node info in sync -42:59.244 [INFO] TestAgent: Synced service: service=a -42:59.244 [DEBUG] TestAgent: Check in sync: check=test -42:59.244 [DEBUG] TestAgent: Check in sync: check=service:a -42:59.245 [DEBUG] TestAgent: Node info in sync -42:59.245 [DEBUG] TestAgent: Service in sync: service=a -42:59.245 [INFO] TestAgent: Synced check: check=test -42:59.245 [DEBUG] TestAgent: Check in sync: check=service:a -42:59.247 [DEBUG] TestAgent: Node info in sync -42:59.248 [INFO] TestAgent: Synced service: service=a -42:59.248 [DEBUG] TestAgent: Check in sync: check=test -42:59.248 [DEBUG] TestAgent: Check in sync: check=service:a -42:59.248 [DEBUG] TestAgent: Node info in sync -42:59.249 [INFO] TestAgent: Synced service: service=a -42:59.249 [DEBUG] TestAgent: Check in sync: check=test -42:59.249 [DEBUG] TestAgent: Check in sync: check=service:a -42:59.249 [INFO] TestAgent: Requesting shutdown -42:59.249 [INFO] TestAgent.server: shutting down server -42:59.249 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:59.249 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:59.249 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:59.249 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:59.249 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:59.250 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:59.250 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:59.250 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -42:59.250 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:59.250 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:59.250 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2022-09-15T21:42:59.250Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:59.250Z [INFO] agent.router.manager: shutting down -42:59.250 [INFO] TestAgent: consul server down -42:59.250 [INFO] TestAgent: shutdown complete -42:59.250 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13346 network=tcp -42:59.250 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13346 network=udp -42:59.250 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13347 network=tcp -42:59.751 [INFO] TestAgent: Waiting for endpoints to shut down -42:59.758 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_RegisterCheck_Scripts (0.78s) +=== CONT TestAgent_RegisterCheck +2021-08-13T15:34:22.606Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:22.656Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:22.670 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:61e3ac71-3a30-3106-527e-0712c2095793 Address:127.0.0.1:24809}]" +34:22.671 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-61e3ac71-3a30-3106-527e-0712c2095793.dc1 127.0.0.1 +34:22.672 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-61e3ac71-3a30-3106-527e-0712c2095793 127.0.0.1 +2021-08-13T15:34:22.672Z [INFO] agent.router: Initializing LAN area manager +34:22.672 [INFO] TestAgent.server: Adding LAN server: server="Node-61e3ac71-3a30-3106-527e-0712c2095793 (Addr: tcp/127.0.0.1:24809) (DC: dc1)" +34:22.672 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-61e3ac71-3a30-3106-527e-0712c2095793.dc1 area=wan +34:22.672 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24818 network=udp +34:22.673 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24818 network=tcp +34:22.673 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24809 [Follower]" leader= +34:22.674 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24819 network=tcp +34:22.674 [INFO] TestAgent: started state syncer +34:22.674 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24838 network=tcp +34:22.733 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:22.734 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24809 [Candidate]" term=2 +34:22.734 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:22.734 [DEBUG] TestAgent.server.raft: vote granted: from=61e3ac71-3a30-3106-527e-0712c2095793 term=2 tally=1 +34:22.734 [INFO] TestAgent.server.raft: election won: tally=1 +34:22.734 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24809 [Leader]" +34:22.734 [INFO] TestAgent.server: cluster leadership acquired +34:22.734 [INFO] TestAgent.server: New leader elected: payload=Node-61e3ac71-3a30-3106-527e-0712c2095793 +34:22.734 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24809 +34:22.735 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:22.738 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:22.738 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:22.738 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:22.738 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:22.738 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:22.738 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-61e3ac71-3a30-3106-527e-0712c2095793 +34:22.738 [INFO] TestAgent.server: member joined, marking health alive: member=Node-61e3ac71-3a30-3106-527e-0712c2095793 +34:22.739 [INFO] TestAgent.server: federation state anti-entropy synced +34:22.813 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:22.814 [INFO] TestAgent: Synced node info +34:22.814 [DEBUG] TestAgent: Node info in sync +34:22.931 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query +34:22.932 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-61e3ac71-3a30-3106-527e-0712c2095793 +34:22.932 [INFO] TestAgent: Requesting shutdown +34:22.932 [INFO] TestAgent.server: shutting down server +34:22.932 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:22.932 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:22.932 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:22.932 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:22.932 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:22.932 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:22.932 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:22.932 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:22.932 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:22.932 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:22.932 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:22.932Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:22.933Z [INFO] agent.router.manager: shutting down +34:22.933 [INFO] TestAgent: consul server down +34:22.933 [INFO] TestAgent: shutdown complete +34:22.933 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24818 network=tcp +34:22.933 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24818 network=udp +34:22.933 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24819 network=tcp +34:23.435 [INFO] TestAgent: Waiting for endpoints to shut down +34:23.435 [INFO] TestAgent: Endpoints down +--- PASS: TestOperator_KeyringListLocalOnly (0.85s) +=== CONT TestOperator_RaftPeer +=== RUN TestOperator_RaftPeer/#00 +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:22.604Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:22.621Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:22.624 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f15c9997-d349-6c0e-0af6-5f63b3c8d910 Address:127.0.0.1:24823}]" +34:22.625 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f15c9997-d349-6c0e-0af6-5f63b3c8d910.dc1 127.0.0.1 +34:22.625 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24823 [Follower]" leader= +34:22.647 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f15c9997-d349-6c0e-0af6-5f63b3c8d910 127.0.0.1 +2021-08-13T15:34:22.647Z [INFO] agent.router: Initializing LAN area manager +34:22.648 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24811 network=udp +34:22.648 [INFO] TestAgent.server: Adding LAN server: server="Node-f15c9997-d349-6c0e-0af6-5f63b3c8d910 (Addr: tcp/127.0.0.1:24823) (DC: dc1)" +34:22.648 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f15c9997-d349-6c0e-0af6-5f63b3c8d910.dc1 area=wan +34:22.648 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24811 network=tcp +34:22.650 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24812 network=tcp +34:22.650 [INFO] TestAgent: started state syncer +34:22.650 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24817 network=tcp +34:22.673 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:22.673 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24823 [Candidate]" term=2 +34:22.674 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:22.674 [DEBUG] TestAgent.server.raft: vote granted: from=f15c9997-d349-6c0e-0af6-5f63b3c8d910 term=2 tally=1 +34:22.674 [INFO] TestAgent.server.raft: election won: tally=1 +34:22.674 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24823 [Leader]" +34:22.674 [INFO] TestAgent.server: cluster leadership acquired +34:22.674 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24823 +34:22.674 [INFO] TestAgent.server: New leader elected: payload=Node-f15c9997-d349-6c0e-0af6-5f63b3c8d910 +34:22.676 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:22.679 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:22.679 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:22.679 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:22.679 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:22.679 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:22.679 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f15c9997-d349-6c0e-0af6-5f63b3c8d910 +34:22.679 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f15c9997-d349-6c0e-0af6-5f63b3c8d910 +34:22.720 [INFO] TestAgent.server: federation state anti-entropy synced +34:22.867 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:22.868 [INFO] TestAgent: Synced node info +34:23.037 [INFO] TestAgent: Requesting shutdown +34:23.037 [INFO] TestAgent.server: shutting down server +34:23.037 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:23.037 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:23.037 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:23.037 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:23.037 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:23.037 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:23.037 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:23.037 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:23.037 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:23.037 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:23.037 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:23.037Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:23.037Z [INFO] agent.router.manager: shutting down +34:23.037 [INFO] TestAgent: consul server down +34:23.038 [INFO] TestAgent: shutdown complete +34:23.038 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24811 network=tcp +34:23.038 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24811 network=udp +34:23.038 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24812 network=tcp +34:23.538 [INFO] TestAgent: Waiting for endpoints to shut down +34:23.538 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_RegisterCheckScriptsExecRemoteDisable (0.95s) +=== CONT TestOperator_RaftConfiguration +2021-08-13T15:34:22.694Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:22.709Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:22.712 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ca8a5dd0-360e-8d51-fcbc-d79f89bb0d60 Address:127.0.0.1:24844}]" +34:22.712 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24844 [Follower]" leader= +34:22.713 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ca8a5dd0-360e-8d51-fcbc-d79f89bb0d60.dc1 127.0.0.1 +34:22.713 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ca8a5dd0-360e-8d51-fcbc-d79f89bb0d60 127.0.0.1 +2021-08-13T15:34:22.713Z [INFO] agent.router: Initializing LAN area manager +34:22.714 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24839 network=udp +34:22.714 [INFO] TestAgent.server: Adding LAN server: server="Node-ca8a5dd0-360e-8d51-fcbc-d79f89bb0d60 (Addr: tcp/127.0.0.1:24844) (DC: dc1)" +34:22.714 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ca8a5dd0-360e-8d51-fcbc-d79f89bb0d60.dc1 area=wan +34:22.714 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24839 network=tcp +34:22.715 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24840 network=tcp +34:22.715 [INFO] TestAgent: started state syncer +34:22.716 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24824 network=tcp +34:22.754 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:22.754 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24844 [Candidate]" term=2 +34:22.754 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:22.754 [DEBUG] TestAgent.server.raft: vote granted: from=ca8a5dd0-360e-8d51-fcbc-d79f89bb0d60 term=2 tally=1 +34:22.754 [INFO] TestAgent.server.raft: election won: tally=1 +34:22.754 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24844 [Leader]" +34:22.754 [INFO] TestAgent.server: cluster leadership acquired +34:22.755 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24844 +34:22.755 [INFO] TestAgent.server: New leader elected: payload=Node-ca8a5dd0-360e-8d51-fcbc-d79f89bb0d60 +34:22.756 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:22.758 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:22.758 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:22.758 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:22.758 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:22.758 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:22.758 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ca8a5dd0-360e-8d51-fcbc-d79f89bb0d60 +34:22.758 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ca8a5dd0-360e-8d51-fcbc-d79f89bb0d60 +34:22.760 [INFO] TestAgent.server: federation state anti-entropy synced +34:22.905 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:22.906 [INFO] TestAgent: Synced node info +34:22.906 [DEBUG] TestAgent: Node info in sync +34:23.066 [INFO] TestAgent: Requesting shutdown +34:23.066 [INFO] TestAgent.server: shutting down server +34:23.066 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:23.066 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:23.066 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:23.066 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:23.066 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:23.066 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:23.066 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:23.066 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:23.066 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:23.066 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:23.066 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:23.066Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:23.066Z [INFO] agent.router.manager: shutting down +34:23.066 [INFO] TestAgent: consul server down +34:23.066 [INFO] TestAgent: shutdown complete +34:23.066 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24839 network=tcp +34:23.066 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24839 network=udp +34:23.066 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24840 network=tcp +34:23.568 [INFO] TestAgent: Waiting for endpoints to shut down +34:23.568 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_RegisterCheckScriptsExecDisable (0.89s) +=== CONT TestAgent_ForceLeavePrune +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:22.811Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:22.828Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:22.831 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0d0b4bb3-3449-aec7-8fd5-7d621fb057fe Address:127.0.0.1:24830}]" +34:22.831 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24830 [Follower]" leader= +34:22.832 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0d0b4bb3-3449-aec7-8fd5-7d621fb057fe.dc1 127.0.0.1 +34:22.833 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0d0b4bb3-3449-aec7-8fd5-7d621fb057fe 127.0.0.1 +2021-08-13T15:34:22.833Z [INFO] agent.router: Initializing LAN area manager +34:22.833 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0d0b4bb3-3449-aec7-8fd5-7d621fb057fe.dc1 area=wan +34:22.833 [INFO] TestAgent.server: Adding LAN server: server="Node-0d0b4bb3-3449-aec7-8fd5-7d621fb057fe (Addr: tcp/127.0.0.1:24830) (DC: dc1)" +34:22.833 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24825 network=udp +34:22.833 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24825 network=tcp +34:22.834 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24826 network=tcp +34:22.834 [INFO] TestAgent: started state syncer +34:22.834 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24859 network=tcp +34:22.878 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:22.878 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24830 [Candidate]" term=2 +34:22.879 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:22.879 [DEBUG] TestAgent.server.raft: vote granted: from=0d0b4bb3-3449-aec7-8fd5-7d621fb057fe term=2 tally=1 +34:22.879 [INFO] TestAgent.server.raft: election won: tally=1 +34:22.879 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24830 [Leader]" +34:22.879 [INFO] TestAgent.server: cluster leadership acquired +34:22.879 [INFO] TestAgent.server: New leader elected: payload=Node-0d0b4bb3-3449-aec7-8fd5-7d621fb057fe +34:22.879 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24830 +34:22.880 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:22.883 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:22.883 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:22.883 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:22.883 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:22.883 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:22.883 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0d0b4bb3-3449-aec7-8fd5-7d621fb057fe +34:22.883 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0d0b4bb3-3449-aec7-8fd5-7d621fb057fe +34:22.884 [INFO] TestAgent.server: federation state anti-entropy synced +34:22.925 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:22.925 [INFO] TestAgent: Synced node info +34:22.925 [DEBUG] TestAgent: Node info in sync +34:23.271 [INFO] TestAgent.server.serf.wan: serf: Received list-keys query +34:23.272 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-0d0b4bb3-3449-aec7-8fd5-7d621fb057fe.dc1 +34:23.272 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query +34:23.272 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-0d0b4bb3-3449-aec7-8fd5-7d621fb057fe +34:23.272 [INFO] TestAgent: Requesting shutdown +34:23.272 [INFO] TestAgent.server: shutting down server +34:23.272 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:23.272 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:23.272 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:23.272 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:23.272 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:23.273 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:23.273 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:23.273 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:23.273 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:23.273 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:23.273 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:23.273Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:23.273Z [INFO] agent.router.manager: shutting down +34:23.273 [INFO] TestAgent: consul server down +34:23.273 [INFO] TestAgent: shutdown complete +34:23.273 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24825 network=tcp +34:23.273 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24825 network=udp +34:23.273 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24826 network=tcp +34:23.773 [INFO] TestAgent: Waiting for endpoints to shut down +34:23.773 [INFO] TestAgent: Endpoints down +--- PASS: TestOperator_KeyringList (0.98s) +=== CONT TestKVSEndpoint_DELETE_ConflictingFlags +2021-08-13T15:34:22.996Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:22.996Z [WARN] agent: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ +2021-08-13T15:34:23.013Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:23.013Z [WARN] agent.auto_config: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ +34:23.016 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3195ceaa-485f-cf45-161c-9e648c720f52 Address:127.0.0.1:24865}]" +34:23.016 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24865 [Follower]" leader= +34:23.016 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3195ceaa-485f-cf45-161c-9e648c720f52.dc1 127.0.0.1 +34:23.017 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3195ceaa-485f-cf45-161c-9e648c720f52 127.0.0.1 +2021-08-13T15:34:23.017Z [INFO] agent.router: Initializing LAN area manager +34:23.018 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24860 network=udp +34:23.018 [INFO] TestAgent.server: Adding LAN server: server="Node-3195ceaa-485f-cf45-161c-9e648c720f52 (Addr: tcp/127.0.0.1:24865) (DC: dc1)" +34:23.018 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3195ceaa-485f-cf45-161c-9e648c720f52.dc1 area=wan +34:23.018 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24860 network=tcp +34:23.019 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24861 network=tcp +34:23.019 [INFO] TestAgent: started state syncer +34:23.019 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24866 network=tcp +34:23.077 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:23.077 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24865 [Candidate]" term=2 +34:23.078 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:23.078 [DEBUG] TestAgent.server.raft: vote granted: from=3195ceaa-485f-cf45-161c-9e648c720f52 term=2 tally=1 +34:23.078 [INFO] TestAgent.server.raft: election won: tally=1 +34:23.078 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24865 [Leader]" +34:23.078 [INFO] TestAgent.server: cluster leadership acquired +34:23.078 [INFO] TestAgent.server: New leader elected: payload=Node-3195ceaa-485f-cf45-161c-9e648c720f52 +34:23.078 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24865 +34:23.080 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:23.083 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:23.083 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:23.083 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:23.083 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:23.083 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:23.083 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3195ceaa-485f-cf45-161c-9e648c720f52 +34:23.084 [INFO] TestAgent.server: federation state anti-entropy synced +34:23.084 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3195ceaa-485f-cf45-161c-9e648c720f52 +34:23.130 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:23.130 [INFO] TestAgent: Synced node info +34:23.260 [DEBUG] TestAgent: Node info in sync +34:23.260 [INFO] TestAgent: Synced check: check=test +34:23.261 [DEBUG] TestAgent: Node info in sync +34:23.262 [INFO] TestAgent: Synced service: service=a +34:23.262 [DEBUG] TestAgent: Check in sync: check=test +34:23.262 [DEBUG] TestAgent: Check in sync: check=service:a +34:23.267 [DEBUG] TestAgent: Node info in sync +34:23.268 [INFO] TestAgent: Synced service: service=a +34:23.268 [DEBUG] TestAgent: Check in sync: check=test +34:23.268 [DEBUG] TestAgent: Check in sync: check=service:a +34:23.269 [DEBUG] TestAgent: Node info in sync +34:23.269 [DEBUG] TestAgent: Service in sync: service=a +34:23.269 [INFO] TestAgent: Synced check: check=test +34:23.269 [DEBUG] TestAgent: Check in sync: check=service:a +34:23.270 [DEBUG] TestAgent: Node info in sync +34:23.271 [INFO] TestAgent: Synced service: service=a +34:23.271 [DEBUG] TestAgent: Check in sync: check=test +34:23.271 [DEBUG] TestAgent: Check in sync: check=service:a +34:23.272 [DEBUG] TestAgent: Node info in sync +34:23.272 [INFO] TestAgent: Synced service: service=a +34:23.272 [DEBUG] TestAgent: Check in sync: check=test +34:23.272 [DEBUG] TestAgent: Check in sync: check=service:a +34:23.273 [DEBUG] TestAgent: Node info in sync +34:23.273 [DEBUG] TestAgent: Service in sync: service=a +34:23.273 [INFO] TestAgent: Synced check: check=test +34:23.273 [DEBUG] TestAgent: Check in sync: check=service:a +34:23.274 [DEBUG] TestAgent: Node info in sync +34:23.275 [INFO] TestAgent: Synced service: service=a +34:23.275 [DEBUG] TestAgent: Check in sync: check=test +34:23.275 [DEBUG] TestAgent: Check in sync: check=service:a +34:23.276 [DEBUG] TestAgent: Node info in sync +34:23.276 [INFO] TestAgent: Synced service: service=a +34:23.276 [DEBUG] TestAgent: Check in sync: check=test +34:23.276 [DEBUG] TestAgent: Check in sync: check=service:a +34:23.276 [INFO] TestAgent: Requesting shutdown +34:23.276 [INFO] TestAgent.server: shutting down server +34:23.276 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:23.276 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:23.276 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:23.276 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:23.277 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:23.277 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:23.277 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:23.277 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:23.277 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:23.277 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:23.277 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:23.277Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:23.277Z [INFO] agent.router.manager: shutting down +34:23.277 [INFO] TestAgent: consul server down +34:23.277 [INFO] TestAgent: shutdown complete +34:23.277 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24860 network=tcp +34:23.277 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24860 network=udp +34:23.277 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24861 network=tcp +34:23.777 [INFO] TestAgent: Waiting for endpoints to shut down +34:23.777 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_RegisterCheck_Scripts (0.80s) --- PASS: TestAgent_RegisterCheck_Scripts/==_Consul_1.0.0_as_node_check (0.00s) --- PASS: TestAgent_RegisterCheck_Scripts/==_Consul_1.0.0_as_top-level_service_check (0.00s) - --- PASS: TestAgent_RegisterCheck_Scripts/==_Consul_1.0.0_as_slice-based_service_check (0.00s) + --- PASS: TestAgent_RegisterCheck_Scripts/==_Consul_1.0.0_as_slice-based_service_check (0.01s) --- PASS: TestAgent_RegisterCheck_Scripts/>_Consul_1.0.0_(fixup)_as_node_check (0.00s) --- PASS: TestAgent_RegisterCheck_Scripts/>_Consul_1.0.0_(fixup)_as_top-level_service_check (0.00s) --- PASS: TestAgent_RegisterCheck_Scripts/>_Consul_1.0.0_(fixup)_as_slice-based_service_check (0.00s) --- PASS: TestAgent_RegisterCheck_Scripts/>_Consul_1.0.0_as_node_check (0.00s) --- PASS: TestAgent_RegisterCheck_Scripts/>_Consul_1.0.0_as_top-level_service_check (0.00s) --- PASS: TestAgent_RegisterCheck_Scripts/>_Consul_1.0.0_as_slice-based_service_check (0.00s) -=== CONT TestAgent_JoinLANNotify -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:59.266Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:59.266Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:59.295Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:59.295Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:59.302 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:44f7d1b3-5976-bbde-ceca-27bdae5d676c Address:127.0.0.1:13400}]" -42:59.302 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13400 [Follower]" leader= -42:59.303 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-44f7d1b3-5976-bbde-ceca-27bdae5d676c.dc1 127.0.0.1 -42:59.310 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-44f7d1b3-5976-bbde-ceca-27bdae5d676c 127.0.0.1 -2022-09-15T21:42:59.310Z [INFO] agent.router: Initializing LAN area manager -42:59.311 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13367 network=udp -42:59.311 [INFO] TestAgent.server: Adding LAN server: server="Node-44f7d1b3-5976-bbde-ceca-27bdae5d676c (Addr: tcp/127.0.0.1:13400) (DC: dc1)" -42:59.311 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-44f7d1b3-5976-bbde-ceca-27bdae5d676c.dc1 area=wan -42:59.311 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13367 network=tcp -42:59.312 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13368 network=tcp -42:59.312 [INFO] TestAgent: started state syncer -42:59.312 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13401 network=tcp -42:59.362 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:59.362 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13400 [Candidate]" term=2 -42:59.362 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -42:59.366 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:59.366 [DEBUG] TestAgent.server.raft: vote granted: from=44f7d1b3-5976-bbde-ceca-27bdae5d676c term=2 tally=1 -42:59.366 [INFO] TestAgent.server.raft: election won: tally=1 -42:59.366 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13400 [Leader]" -42:59.366 [INFO] TestAgent.server: cluster leadership acquired -42:59.366 [INFO] TestAgent.server: initializing acls -42:59.367 [INFO] TestAgent.server: New leader elected: payload=Node-44f7d1b3-5976-bbde-ceca-27bdae5d676c -42:59.368 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="rpc error making call: ACL not found" -42:59.369 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:59.369 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:59.370 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:59.370 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:59.371 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:59.371 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:59.371 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-44f7d1b3-5976-bbde-ceca-27bdae5d676c -42:59.372 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-44f7d1b3-5976-bbde-ceca-27bdae5d676c.dc1 -42:59.373 [INFO] TestAgent.server: Updating LAN server: server="Node-44f7d1b3-5976-bbde-ceca-27bdae5d676c (Addr: tcp/127.0.0.1:13400) (DC: dc1)" -42:59.373 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-44f7d1b3-5976-bbde-ceca-27bdae5d676c.dc1 area=wan -42:59.376 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:59.378 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:59.378 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:59.378 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:59.378 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:59.378 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:59.378 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-44f7d1b3-5976-bbde-ceca-27bdae5d676c -42:59.378 [INFO] TestAgent.server: member joined, marking health alive: member=Node-44f7d1b3-5976-bbde-ceca-27bdae5d676c -42:59.379 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-44f7d1b3-5976-bbde-ceca-27bdae5d676c -42:59.380 [INFO] TestAgent.server: federation state anti-entropy synced -42:59.398 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-44f7d1b3-5976-bbde-ceca-27bdae5d676c -42:59.398 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-44f7d1b3-5976-bbde-ceca-27bdae5d676c -42:59.401 [INFO] TestAgent: Force leaving node: node=Node-44f7d1b3-5976-bbde-ceca-27bdae5d676c -42:59.401 [DEBUG] TestAgent.server.serf.lan: serf: Refuting an older leave intent -42:59.401 [DEBUG] TestAgent.server.serf.wan: serf: Refuting an older leave intent -42:59.401 [INFO] TestAgent: Requesting shutdown -42:59.405 [INFO] TestAgent.server: shutting down server -42:59.405 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:59.405 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:59.405 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:59.405 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:59.405 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:59.405 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:59.405 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:59.405 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:59.405 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:59.405 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:59.405 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:59.405 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:59.405 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:59.405 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:59.405 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:59.406Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:59.406Z [INFO] agent.router.manager: shutting down -42:59.407 [INFO] TestAgent: consul server down -42:59.407 [INFO] TestAgent: shutdown complete -42:59.407 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13367 network=tcp -42:59.407 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13367 network=udp -42:59.407 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13368 network=tcp -42:59.908 [INFO] TestAgent: Waiting for endpoints to shut down -42:59.908 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_ForceLeave_ACLDeny (0.66s) +=== CONT TestAgent_ForceLeave_ACLDeny +2021-08-13T15:34:23.043Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:23.060Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:23.063 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e35091fb-9033-c1e7-a81b-7675ca25a86d Address:127.0.0.1:24872}]" +34:23.063 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24872 [Follower]" leader= +34:23.064 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e35091fb-9033-c1e7-a81b-7675ca25a86d.dc1 127.0.0.1 +34:23.065 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e35091fb-9033-c1e7-a81b-7675ca25a86d 127.0.0.1 +2021-08-13T15:34:23.065Z [INFO] agent.router: Initializing LAN area manager +34:23.065 [INFO] TestAgent.server: Adding LAN server: server="Node-e35091fb-9033-c1e7-a81b-7675ca25a86d (Addr: tcp/127.0.0.1:24872) (DC: dc1)" +34:23.065 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e35091fb-9033-c1e7-a81b-7675ca25a86d.dc1 area=wan +34:23.065 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24867 network=tcp +34:23.066 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24867 network=udp +34:23.066 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24868 network=tcp +34:23.066 [INFO] TestAgent: started state syncer +34:23.067 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24845 network=tcp +34:23.111 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:23.111 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24872 [Candidate]" term=2 +34:23.112 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:23.112 [DEBUG] TestAgent.server.raft: vote granted: from=e35091fb-9033-c1e7-a81b-7675ca25a86d term=2 tally=1 +34:23.112 [INFO] TestAgent.server.raft: election won: tally=1 +34:23.112 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24872 [Leader]" +34:23.112 [INFO] TestAgent.server: cluster leadership acquired +34:23.112 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24872 +34:23.113 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:23.115 [INFO] TestAgent.server: New leader elected: payload=Node-e35091fb-9033-c1e7-a81b-7675ca25a86d +34:23.116 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:23.116 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:23.116 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:23.116 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:23.116 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:23.116 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e35091fb-9033-c1e7-a81b-7675ca25a86d +34:23.116 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e35091fb-9033-c1e7-a81b-7675ca25a86d +34:23.118 [INFO] TestAgent.server: federation state anti-entropy synced +34:23.231 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:23.232 [INFO] TestAgent: Synced node info +34:23.278 [INFO] TestAgent.server.serf.wan: serf: Received install-key query +34:23.279 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-e35091fb-9033-c1e7-a81b-7675ca25a86d.dc1 +34:23.279 [INFO] TestAgent.server.serf.lan: serf: Received install-key query +34:23.279 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-e35091fb-9033-c1e7-a81b-7675ca25a86d +34:23.280 [INFO] TestAgent.server.serf.wan: serf: Received list-keys query +34:23.280 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-e35091fb-9033-c1e7-a81b-7675ca25a86d.dc1 +34:23.280 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query +34:23.280 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-e35091fb-9033-c1e7-a81b-7675ca25a86d +34:23.280 [INFO] TestAgent: Requesting shutdown +34:23.280 [INFO] TestAgent.server: shutting down server +34:23.280 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:23.281 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:23.281 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:23.281 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:23.281 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:23.281 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:23.281 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:23.281 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:23.281 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:23.281 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:23.281 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:23.281Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:23.281Z [INFO] agent.router.manager: shutting down +34:23.281 [INFO] TestAgent: consul server down +34:23.281 [INFO] TestAgent: shutdown complete +34:23.281 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24867 network=tcp +34:23.281 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24867 network=udp +34:23.281 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24868 network=tcp +34:23.782 [INFO] TestAgent: Waiting for endpoints to shut down +34:23.782 [INFO] TestAgent: Endpoints down +--- PASS: TestOperator_KeyringInstall (0.76s) +=== CONT TestKVSEndpoint_PUT_ConflictingFlags +2021-08-13T15:34:23.220Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:23.237Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:23.240 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0d0ff00f-9ad0-d691-6253-60bbe04f8f94 Address:127.0.0.1:24851}]" +34:23.241 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0d0ff00f-9ad0-d691-6253-60bbe04f8f94.dc1 127.0.0.1 +34:23.241 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0d0ff00f-9ad0-d691-6253-60bbe04f8f94 127.0.0.1 +2021-08-13T15:34:23.241Z [INFO] agent.router: Initializing LAN area manager +34:23.242 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24851 [Follower]" leader= +34:23.242 [INFO] TestAgent.server: Adding LAN server: server="Node-0d0ff00f-9ad0-d691-6253-60bbe04f8f94 (Addr: tcp/127.0.0.1:24851) (DC: dc1)" +34:23.242 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0d0ff00f-9ad0-d691-6253-60bbe04f8f94.dc1 area=wan +34:23.242 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24846 network=tcp +34:23.243 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24846 network=udp +34:23.244 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24847 network=tcp +34:23.244 [INFO] TestAgent: started state syncer +34:23.244 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24831 network=tcp +34:23.290 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:23.290 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24851 [Candidate]" term=2 +34:23.290 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:23.290 [DEBUG] TestAgent.server.raft: vote granted: from=0d0ff00f-9ad0-d691-6253-60bbe04f8f94 term=2 tally=1 +34:23.290 [INFO] TestAgent.server.raft: election won: tally=1 +34:23.290 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24851 [Leader]" +34:23.290 [INFO] TestAgent.server: cluster leadership acquired +34:23.290 [INFO] TestAgent.server: New leader elected: payload=Node-0d0ff00f-9ad0-d691-6253-60bbe04f8f94 +34:23.290 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24851 +34:23.292 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:23.294 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:23.294 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:23.294 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:23.294 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:23.294 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:23.294 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0d0ff00f-9ad0-d691-6253-60bbe04f8f94 +34:23.294 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0d0ff00f-9ad0-d691-6253-60bbe04f8f94 +34:23.295 [INFO] TestAgent.server: federation state anti-entropy synced +34:23.442 [INFO] TestAgent: Requesting shutdown +34:23.442 [INFO] TestAgent.server: shutting down server +34:23.442 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:23.442 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:23.442 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:23.442 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:23.442 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:23.442 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:23.442 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:23.442 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:23.442 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:23.442 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:23.442 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:23.442 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +34:23.443 [INFO] TestAgent: consul server down +34:23.443 [INFO] TestAgent: shutdown complete +34:23.443 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24846 network=tcp +34:23.443 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24846 network=udp +34:23.443 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24847 network=tcp +2021-08-13T15:34:23.443Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:23.443Z [INFO] agent.router.manager: shutting down +34:23.944 [INFO] TestAgent: Waiting for endpoints to shut down +34:23.944 [INFO] TestAgent: Endpoints down +--- PASS: TestPreparedQuery_Execute (6.72s) + --- PASS: TestPreparedQuery_Execute/#00 (0.87s) + --- PASS: TestPreparedQuery_Execute/#01 (0.88s) + --- PASS: TestPreparedQuery_Execute/#02 (0.80s) + --- PASS: TestPreparedQuery_Execute/#03 (0.80s) + --- PASS: TestPreparedQuery_Execute/#04 (0.91s) + --- PASS: TestPreparedQuery_Execute/#05 (0.80s) + --- PASS: TestPreparedQuery_Execute/#06 (0.92s) + --- PASS: TestPreparedQuery_Execute/#07 (0.74s) +=== CONT TestOpenMetricsMimeTypeHeaders +--- PASS: TestOpenMetricsMimeTypeHeaders (0.00s) +=== CONT TestKVSEndpoint_GET_Raw +[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestAgent_ForceLeave_ACLDeny/no_token +=== RUN TestAgent_ForceLeave_ACLDeny/agent_master_token +=== RUN TestAgent_ForceLeave_ACLDeny/read-only_token +=== RUN TestAgent_ForceLeave_ACLDeny/operator_write_token +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:23.429Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:23.470Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:23.473 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a73338ab-2b3e-5481-acc1-99f2fd2d9cd7 Address:127.0.0.1:24837}]" +34:23.474 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24837 [Follower]" leader= +34:23.475 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a73338ab-2b3e-5481-acc1-99f2fd2d9cd7.dc1 127.0.0.1 +34:23.475 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a73338ab-2b3e-5481-acc1-99f2fd2d9cd7 127.0.0.1 +2021-08-13T15:34:23.476Z [INFO] agent.router: Initializing LAN area manager +34:23.476 [INFO] TestAgent.server: Adding LAN server: server="Node-a73338ab-2b3e-5481-acc1-99f2fd2d9cd7 (Addr: tcp/127.0.0.1:24837) (DC: dc1)" +34:23.476 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a73338ab-2b3e-5481-acc1-99f2fd2d9cd7.dc1 area=wan +34:23.476 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24832 network=tcp +34:23.476 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24832 network=udp +34:23.477 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24833 network=tcp +34:23.477 [INFO] TestAgent: started state syncer +34:23.477 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24852 network=tcp +34:23.536 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:23.537 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24837 [Candidate]" term=2 +34:23.537 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:23.537 [DEBUG] TestAgent.server.raft: vote granted: from=a73338ab-2b3e-5481-acc1-99f2fd2d9cd7 term=2 tally=1 +34:23.537 [INFO] TestAgent.server.raft: election won: tally=1 +34:23.537 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24837 [Leader]" +34:23.537 [INFO] TestAgent.server: cluster leadership acquired +34:23.537 [INFO] TestAgent.server: New leader elected: payload=Node-a73338ab-2b3e-5481-acc1-99f2fd2d9cd7 +34:23.537 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24837 +34:23.539 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:23.541 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:23.541 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:23.541 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:23.541 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:23.541 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:23.541 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a73338ab-2b3e-5481-acc1-99f2fd2d9cd7 +34:23.542 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a73338ab-2b3e-5481-acc1-99f2fd2d9cd7 +34:23.543 [INFO] TestAgent.server: federation state anti-entropy synced +34:23.788 [INFO] TestAgent: Synced node info +34:23.789 [INFO] TestAgent: Synced check: check=test +34:23.789 [INFO] TestAgent: Requesting shutdown +34:23.789 [INFO] TestAgent.server: shutting down server +34:23.789 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:23.789 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:23.789 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:23.789 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:23.789 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:23.789 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:23.789 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:23.789 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:23.789 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:23.789 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:23.789 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:23.789 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:23.789Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:23.789Z [INFO] agent.router.manager: shutting down +34:23.789 [INFO] TestAgent: consul server down +34:23.789 [INFO] TestAgent: shutdown complete +34:23.789 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24832 network=tcp +34:23.790 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24832 network=udp +34:23.790 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24833 network=tcp +34:24.290 [INFO] TestAgent: Waiting for endpoints to shut down +34:24.290 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_RegisterCheck (0.91s) +=== CONT TestAgent_ForceLeave +2021-08-13T15:34:23.473Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:23.488Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:23.491 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d34ba8f5-24da-3ff9-915a-ea74648e7437 Address:127.0.0.1:24858}]" +34:23.491 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24858 [Follower]" leader= +34:23.492 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d34ba8f5-24da-3ff9-915a-ea74648e7437.dc1 127.0.0.1 +34:23.493 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d34ba8f5-24da-3ff9-915a-ea74648e7437 127.0.0.1 +2021-08-13T15:34:23.493Z [INFO] agent.router: Initializing LAN area manager +34:23.493 [INFO] TestAgent.server: Adding LAN server: server="Node-d34ba8f5-24da-3ff9-915a-ea74648e7437 (Addr: tcp/127.0.0.1:24858) (DC: dc1)" +34:23.493 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d34ba8f5-24da-3ff9-915a-ea74648e7437.dc1 area=wan +34:23.493 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24853 network=tcp +34:23.494 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24853 network=udp +34:23.494 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24854 network=tcp +34:23.495 [INFO] TestAgent: started state syncer +34:23.495 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24894 network=tcp +34:23.560 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:23.560 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24858 [Candidate]" term=2 +34:23.561 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:23.561 [DEBUG] TestAgent.server.raft: vote granted: from=d34ba8f5-24da-3ff9-915a-ea74648e7437 term=2 tally=1 +34:23.561 [INFO] TestAgent.server.raft: election won: tally=1 +34:23.561 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24858 [Leader]" +34:23.561 [INFO] TestAgent.server: cluster leadership acquired +34:23.561 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24858 +34:23.562 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:23.565 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:23.565 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:23.565 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:23.565 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:23.565 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:23.565 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d34ba8f5-24da-3ff9-915a-ea74648e7437 +34:23.565 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d34ba8f5-24da-3ff9-915a-ea74648e7437 +34:23.566 [INFO] TestAgent.server: New leader elected: payload=Node-d34ba8f5-24da-3ff9-915a-ea74648e7437 +34:23.566 [INFO] TestAgent.server: federation state anti-entropy synced +34:23.568 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:23.569 [INFO] TestAgent: Synced node info +34:23.569 [DEBUG] TestAgent: Node info in sync +34:23.790 [INFO] TestAgent: Requesting shutdown +34:23.790 [INFO] TestAgent.server: shutting down server +34:23.790 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:23.790 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:23.790 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:23.790 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:23.790 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:23.790 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:23.790 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:23.790 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:23.790 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:23.790 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:23.790 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:23.790Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:23.790Z [INFO] agent.router.manager: shutting down +34:23.791 [INFO] TestAgent: consul server down +34:23.791 [INFO] TestAgent: shutdown complete +34:23.791 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24853 network=tcp +34:23.791 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24853 network=udp +34:23.791 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24854 network=tcp +34:24.291 [INFO] TestAgent: Waiting for endpoints to shut down +34:24.291 [INFO] TestAgent: Endpoints down +=== RUN TestOperator_RaftPeer/#01 +2021-08-13T15:34:23.555Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:23.571Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:23.574 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5ce99a3f-f2fe-e12f-dbc3-e46ccd4dedaf Address:127.0.0.1:24886}]" +34:23.574 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24886 [Follower]" leader= +34:23.574 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5ce99a3f-f2fe-e12f-dbc3-e46ccd4dedaf.dc1 127.0.0.1 +34:23.575 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5ce99a3f-f2fe-e12f-dbc3-e46ccd4dedaf 127.0.0.1 +2021-08-13T15:34:23.575Z [INFO] agent.router: Initializing LAN area manager +34:23.576 [INFO] TestAgent.server: Adding LAN server: server="Node-5ce99a3f-f2fe-e12f-dbc3-e46ccd4dedaf (Addr: tcp/127.0.0.1:24886) (DC: dc1)" +34:23.576 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5ce99a3f-f2fe-e12f-dbc3-e46ccd4dedaf.dc1 area=wan +34:23.576 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24895 network=udp +34:23.576 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24895 network=tcp +34:23.577 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24896 network=tcp +34:23.577 [INFO] TestAgent: started state syncer +34:23.577 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24880 network=tcp +34:23.629 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:23.629 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24886 [Candidate]" term=2 +34:23.629 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:23.629 [DEBUG] TestAgent.server.raft: vote granted: from=5ce99a3f-f2fe-e12f-dbc3-e46ccd4dedaf term=2 tally=1 +34:23.629 [INFO] TestAgent.server.raft: election won: tally=1 +34:23.629 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24886 [Leader]" +34:23.629 [INFO] TestAgent.server: cluster leadership acquired +34:23.630 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24886 +34:23.631 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:23.631 [INFO] TestAgent.server: New leader elected: payload=Node-5ce99a3f-f2fe-e12f-dbc3-e46ccd4dedaf +34:23.633 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:23.633 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:23.633 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:23.633 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:23.633 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:23.634 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5ce99a3f-f2fe-e12f-dbc3-e46ccd4dedaf +34:23.634 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5ce99a3f-f2fe-e12f-dbc3-e46ccd4dedaf +34:23.635 [INFO] TestAgent.server: federation state anti-entropy synced +34:23.865 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:23.866 [INFO] TestAgent: Synced node info +34:23.920 [INFO] TestAgent: Requesting shutdown +34:23.920 [INFO] TestAgent.server: shutting down server +34:23.920 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:23.920 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:23.920 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:23.920 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:23.920 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:23.921 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:23.921 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:23.921 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:23.921 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:23.921 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:23.921 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:23.921Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:23.921Z [INFO] agent.router.manager: shutting down +34:23.921 [INFO] TestAgent: consul server down +34:23.921 [INFO] TestAgent: shutdown complete +34:23.921 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24895 network=tcp +34:23.921 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24895 network=udp +34:23.921 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24896 network=tcp +34:24.421 [INFO] TestAgent: Waiting for endpoints to shut down +34:24.421 [INFO] TestAgent: Endpoints down +--- PASS: TestOperator_RaftConfiguration (0.88s) +=== CONT TestKVSEndpoint_Recurse +2021-08-13T15:34:23.802Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:23.826Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:23.837 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:49b4f740-c23e-ff69-6add-e2ef7451a7b7 Address:127.0.0.1:24900}]" +34:23.837 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24900 [Follower]" leader= +34:23.838 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-49b4f740-c23e-ff69-6add-e2ef7451a7b7.dc1 127.0.0.1 +34:23.865 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-49b4f740-c23e-ff69-6add-e2ef7451a7b7 127.0.0.1 +2021-08-13T15:34:23.865Z [INFO] agent.router: Initializing LAN area manager +34:23.866 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24888 network=udp +34:23.866 [INFO] TestAgent.server: Adding LAN server: server="Node-49b4f740-c23e-ff69-6add-e2ef7451a7b7 (Addr: tcp/127.0.0.1:24900) (DC: dc1)" +34:23.866 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-49b4f740-c23e-ff69-6add-e2ef7451a7b7.dc1 area=wan +34:23.866 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24888 network=tcp +34:23.869 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24889 network=tcp +34:23.869 [INFO] TestAgent: started state syncer +34:23.869 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24873 network=tcp +34:23.889 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:23.889 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24900 [Candidate]" term=2 +34:23.889 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:23.889 [DEBUG] TestAgent.server.raft: vote granted: from=49b4f740-c23e-ff69-6add-e2ef7451a7b7 term=2 tally=1 +34:23.889 [INFO] TestAgent.server.raft: election won: tally=1 +34:23.889 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24900 [Leader]" +34:23.889 [INFO] TestAgent.server: cluster leadership acquired +34:23.890 [INFO] TestAgent.server: New leader elected: payload=Node-49b4f740-c23e-ff69-6add-e2ef7451a7b7 +34:23.890 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24900 +34:23.891 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:23.894 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:23.894 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:23.894 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:23.894 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:23.894 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:23.894 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-49b4f740-c23e-ff69-6add-e2ef7451a7b7 +34:23.894 [INFO] TestAgent.server: member joined, marking health alive: member=Node-49b4f740-c23e-ff69-6add-e2ef7451a7b7 +34:23.895 [INFO] TestAgent.server: federation state anti-entropy synced +34:23.972 [INFO] TestAgent: Requesting shutdown +34:23.972 [INFO] TestAgent.server: shutting down server +34:23.972 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:23.972 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:23.972 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:23.972 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:23.972 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:23.972 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:23.972 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:23.972 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:23.972 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:23.972 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:23.972 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:23.972 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:23.972Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:23.972Z [INFO] agent.router.manager: shutting down +34:23.972 [INFO] TestAgent: consul server down +34:23.972 [INFO] TestAgent: shutdown complete +34:23.972 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24888 network=tcp +34:23.972 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24888 network=udp +34:23.972 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24889 network=tcp +34:24.473 [INFO] TestAgent: Waiting for endpoints to shut down +34:24.473 [INFO] TestAgent: Endpoints down +--- PASS: TestKVSEndpoint_PUT_ConflictingFlags (0.69s) +=== CONT TestAgent_Leave_ACLDeny +[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:23.795Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:23.795Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:23.811Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:23.811Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:23.814 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5deaee5c-8f32-cd7d-56df-289c55000506 Address:127.0.0.1:24914}]" +34:23.815 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5deaee5c-8f32-cd7d-56df-289c55000506.dc1 127.0.0.1 +34:23.815 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24914 [Follower]" leader= +34:23.816 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5deaee5c-8f32-cd7d-56df-289c55000506 127.0.0.1 +2021-08-13T15:34:23.816Z [INFO] agent.router: Initializing LAN area manager +34:23.816 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24909 network=udp +34:23.816 [INFO] TestAgent.server: Adding LAN server: server="Node-5deaee5c-8f32-cd7d-56df-289c55000506 (Addr: tcp/127.0.0.1:24914) (DC: dc1)" +34:23.816 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5deaee5c-8f32-cd7d-56df-289c55000506.dc1 area=wan +34:23.817 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24909 network=tcp +34:23.817 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24910 network=tcp +34:23.818 [INFO] TestAgent: started state syncer +34:23.818 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24887 network=tcp +34:23.870 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:23.870 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:23.870 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24914 [Candidate]" term=2 +34:23.871 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:23.871 [DEBUG] TestAgent.server.raft: vote granted: from=5deaee5c-8f32-cd7d-56df-289c55000506 term=2 tally=1 +34:23.871 [INFO] TestAgent.server.raft: election won: tally=1 +34:23.871 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24914 [Leader]" +34:23.871 [INFO] TestAgent.server: cluster leadership acquired +34:23.872 [INFO] TestAgent.server: initializing acls +34:23.873 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:23.873 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:23.874 [INFO] TestAgent.server: New leader elected: payload=Node-5deaee5c-8f32-cd7d-56df-289c55000506 +34:23.875 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:23.876 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:23.876 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:23.876 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:23.876 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-5deaee5c-8f32-cd7d-56df-289c55000506 +34:23.876 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-5deaee5c-8f32-cd7d-56df-289c55000506.dc1 +34:23.878 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:23.879 [INFO] TestAgent.server: Updating LAN server: server="Node-5deaee5c-8f32-cd7d-56df-289c55000506 (Addr: tcp/127.0.0.1:24914) (DC: dc1)" +34:23.879 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-5deaee5c-8f32-cd7d-56df-289c55000506.dc1 area=wan +34:23.881 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:23.881 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:23.881 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:23.881 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:23.881 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:23.881 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5deaee5c-8f32-cd7d-56df-289c55000506 +34:23.881 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5deaee5c-8f32-cd7d-56df-289c55000506 +34:23.883 [INFO] TestAgent.server: federation state anti-entropy synced +34:23.883 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5deaee5c-8f32-cd7d-56df-289c55000506 +34:24.056 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-5deaee5c-8f32-cd7d-56df-289c55000506 +34:24.056 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-5deaee5c-8f32-cd7d-56df-289c55000506 +34:24.060 [INFO] TestAgent: Force leaving node: node=Node-5deaee5c-8f32-cd7d-56df-289c55000506 +34:24.060 [DEBUG] TestAgent.server.serf.lan: serf: Refuting an older leave intent +34:24.060 [DEBUG] TestAgent.server.serf.wan: serf: Refuting an older leave intent +34:24.060 [INFO] TestAgent: Requesting shutdown +34:24.060 [INFO] TestAgent.server: shutting down server +34:24.060 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:24.060 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:24.060 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:24.060 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:24.060 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:24.060 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:24.060 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:24.060 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:24.060 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:24.061 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:24.061 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:24.061 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:24.061 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:24.061 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:24.061 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:24.061 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:24.061Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:24.061Z [INFO] agent.router.manager: shutting down +34:24.061 [INFO] TestAgent: consul server down +34:24.061 [INFO] TestAgent: shutdown complete +34:24.061 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24909 network=tcp +34:24.061 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24909 network=udp +34:24.061 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24910 network=tcp +34:24.561 [INFO] TestAgent: Waiting for endpoints to shut down +34:24.561 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_ForceLeave_ACLDeny (0.78s) --- PASS: TestAgent_ForceLeave_ACLDeny/no_token (0.00s) --- PASS: TestAgent_ForceLeave_ACLDeny/agent_master_token (0.00s) --- PASS: TestAgent_ForceLeave_ACLDeny/read-only_token (0.00s) --- PASS: TestAgent_ForceLeave_ACLDeny/operator_write_token (0.00s) +=== CONT TestKVSEndpoint_AcquireRelease +2021-08-13T15:34:23.792Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:23.833Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:23.855 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0ee4da31-7478-054b-9503-33dc3362f936 Address:127.0.0.1:24879}]" +34:23.855 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24879 [Follower]" leader= +34:23.865 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0ee4da31-7478-054b-9503-33dc3362f936.dc1 127.0.0.1 +34:23.867 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0ee4da31-7478-054b-9503-33dc3362f936 127.0.0.1 +2021-08-13T15:34:23.867Z [INFO] agent.router: Initializing LAN area manager +34:23.870 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24902 network=udp +34:23.870 [INFO] TestAgent.server: Adding LAN server: server="Node-0ee4da31-7478-054b-9503-33dc3362f936 (Addr: tcp/127.0.0.1:24879) (DC: dc1)" +34:23.870 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0ee4da31-7478-054b-9503-33dc3362f936.dc1 area=wan +34:23.870 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24902 network=tcp +34:23.873 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24903 network=tcp +34:23.873 [INFO] TestAgent: started state syncer +34:23.873 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24908 network=tcp +34:23.922 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:23.922 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24879 [Candidate]" term=2 +34:23.922 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:23.922 [DEBUG] TestAgent.server.raft: vote granted: from=0ee4da31-7478-054b-9503-33dc3362f936 term=2 tally=1 +34:23.922 [INFO] TestAgent.server.raft: election won: tally=1 +34:23.922 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24879 [Leader]" +34:23.923 [INFO] TestAgent.server: cluster leadership acquired +34:23.923 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24879 +34:23.923 [INFO] TestAgent.server: New leader elected: payload=Node-0ee4da31-7478-054b-9503-33dc3362f936 +34:23.924 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:23.927 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:23.927 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:23.927 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:23.927 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:23.927 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:23.927 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0ee4da31-7478-054b-9503-33dc3362f936 +34:23.927 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0ee4da31-7478-054b-9503-33dc3362f936 +34:23.927 [INFO] TestAgent.server: federation state anti-entropy synced +34:24.138 [INFO] TestAgent: Requesting shutdown +34:24.138 [INFO] TestAgent.server: shutting down server +34:24.138 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:24.138 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:24.138 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:24.138 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:24.138 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:24.138 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:24.138 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:24.138 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:24.138 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:24.138 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:24.138 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:24.138 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:24.138Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:24.138Z [INFO] agent.router.manager: shutting down +34:24.139 [INFO] TestAgent: consul server down +34:24.139 [INFO] TestAgent: shutdown complete +34:24.139 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24902 network=tcp +34:24.139 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24902 network=udp +34:24.139 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24903 network=tcp +34:24.639 [INFO] TestAgent: Waiting for endpoints to shut down +34:24.639 [INFO] TestAgent: Endpoints down +--- PASS: TestKVSEndpoint_DELETE_ConflictingFlags (0.87s) +=== CONT TestAgent_Leave +=== RUN TestAgent_Leave_ACLDeny/no_token +=== RUN TestAgent_Leave_ACLDeny/read-only_token +=== RUN TestAgent_Leave_ACLDeny/agent_master_token +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:23.961Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:23.977Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:23.979 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b44a47a7-6d6c-e9c6-e2ed-6f9bf3a02dd9 Address:127.0.0.1:24921}]" +34:23.979 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24921 [Follower]" leader= +34:23.980 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b44a47a7-6d6c-e9c6-e2ed-6f9bf3a02dd9.dc1 127.0.0.1 +34:23.981 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b44a47a7-6d6c-e9c6-e2ed-6f9bf3a02dd9 127.0.0.1 +2021-08-13T15:34:23.981Z [INFO] agent.router: Initializing LAN area manager +34:23.981 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b44a47a7-6d6c-e9c6-e2ed-6f9bf3a02dd9.dc1 area=wan +34:23.981 [INFO] TestAgent.server: Adding LAN server: server="Node-b44a47a7-6d6c-e9c6-e2ed-6f9bf3a02dd9 (Addr: tcp/127.0.0.1:24921) (DC: dc1)" +34:23.981 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24916 network=tcp +34:23.982 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24916 network=udp +34:23.982 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24917 network=tcp +34:23.982 [INFO] TestAgent: started state syncer +34:23.982 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24929 network=tcp +34:24.038 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:24.038 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24921 [Candidate]" term=2 +34:24.039 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:24.039 [DEBUG] TestAgent.server.raft: vote granted: from=b44a47a7-6d6c-e9c6-e2ed-6f9bf3a02dd9 term=2 tally=1 +34:24.039 [INFO] TestAgent.server.raft: election won: tally=1 +34:24.039 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24921 [Leader]" +34:24.039 [INFO] TestAgent.server: cluster leadership acquired +34:24.039 [INFO] TestAgent.server: New leader elected: payload=Node-b44a47a7-6d6c-e9c6-e2ed-6f9bf3a02dd9 +34:24.039 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24921 +34:24.040 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:24.043 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:24.043 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:24.043 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:24.043 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:24.043 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:24.043 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b44a47a7-6d6c-e9c6-e2ed-6f9bf3a02dd9 +34:24.043 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b44a47a7-6d6c-e9c6-e2ed-6f9bf3a02dd9 +34:24.043 [INFO] TestAgent.server: federation state anti-entropy synced +34:24.062 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:24.062 [INFO] TestAgent: Synced node info +34:24.418 [INFO] TestAgent: Requesting shutdown +34:24.418 [INFO] TestAgent.server: shutting down server +34:24.418 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:24.418 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:24.418 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:24.418 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:24.418 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:24.418 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:24.418 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:24.418 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:24.418 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:24.418 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:24.418 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:24.418Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:24.418Z [INFO] agent.router.manager: shutting down +34:24.419 [INFO] TestAgent: consul server down +34:24.419 [INFO] TestAgent: shutdown complete +34:24.419 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24916 network=tcp +34:24.419 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24916 network=udp +34:24.419 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24917 network=tcp +34:24.919 [INFO] TestAgent: Waiting for endpoints to shut down +34:24.919 [INFO] TestAgent: Endpoints down +--- PASS: TestKVSEndpoint_GET_Raw (0.97s) +=== CONT TestKVSEndpoint_ListKeys +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:24.306Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:24.322Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:24.324 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6c27adcd-308c-167d-e1b9-f8d502b0bd07 Address:127.0.0.1:24928}]" +34:24.324 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24928 [Follower]" leader= +34:24.325 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6c27adcd-308c-167d-e1b9-f8d502b0bd07.dc1 127.0.0.1 +34:24.326 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6c27adcd-308c-167d-e1b9-f8d502b0bd07 127.0.0.1 +2021-08-13T15:34:24.326Z [INFO] agent.router: Initializing LAN area manager +34:24.326 [INFO] TestAgent.server: Adding LAN server: server="Node-6c27adcd-308c-167d-e1b9-f8d502b0bd07 (Addr: tcp/127.0.0.1:24928) (DC: dc1)" +34:24.326 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6c27adcd-308c-167d-e1b9-f8d502b0bd07.dc1 area=wan +34:24.326 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24923 network=udp +34:24.327 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24923 network=tcp +34:24.327 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24924 network=tcp +34:24.327 [INFO] TestAgent: started state syncer +34:24.328 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24943 network=tcp +34:24.363 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:24.363 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24928 [Candidate]" term=2 +34:24.363 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:24.363 [DEBUG] TestAgent.server.raft: vote granted: from=6c27adcd-308c-167d-e1b9-f8d502b0bd07 term=2 tally=1 +34:24.363 [INFO] TestAgent.server.raft: election won: tally=1 +34:24.363 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24928 [Leader]" +34:24.363 [INFO] TestAgent.server: cluster leadership acquired +34:24.363 [INFO] TestAgent.server: New leader elected: payload=Node-6c27adcd-308c-167d-e1b9-f8d502b0bd07 +34:24.363 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24928 +34:24.365 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:24.367 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:24.367 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:24.367 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:24.367 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:24.367 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:24.367 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6c27adcd-308c-167d-e1b9-f8d502b0bd07 +34:24.367 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6c27adcd-308c-167d-e1b9-f8d502b0bd07 +34:24.368 [INFO] TestAgent.server: federation state anti-entropy synced +34:24.575 [INFO] TestAgent: Requesting shutdown +34:24.575 [INFO] TestAgent.server: shutting down server +34:24.575 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:24.575 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:24.575 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:24.575 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:24.575 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:24.575 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:24.575 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:24.575 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:24.575 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:24.575 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:24.575 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:24.575 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:24.575Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:24.575Z [INFO] agent.router.manager: shutting down +34:24.576 [INFO] TestAgent: consul server down +34:24.576 [INFO] TestAgent: shutdown complete +34:24.576 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24923 network=tcp +34:24.576 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24923 network=udp +34:24.576 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24924 network=tcp +34:25.076 [INFO] TestAgent: Waiting for endpoints to shut down +34:25.076 [INFO] TestAgent: Endpoints down +--- PASS: TestOperator_RaftPeer (1.64s) + --- PASS: TestOperator_RaftPeer/#00 (0.86s) + --- PASS: TestOperator_RaftPeer/#01 (0.78s) +=== CONT TestAgent_JoinLANNotify +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:24.439Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:24.455Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:24.458 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a090bfaa-344e-d6d2-b3ae-86796da10097 Address:127.0.0.1:24970}]" +34:24.458 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24970 [Follower]" leader= +34:24.459 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a090bfaa-344e-d6d2-b3ae-86796da10097.dc1 127.0.0.1 +34:24.460 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a090bfaa-344e-d6d2-b3ae-86796da10097 127.0.0.1 +2021-08-13T15:34:24.460Z [INFO] agent.router: Initializing LAN area manager +34:24.460 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a090bfaa-344e-d6d2-b3ae-86796da10097.dc1 area=wan +34:24.460 [INFO] TestAgent.server: Adding LAN server: server="Node-a090bfaa-344e-d6d2-b3ae-86796da10097 (Addr: tcp/127.0.0.1:24970) (DC: dc1)" +34:24.460 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24944 network=udp +34:24.460 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24944 network=tcp +34:24.461 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24945 network=tcp +34:24.461 [INFO] TestAgent: started state syncer +34:24.461 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24950 network=tcp +34:24.519 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:24.519 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24970 [Candidate]" term=2 +34:24.519 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:24.519 [DEBUG] TestAgent.server.raft: vote granted: from=a090bfaa-344e-d6d2-b3ae-86796da10097 term=2 tally=1 +34:24.519 [INFO] TestAgent.server.raft: election won: tally=1 +34:24.519 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24970 [Leader]" +34:24.520 [INFO] TestAgent.server: cluster leadership acquired +34:24.520 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24970 +34:24.522 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:24.523 [INFO] TestAgent.server: New leader elected: payload=Node-a090bfaa-344e-d6d2-b3ae-86796da10097 +34:24.525 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:24.525 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:24.525 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:24.525 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:24.525 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:24.525 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a090bfaa-344e-d6d2-b3ae-86796da10097 +34:24.525 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a090bfaa-344e-d6d2-b3ae-86796da10097 +34:24.527 [INFO] TestAgent.server: federation state anti-entropy synced +34:24.765 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:24.766 [INFO] TestAgent: Synced node info +34:24.766 [DEBUG] TestAgent: Node info in sync +34:24.825 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:24.825 [DEBUG] TestAgent: Node info in sync +34:24.896 [INFO] TestAgent: Requesting shutdown +34:24.896 [INFO] TestAgent.server: shutting down server +34:24.896 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:24.896 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:24.896 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:24.896 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:24.896 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:24.896 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:24.896 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:24.896 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:24.896 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:24.896 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:24.896 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:24.896Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:24.896Z [INFO] agent.router.manager: shutting down +34:24.897 [INFO] TestAgent: consul server down +34:24.897 [INFO] TestAgent: shutdown complete +34:24.897 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24944 network=tcp +34:24.897 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24944 network=udp +34:24.897 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24945 network=tcp +34:25.397 [INFO] TestAgent: Waiting for endpoints to shut down +34:25.397 [INFO] TestAgent: Endpoints down +--- PASS: TestKVSEndpoint_Recurse (0.98s) +=== CONT TestKVSEndpoint_CAS +2021-08-13T15:34:24.579Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:24.595Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:24.598 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:478f7ae2-0cae-ce6b-46b1-8a18d95218ed Address:127.0.0.1:24956}]" +34:24.598 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24956 [Follower]" leader= +34:24.599 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-478f7ae2-0cae-ce6b-46b1-8a18d95218ed.dc1 127.0.0.1 +34:24.600 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-478f7ae2-0cae-ce6b-46b1-8a18d95218ed 127.0.0.1 +2021-08-13T15:34:24.600Z [INFO] agent.router: Initializing LAN area manager +34:24.600 [INFO] TestAgent.server: Adding LAN server: server="Node-478f7ae2-0cae-ce6b-46b1-8a18d95218ed (Addr: tcp/127.0.0.1:24956) (DC: dc1)" +34:24.600 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-478f7ae2-0cae-ce6b-46b1-8a18d95218ed.dc1 area=wan +34:24.600 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24937 network=udp +34:24.600 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24937 network=tcp +34:24.601 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24938 network=tcp +34:24.601 [INFO] TestAgent: started state syncer +34:24.601 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24971 network=tcp +34:24.649 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:24.649 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24956 [Candidate]" term=2 +34:24.649 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:24.649 [DEBUG] TestAgent.server.raft: vote granted: from=478f7ae2-0cae-ce6b-46b1-8a18d95218ed term=2 tally=1 +34:24.649 [INFO] TestAgent.server.raft: election won: tally=1 +34:24.649 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24956 [Leader]" +34:24.650 [INFO] TestAgent.server: cluster leadership acquired +34:24.650 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24956 +34:24.650 [INFO] TestAgent.server: New leader elected: payload=Node-478f7ae2-0cae-ce6b-46b1-8a18d95218ed +34:24.651 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:24.653 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:24.653 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:24.653 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:24.653 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:24.653 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:24.653 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-478f7ae2-0cae-ce6b-46b1-8a18d95218ed +34:24.653 [INFO] TestAgent.server: member joined, marking health alive: member=Node-478f7ae2-0cae-ce6b-46b1-8a18d95218ed +34:24.655 [INFO] TestAgent.server: federation state anti-entropy synced +34:24.934 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:24.934 [INFO] TestAgent: Synced node info +34:24.940 [INFO] TestAgent: Requesting shutdown +34:24.940 [INFO] TestAgent.server: shutting down server +34:24.940 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:24.940 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:24.940 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:24.940 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:24.940 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:24.940 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:24.940 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:24.940 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:24.940 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:24.940 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:24.940 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:24.940Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:24.940Z [INFO] agent.router.manager: shutting down +34:24.941 [INFO] TestAgent: consul server down +34:24.941 [INFO] TestAgent: shutdown complete +34:24.941 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24937 network=tcp +34:24.941 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24937 network=udp +34:24.941 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24938 network=tcp +34:25.441 [INFO] TestAgent: Waiting for endpoints to shut down +34:25.441 [INFO] TestAgent: Endpoints down +--- PASS: TestKVSEndpoint_AcquireRelease (0.88s) === CONT TestAgent_Join_ACLDeny -2022-09-15T21:42:47.900Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:47.962Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:47.969 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:634e78a1-89bc-8b31-1792-a4a2a1344cd3 Address:127.0.0.1:13842}]" -42:47.970 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-634e78a1-89bc-8b31-1792-a4a2a1344cd3.dc1 127.0.0.1 -42:47.972 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-634e78a1-89bc-8b31-1792-a4a2a1344cd3 127.0.0.1 -2022-09-15T21:42:47.972Z [INFO] agent.router: Initializing LAN area manager -42:47.972 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13842 [Follower]" leader= -42:47.973 [INFO] TestAgent.server: Adding LAN server: server="Node-634e78a1-89bc-8b31-1792-a4a2a1344cd3 (Addr: tcp/127.0.0.1:13842) (DC: dc1)" -42:47.973 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-634e78a1-89bc-8b31-1792-a4a2a1344cd3.dc1 area=wan -42:47.973 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13865 network=udp -42:47.973 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13865 network=tcp -42:47.980 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13866 network=tcp -42:47.980 [INFO] TestAgent: started state syncer -42:47.980 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13843 network=tcp -42:48.048 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:48.048 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13842 [Candidate]" term=2 -42:48.048 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:48.048 [DEBUG] TestAgent.server.raft: vote granted: from=634e78a1-89bc-8b31-1792-a4a2a1344cd3 term=2 tally=1 -42:48.048 [INFO] TestAgent.server.raft: election won: tally=1 -42:48.048 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13842 [Leader]" -42:48.049 [INFO] TestAgent.server: cluster leadership acquired -42:48.049 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13842 -42:48.050 [INFO] TestAgent.server: New leader elected: payload=Node-634e78a1-89bc-8b31-1792-a4a2a1344cd3 -42:48.059 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:48.068 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:48.068 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:48.068 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:48.068 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:48.068 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:48.068 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-634e78a1-89bc-8b31-1792-a4a2a1344cd3 -42:48.069 [INFO] TestAgent.server: member joined, marking health alive: member=Node-634e78a1-89bc-8b31-1792-a4a2a1344cd3 -42:48.077 [INFO] TestAgent.server: federation state anti-entropy synced -42:48.150 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:48.151 [INFO] TestAgent: Synced node info -42:48.151 [DEBUG] TestAgent: Node info in sync -42:48.402 [INFO] TestAgent.server: server starting leave -42:48.402 [INFO] TestAgent.server.serf.wan: serf: EventMemberLeave: Node-634e78a1-89bc-8b31-1792-a4a2a1344cd3.dc1 127.0.0.1 -42:48.402 [INFO] TestAgent.server: Handled event for server in area: event=member-leave server=Node-634e78a1-89bc-8b31-1792-a4a2a1344cd3.dc1 area=wan -2022-09-15T21:42:48.402Z [INFO] agent.router.manager: shutting down -42:51.038 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:51.038 [DEBUG] TestAgent: Node info in sync -42:51.406 [INFO] TestAgent.server.serf.lan: serf: EventMemberLeave: Node-634e78a1-89bc-8b31-1792-a4a2a1344cd3 127.0.0.1 -42:51.406 [INFO] TestAgent.server: Removing LAN server: server="Node-634e78a1-89bc-8b31-1792-a4a2a1344cd3 (Addr: tcp/127.0.0.1:13842) (DC: dc1)" -42:51.406 [WARN] TestAgent.server: deregistering self should be done by follower: name=Node-634e78a1-89bc-8b31-1792-a4a2a1344cd3 -42:52.054 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" -42:54.054 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" -42:54.406 [INFO] TestAgent.server: Waiting to drain RPC traffic: drain_time=5s -42:56.054 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" -42:58.050 [ERROR] TestAgent.server.autopilot: Error promoting servers: error="error getting server raft protocol versions: No servers found" -42:58.054 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" -42:59.406 [INFO] TestAgent: Requesting shutdown -42:59.406 [INFO] TestAgent.server: shutting down server -42:59.406 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:59.406 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:59.406 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:59.406 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:59.406 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:59.407 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:59.407 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:59.407 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:59.407 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -2022-09-15T21:42:59.407Z [INFO] agent.router.manager: shutting down -42:59.408 [INFO] TestAgent: consul server down -42:59.408 [INFO] TestAgent: shutdown complete -42:59.408 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13865 network=tcp -42:59.408 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13865 network=udp -42:59.408 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13866 network=tcp -42:59.908 [INFO] TestAgent: Waiting for endpoints to shut down -42:59.908 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_StartStop (12.03s) -=== CONT TestAgent_Join_WAN -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 4 out of 4 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! === RUN TestAgent_Join_ACLDeny/no_token === RUN TestAgent_Join_ACLDeny/agent_master_token === RUN TestAgent_Join_ACLDeny/read-only_token -[WARN] freeport: 4 out of 4 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 4 out of 4 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:52.838Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:52.857Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:52.860 [INFO] dc2.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:707975bd-eb4c-52b5-e3cc-e14e05407b6d Address:127.0.0.1:13064}]" -42:52.861 [INFO] dc2.server.serf.wan: serf: EventMemberJoin: Node-707975bd-eb4c-52b5-e3cc-e14e05407b6d.dc2 127.0.0.1 -42:52.862 [INFO] dc2.server.raft: entering follower state: follower="Node at 127.0.0.1:13064 [Follower]" leader= -42:52.862 [INFO] dc2.server.serf.lan: serf: EventMemberJoin: Node-707975bd-eb4c-52b5-e3cc-e14e05407b6d 127.0.0.1 -2022-09-15T21:42:52.862Z [INFO] agent.router: Initializing LAN area manager -42:52.862 [INFO] dc2.server: Adding LAN server: server="Node-707975bd-eb4c-52b5-e3cc-e14e05407b6d (Addr: tcp/127.0.0.1:13064) (DC: dc2)" -42:52.862 [INFO] dc2.server: Handled event for server in area: event=member-join server=Node-707975bd-eb4c-52b5-e3cc-e14e05407b6d.dc2 area=wan -42:52.862 [INFO] dc2: Started DNS server: address=127.0.0.1:14079 network=udp -42:52.862 [INFO] dc2: Started DNS server: address=127.0.0.1:14079 network=tcp -42:52.863 [INFO] dc2: Started HTTP server: address=127.0.0.1:14080 network=tcp -42:52.863 [INFO] dc2: started state syncer -42:52.864 [INFO] dc2: Started gRPC server: address=127.0.0.1:13065 network=tcp -42:52.910 [WARN] dc2.server.raft: heartbeat timeout reached, starting election: last-leader= -42:52.910 [INFO] dc2.server.raft: entering candidate state: node="Node at 127.0.0.1:13064 [Candidate]" term=2 -42:52.910 [DEBUG] dc2.server.raft: votes: needed=1 -42:52.910 [DEBUG] dc2.server.raft: vote granted: from=707975bd-eb4c-52b5-e3cc-e14e05407b6d term=2 tally=1 -42:52.910 [INFO] dc2.server.raft: election won: tally=1 -42:52.910 [INFO] dc2.server.raft: entering leader state: leader="Node at 127.0.0.1:13064 [Leader]" -42:52.910 [INFO] dc2.server: cluster leadership acquired -42:52.911 [DEBUG] dc2.server: Cannot upgrade to new ACLs, servers in acl datacenter are not yet upgraded: ACLDatacenter=dc1 mode=1 found=false -42:52.912 [WARN] dc2.server.connect: primary datacenter is configured but unreachable - deferring initialization of the secondary datacenter CA -42:52.912 [INFO] dc2.leader: started routine: routine="config entry replication" -42:52.912 [INFO] dc2.leader: started routine: routine="federation state replication" -42:52.912 [INFO] dc2.leader: started routine: routine="federation state anti-entropy" -42:52.912 [INFO] dc2.leader: started routine: routine="secondary CA roots watch" -42:52.912 [INFO] dc2.leader: started routine: routine="intention replication" -42:52.912 [INFO] dc2.leader: started routine: routine="intermediate cert renew watch" -42:52.912 [INFO] dc2.leader: started routine: routine="CA root pruning" -42:52.912 [DEBUG] dc2.server: Skipping self join check for node since the cluster is too small: node=Node-707975bd-eb4c-52b5-e3cc-e14e05407b6d -42:52.912 [INFO] dc2.server: member joined, marking health alive: member=Node-707975bd-eb4c-52b5-e3cc-e14e05407b6d -42:52.913 [INFO] dc2.server: New leader elected: payload=Node-707975bd-eb4c-52b5-e3cc-e14e05407b6d -42:52.913 [WARN] dc2.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ConfigEntry.ListAll -42:52.913 [DEBUG] dc2.server: federation states are not enabled in the primary dc -42:52.913 [DEBUG] dc2.server: federation states are not enabled in the primary dc -42:52.913 [DEBUG] dc2.server: federation states are not enabled in the primary dc -42:52.913 [DEBUG] dc2.server: federation states are not enabled in the primary dc -42:52.913 [DEBUG] dc2.server: federation states are not enabled in the primary dc -42:52.913 [DEBUG] dc2.server: federation states are not enabled in the primary dc -42:52.913 [DEBUG] dc2.server.connect: starting Connect CA root replication from primary datacenter: primary=dc1 -42:52.913 [WARN] dc2.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ConnectCA.Roots -42:52.913 [ERROR] dc2.server.connect: CA root replication failed, will retry: routine="secondary CA roots watch" error="Error retrieving the primary datacenter's roots: No path to datacenter" -42:52.913 [DEBUG] dc2.server.connect: starting Connect intention replication from primary datacenter: primary=dc1 -42:52.913 [WARN] dc2.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=Intention.List -42:52.913 [ERROR] dc2.server.connect: error replicating intentions: routine="intention replication" error="No path to datacenter" -42:52.922 [INFO] dc2: (WAN) joining: wan_addresses=[127.0.0.1:14069] -42:52.922 [DEBUG] dc2.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:14069 -42:52.924 [INFO] dc2.server.serf.wan: serf: EventMemberJoin: Node-5c410c28-1cd7-a770-03ad-177be0697cdc.dc1 127.0.0.1 -42:52.924 [INFO] dc2: (WAN) joined: number_of_nodes=1 -42:52.924 [INFO] dc2.server: Handled event for server in area: event=member-join server=Node-5c410c28-1cd7-a770-03ad-177be0697cdc.dc1 area=wan -42:53.178 [DEBUG] dc2: Skipping remote check since it is managed automatically: check=serfHealth -42:53.179 [INFO] dc2: Synced node info -42:53.682 [DEBUG] dc2.server.serf.wan: serf: messageJoinType: Node-707975bd-eb4c-52b5-e3cc-e14e05407b6d.dc2 -42:53.969 [DEBUG] dc2.server: finished fetching remote objects: amount=1 -42:53.969 [DEBUG] dc2.server: diffing replication state: local_amount=0 remote_amount=1 -42:53.969 [DEBUG] dc2.server: diffed replication state: deletions=0 updates=1 -42:53.969 [DEBUG] dc2.server: performing updates: updates=1 -42:53.987 [DEBUG] dc2.server: finished updates -42:53.987 [DEBUG] dc2.replication.federation_state: replication completed through remote index: index=11 -42:53.988 [DEBUG] dc2.server.replication.config_entry: finished fetching config entries: amount=0 -42:53.988 [DEBUG] dc2.server.replication.config_entry: Config Entry replication: local=0 remote=0 -42:53.988 [DEBUG] dc2.server.replication.config_entry: Config Entry replication: deletions=0 updates=0 -42:53.988 [DEBUG] dc2.server.replication.config_entry: replication completed through remote index: index=1 -42:54.182 [DEBUG] dc2.server.serf.wan: serf: messageJoinType: Node-707975bd-eb4c-52b5-e3cc-e14e05407b6d.dc2 -42:54.686 [DEBUG] dc2.server.serf.wan: serf: messageJoinType: Node-707975bd-eb4c-52b5-e3cc-e14e05407b6d.dc2 -42:54.723 [DEBUG] dc2: Skipping remote check since it is managed automatically: check=serfHealth -42:54.723 [DEBUG] dc2: Node info in sync -42:54.723 [DEBUG] dc2: Node info in sync -42:54.916 [DEBUG] connect.ca.consul: consul CA provider configured: id=ad:4a:c6:ab:ef:63:c9:60:1a:51:7f:19:62:e3:e9:d9:0e:76:55:10:6e:74:24:69:28:a1:6c:b8:b9:8f:fd:89 is_primary=false -42:54.923 [INFO] dc2.server.connect: received new intermediate certificate from primary datacenter -42:54.924 [INFO] dc2.server.connect: updated root certificates from primary datacenter -42:54.945 [DEBUG] dc2: Node info in sync -42:54.946 [INFO] dc2: Synced service: service=foo -42:54.946 [DEBUG] dc2: Check in sync: check=service:foo -42:54.950 [DEBUG] dc2: Node info in sync -42:54.950 [DEBUG] dc2: Service in sync: service=foo -42:54.950 [DEBUG] dc2: Check in sync: check=service:foo -42:54.950 [DEBUG] dc2: Node info in sync -42:54.950 [DEBUG] dc2: Service in sync: service=foo -42:54.950 [DEBUG] dc2: Check in sync: check=service:foo -42:55.186 [DEBUG] dc2.server.serf.wan: serf: messageJoinType: Node-707975bd-eb4c-52b5-e3cc-e14e05407b6d.dc2 -42:57.922 [DEBUG] dc2.server: finished fetching remote objects: amount=2 -42:57.922 [DEBUG] dc2.server: diffing replication state: local_amount=1 remote_amount=2 -42:57.922 [DEBUG] dc2.server: diffed replication state: deletions=0 updates=1 -42:57.922 [DEBUG] dc2.server: performing updates: updates=1 -42:57.922 [DEBUG] dc2.server: finished updates -42:57.922 [DEBUG] dc2.replication.federation_state: replication completed through remote index: index=19 -42:57.922 [INFO] dc2.server: federation state anti-entropy synced -42:59.966 [INFO] dc2.server.connect: received new intermediate certificate from primary datacenter -42:59.967 [INFO] dc2.server.connect: updated root certificates from primary datacenter -42:59.989 [INFO] dc2: Requesting shutdown -42:59.989 [INFO] dc2.server: shutting down server -42:59.989 [DEBUG] dc2.leader: stopping routine: routine="intermediate cert renew watch" -42:59.989 [DEBUG] dc2.leader: stopping routine: routine="CA root pruning" -42:59.989 [DEBUG] dc2.leader: stopping routine: routine="config entry replication" -42:59.989 [DEBUG] dc2.leader: stopping routine: routine="federation state replication" -42:59.989 [DEBUG] dc2.leader: stopping routine: routine="federation state anti-entropy" -42:59.989 [DEBUG] dc2.leader: stopping routine: routine="secondary CA roots watch" -42:59.989 [DEBUG] dc2.leader: stopping routine: routine="intention replication" -42:59.989 [WARN] dc2.server.serf.lan: serf: Shutdown without a Leave -42:59.989 [DEBUG] dc2.leader: stopped routine: routine="intermediate cert renew watch" -42:59.989 [DEBUG] dc2.leader: stopped routine: routine="CA root pruning" -42:59.989 [DEBUG] dc2.leader: stopped routine: routine="federation state anti-entropy" -42:59.989 [WARN] dc2.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:59.990Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:59.990Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:59.990Z [INFO] agent.router.manager: shutting down -42:59.990 [INFO] dc2: consul server down -42:59.990 [INFO] dc2: shutdown complete -42:59.990 [INFO] dc2: Stopping server: protocol=DNS address=127.0.0.1:14079 network=tcp -42:59.990 [INFO] dc2: Stopping server: protocol=DNS address=127.0.0.1:14079 network=udp -42:59.991 [INFO] dc2: Stopping server: protocol=HTTP address=127.0.0.1:14080 network=tcp -42:59.991 [ERROR] dc2.server.rpc: RPC failed to server in DC: server=127.0.0.1:14077 datacenter=dc1 method=ConnectCA.Roots error="rpc error making call: EOF" -42:59.991 [ERROR] dc2.server.connect: CA root replication failed, will retry: routine="secondary CA roots watch" error="Error retrieving the primary datacenter's roots: rpc error making call: EOF" -42:59.991 [DEBUG] dc2.leader: stopped routine: routine="secondary CA roots watch" -42:59.991 [ERROR] dc2.server.rpc: RPC failed to server in DC: server=127.0.0.1:14077 datacenter=dc1 method=ConfigEntry.ListAll error="rpc error making call: EOF" -42:59.991 [INFO] dc2.server.replication.config_entry: stopped replication -42:59.991 [DEBUG] dc2.leader: stopped routine: routine="config entry replication" -42:59.991 [ERROR] dc2.server.rpc: RPC failed to server in DC: server=127.0.0.1:14077 datacenter=dc1 method=FederationState.List error="rpc error making call: EOF" -42:59.991 [WARN] dc2.replication.federation_state: replication error (will retry if still leader): error="failed to retrieve federation states: rpc error making call: EOF" -42:59.991 [INFO] dc2.replication.federation_state: stopped replication -42:59.991 [DEBUG] dc2.leader: stopped routine: routine="federation state replication" -42:59.991 [ERROR] dc2.server.rpc: RPC failed to server in DC: server=127.0.0.1:14077 datacenter=dc1 method=Intention.List error="rpc error making call: EOF" -42:59.991 [ERROR] dc2.server.connect: error replicating intentions: routine="intention replication" error="rpc error making call: EOF" -42:59.991 [DEBUG] dc2.leader: stopped routine: routine="intention replication" -43:00.491 [INFO] dc2: Waiting for endpoints to shut down -43:00.491 [INFO] dc2: Endpoints down -2022-09-15T21:42:52.651Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:52.673Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:52.680 [INFO] dc1.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5c410c28-1cd7-a770-03ad-177be0697cdc Address:127.0.0.1:14077}]" -42:52.681 [INFO] dc1.server.serf.wan: serf: EventMemberJoin: Node-5c410c28-1cd7-a770-03ad-177be0697cdc.dc1 127.0.0.1 -42:52.682 [INFO] dc1.server.raft: entering follower state: follower="Node at 127.0.0.1:14077 [Follower]" leader= -42:52.683 [INFO] dc1.server.serf.lan: serf: EventMemberJoin: Node-5c410c28-1cd7-a770-03ad-177be0697cdc 127.0.0.1 -2022-09-15T21:42:52.683Z [INFO] agent.router: Initializing LAN area manager -42:52.684 [INFO] dc1: Started DNS server: address=127.0.0.1:14065 network=udp -42:52.684 [INFO] dc1.server: Adding LAN server: server="Node-5c410c28-1cd7-a770-03ad-177be0697cdc (Addr: tcp/127.0.0.1:14077) (DC: dc1)" -42:52.684 [INFO] dc1.server: Handled event for server in area: event=member-join server=Node-5c410c28-1cd7-a770-03ad-177be0697cdc.dc1 area=wan -42:52.684 [INFO] dc1: Started DNS server: address=127.0.0.1:14065 network=tcp -42:52.685 [INFO] dc1: Started HTTP server: address=127.0.0.1:14066 network=tcp -42:52.685 [INFO] dc1: started state syncer -42:52.685 [INFO] dc1: Started gRPC server: address=127.0.0.1:14078 network=tcp -42:52.737 [WARN] dc1.server.raft: heartbeat timeout reached, starting election: last-leader= -42:52.737 [INFO] dc1.server.raft: entering candidate state: node="Node at 127.0.0.1:14077 [Candidate]" term=2 -42:52.738 [DEBUG] dc1.server.raft: votes: needed=1 -42:52.738 [DEBUG] dc1.server.raft: vote granted: from=5c410c28-1cd7-a770-03ad-177be0697cdc term=2 tally=1 -42:52.738 [INFO] dc1.server.raft: election won: tally=1 -42:52.738 [INFO] dc1.server.raft: entering leader state: leader="Node at 127.0.0.1:14077 [Leader]" -42:52.738 [INFO] dc1.server: cluster leadership acquired -42:52.738 [INFO] dc1.server: New leader elected: payload=Node-5c410c28-1cd7-a770-03ad-177be0697cdc -42:52.738 [DEBUG] dc1.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14077 -42:52.740 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:52.743 [INFO] dc1.server.connect: initialized primary datacenter CA with provider: provider=consul -42:52.744 [INFO] dc1.leader: started routine: routine="federation state anti-entropy" -42:52.744 [INFO] dc1.leader: started routine: routine="federation state pruning" -42:52.744 [INFO] dc1.leader: started routine: routine="intermediate cert renew watch" -42:52.744 [INFO] dc1.leader: started routine: routine="CA root pruning" -42:52.744 [DEBUG] dc1.server: Skipping self join check for node since the cluster is too small: node=Node-5c410c28-1cd7-a770-03ad-177be0697cdc -42:52.744 [INFO] dc1.server: member joined, marking health alive: member=Node-5c410c28-1cd7-a770-03ad-177be0697cdc -42:52.745 [INFO] dc1.server: federation state anti-entropy synced -42:52.755 [DEBUG] dc1: Skipping remote check since it is managed automatically: check=serfHealth -42:52.756 [INFO] dc1: Synced node info -42:52.774 [DEBUG] dc1: Skipping remote check since it is managed automatically: check=serfHealth -42:52.774 [DEBUG] dc1: Node info in sync -42:52.774 [DEBUG] dc1: Node info in sync -42:52.923 [DEBUG] dc1.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:54538 -42:52.924 [INFO] dc1.server.serf.wan: serf: EventMemberJoin: Node-707975bd-eb4c-52b5-e3cc-e14e05407b6d.dc2 127.0.0.1 -42:52.924 [INFO] dc1.server: Handled event for server in area: event=member-join server=Node-707975bd-eb4c-52b5-e3cc-e14e05407b6d.dc2 area=wan -42:52.926 [DEBUG] connect.ca.consul: consul CA provider configured: id=05:e1:cd:30:d2:da:b2:b2:7a:d4:32:15:dd:1a:db:2e:dc:df:f9:e2:92:87:d0:89:58:00:c8:4d:0d:49:8d:0a is_primary=true -42:52.929 [INFO] dc1.server.connect: CA rotated to new root under provider: provider=consul -42:53.363 [DEBUG] dc1.server.serf.wan: serf: messageJoinType: Node-707975bd-eb4c-52b5-e3cc-e14e05407b6d.dc2 -42:53.862 [DEBUG] dc1.server.serf.wan: serf: messageJoinType: Node-707975bd-eb4c-52b5-e3cc-e14e05407b6d.dc2 -42:54.362 [DEBUG] dc1.server.serf.wan: serf: messageJoinType: Node-707975bd-eb4c-52b5-e3cc-e14e05407b6d.dc2 -42:54.865 [DEBUG] dc1.server.serf.wan: serf: messageJoinType: Node-707975bd-eb4c-52b5-e3cc-e14e05407b6d.dc2 -42:59.956 [DEBUG] connect.ca.consul: consul CA provider configured: id=01:39:8a:52:37:10:81:fa:60:2f:ad:7e:14:58:fd:34:46:51:23:da:e3:e4:17:7b:c3:dc:ce:6c:3f:6a:cf:77 is_primary=true -42:59.960 [INFO] dc1.server.connect: CA rotated to new root under provider: provider=consul -43:00.491 [INFO] dc1: Requesting shutdown -43:00.491 [INFO] dc1.server: shutting down server -43:00.491 [DEBUG] dc1.leader: stopping routine: routine="federation state anti-entropy" -43:00.491 [DEBUG] dc1.leader: stopping routine: routine="federation state pruning" -43:00.491 [DEBUG] dc1.leader: stopping routine: routine="intermediate cert renew watch" -43:00.491 [DEBUG] dc1.leader: stopping routine: routine="CA root pruning" -43:00.491 [WARN] dc1.server.serf.lan: serf: Shutdown without a Leave -43:00.491 [ERROR] dc1.server: error performing anti-entropy sync of federation state: error="context canceled" -43:00.491 [DEBUG] dc1.leader: stopped routine: routine="federation state anti-entropy" -43:00.491 [DEBUG] dc1.leader: stopped routine: routine="federation state pruning" -43:00.492 [DEBUG] dc1.leader: stopped routine: routine="intermediate cert renew watch" -43:00.492 [DEBUG] dc1.leader: stopped routine: routine="CA root pruning" -43:00.492 [WARN] dc1.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:00.492Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:00.492Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:00.492Z [INFO] agent.router.manager: shutting down -43:00.492 [INFO] dc1: consul server down -43:00.492 [INFO] dc1: shutdown complete -43:00.492 [INFO] dc1: Stopping server: protocol=DNS address=127.0.0.1:14065 network=tcp -43:00.492 [INFO] dc1: Stopping server: protocol=DNS address=127.0.0.1:14065 network=udp -43:00.492 [INFO] dc1: Stopping server: protocol=HTTP address=127.0.0.1:14066 network=tcp -43:00.994 [INFO] dc1: Waiting for endpoints to shut down -43:00.994 [INFO] dc1: Endpoints down ---- PASS: TestAgentConnectCALeafCert_secondaryDC_good (8.40s) -=== CONT TestParseConsistency -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:00.042Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:00.060Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:00.063 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:135015e1-930a-46ea-82ed-b7903cd62ee7 Address:127.0.0.1:13421}]" -43:00.064 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-135015e1-930a-46ea-82ed-b7903cd62ee7.dc1 127.0.0.1 -43:00.064 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13421 [Follower]" leader= -43:00.065 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-135015e1-930a-46ea-82ed-b7903cd62ee7 127.0.0.1 -2022-09-15T21:43:00.065Z [INFO] agent.router: Initializing LAN area manager -43:00.065 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13486 network=udp -43:00.065 [INFO] TestAgent.server: Adding LAN server: server="Node-135015e1-930a-46ea-82ed-b7903cd62ee7 (Addr: tcp/127.0.0.1:13421) (DC: dc1)" -43:00.065 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-135015e1-930a-46ea-82ed-b7903cd62ee7.dc1 area=wan -43:00.066 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13486 network=tcp -43:00.067 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13487 network=tcp -43:00.067 [INFO] TestAgent: started state syncer -43:00.067 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13422 network=tcp -43:00.107 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:00.107 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13421 [Candidate]" term=2 -43:00.107 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:00.107 [DEBUG] TestAgent.server.raft: vote granted: from=135015e1-930a-46ea-82ed-b7903cd62ee7 term=2 tally=1 -43:00.107 [INFO] TestAgent.server.raft: election won: tally=1 -43:00.107 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13421 [Leader]" -43:00.107 [INFO] TestAgent.server: cluster leadership acquired -43:00.107 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13421 -43:00.109 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:00.110 [INFO] TestAgent.server: New leader elected: payload=Node-135015e1-930a-46ea-82ed-b7903cd62ee7 -43:00.111 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:00.111 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:00.111 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:00.111 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:00.111 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:00.112 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-135015e1-930a-46ea-82ed-b7903cd62ee7 -43:00.112 [INFO] TestAgent.server: member joined, marking health alive: member=Node-135015e1-930a-46ea-82ed-b7903cd62ee7 -43:00.113 [INFO] TestAgent.server: federation state anti-entropy synced -43:00.115 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:41876 -43:00.115 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1ad6f2eb-a76b-e705-f807-14ca5eb62d58.dc1 127.0.0.1 -43:00.116 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1ad6f2eb-a76b-e705-f807-14ca5eb62d58.dc1 area=wan -43:00.116 [INFO] TestAgent: Requesting shutdown -43:00.116 [INFO] TestAgent.server: shutting down server -43:00.116 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:00.116 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:00.116 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:00.116 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:00.116 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:00.116 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:00.116 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:00.116 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:00.116 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:00.116 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:00.116 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:00.117 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:00.117Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:00.117Z [INFO] agent.router.manager: shutting down -43:00.117 [INFO] TestAgent: consul server down -43:00.117 [INFO] TestAgent: shutdown complete -43:00.117 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13486 network=tcp -43:00.117 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13486 network=udp -43:00.117 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13487 network=tcp -43:00.618 [INFO] TestAgent: Waiting for endpoints to shut down -43:00.618 [INFO] TestAgent: Endpoints down -2022-09-15T21:42:59.926Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:59.946Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:59.948 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1ad6f2eb-a76b-e705-f807-14ca5eb62d58 Address:127.0.0.1:13428}]" -42:59.949 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1ad6f2eb-a76b-e705-f807-14ca5eb62d58.dc1 127.0.0.1 -42:59.950 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13428 [Follower]" leader= -42:59.954 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1ad6f2eb-a76b-e705-f807-14ca5eb62d58 127.0.0.1 -2022-09-15T21:42:59.955Z [INFO] agent.router: Initializing LAN area manager -42:59.956 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13458 network=udp -42:59.956 [INFO] TestAgent.server: Adding LAN server: server="Node-1ad6f2eb-a76b-e705-f807-14ca5eb62d58 (Addr: tcp/127.0.0.1:13428) (DC: dc1)" -42:59.956 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1ad6f2eb-a76b-e705-f807-14ca5eb62d58.dc1 area=wan -42:59.956 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13458 network=tcp -42:59.957 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13459 network=tcp -42:59.957 [INFO] TestAgent: started state syncer -42:59.957 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13429 network=tcp -43:00.007 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:00.008 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13428 [Candidate]" term=2 -43:00.008 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:00.008 [DEBUG] TestAgent.server.raft: vote granted: from=1ad6f2eb-a76b-e705-f807-14ca5eb62d58 term=2 tally=1 -43:00.008 [INFO] TestAgent.server.raft: election won: tally=1 -43:00.008 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13428 [Leader]" -43:00.008 [INFO] TestAgent.server: cluster leadership acquired -43:00.008 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13428 -43:00.009 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:00.012 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:00.012 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:00.012 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:00.012 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:00.012 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:00.012 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1ad6f2eb-a76b-e705-f807-14ca5eb62d58 -43:00.012 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1ad6f2eb-a76b-e705-f807-14ca5eb62d58 -43:00.013 [INFO] TestAgent.server: New leader elected: payload=Node-1ad6f2eb-a76b-e705-f807-14ca5eb62d58 -43:00.013 [INFO] TestAgent.server: federation state anti-entropy synced -43:00.104 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:00.105 [INFO] TestAgent: Synced node info -43:00.105 [DEBUG] TestAgent: Node info in sync -43:00.113 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:13490] -43:00.114 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:13490 -43:00.116 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-135015e1-930a-46ea-82ed-b7903cd62ee7.dc1 127.0.0.1 -43:00.116 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 -43:00.116 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-135015e1-930a-46ea-82ed-b7903cd62ee7.dc1 area=wan -43:00.618 [INFO] TestAgent: Requesting shutdown -43:00.618 [INFO] TestAgent.server: shutting down server -43:00.618 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:00.618 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:00.618 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:00.618 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:00.618 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:00.619 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:00.619 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:00.619 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:00.619 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:00.619 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:00.619 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:00.619Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:00.619Z [INFO] agent.router.manager: shutting down -43:00.619 [INFO] TestAgent: consul server down -43:00.619 [INFO] TestAgent: shutdown complete -43:00.619 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13458 network=tcp -43:00.619 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13458 network=udp -43:00.619 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13459 network=tcp -43:01.127 [INFO] TestAgent: Waiting for endpoints to shut down -43:01.127 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Join_WAN (1.22s) -=== CONT TestHTTPServer_HandshakeTimeout -2022/09/15 21:43:01 http: TLS handshake error from 127.0.0.1:33948: read tcp 127.0.0.1:13446->127.0.0.1:33948: i/o timeout -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:00.141Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:00.159Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:00.167 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ddda13d9-5cdc-9a80-0f2e-10418e12a6c6 Address:127.0.0.1:13456}]" -43:00.168 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ddda13d9-5cdc-9a80-0f2e-10418e12a6c6.dc1 127.0.0.1 -43:00.168 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13456 [Follower]" leader= -43:00.169 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ddda13d9-5cdc-9a80-0f2e-10418e12a6c6 127.0.0.1 -2022-09-15T21:43:00.169Z [INFO] agent.router: Initializing LAN area manager -43:00.169 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13423 network=udp -43:00.169 [INFO] TestAgent.server: Adding LAN server: server="Node-ddda13d9-5cdc-9a80-0f2e-10418e12a6c6 (Addr: tcp/127.0.0.1:13456) (DC: dc1)" -43:00.169 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ddda13d9-5cdc-9a80-0f2e-10418e12a6c6.dc1 area=wan -43:00.170 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13423 network=tcp -43:00.171 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13424 network=tcp -43:00.171 [INFO] TestAgent: started state syncer -43:00.171 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13457 network=tcp -43:00.211 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:00.211 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13456 [Candidate]" term=2 -43:00.211 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:00.211 [DEBUG] TestAgent.server.raft: vote granted: from=ddda13d9-5cdc-9a80-0f2e-10418e12a6c6 term=2 tally=1 -43:00.211 [INFO] TestAgent.server.raft: election won: tally=1 -43:00.211 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13456 [Leader]" -43:00.211 [INFO] TestAgent.server: cluster leadership acquired -43:00.212 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13456 -43:00.213 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:00.216 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:00.216 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:00.216 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:00.216 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:00.216 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:00.216 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ddda13d9-5cdc-9a80-0f2e-10418e12a6c6 -43:00.216 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ddda13d9-5cdc-9a80-0f2e-10418e12a6c6 -43:00.217 [INFO] TestAgent.server: New leader elected: payload=Node-ddda13d9-5cdc-9a80-0f2e-10418e12a6c6 -43:00.217 [INFO] TestAgent.server: federation state anti-entropy synced -43:00.280 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:58250 -43:00.281 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bed24f7e-5da9-5b20-55a6-53891a903e63 127.0.0.1 -43:00.281 [INFO] TestAgent.server: Adding LAN server: server="Node-bed24f7e-5da9-5b20-55a6-53891a903e63 (Addr: tcp/127.0.0.1:13442) (DC: dc1)" -43:00.281 [INFO] TestAgent.server: New leader elected: payload=Node-bed24f7e-5da9-5b20-55a6-53891a903e63 -43:00.281 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-bed24f7e-5da9-5b20-55a6-53891a903e63 other=Node-ddda13d9-5cdc-9a80-0f2e-10418e12a6c6 -43:00.281 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bed24f7e-5da9-5b20-55a6-53891a903e63 -43:00.290 [INFO] TestAgent: Requesting shutdown -43:00.290 [INFO] TestAgent.server: shutting down server -43:00.290 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:00.290 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:00.290 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:00.290 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:00.290 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:00.291 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:00.291 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:00.291 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:00.291 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:00.291 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:00.291 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:00.291 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:00.291Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:00.291Z [INFO] agent.router.manager: shutting down -43:00.291 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: Node-bed24f7e-5da9-5b20-55a6-53891a903e63.dc1 127.0.0.1:13406 -43:00.292 [INFO] TestAgent: consul server down -43:00.292 [INFO] TestAgent: shutdown complete -43:00.292 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13423 network=tcp -43:00.292 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13423 network=udp -43:00.292 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13424 network=tcp -43:00.293 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bed24f7e-5da9-5b20-55a6-53891a903e63.dc1 127.0.0.1 -43:00.293 [DEBUG] TestAgent.server: Successfully performed flood-join for server at address: server=Node-bed24f7e-5da9-5b20-55a6-53891a903e63.dc1 address=127.0.0.1:13406 -43:00.793 [INFO] TestAgent: Waiting for endpoints to shut down -43:00.793 [INFO] TestAgent: Endpoints down -2022-09-15T21:42:59.927Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:59.927Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:59.944Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:59.944Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:59.947 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bed24f7e-5da9-5b20-55a6-53891a903e63 Address:127.0.0.1:13442}]" -42:59.948 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bed24f7e-5da9-5b20-55a6-53891a903e63.dc1 127.0.0.1 -42:59.949 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bed24f7e-5da9-5b20-55a6-53891a903e63 127.0.0.1 -2022-09-15T21:42:59.949Z [INFO] agent.router: Initializing LAN area manager -42:59.949 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13402 network=udp -42:59.949 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13442 [Follower]" leader= -42:59.950 [INFO] TestAgent.server: Adding LAN server: server="Node-bed24f7e-5da9-5b20-55a6-53891a903e63 (Addr: tcp/127.0.0.1:13442) (DC: dc1)" -42:59.950 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bed24f7e-5da9-5b20-55a6-53891a903e63.dc1 area=wan -42:59.950 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13402 network=tcp -42:59.951 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13403 network=tcp -42:59.951 [INFO] TestAgent: started state syncer -42:59.951 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13443 network=tcp -43:00.000 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -43:00.013 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:00.013 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13442 [Candidate]" term=2 -43:00.014 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:00.014 [DEBUG] TestAgent.server.raft: vote granted: from=bed24f7e-5da9-5b20-55a6-53891a903e63 term=2 tally=1 -43:00.014 [INFO] TestAgent.server.raft: election won: tally=1 -43:00.014 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13442 [Leader]" -43:00.014 [INFO] TestAgent.server: cluster leadership acquired -43:00.014 [INFO] TestAgent.server: initializing acls -43:00.014 [INFO] TestAgent.server: Created ACL 'global-management' policy -43:00.015 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -43:00.015 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -43:00.016 [INFO] TestAgent.server: Created ACL anonymous token from configuration -43:00.016 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -43:00.016 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -43:00.016 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-bed24f7e-5da9-5b20-55a6-53891a903e63 -43:00.016 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-bed24f7e-5da9-5b20-55a6-53891a903e63.dc1 -43:00.017 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:00.018 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-bed24f7e-5da9-5b20-55a6-53891a903e63.dc1 area=wan -43:00.018 [INFO] TestAgent.server: New leader elected: payload=Node-bed24f7e-5da9-5b20-55a6-53891a903e63 -43:00.018 [INFO] TestAgent.server: Updating LAN server: server="Node-bed24f7e-5da9-5b20-55a6-53891a903e63 (Addr: tcp/127.0.0.1:13442) (DC: dc1)" -43:00.020 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:00.020 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:00.020 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:00.020 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:00.020 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:00.020 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bed24f7e-5da9-5b20-55a6-53891a903e63 -43:00.020 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bed24f7e-5da9-5b20-55a6-53891a903e63 -43:00.021 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bed24f7e-5da9-5b20-55a6-53891a903e63 -43:00.021 [INFO] TestAgent.server: federation state anti-entropy synced -43:00.123 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-bed24f7e-5da9-5b20-55a6-53891a903e63 -43:00.279 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-bed24f7e-5da9-5b20-55a6-53891a903e63 -43:00.279 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:13426] -43:00.279 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:13426 -43:00.282 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ddda13d9-5cdc-9a80-0f2e-10418e12a6c6 127.0.0.1 -43:00.282 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 -43:00.282 [DEBUG] TestAgent: systemd notify failed: error="No socket" -43:00.285 [INFO] TestAgent.server: Adding LAN server: server="Node-ddda13d9-5cdc-9a80-0f2e-10418e12a6c6 (Addr: tcp/127.0.0.1:13456) (DC: dc1)" -43:00.285 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-ddda13d9-5cdc-9a80-0f2e-10418e12a6c6 other=Node-bed24f7e-5da9-5b20-55a6-53891a903e63 -43:00.285 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ddda13d9-5cdc-9a80-0f2e-10418e12a6c6 -43:00.292 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:34264 -43:00.293 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:13427: connect: connection refused -43:00.293 [DEBUG] TestAgent.server: Failed to flood-join server at address: server=Node-ddda13d9-5cdc-9a80-0f2e-10418e12a6c6.dc1 address=127.0.0.1:13427 error="1 error occurred: - * Failed to join 127.0.0.1: dial tcp 127.0.0.1:13427: connect: connection refused +2021-08-13T15:34:24.941Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:24.957Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:24.960 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ea002212-1203-d64b-6efc-71e52523f4dd Address:127.0.0.1:24991}]" +34:24.960 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24991 [Follower]" leader= +34:24.961 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ea002212-1203-d64b-6efc-71e52523f4dd.dc1 127.0.0.1 +34:24.962 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ea002212-1203-d64b-6efc-71e52523f4dd 127.0.0.1 +2021-08-13T15:34:24.962Z [INFO] agent.router: Initializing LAN area manager +34:24.962 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24986 network=udp +34:24.962 [INFO] TestAgent.server: Adding LAN server: server="Node-ea002212-1203-d64b-6efc-71e52523f4dd (Addr: tcp/127.0.0.1:24991) (DC: dc1)" +34:24.962 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ea002212-1203-d64b-6efc-71e52523f4dd.dc1 area=wan +34:24.962 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24986 network=tcp +34:24.963 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24987 network=tcp +34:24.963 [INFO] TestAgent: started state syncer +34:24.963 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24992 network=tcp +34:25.019 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:25.019 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24991 [Candidate]" term=2 +34:25.020 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:25.020 [DEBUG] TestAgent.server.raft: vote granted: from=ea002212-1203-d64b-6efc-71e52523f4dd term=2 tally=1 +34:25.020 [INFO] TestAgent.server.raft: election won: tally=1 +34:25.020 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24991 [Leader]" +34:25.020 [INFO] TestAgent.server: cluster leadership acquired +34:25.020 [INFO] TestAgent.server: New leader elected: payload=Node-ea002212-1203-d64b-6efc-71e52523f4dd +34:25.020 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24991 +34:25.021 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:25.024 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:25.024 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:25.024 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:25.024 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:25.024 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:25.024 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ea002212-1203-d64b-6efc-71e52523f4dd +34:25.024 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ea002212-1203-d64b-6efc-71e52523f4dd +34:25.024 [INFO] TestAgent.server: federation state anti-entropy synced +34:25.092 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:25.093 [INFO] TestAgent: Synced node info +34:25.093 [DEBUG] TestAgent: Node info in sync +34:25.362 [INFO] TestAgent: Requesting shutdown +34:25.362 [INFO] TestAgent.server: shutting down server +34:25.362 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:25.362 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:25.362 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:25.362 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:25.362 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:25.363 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:25.363 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:25.363 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:25.363 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +34:25.363 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:25.363 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2021-08-13T15:34:25.363Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:25.363Z [INFO] agent.router.manager: shutting down +34:25.364 [INFO] TestAgent: consul server down +34:25.364 [INFO] TestAgent: shutdown complete +34:25.364 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24986 network=tcp +34:25.364 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24986 network=udp +34:25.364 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24987 network=tcp +34:25.864 [INFO] TestAgent: Waiting for endpoints to shut down +34:25.864 [INFO] TestAgent: Endpoints down +--- PASS: TestKVSEndpoint_ListKeys (0.95s) +=== CONT TestKVSEndpoint_DELETE_CAS +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:25.413Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:25.429Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:25.431 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d15fb352-be3e-5676-212a-59ee6e79d2eb Address:127.0.0.1:25005}]" +34:25.432 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d15fb352-be3e-5676-212a-59ee6e79d2eb.dc1 127.0.0.1 +34:25.433 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d15fb352-be3e-5676-212a-59ee6e79d2eb 127.0.0.1 +2021-08-13T15:34:25.433Z [INFO] agent.router: Initializing LAN area manager +34:25.434 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25014 network=udp +34:25.434 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25005 [Follower]" leader= +34:25.434 [INFO] TestAgent.server: Adding LAN server: server="Node-d15fb352-be3e-5676-212a-59ee6e79d2eb (Addr: tcp/127.0.0.1:25005) (DC: dc1)" +34:25.434 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d15fb352-be3e-5676-212a-59ee6e79d2eb.dc1 area=wan +34:25.434 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25014 network=tcp +34:25.435 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25015 network=tcp +34:25.436 [INFO] TestAgent: started state syncer +34:25.436 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25027 network=tcp +34:25.500 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:25.500 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25005 [Candidate]" term=2 +34:25.500 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:25.500 [DEBUG] TestAgent.server.raft: vote granted: from=d15fb352-be3e-5676-212a-59ee6e79d2eb term=2 tally=1 +34:25.500 [INFO] TestAgent.server.raft: election won: tally=1 +34:25.500 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25005 [Leader]" +34:25.500 [INFO] TestAgent.server: cluster leadership acquired +34:25.500 [INFO] TestAgent.server: New leader elected: payload=Node-d15fb352-be3e-5676-212a-59ee6e79d2eb +34:25.501 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25005 +34:25.502 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:25.504 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:25.504 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:25.504 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:25.504 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:25.504 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:25.504 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d15fb352-be3e-5676-212a-59ee6e79d2eb +34:25.504 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d15fb352-be3e-5676-212a-59ee6e79d2eb +34:25.505 [INFO] TestAgent.server: federation state anti-entropy synced +34:25.611 [INFO] TestAgent: Requesting shutdown +34:25.611 [INFO] TestAgent.server: shutting down server +34:25.611 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:25.611 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:25.611 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:25.611 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:25.611 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:25.611 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:25.612 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:25.612 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:25.612 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:25.612 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:25.612 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:25.612 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:25.612Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:25.612Z [INFO] agent.router.manager: shutting down +34:25.612 [INFO] TestAgent: consul server down +34:25.612 [INFO] TestAgent: shutdown complete +34:25.612 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25014 network=tcp +34:25.612 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25014 network=udp +34:25.612 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25015 network=tcp +34:26.112 [INFO] TestAgent: Waiting for endpoints to shut down +34:26.112 [INFO] TestAgent: Endpoints down +--- PASS: TestKVSEndpoint_CAS (0.72s) +=== CONT TestAgent_Join_WAN +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:25.881Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:25.896Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:25.898 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8af264e9-5b54-cce3-a322-c02232bf1653 Address:127.0.0.1:24639}]" +34:25.898 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24639 [Follower]" leader= +34:25.899 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8af264e9-5b54-cce3-a322-c02232bf1653.dc1 127.0.0.1 +34:25.900 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8af264e9-5b54-cce3-a322-c02232bf1653 127.0.0.1 +2021-08-13T15:34:25.900Z [INFO] agent.router: Initializing LAN area manager +34:25.900 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8af264e9-5b54-cce3-a322-c02232bf1653.dc1 area=wan +34:25.900 [INFO] TestAgent.server: Adding LAN server: server="Node-8af264e9-5b54-cce3-a322-c02232bf1653 (Addr: tcp/127.0.0.1:24639) (DC: dc1)" +34:25.900 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24488 network=tcp +34:25.900 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24488 network=udp +34:25.901 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24486 network=tcp +34:25.901 [INFO] TestAgent: started state syncer +34:25.901 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24640 network=tcp +34:25.951 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:25.951 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24639 [Candidate]" term=2 +34:25.951 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:25.951 [DEBUG] TestAgent.server.raft: vote granted: from=8af264e9-5b54-cce3-a322-c02232bf1653 term=2 tally=1 +34:25.951 [INFO] TestAgent.server.raft: election won: tally=1 +34:25.951 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24639 [Leader]" +34:25.951 [INFO] TestAgent.server: cluster leadership acquired +34:25.951 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24639 +34:25.952 [INFO] TestAgent.server: New leader elected: payload=Node-8af264e9-5b54-cce3-a322-c02232bf1653 +34:25.953 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:25.955 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:25.955 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:25.955 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:25.955 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:25.955 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:25.955 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8af264e9-5b54-cce3-a322-c02232bf1653 +34:25.955 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8af264e9-5b54-cce3-a322-c02232bf1653 +34:25.956 [INFO] TestAgent.server: federation state anti-entropy synced +34:25.986 [INFO] TestAgent: Requesting shutdown +34:25.986 [INFO] TestAgent.server: shutting down server +34:25.986 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:25.986 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:25.986 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:25.986 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:25.986 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:25.986 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:25.986 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:25.986 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:25.986 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:25.986 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:25.986 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:25.986 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:25.987Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:25.987Z [INFO] agent.router.manager: shutting down +34:25.987 [INFO] TestAgent: consul server down +34:25.987 [INFO] TestAgent: shutdown complete +34:25.987 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24488 network=tcp +34:25.987 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24488 network=udp +34:25.987 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24486 network=tcp +34:26.487 [INFO] TestAgent: Waiting for endpoints to shut down +34:26.487 [INFO] TestAgent: Endpoints down +--- PASS: TestKVSEndpoint_DELETE_CAS (0.62s) +=== CONT TestAgent_Join +34:25.504 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-5d4d103c-234e-a97f-15de-53c8ff247e4d 127.0.0.1 +2021-08-13T15:34:25.504Z [INFO] agent.router: Initializing LAN area manager +34:25.504 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24979 network=tcp +34:25.504 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24979 network=udp +34:25.505 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24980 network=tcp +34:25.505 [INFO] TestAgent: started state syncer +34:25.505 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25006 network=tcp +2021-08-13T15:34:25.505Z [WARN] agent.router.manager: No servers available +34:25.505 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" +34:25.506 [DEBUG] TestAgent.client.memberlist.lan: memberlist: Stream connection from=127.0.0.1:36174 +34:25.507 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-6ebc2cc2-a10b-9b0c-5353-39d24ecfd1c8 127.0.0.1 +34:25.507 [INFO] TestAgent: Requesting shutdown +34:25.507 [INFO] TestAgent.client: adding server: server="Node-6ebc2cc2-a10b-9b0c-5353-39d24ecfd1c8 (Addr: tcp/127.0.0.1:24998) (DC: dc1)" +34:25.507 [INFO] TestAgent.client: New leader elected: payload=Node-6ebc2cc2-a10b-9b0c-5353-39d24ecfd1c8 +34:25.507 [INFO] TestAgent.client: shutting down client +34:25.507 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave +34:25.507 [INFO] TestAgent: consul client down +34:25.507 [INFO] TestAgent: shutdown complete +34:25.507 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24979 network=tcp +34:25.508 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24979 network=udp +34:25.508 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24980 network=tcp +34:26.008 [INFO] TestAgent: Waiting for endpoints to shut down +34:26.008 [INFO] TestAgent: Endpoints down +2021-08-13T15:34:25.093Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:25.109Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:25.112 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6ebc2cc2-a10b-9b0c-5353-39d24ecfd1c8 Address:127.0.0.1:24998}]" +34:25.112 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24998 [Follower]" leader= +34:25.113 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6ebc2cc2-a10b-9b0c-5353-39d24ecfd1c8.dc1 127.0.0.1 +34:25.114 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6ebc2cc2-a10b-9b0c-5353-39d24ecfd1c8 127.0.0.1 +2021-08-13T15:34:25.114Z [INFO] agent.router: Initializing LAN area manager +34:25.114 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6ebc2cc2-a10b-9b0c-5353-39d24ecfd1c8.dc1 area=wan +34:25.114 [INFO] TestAgent.server: Adding LAN server: server="Node-6ebc2cc2-a10b-9b0c-5353-39d24ecfd1c8 (Addr: tcp/127.0.0.1:24998) (DC: dc1)" +34:25.115 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24993 network=tcp +34:25.115 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24993 network=udp +34:25.115 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24994 network=tcp +34:25.116 [INFO] TestAgent: started state syncer +34:25.116 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25013 network=tcp +34:25.175 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:25.175 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24998 [Candidate]" term=2 +34:25.176 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:25.176 [DEBUG] TestAgent.server.raft: vote granted: from=6ebc2cc2-a10b-9b0c-5353-39d24ecfd1c8 term=2 tally=1 +34:25.176 [INFO] TestAgent.server.raft: election won: tally=1 +34:25.176 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24998 [Leader]" +34:25.176 [INFO] TestAgent.server: cluster leadership acquired +34:25.176 [INFO] TestAgent.server: New leader elected: payload=Node-6ebc2cc2-a10b-9b0c-5353-39d24ecfd1c8 +34:25.176 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24998 +34:25.177 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:25.180 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:25.180 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:25.180 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:25.180 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:25.180 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:25.180 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6ebc2cc2-a10b-9b0c-5353-39d24ecfd1c8 +34:25.180 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6ebc2cc2-a10b-9b0c-5353-39d24ecfd1c8 +34:25.180 [INFO] TestAgent.server: federation state anti-entropy synced +34:25.506 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:24982] +34:25.506 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:24982 +34:25.507 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5d4d103c-234e-a97f-15de-53c8ff247e4d 127.0.0.1 +34:25.507 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 +34:25.507 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5d4d103c-234e-a97f-15de-53c8ff247e4d +34:25.542 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:25.542 [INFO] TestAgent: Synced node info +34:26.008 [INFO] TestAgent: Requesting shutdown +34:26.008 [INFO] TestAgent.server: shutting down server +34:26.008 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:26.008 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:26.008 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:26.008 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:26.008 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:26.008 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:26.008 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:26.008 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:26.008 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:26.008 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:26.008 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:26.008Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:26.008Z [INFO] agent.router.manager: shutting down +34:26.008 [INFO] TestAgent: consul server down +34:26.009 [INFO] TestAgent: shutdown complete +34:26.009 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24993 network=tcp +34:26.009 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24993 network=udp +34:26.009 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24994 network=tcp +34:26.511 [INFO] TestAgent: Waiting for endpoints to shut down +34:26.511 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_JoinLANNotify (1.44s) +=== CONT TestKVSEndpoint_PUT_GET_DELETE +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:25.627Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:25.643Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:25.645 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:aff32e90-e5dc-67a3-6db3-4409879c6529 Address:127.0.0.1:25026}]" +34:25.646 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25026 [Follower]" leader= +34:25.646 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-aff32e90-e5dc-67a3-6db3-4409879c6529.dc1 127.0.0.1 +34:25.647 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-aff32e90-e5dc-67a3-6db3-4409879c6529 127.0.0.1 +2021-08-13T15:34:25.647Z [INFO] agent.router: Initializing LAN area manager +34:25.647 [INFO] TestAgent.server: Adding LAN server: server="Node-aff32e90-e5dc-67a3-6db3-4409879c6529 (Addr: tcp/127.0.0.1:25026) (DC: dc1)" +34:25.647 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-aff32e90-e5dc-67a3-6db3-4409879c6529.dc1 area=wan +34:25.648 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25007 network=tcp +34:25.648 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25007 network=udp +34:25.648 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25008 network=tcp +34:25.649 [INFO] TestAgent: started state syncer +34:25.649 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24487 network=tcp +34:25.694 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:25.694 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25026 [Candidate]" term=2 +34:25.694 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:25.694 [DEBUG] TestAgent.server.raft: vote granted: from=aff32e90-e5dc-67a3-6db3-4409879c6529 term=2 tally=1 +34:25.694 [INFO] TestAgent.server.raft: election won: tally=1 +34:25.694 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25026 [Leader]" +34:25.694 [INFO] TestAgent.server: cluster leadership acquired +34:25.694 [INFO] TestAgent.server: New leader elected: payload=Node-aff32e90-e5dc-67a3-6db3-4409879c6529 +34:25.695 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25026 +34:25.696 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:25.698 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:25.698 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:25.698 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:25.698 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:25.698 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:25.698 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-aff32e90-e5dc-67a3-6db3-4409879c6529 +34:25.698 [INFO] TestAgent.server: member joined, marking health alive: member=Node-aff32e90-e5dc-67a3-6db3-4409879c6529 +34:25.699 [INFO] TestAgent.server: federation state anti-entropy synced +34:25.835 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:57762 +34:25.836 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f873eb52-1e20-b005-573b-8ab12537fa81 127.0.0.1 +34:25.837 [INFO] TestAgent.server: Adding LAN server: server="Node-f873eb52-1e20-b005-573b-8ab12537fa81 (Addr: tcp/127.0.0.1:25019) (DC: dc1)" +34:25.837 [INFO] TestAgent.server: New leader elected: payload=Node-f873eb52-1e20-b005-573b-8ab12537fa81 +34:25.837 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-f873eb52-1e20-b005-573b-8ab12537fa81 other=Node-aff32e90-e5dc-67a3-6db3-4409879c6529 +34:25.837 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f873eb52-1e20-b005-573b-8ab12537fa81 +34:25.838 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: Node-f873eb52-1e20-b005-573b-8ab12537fa81.dc1 127.0.0.1:25018 +34:25.838 [INFO] TestAgent: Requesting shutdown +34:25.838 [INFO] TestAgent.server: shutting down server +34:25.838 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:25.838 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:25.838 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:25.838 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:25.838 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:25.839 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:25.839 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:25.839 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:25.839 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:25.839 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:25.839 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:25.839 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:25.839Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:25.839Z [INFO] agent.router.manager: shutting down +34:25.840 [INFO] TestAgent: consul server down +34:25.840 [INFO] TestAgent: shutdown complete +34:25.840 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25007 network=tcp +34:25.840 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25007 network=udp +34:25.840 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25008 network=tcp +34:25.841 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f873eb52-1e20-b005-573b-8ab12537fa81.dc1 127.0.0.1 +34:25.841 [DEBUG] TestAgent.server: Successfully performed flood-join for server at address: server=Node-f873eb52-1e20-b005-573b-8ab12537fa81.dc1 address=127.0.0.1:25018 +34:26.340 [INFO] TestAgent: Waiting for endpoints to shut down +34:26.340 [INFO] TestAgent: Endpoints down +2021-08-13T15:34:25.458Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:25.458Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:25.474Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:25.474Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:25.477 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f873eb52-1e20-b005-573b-8ab12537fa81 Address:127.0.0.1:25019}]" +34:25.477 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25019 [Follower]" leader= +34:25.477 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f873eb52-1e20-b005-573b-8ab12537fa81.dc1 127.0.0.1 +34:25.478 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f873eb52-1e20-b005-573b-8ab12537fa81 127.0.0.1 +2021-08-13T15:34:25.478Z [INFO] agent.router: Initializing LAN area manager +34:25.479 [INFO] TestAgent.server: Adding LAN server: server="Node-f873eb52-1e20-b005-573b-8ab12537fa81 (Addr: tcp/127.0.0.1:25019) (DC: dc1)" +34:25.479 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f873eb52-1e20-b005-573b-8ab12537fa81.dc1 area=wan +34:25.479 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25028 network=udp +34:25.479 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25028 network=tcp +34:25.480 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25029 network=tcp +34:25.480 [INFO] TestAgent: started state syncer +34:25.480 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24978 network=tcp +34:25.517 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:25.517 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25019 [Candidate]" term=2 +34:25.517 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:25.517 [DEBUG] TestAgent.server.raft: vote granted: from=f873eb52-1e20-b005-573b-8ab12537fa81 term=2 tally=1 +34:25.517 [INFO] TestAgent.server.raft: election won: tally=1 +34:25.517 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25019 [Leader]" +34:25.517 [INFO] TestAgent.server: cluster leadership acquired +34:25.518 [INFO] TestAgent.server: New leader elected: payload=Node-f873eb52-1e20-b005-573b-8ab12537fa81 +34:25.518 [INFO] TestAgent.server: initializing acls +34:25.518 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:25.518 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:25.519 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:25.519 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:25.519 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:25.519 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:25.520 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-f873eb52-1e20-b005-573b-8ab12537fa81 +34:25.520 [INFO] TestAgent.server: Updating LAN server: server="Node-f873eb52-1e20-b005-573b-8ab12537fa81 (Addr: tcp/127.0.0.1:25019) (DC: dc1)" +34:25.520 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-f873eb52-1e20-b005-573b-8ab12537fa81.dc1 +34:25.520 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-f873eb52-1e20-b005-573b-8ab12537fa81.dc1 area=wan +34:25.521 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:25.523 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:25.523 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:25.523 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:25.523 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:25.523 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:25.523 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f873eb52-1e20-b005-573b-8ab12537fa81 +34:25.523 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f873eb52-1e20-b005-573b-8ab12537fa81 +34:25.524 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f873eb52-1e20-b005-573b-8ab12537fa81 +34:25.524 [INFO] TestAgent.server: federation state anti-entropy synced +34:25.610 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-f873eb52-1e20-b005-573b-8ab12537fa81 +34:25.835 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-f873eb52-1e20-b005-573b-8ab12537fa81 +34:25.835 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:25024] +34:25.835 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:25024 +34:25.837 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-aff32e90-e5dc-67a3-6db3-4409879c6529 127.0.0.1 +34:25.837 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 +34:25.837 [DEBUG] TestAgent: systemd notify failed: error="No socket" +34:25.839 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:52980 +34:25.839 [INFO] TestAgent.server: Adding LAN server: server="Node-aff32e90-e5dc-67a3-6db3-4409879c6529 (Addr: tcp/127.0.0.1:25026) (DC: dc1)" +34:25.839 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-aff32e90-e5dc-67a3-6db3-4409879c6529 other=Node-f873eb52-1e20-b005-573b-8ab12537fa81 +34:25.839 [INFO] TestAgent.server: member joined, marking health alive: member=Node-aff32e90-e5dc-67a3-6db3-4409879c6529 +34:25.840 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:25025: connect: connection refused +34:25.840 [DEBUG] TestAgent.server: Failed to flood-join server at address: server=Node-aff32e90-e5dc-67a3-6db3-4409879c6529.dc1 address=127.0.0.1:25025 error="1 error occurred: + * Failed to join 127.0.0.1: dial tcp 127.0.0.1:25025: connect: connection refused " -43:00.293 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ddda13d9-5cdc-9a80-0f2e-10418e12a6c6.dc1 127.0.0.1 -43:00.293 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ddda13d9-5cdc-9a80-0f2e-10418e12a6c6.dc1 area=wan -43:00.372 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:00.373 [INFO] TestAgent: Synced node info -43:00.373 [DEBUG] TestAgent: Node info in sync -43:00.793 [INFO] TestAgent: Requesting shutdown -43:00.793 [INFO] TestAgent.server: shutting down server -43:00.793 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:00.793 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:00.793 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:00.793 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:00.793 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -43:00.793 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -43:00.793 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:00.794 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:00.794 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:00.794 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:00.794 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:00.794 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:00.794 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -43:00.794 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -43:00.794 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:00.794Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:00.794Z [INFO] agent.router.manager: shutting down -43:00.794 [INFO] TestAgent: consul server down -43:00.794 [INFO] TestAgent: shutdown complete -43:00.794 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13402 network=tcp -43:00.794 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13402 network=udp -43:00.794 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13403 network=tcp -43:01.294 [INFO] TestAgent: Waiting for endpoints to shut down -43:01.294 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Join_ACLDeny (1.39s) +34:25.840 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-aff32e90-e5dc-67a3-6db3-4409879c6529.dc1 127.0.0.1 +34:25.840 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-aff32e90-e5dc-67a3-6db3-4409879c6529.dc1 area=wan +34:25.856 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:25.857 [INFO] TestAgent: Synced node info +34:26.340 [INFO] TestAgent: Requesting shutdown +34:26.340 [INFO] TestAgent.server: shutting down server +34:26.340 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:26.340 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:26.340 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:26.340 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:26.340 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:26.340 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:26.340 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:26.340 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:26.340 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:26.340 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:26.340 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:26.340 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:26.340 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:26.340 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:26.340 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:26.340Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:26.340Z [INFO] agent.router.manager: shutting down +34:26.341 [INFO] TestAgent: consul server down +34:26.341 [INFO] TestAgent: shutdown complete +34:26.341 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25028 network=tcp +34:26.341 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25028 network=udp +34:26.341 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25029 network=tcp +34:26.841 [INFO] TestAgent: Waiting for endpoints to shut down +34:26.841 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Join_ACLDeny (1.40s) --- PASS: TestAgent_Join_ACLDeny/no_token (0.00s) --- PASS: TestAgent_Join_ACLDeny/agent_master_token (0.00s) - --- PASS: TestAgent_Join_ACLDeny/read-only_token (0.01s) -=== CONT TestEnableWebUI -43:00.332 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-946489c1-8edd-ca73-6756-7aa6d0cfff0c 127.0.0.1 -2022-09-15T21:43:00.332Z [INFO] agent.router: Initializing LAN area manager -43:00.332 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13409 network=udp -43:00.332 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13409 network=tcp -43:00.333 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13410 network=tcp -43:00.333 [INFO] TestAgent: started state syncer -43:00.334 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13464 network=tcp -2022-09-15T21:43:00.334Z [WARN] agent.router.manager: No servers available -43:00.334 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" -43:00.335 [DEBUG] TestAgent.client.memberlist.lan: memberlist: Stream connection from=127.0.0.1:60940 -43:00.336 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-7cd42364-4014-b517-4751-517d216f1f5e 127.0.0.1 -43:00.336 [INFO] TestAgent: Requesting shutdown -43:00.336 [INFO] TestAgent.client: shutting down client -43:00.336 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave -43:00.336 [INFO] TestAgent: consul client down -43:00.336 [INFO] TestAgent: shutdown complete -43:00.336 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13409 network=tcp -43:00.336 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13409 network=udp -43:00.336 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13410 network=tcp -43:00.837 [INFO] TestAgent: Waiting for endpoints to shut down -43:00.837 [INFO] TestAgent: Endpoints down -2022-09-15T21:42:59.860Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:59.899Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:59.902 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7cd42364-4014-b517-4751-517d216f1f5e Address:127.0.0.1:13393}]" -42:59.902 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13393 [Follower]" leader= -42:59.910 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7cd42364-4014-b517-4751-517d216f1f5e.dc1 127.0.0.1 -42:59.911 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7cd42364-4014-b517-4751-517d216f1f5e 127.0.0.1 -2022-09-15T21:42:59.911Z [INFO] agent.router: Initializing LAN area manager -42:59.912 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13381 network=udp -42:59.912 [INFO] TestAgent.server: Adding LAN server: server="Node-7cd42364-4014-b517-4751-517d216f1f5e (Addr: tcp/127.0.0.1:13393) (DC: dc1)" -42:59.912 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7cd42364-4014-b517-4751-517d216f1f5e.dc1 area=wan -42:59.912 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13381 network=tcp -42:59.913 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13382 network=tcp -42:59.913 [INFO] TestAgent: started state syncer -42:59.913 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13394 network=tcp -42:59.954 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:59.954 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13393 [Candidate]" term=2 -42:59.954 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:59.954 [DEBUG] TestAgent.server.raft: vote granted: from=7cd42364-4014-b517-4751-517d216f1f5e term=2 tally=1 -42:59.954 [INFO] TestAgent.server.raft: election won: tally=1 -42:59.954 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13393 [Leader]" -42:59.954 [INFO] TestAgent.server: cluster leadership acquired -42:59.954 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13393 -42:59.955 [INFO] TestAgent.server: New leader elected: payload=Node-7cd42364-4014-b517-4751-517d216f1f5e -42:59.956 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:59.960 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:59.960 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:59.960 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:59.960 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:59.960 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:59.960 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7cd42364-4014-b517-4751-517d216f1f5e -42:59.960 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7cd42364-4014-b517-4751-517d216f1f5e -42:59.964 [INFO] TestAgent.server: federation state anti-entropy synced -43:00.163 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:00.164 [INFO] TestAgent: Synced node info -43:00.334 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:13412] -43:00.335 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:13412 -43:00.336 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-946489c1-8edd-ca73-6756-7aa6d0cfff0c 127.0.0.1 -43:00.336 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 -43:00.336 [INFO] TestAgent.server: member joined, marking health alive: member=Node-946489c1-8edd-ca73-6756-7aa6d0cfff0c -43:00.837 [INFO] TestAgent: Requesting shutdown -43:00.837 [INFO] TestAgent.server: shutting down server -43:00.837 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:00.837 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:00.837 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:00.837 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:00.837 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:00.837 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:00.838 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:00.838 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:00.838 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:00.838 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:00.838 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:00.838Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:00.838Z [INFO] agent.router.manager: shutting down -43:00.838 [INFO] TestAgent: consul server down -43:00.838 [INFO] TestAgent: shutdown complete -43:00.838 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13381 network=tcp -43:00.838 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13381 network=udp -43:00.838 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13382 network=tcp -43:01.341 [INFO] TestAgent: Waiting for endpoints to shut down -43:01.341 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_JoinLANNotify (1.58s) -=== CONT TestACLResolution -[WARN] freeport: 4 out of 32 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 4 out of 4 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:01.149Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:01.167Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:01.171 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:30ed154d-05f3-12dd-0047-400ce421053e Address:127.0.0.1:13435}]" -43:01.171 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13435 [Follower]" leader= -43:01.172 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-30ed154d-05f3-12dd-0047-400ce421053e.dc1 127.0.0.1 -43:01.173 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-30ed154d-05f3-12dd-0047-400ce421053e 127.0.0.1 -2022-09-15T21:43:01.173Z [INFO] agent.router: Initializing LAN area manager -43:01.173 [INFO] TestAgent.server: Adding LAN server: server="Node-30ed154d-05f3-12dd-0047-400ce421053e (Addr: tcp/127.0.0.1:13435) (DC: dc1)" -43:01.173 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-30ed154d-05f3-12dd-0047-400ce421053e.dc1 area=wan -43:01.174 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13444 network=udp -43:01.174 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13444 network=tcp -43:01.174 [INFO] TestAgent: Started HTTPS server: address=127.0.0.1:13446 network=tcp -43:01.175 [INFO] TestAgent: started state syncer -43:01.175 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13436 network=tcp -43:01.234 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:01.234 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13435 [Candidate]" term=2 -43:01.234 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:01.234 [DEBUG] TestAgent.server.raft: vote granted: from=30ed154d-05f3-12dd-0047-400ce421053e term=2 tally=1 -43:01.234 [INFO] TestAgent.server.raft: election won: tally=1 -43:01.234 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13435 [Leader]" -43:01.235 [INFO] TestAgent.server: cluster leadership acquired -43:01.235 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13435 -43:01.235 [INFO] TestAgent.server: New leader elected: payload=Node-30ed154d-05f3-12dd-0047-400ce421053e -43:01.236 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:01.239 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:01.239 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:01.239 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:01.239 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:01.239 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:01.239 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-30ed154d-05f3-12dd-0047-400ce421053e -43:01.239 [INFO] TestAgent.server: member joined, marking health alive: member=Node-30ed154d-05f3-12dd-0047-400ce421053e -43:01.240 [INFO] TestAgent.server: federation state anti-entropy synced -43:01.339 [INFO] TestAgent: Requesting shutdown -43:01.339 [INFO] TestAgent.server: shutting down server -43:01.339 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:01.339 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:01.339 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:01.339 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:01.339 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:01.339 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:01.339 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:01.339 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:01.339 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:01.339 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:01.339 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:01.339 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:01.340Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:01.340Z [INFO] agent.router.manager: shutting down -43:01.340 [INFO] TestAgent: consul server down -43:01.340 [INFO] TestAgent: shutdown complete -43:01.340 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13444 network=tcp -43:01.340 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13444 network=udp -43:01.340 [INFO] TestAgent: Stopping server: protocol=HTTPS address=127.0.0.1:13446 network=tcp -43:01.841 [INFO] TestAgent: Waiting for endpoints to shut down -43:01.841 [INFO] TestAgent: Endpoints down ---- PASS: TestHTTPServer_HandshakeTimeout (0.71s) -=== CONT TestParseConsistency_Invalid -2022-09-15T21:43:01.051Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:01.068Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:01.071 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b0b41ba4-c23e-a6c0-27d8-b7100df8275e Address:127.0.0.1:13484}]" -43:01.071 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13484 [Follower]" leader= -43:01.072 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b0b41ba4-c23e-a6c0-27d8-b7100df8275e.dc1 127.0.0.1 -43:01.073 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b0b41ba4-c23e-a6c0-27d8-b7100df8275e 127.0.0.1 -2022-09-15T21:43:01.073Z [INFO] agent.router: Initializing LAN area manager -43:01.073 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13465 network=udp -43:01.074 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b0b41ba4-c23e-a6c0-27d8-b7100df8275e.dc1 area=wan -43:01.074 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13465 network=tcp -43:01.075 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13466 network=tcp -43:01.075 [INFO] TestAgent: started state syncer -43:01.075 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13485 network=tcp -43:01.076 [INFO] TestAgent.server: Adding LAN server: server="Node-b0b41ba4-c23e-a6c0-27d8-b7100df8275e (Addr: tcp/127.0.0.1:13484) (DC: dc1)" -43:01.112 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:01.112 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13484 [Candidate]" term=2 -43:01.112 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:01.112 [DEBUG] TestAgent.server.raft: vote granted: from=b0b41ba4-c23e-a6c0-27d8-b7100df8275e term=2 tally=1 -43:01.112 [INFO] TestAgent.server.raft: election won: tally=1 -43:01.112 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13484 [Leader]" -43:01.112 [INFO] TestAgent.server: cluster leadership acquired -43:01.113 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13484 -43:01.114 [INFO] TestAgent.server: New leader elected: payload=Node-b0b41ba4-c23e-a6c0-27d8-b7100df8275e -43:01.114 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:01.135 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:01.135 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:01.135 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:01.135 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:01.135 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:01.135 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b0b41ba4-c23e-a6c0-27d8-b7100df8275e -43:01.135 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b0b41ba4-c23e-a6c0-27d8-b7100df8275e -43:01.136 [INFO] TestAgent.server: federation state anti-entropy synced -43:01.251 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:01.251 [INFO] TestAgent: Synced node info -43:01.340 [INFO] TestAgent: Requesting shutdown -43:01.340 [INFO] TestAgent.server: shutting down server -43:01.340 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:01.340 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:01.340 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:01.340 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:01.340 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:01.340 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:01.340 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:01.340 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:01.341 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:01.341 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:01.341 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:01.341Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:01.341Z [INFO] agent.router.manager: shutting down -43:01.341 [INFO] TestAgent: consul server down -43:01.341 [INFO] TestAgent: shutdown complete -43:01.341 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13465 network=tcp -43:01.341 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13465 network=udp -43:01.341 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13466 network=tcp -43:01.841 [INFO] TestAgent: Waiting for endpoints to shut down -43:01.841 [INFO] TestAgent: Endpoints down ---- PASS: TestParseConsistency (0.85s) -=== CONT TestACL_filterChecks -2022-09-15T21:43:01.859Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:01.859Z [WARN] agent: bootstrap = true: do not enable unless necessary -43:01.860 [DEBUG] TestACL_filterChecks: dropping check from result due to ACLs: check=my-other -43:01.860 [DEBUG] TestACL_filterChecks: dropping check from result due to ACLs: check=my-node -43:01.860 [DEBUG] TestACL_filterChecks: dropping check from result due to ACLs: check=my-service -43:01.860 [DEBUG] TestACL_filterChecks: dropping check from result due to ACLs: check=my-other ---- PASS: TestACL_filterChecks (0.02s) + --- PASS: TestAgent_Join_ACLDeny/read-only_token (0.00s) +=== CONT TestAgentKeyring_ACL +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:26.536Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:26.556Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:26.559 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7407137c-d11d-375f-31f4-8d6019040904 Address:127.0.0.1:25030}]" +34:26.559 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25030 [Follower]" leader= +34:26.560 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7407137c-d11d-375f-31f4-8d6019040904.dc1 127.0.0.1 +34:26.561 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7407137c-d11d-375f-31f4-8d6019040904 127.0.0.1 +2021-08-13T15:34:26.561Z [INFO] agent.router: Initializing LAN area manager +34:26.561 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25011 network=udp +34:26.561 [INFO] TestAgent.server: Adding LAN server: server="Node-7407137c-d11d-375f-31f4-8d6019040904 (Addr: tcp/127.0.0.1:25030) (DC: dc1)" +34:26.561 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7407137c-d11d-375f-31f4-8d6019040904.dc1 area=wan +34:26.562 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25011 network=tcp +34:26.562 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25012 network=tcp +34:26.563 [INFO] TestAgent: started state syncer +34:26.563 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25031 network=tcp +34:26.629 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:26.629 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25030 [Candidate]" term=2 +34:26.629 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:26.629 [DEBUG] TestAgent.server.raft: vote granted: from=7407137c-d11d-375f-31f4-8d6019040904 term=2 tally=1 +34:26.629 [INFO] TestAgent.server.raft: election won: tally=1 +34:26.629 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25030 [Leader]" +34:26.629 [INFO] TestAgent.server: cluster leadership acquired +34:26.630 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25030 +34:26.631 [INFO] TestAgent.server: New leader elected: payload=Node-7407137c-d11d-375f-31f4-8d6019040904 +34:26.631 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:26.634 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:26.634 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:26.634 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:26.634 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:26.634 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:26.634 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7407137c-d11d-375f-31f4-8d6019040904 +34:26.634 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7407137c-d11d-375f-31f4-8d6019040904 +34:26.635 [INFO] TestAgent.server: federation state anti-entropy synced +34:26.729 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:26.730 [INFO] TestAgent: Synced node info +34:26.730 [DEBUG] TestAgent: Node info in sync +34:26.859 [INFO] TestAgent: Requesting shutdown +34:26.859 [INFO] TestAgent.server: shutting down server +34:26.859 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:26.859 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:26.859 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:26.859 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:26.859 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:26.859 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:26.859 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:26.859 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:26.859 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:26.859 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:26.859 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:26.859Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:26.859Z [INFO] agent.router.manager: shutting down +34:26.860 [INFO] TestAgent: consul server down +34:26.860 [INFO] TestAgent: shutdown complete +34:26.860 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25011 network=tcp +34:26.860 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25011 network=udp +34:26.860 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25012 network=tcp +34:27.360 [INFO] TestAgent: Waiting for endpoints to shut down +34:27.360 [INFO] TestAgent: Endpoints down +--- PASS: TestKVSEndpoint_PUT_GET_DELETE (0.85s) +=== CONT TestParseConsistency +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:26.509Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:26.525Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:26.528 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4595772f-1869-7714-63fb-6bdb03734739 Address:127.0.0.1:25009}]" +34:26.529 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4595772f-1869-7714-63fb-6bdb03734739.dc1 127.0.0.1 +34:26.530 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4595772f-1869-7714-63fb-6bdb03734739 127.0.0.1 +2021-08-13T15:34:26.530Z [INFO] agent.router: Initializing LAN area manager +34:26.530 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24508 network=udp +34:26.530 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25009 [Follower]" leader= +34:26.531 [INFO] TestAgent.server: Adding LAN server: server="Node-4595772f-1869-7714-63fb-6bdb03734739 (Addr: tcp/127.0.0.1:25009) (DC: dc1)" +34:26.531 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4595772f-1869-7714-63fb-6bdb03734739.dc1 area=wan +34:26.531 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24508 network=tcp +34:26.532 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24509 network=tcp +34:26.532 [INFO] TestAgent: started state syncer +34:26.532 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25010 network=tcp +34:26.594 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:26.595 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25009 [Candidate]" term=2 +34:26.595 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:26.595 [DEBUG] TestAgent.server.raft: vote granted: from=4595772f-1869-7714-63fb-6bdb03734739 term=2 tally=1 +34:26.595 [INFO] TestAgent.server.raft: election won: tally=1 +34:26.595 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25009 [Leader]" +34:26.595 [INFO] TestAgent.server: cluster leadership acquired +34:26.595 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25009 +34:26.597 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:26.598 [INFO] TestAgent.server: New leader elected: payload=Node-4595772f-1869-7714-63fb-6bdb03734739 +34:26.605 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:26.605 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:26.605 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:26.605 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:26.605 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:26.605 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4595772f-1869-7714-63fb-6bdb03734739 +34:26.605 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4595772f-1869-7714-63fb-6bdb03734739 +34:26.606 [INFO] TestAgent.server: federation state anti-entropy synced +34:26.737 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:37354 +34:26.737 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-46ea8226-9b6c-059d-dfdb-20e42bc6a757.dc1 127.0.0.1 +34:26.738 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-46ea8226-9b6c-059d-dfdb-20e42bc6a757.dc1 area=wan +34:26.738 [INFO] TestAgent: Requesting shutdown +34:26.738 [INFO] TestAgent.server: shutting down server +34:26.738 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:26.738 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:26.738 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:26.738 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:26.738 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:26.738 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:26.738 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:26.738 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:26.738 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:26.739 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:26.739 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:26.739 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:26.739Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:26.739Z [INFO] agent.router.manager: shutting down +34:26.739 [INFO] TestAgent: consul server down +34:26.739 [INFO] TestAgent: shutdown complete +34:26.739 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24508 network=tcp +34:26.739 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24508 network=udp +34:26.739 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24509 network=tcp +34:27.243 [INFO] TestAgent: Waiting for endpoints to shut down +34:27.243 [INFO] TestAgent: Endpoints down +2021-08-13T15:34:26.129Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:26.174Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:26.183 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:46ea8226-9b6c-059d-dfdb-20e42bc6a757 Address:127.0.0.1:24632}]" +34:26.183 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24632 [Follower]" leader= +34:26.184 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-46ea8226-9b6c-059d-dfdb-20e42bc6a757.dc1 127.0.0.1 +34:26.186 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-46ea8226-9b6c-059d-dfdb-20e42bc6a757 127.0.0.1 +2021-08-13T15:34:26.186Z [INFO] agent.router: Initializing LAN area manager +34:26.187 [INFO] TestAgent.server: Adding LAN server: server="Node-46ea8226-9b6c-059d-dfdb-20e42bc6a757 (Addr: tcp/127.0.0.1:24632) (DC: dc1)" +34:26.187 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24641 network=udp +34:26.187 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24641 network=tcp +34:26.188 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24642 network=tcp +34:26.188 [INFO] TestAgent: started state syncer +34:26.188 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24633 network=tcp +34:26.191 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-46ea8226-9b6c-059d-dfdb-20e42bc6a757.dc1 area=wan +34:26.226 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:26.226 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24632 [Candidate]" term=2 +34:26.226 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:26.226 [DEBUG] TestAgent.server.raft: vote granted: from=46ea8226-9b6c-059d-dfdb-20e42bc6a757 term=2 tally=1 +34:26.226 [INFO] TestAgent.server.raft: election won: tally=1 +34:26.226 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24632 [Leader]" +34:26.226 [INFO] TestAgent.server: cluster leadership acquired +34:26.226 [INFO] TestAgent.server: New leader elected: payload=Node-46ea8226-9b6c-059d-dfdb-20e42bc6a757 +34:26.226 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24632 +34:26.228 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:26.230 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:26.230 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:26.230 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:26.230 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:26.230 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:26.230 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-46ea8226-9b6c-059d-dfdb-20e42bc6a757 +34:26.230 [INFO] TestAgent.server: member joined, marking health alive: member=Node-46ea8226-9b6c-059d-dfdb-20e42bc6a757 +34:26.231 [INFO] TestAgent.server: federation state anti-entropy synced +34:26.398 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:26.398 [INFO] TestAgent: Synced node info +34:26.736 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:25001] +34:26.736 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:25001 +34:26.738 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4595772f-1869-7714-63fb-6bdb03734739.dc1 127.0.0.1 +34:26.738 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 +34:26.738 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4595772f-1869-7714-63fb-6bdb03734739.dc1 area=wan +34:27.102 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:27.102 [DEBUG] TestAgent: Node info in sync +34:27.102 [DEBUG] TestAgent: Node info in sync +34:27.243 [INFO] TestAgent: Requesting shutdown +34:27.243 [INFO] TestAgent.server: shutting down server +34:27.243 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:27.243 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:27.243 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:27.243 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:27.243 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:27.243 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:27.243 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:27.243 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:27.243 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:27.243 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:27.243 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:27.244Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:27.244Z [INFO] agent.router.manager: shutting down +34:27.244 [INFO] TestAgent: consul server down +34:27.244 [INFO] TestAgent: shutdown complete +34:27.244 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24641 network=tcp +34:27.244 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24641 network=udp +34:27.244 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24642 network=tcp +34:27.744 [INFO] TestAgent: Waiting for endpoints to shut down +34:27.744 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Join_WAN (1.63s) +=== CONT TestAgent_InitKeyring +--- PASS: TestAgent_InitKeyring (0.00s) === CONT TestAgent_Members_ACLFilter -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:26.860Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:26.860Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:26.878Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:26.878Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:26.881 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f242a10a-b463-6137-ceeb-fdb0a127c29a Address:127.0.0.1:25051}]" +34:26.881 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25051 [Follower]" leader= +34:26.882 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f242a10a-b463-6137-ceeb-fdb0a127c29a.dc1 127.0.0.1 +34:26.883 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f242a10a-b463-6137-ceeb-fdb0a127c29a 127.0.0.1 +2021-08-13T15:34:26.883Z [INFO] agent.router: Initializing LAN area manager +34:26.883 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f242a10a-b463-6137-ceeb-fdb0a127c29a.dc1 area=wan +34:26.883 [INFO] TestAgent.server: Adding LAN server: server="Node-f242a10a-b463-6137-ceeb-fdb0a127c29a (Addr: tcp/127.0.0.1:25051) (DC: dc1)" +34:26.883 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25032 network=tcp +34:26.883 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25032 network=udp +34:26.884 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25033 network=tcp +34:26.884 [INFO] TestAgent: started state syncer +34:26.884 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25052 network=tcp +34:26.923 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:26.923 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25051 [Candidate]" term=2 +34:26.924 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:26.924 [DEBUG] TestAgent.server.raft: vote granted: from=f242a10a-b463-6137-ceeb-fdb0a127c29a term=2 tally=1 +34:26.924 [INFO] TestAgent.server.raft: election won: tally=1 +34:26.924 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25051 [Leader]" +34:26.924 [INFO] TestAgent.server: cluster leadership acquired +34:26.924 [INFO] TestAgent.server: New leader elected: payload=Node-f242a10a-b463-6137-ceeb-fdb0a127c29a +34:26.924 [INFO] TestAgent.server: initializing acls +34:26.925 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:26.925 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:26.925 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:26.926 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:26.926 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:26.926 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:26.926 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-f242a10a-b463-6137-ceeb-fdb0a127c29a +34:26.927 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-f242a10a-b463-6137-ceeb-fdb0a127c29a.dc1 +34:26.927 [INFO] TestAgent.server: Updating LAN server: server="Node-f242a10a-b463-6137-ceeb-fdb0a127c29a (Addr: tcp/127.0.0.1:25051) (DC: dc1)" +34:26.927 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-f242a10a-b463-6137-ceeb-fdb0a127c29a.dc1 area=wan +34:26.928 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:26.931 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:26.931 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:26.931 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:26.931 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:26.931 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:26.931 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f242a10a-b463-6137-ceeb-fdb0a127c29a +34:26.931 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f242a10a-b463-6137-ceeb-fdb0a127c29a +34:26.932 [INFO] TestAgent.server: federation state anti-entropy synced +34:26.932 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f242a10a-b463-6137-ceeb-fdb0a127c29a +34:27.208 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:27.209 [INFO] TestAgent: Synced node info +34:27.209 [DEBUG] TestAgent: Node info in sync +34:27.305 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-f242a10a-b463-6137-ceeb-fdb0a127c29a +34:27.305 [INFO] TestAgent.server.serf.wan: serf: Received list-keys query +34:27.306 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-f242a10a-b463-6137-ceeb-fdb0a127c29a.dc1 +34:27.306 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query +34:27.306 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-f242a10a-b463-6137-ceeb-fdb0a127c29a +34:27.307 [INFO] TestAgent.server.serf.wan: serf: Received install-key query +34:27.311 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-f242a10a-b463-6137-ceeb-fdb0a127c29a.dc1 +34:27.311 [INFO] TestAgent.server.serf.lan: serf: Received install-key query +34:27.312 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-f242a10a-b463-6137-ceeb-fdb0a127c29a +34:27.312 [INFO] TestAgent.server.serf.wan: serf: Received use-key query +34:27.312 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-f242a10a-b463-6137-ceeb-fdb0a127c29a.dc1 +34:27.312 [INFO] TestAgent.server.serf.lan: serf: Received use-key query +34:27.313 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-f242a10a-b463-6137-ceeb-fdb0a127c29a +34:27.313 [INFO] TestAgent.server.serf.wan: serf: Received remove-key query +34:27.313 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-f242a10a-b463-6137-ceeb-fdb0a127c29a.dc1 +34:27.314 [INFO] TestAgent.server.serf.lan: serf: Received remove-key query +34:27.314 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-f242a10a-b463-6137-ceeb-fdb0a127c29a +34:27.314 [INFO] TestAgent: Requesting shutdown +34:27.314 [INFO] TestAgent.server: shutting down server +34:27.314 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:27.314 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:27.314 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:27.314 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:27.314 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:27.314 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:27.314 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:27.314 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:27.314 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:27.314 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:27.314 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:27.314 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:27.314 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:27.314 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:27.315 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:27.315Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:27.315Z [INFO] agent.router.manager: shutting down +34:27.315 [INFO] TestAgent: consul server down +34:27.315 [INFO] TestAgent: shutdown complete +34:27.315 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25032 network=tcp +34:27.315 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25032 network=udp +34:27.315 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25033 network=tcp +34:27.815 [INFO] TestAgent: Waiting for endpoints to shut down +34:27.815 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentKeyring_ACL (0.97s) +=== CONT TestAgent_InmemKeyrings +=== RUN TestAgent_InmemKeyrings/no_keys === RUN TestAgent_Members_ACLFilter/no_token === RUN TestAgent_Members_ACLFilter/root_token -2022-09-15T21:43:01.313Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:01.337Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:01.340 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8433f175-c78d-48cd-2f67-ff7a9d028245 Address:127.0.0.1:13407}]" -43:01.341 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13407 [Follower]" leader= -43:01.342 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8433f175-c78d-48cd-2f67-ff7a9d028245.dc1 127.0.0.1 -43:01.343 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8433f175-c78d-48cd-2f67-ff7a9d028245 127.0.0.1 -2022-09-15T21:43:01.343Z [INFO] agent.router: Initializing LAN area manager -43:01.344 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13437 network=udp -43:01.344 [INFO] TestAgent.server: Adding LAN server: server="Node-8433f175-c78d-48cd-2f67-ff7a9d028245 (Addr: tcp/127.0.0.1:13407) (DC: dc1)" -43:01.344 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8433f175-c78d-48cd-2f67-ff7a9d028245.dc1 area=wan -43:01.345 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13437 network=tcp -43:01.346 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13438 network=tcp -43:01.346 [INFO] TestAgent: started state syncer -43:01.346 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13408 network=tcp -43:01.378 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:01.378 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13407 [Candidate]" term=2 -43:01.378 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:01.378 [DEBUG] TestAgent.server.raft: vote granted: from=8433f175-c78d-48cd-2f67-ff7a9d028245 term=2 tally=1 -43:01.378 [INFO] TestAgent.server.raft: election won: tally=1 -43:01.378 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13407 [Leader]" -43:01.378 [INFO] TestAgent.server: cluster leadership acquired -43:01.379 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13407 -43:01.380 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:01.381 [INFO] TestAgent.server: New leader elected: payload=Node-8433f175-c78d-48cd-2f67-ff7a9d028245 -43:01.383 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:01.383 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:01.383 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:01.383 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:01.383 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:01.383 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8433f175-c78d-48cd-2f67-ff7a9d028245 -43:01.383 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8433f175-c78d-48cd-2f67-ff7a9d028245 -43:01.384 [INFO] TestAgent.server: federation state anti-entropy synced -43:01.759 [INFO] TestAgent: Requesting shutdown -43:01.759 [INFO] TestAgent.server: shutting down server -43:01.759 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:01.759 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:01.759 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:01.759 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:01.759 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:01.760 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:01.760 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:01.760 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:01.760 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:01.760 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:01.760 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:01.760 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:01.760Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:01.760Z [INFO] agent.router.manager: shutting down -43:01.760 [INFO] TestAgent: consul server down -43:01.760 [INFO] TestAgent: shutdown complete -43:01.760 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13437 network=tcp -43:01.760 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13437 network=udp -43:01.760 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13438 network=tcp -43:02.261 [INFO] TestAgent: Waiting for endpoints to shut down -43:02.261 [INFO] TestAgent: Endpoints down ---- PASS: TestEnableWebUI (0.97s) +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:26.980Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:26.997Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:27.000 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2c4ae367-dbce-d96d-8559-c0d65b28b76c Address:127.0.0.1:25058}]" +34:27.000 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25058 [Follower]" leader= +34:27.001 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2c4ae367-dbce-d96d-8559-c0d65b28b76c.dc1 127.0.0.1 +34:27.002 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2c4ae367-dbce-d96d-8559-c0d65b28b76c 127.0.0.1 +2021-08-13T15:34:27.002Z [INFO] agent.router: Initializing LAN area manager +34:27.002 [INFO] TestAgent.server: Adding LAN server: server="Node-2c4ae367-dbce-d96d-8559-c0d65b28b76c (Addr: tcp/127.0.0.1:25058) (DC: dc1)" +34:27.002 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2c4ae367-dbce-d96d-8559-c0d65b28b76c.dc1 area=wan +34:27.002 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25053 network=udp +34:27.002 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25053 network=tcp +34:27.003 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25054 network=tcp +34:27.003 [INFO] TestAgent: started state syncer +34:27.003 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25059 network=tcp +34:27.061 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:27.061 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25058 [Candidate]" term=2 +34:27.061 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:27.061 [DEBUG] TestAgent.server.raft: vote granted: from=2c4ae367-dbce-d96d-8559-c0d65b28b76c term=2 tally=1 +34:27.061 [INFO] TestAgent.server.raft: election won: tally=1 +34:27.061 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25058 [Leader]" +34:27.061 [INFO] TestAgent.server: cluster leadership acquired +34:27.062 [INFO] TestAgent.server: New leader elected: payload=Node-2c4ae367-dbce-d96d-8559-c0d65b28b76c +34:27.062 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25058 +34:27.063 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:27.066 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:27.066 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:27.066 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:27.066 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:27.066 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:27.066 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2c4ae367-dbce-d96d-8559-c0d65b28b76c +34:27.066 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2c4ae367-dbce-d96d-8559-c0d65b28b76c +34:27.067 [INFO] TestAgent.server: federation state anti-entropy synced +34:27.251 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:27.252 [INFO] TestAgent: Synced node info +34:27.302 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:52580 +34:27.303 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6da83a1a-9178-ed3f-6381-639ef1e17f8e 127.0.0.1 +34:27.304 [INFO] TestAgent.server: Adding LAN server: server="Node-6da83a1a-9178-ed3f-6381-639ef1e17f8e (Addr: tcp/127.0.0.1:24506) (DC: dc1)" +34:27.304 [INFO] TestAgent: Requesting shutdown +34:27.304 [INFO] TestAgent.server: New leader elected: payload=Node-6da83a1a-9178-ed3f-6381-639ef1e17f8e +34:27.304 [INFO] TestAgent.server: shutting down server +34:27.304 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-6da83a1a-9178-ed3f-6381-639ef1e17f8e other=Node-2c4ae367-dbce-d96d-8559-c0d65b28b76c +34:27.304 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:27.304 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:27.304 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:27.304 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6da83a1a-9178-ed3f-6381-639ef1e17f8e +34:27.304 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:27.304 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:27.304 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:27.304 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:27.304 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:27.306 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:27.306 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:27.306 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +34:27.306 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:36036 +2021-08-13T15:34:27.306Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:27.306Z [INFO] agent.router.manager: shutting down +34:27.306 [INFO] TestAgent: consul server down +34:27.307 [INFO] TestAgent: shutdown complete +34:27.307 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25053 network=tcp +34:27.307 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25053 network=udp +34:27.307 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25054 network=tcp +34:27.307 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: Node-6da83a1a-9178-ed3f-6381-639ef1e17f8e.dc1 127.0.0.1:24505 +34:27.308 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6da83a1a-9178-ed3f-6381-639ef1e17f8e.dc1 127.0.0.1 +34:27.312 [DEBUG] TestAgent.server: Successfully performed flood-join for server at address: server=Node-6da83a1a-9178-ed3f-6381-639ef1e17f8e.dc1 address=127.0.0.1:24505 +34:27.807 [INFO] TestAgent: Waiting for endpoints to shut down +34:27.807 [INFO] TestAgent: Endpoints down +2021-08-13T15:34:26.503Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:26.539Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:26.541 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6da83a1a-9178-ed3f-6381-639ef1e17f8e Address:127.0.0.1:24506}]" +34:26.542 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24506 [Follower]" leader= +34:26.542 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6da83a1a-9178-ed3f-6381-639ef1e17f8e.dc1 127.0.0.1 +34:26.543 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6da83a1a-9178-ed3f-6381-639ef1e17f8e 127.0.0.1 +2021-08-13T15:34:26.543Z [INFO] agent.router: Initializing LAN area manager +34:26.543 [INFO] TestAgent.server: Adding LAN server: server="Node-6da83a1a-9178-ed3f-6381-639ef1e17f8e (Addr: tcp/127.0.0.1:24506) (DC: dc1)" +34:26.543 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6da83a1a-9178-ed3f-6381-639ef1e17f8e.dc1 area=wan +34:26.544 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24634 network=udp +34:26.544 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24634 network=tcp +34:26.544 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24635 network=tcp +34:26.545 [INFO] TestAgent: started state syncer +34:26.545 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24507 network=tcp +34:26.599 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:26.599 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24506 [Candidate]" term=2 +34:26.599 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:26.599 [DEBUG] TestAgent.server.raft: vote granted: from=6da83a1a-9178-ed3f-6381-639ef1e17f8e term=2 tally=1 +34:26.599 [INFO] TestAgent.server.raft: election won: tally=1 +34:26.599 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24506 [Leader]" +34:26.599 [INFO] TestAgent.server: cluster leadership acquired +34:26.600 [INFO] TestAgent.server: New leader elected: payload=Node-6da83a1a-9178-ed3f-6381-639ef1e17f8e +34:26.600 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24506 +34:26.601 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:26.603 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:26.603 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:26.603 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:26.603 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:26.603 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:26.603 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6da83a1a-9178-ed3f-6381-639ef1e17f8e +34:26.603 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6da83a1a-9178-ed3f-6381-639ef1e17f8e +34:26.604 [INFO] TestAgent.server: federation state anti-entropy synced +34:26.623 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:26.623 [INFO] TestAgent: Synced node info +34:27.302 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:25070] +34:27.302 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:25070 +34:27.303 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2c4ae367-dbce-d96d-8559-c0d65b28b76c 127.0.0.1 +34:27.304 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 +34:27.304 [DEBUG] TestAgent: systemd notify failed: error="No socket" +34:27.304 [INFO] TestAgent.server: Adding LAN server: server="Node-2c4ae367-dbce-d96d-8559-c0d65b28b76c (Addr: tcp/127.0.0.1:25058) (DC: dc1)" +34:27.304 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-2c4ae367-dbce-d96d-8559-c0d65b28b76c other=Node-6da83a1a-9178-ed3f-6381-639ef1e17f8e +34:27.304 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2c4ae367-dbce-d96d-8559-c0d65b28b76c +34:27.306 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: Node-2c4ae367-dbce-d96d-8559-c0d65b28b76c.dc1 127.0.0.1:25071 +34:27.307 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:40040 +34:27.307 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2c4ae367-dbce-d96d-8559-c0d65b28b76c.dc1 127.0.0.1 +34:27.307 [DEBUG] TestAgent.server: Successfully performed flood-join for server at address: server=Node-2c4ae367-dbce-d96d-8559-c0d65b28b76c.dc1 address=127.0.0.1:25071 +34:27.307 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2c4ae367-dbce-d96d-8559-c0d65b28b76c.dc1 area=wan +34:27.807 [INFO] TestAgent: Requesting shutdown +34:27.807 [INFO] TestAgent.server: shutting down server +34:27.807 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:27.807 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:27.807 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:27.807 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:27.807 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:27.807 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:27.807 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:27.807 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:27.807 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:27.807 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:27.807 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:27.808Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:27.808Z [INFO] agent.router.manager: shutting down +34:27.808 [INFO] TestAgent: consul server down +34:27.808 [INFO] TestAgent: shutdown complete +34:27.808 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24634 network=tcp +34:27.808 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24634 network=udp +34:27.808 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24635 network=tcp +34:28.044 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Failed ping: Node-2c4ae367-dbce-d96d-8559-c0d65b28b76c (timeout reached) +34:28.308 [INFO] TestAgent: Waiting for endpoints to shut down +34:28.308 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Join (1.82s) === CONT TestAgent_Members_WAN -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:01.367Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:01.392Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:01.395 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8ec3d15c-6673-70e1-40fb-98917196beef Address:127.0.0.1:13481}]" -43:01.396 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8ec3d15c-6673-70e1-40fb-98917196beef.dc1 127.0.0.1 -43:01.397 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8ec3d15c-6673-70e1-40fb-98917196beef 127.0.0.1 -2022-09-15T21:43:01.397Z [INFO] agent.router: Initializing LAN area manager -43:01.397 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13430 network=udp -43:01.397 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13481 [Follower]" leader= -43:01.398 [INFO] TestAgent.server: Adding LAN server: server="Node-8ec3d15c-6673-70e1-40fb-98917196beef (Addr: tcp/127.0.0.1:13481) (DC: dc1)" -43:01.398 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8ec3d15c-6673-70e1-40fb-98917196beef.dc1 area=wan -43:01.398 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13430 network=tcp -43:01.399 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13431 network=tcp -43:01.399 [INFO] TestAgent: started state syncer -43:01.399 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13482 network=tcp -43:01.445 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:01.445 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13481 [Candidate]" term=2 -43:01.445 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:01.445 [DEBUG] TestAgent.server.raft: vote granted: from=8ec3d15c-6673-70e1-40fb-98917196beef term=2 tally=1 -43:01.445 [INFO] TestAgent.server.raft: election won: tally=1 -43:01.445 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13481 [Leader]" -43:01.445 [INFO] TestAgent.server: cluster leadership acquired -43:01.445 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13481 -43:01.446 [INFO] TestAgent.server: New leader elected: payload=Node-8ec3d15c-6673-70e1-40fb-98917196beef -43:01.447 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:01.450 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:01.450 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:01.450 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:01.450 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:01.450 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:01.450 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8ec3d15c-6673-70e1-40fb-98917196beef -43:01.450 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8ec3d15c-6673-70e1-40fb-98917196beef -43:01.451 [INFO] TestAgent.server: federation state anti-entropy synced -43:01.473 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:01.473 [INFO] TestAgent: Synced node info -43:01.473 [DEBUG] TestAgent: Node info in sync -43:01.832 [INFO] TestAgent: Requesting shutdown -43:01.832 [INFO] TestAgent.server: shutting down server -43:01.832 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:01.832 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:01.832 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:01.832 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:01.832 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:01.832 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:01.832 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:01.832 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:01.832 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:01.832 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:01.832 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:01.832Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:01.833Z [INFO] agent.router.manager: shutting down -43:01.833 [INFO] TestAgent: consul server down -43:01.833 [INFO] TestAgent: shutdown complete -43:01.833 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13430 network=tcp -43:01.833 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13430 network=udp -43:01.833 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13431 network=tcp -43:02.333 [INFO] TestAgent: Waiting for endpoints to shut down -43:02.333 [INFO] TestAgent: Endpoints down ---- PASS: TestACLResolution (0.99s) -=== CONT TestAgent_Members -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:01.882Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:01.882Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:01.963Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:01.963Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:01.967 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a5be3fe0-28fd-f6c9-7425-478dad5280db Address:127.0.0.1:13453}]" -43:01.968 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13453 [Follower]" leader= -43:01.969 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a5be3fe0-28fd-f6c9-7425-478dad5280db.dc1 127.0.0.1 -43:01.970 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a5be3fe0-28fd-f6c9-7425-478dad5280db 127.0.0.1 -2022-09-15T21:43:01.970Z [INFO] agent.router: Initializing LAN area manager -43:01.971 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13476 network=udp -43:01.971 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a5be3fe0-28fd-f6c9-7425-478dad5280db.dc1 area=wan -43:01.971 [INFO] TestAgent.server: Adding LAN server: server="Node-a5be3fe0-28fd-f6c9-7425-478dad5280db (Addr: tcp/127.0.0.1:13453) (DC: dc1)" -43:01.971 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13476 network=tcp -43:01.973 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13498 network=tcp -43:01.973 [INFO] TestAgent: started state syncer -43:01.973 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13454 network=tcp -43:02.006 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:02.006 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13453 [Candidate]" term=2 -43:02.007 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:02.007 [DEBUG] TestAgent.server.raft: vote granted: from=a5be3fe0-28fd-f6c9-7425-478dad5280db term=2 tally=1 -43:02.007 [INFO] TestAgent.server.raft: election won: tally=1 -43:02.007 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13453 [Leader]" -43:02.007 [INFO] TestAgent.server: cluster leadership acquired -43:02.007 [INFO] TestAgent.server: initializing acls -43:02.008 [INFO] TestAgent.server: Created ACL 'global-management' policy -43:02.008 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -43:02.008 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -43:02.009 [INFO] TestAgent.server: Created ACL anonymous token from configuration -43:02.009 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -43:02.009 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -43:02.009 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-a5be3fe0-28fd-f6c9-7425-478dad5280db -43:02.009 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-a5be3fe0-28fd-f6c9-7425-478dad5280db.dc1 -43:02.010 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-a5be3fe0-28fd-f6c9-7425-478dad5280db.dc1 area=wan -43:02.010 [INFO] TestAgent.server: New leader elected: payload=Node-a5be3fe0-28fd-f6c9-7425-478dad5280db -43:02.010 [INFO] TestAgent.server: Updating LAN server: server="Node-a5be3fe0-28fd-f6c9-7425-478dad5280db (Addr: tcp/127.0.0.1:13453) (DC: dc1)" -43:02.010 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:02.013 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:02.013 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:02.013 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:02.013 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:02.013 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:02.013 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a5be3fe0-28fd-f6c9-7425-478dad5280db -43:02.013 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a5be3fe0-28fd-f6c9-7425-478dad5280db -43:02.014 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a5be3fe0-28fd-f6c9-7425-478dad5280db -43:02.015 [INFO] TestAgent.server: federation state anti-entropy synced -43:02.112 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-a5be3fe0-28fd-f6c9-7425-478dad5280db -43:02.112 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-a5be3fe0-28fd-f6c9-7425-478dad5280db -43:02.112 [DEBUG] TestAgent: dropping node from result due to ACLs: node=Node-a5be3fe0-28fd-f6c9-7425-478dad5280db accessorID=00000000-0000-0000-0000-000000000002 -43:02.113 [INFO] TestAgent: Requesting shutdown -43:02.113 [INFO] TestAgent.server: shutting down server -43:02.113 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:02.113 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:02.113 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -43:02.113 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -43:02.113 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:02.113 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:02.113 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:02.114 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:02.114 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -43:02.114 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:02.113 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:02.113 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:02.113 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:02.113 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -43:02.114 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:02.114 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:02.114Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:02.114Z [INFO] agent.router.manager: shutting down -43:02.114 [INFO] TestAgent: consul server down -43:02.114 [INFO] TestAgent: shutdown complete -43:02.114 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13476 network=tcp -43:02.114 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13476 network=udp -43:02.114 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13498 network=tcp -43:02.615 [INFO] TestAgent: Waiting for endpoints to shut down -43:02.615 [INFO] TestAgent: Endpoints down +2021-08-13T15:34:27.377Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:27.450Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:27.453 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5c92314d-7aed-58e2-cda1-db59bc344d11 Address:127.0.0.1:25037}]" +34:27.454 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5c92314d-7aed-58e2-cda1-db59bc344d11.dc1 127.0.0.1 +34:27.454 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5c92314d-7aed-58e2-cda1-db59bc344d11 127.0.0.1 +2021-08-13T15:34:27.454Z [INFO] agent.router: Initializing LAN area manager +34:27.455 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25060 network=udp +34:27.455 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25037 [Follower]" leader= +34:27.455 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5c92314d-7aed-58e2-cda1-db59bc344d11.dc1 area=wan +34:27.455 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25060 network=tcp +34:27.455 [INFO] TestAgent.server: Adding LAN server: server="Node-5c92314d-7aed-58e2-cda1-db59bc344d11 (Addr: tcp/127.0.0.1:25037) (DC: dc1)" +34:27.456 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25061 network=tcp +34:27.456 [INFO] TestAgent: started state syncer +34:27.456 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25038 network=tcp +34:27.497 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:27.497 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25037 [Candidate]" term=2 +34:27.497 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:27.497 [DEBUG] TestAgent.server.raft: vote granted: from=5c92314d-7aed-58e2-cda1-db59bc344d11 term=2 tally=1 +34:27.497 [INFO] TestAgent.server.raft: election won: tally=1 +34:27.497 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25037 [Leader]" +34:27.497 [INFO] TestAgent.server: cluster leadership acquired +34:27.497 [INFO] TestAgent.server: New leader elected: payload=Node-5c92314d-7aed-58e2-cda1-db59bc344d11 +34:27.497 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25037 +34:27.499 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:27.501 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:27.501 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:27.501 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:27.501 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:27.501 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:27.501 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5c92314d-7aed-58e2-cda1-db59bc344d11 +34:27.501 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5c92314d-7aed-58e2-cda1-db59bc344d11 +34:27.502 [INFO] TestAgent.server: federation state anti-entropy synced +34:27.647 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:27.647 [INFO] TestAgent: Synced node info +34:27.647 [DEBUG] TestAgent: Node info in sync +34:27.883 [INFO] TestAgent: Requesting shutdown +34:27.883 [INFO] TestAgent.server: shutting down server +34:27.883 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:27.883 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:27.883 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:27.883 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:27.883 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:27.883 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:27.883 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:27.883 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:27.883 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:27.883 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:27.883 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:27.883Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:27.883Z [INFO] agent.router.manager: shutting down +34:27.883 [INFO] TestAgent: consul server down +34:27.883 [INFO] TestAgent: shutdown complete +34:27.883 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25060 network=tcp +34:27.883 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25060 network=udp +34:27.883 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25061 network=tcp +34:28.384 [INFO] TestAgent: Waiting for endpoints to shut down +34:28.384 [INFO] TestAgent: Endpoints down +--- PASS: TestParseConsistency (1.02s) +=== CONT TestAgent_LoadKeyrings +=== RUN TestAgent_LoadKeyrings/no_keys +2021-08-13T15:34:27.762Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:27.762Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:27.778Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:27.778Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:27.781 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d3e124b8-08a6-abca-5d12-2dc9f135efe2 Address:127.0.0.1:25079}]" +34:27.781 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25079 [Follower]" leader= +34:27.782 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d3e124b8-08a6-abca-5d12-2dc9f135efe2.dc1 127.0.0.1 +34:27.782 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d3e124b8-08a6-abca-5d12-2dc9f135efe2 127.0.0.1 +2021-08-13T15:34:27.782Z [INFO] agent.router: Initializing LAN area manager +34:27.783 [INFO] TestAgent.server: Adding LAN server: server="Node-d3e124b8-08a6-abca-5d12-2dc9f135efe2 (Addr: tcp/127.0.0.1:25079) (DC: dc1)" +34:27.783 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d3e124b8-08a6-abca-5d12-2dc9f135efe2.dc1 area=wan +34:27.783 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25039 network=tcp +34:27.783 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25039 network=udp +34:27.784 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25040 network=tcp +34:27.784 [INFO] TestAgent: started state syncer +34:27.784 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25080 network=tcp +34:27.833 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:27.847 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:27.847 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25079 [Candidate]" term=2 +34:27.847 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:27.847 [DEBUG] TestAgent.server.raft: vote granted: from=d3e124b8-08a6-abca-5d12-2dc9f135efe2 term=2 tally=1 +34:27.847 [INFO] TestAgent.server.raft: election won: tally=1 +34:27.847 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25079 [Leader]" +34:27.847 [INFO] TestAgent.server: cluster leadership acquired +34:27.848 [INFO] TestAgent.server: initializing acls +34:27.848 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:27.848 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:27.849 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:27.849 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:27.849 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:27.849 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:27.849 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-d3e124b8-08a6-abca-5d12-2dc9f135efe2 +34:27.850 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-d3e124b8-08a6-abca-5d12-2dc9f135efe2.dc1 +34:27.850 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:27.851 [INFO] TestAgent.server: New leader elected: payload=Node-d3e124b8-08a6-abca-5d12-2dc9f135efe2 +34:27.851 [INFO] TestAgent.server: Updating LAN server: server="Node-d3e124b8-08a6-abca-5d12-2dc9f135efe2 (Addr: tcp/127.0.0.1:25079) (DC: dc1)" +34:27.851 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-d3e124b8-08a6-abca-5d12-2dc9f135efe2.dc1 area=wan +34:27.854 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:27.854 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:27.854 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:27.854 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:27.854 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:27.854 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d3e124b8-08a6-abca-5d12-2dc9f135efe2 +34:27.854 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d3e124b8-08a6-abca-5d12-2dc9f135efe2 +34:27.855 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d3e124b8-08a6-abca-5d12-2dc9f135efe2 +34:27.855 [INFO] TestAgent.server: federation state anti-entropy synced +34:27.988 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-d3e124b8-08a6-abca-5d12-2dc9f135efe2 +34:27.989 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-d3e124b8-08a6-abca-5d12-2dc9f135efe2 +34:27.989 [DEBUG] TestAgent: dropping node from result due to ACLs: node=Node-d3e124b8-08a6-abca-5d12-2dc9f135efe2 accessorID=00000000-0000-0000-0000-000000000002 +34:27.990 [INFO] TestAgent: Requesting shutdown +34:27.990 [INFO] TestAgent.server: shutting down server +34:27.990 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:27.990 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:27.990 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:27.990 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:27.990 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:27.990 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:27.990 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:27.990 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:27.990 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:27.990 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:27.990 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:27.990 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:27.990 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:27.990 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:27.990 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:27.990 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:27.990Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:27.990Z [INFO] agent.router.manager: shutting down +34:27.990 [INFO] TestAgent: consul server down +34:27.990 [INFO] TestAgent: shutdown complete +34:27.990 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25039 network=tcp +34:27.990 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25039 network=udp +34:27.991 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25040 network=tcp +34:28.491 [INFO] TestAgent: Waiting for endpoints to shut down +34:28.491 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_Members_ACLFilter (0.75s) --- PASS: TestAgent_Members_ACLFilter/no_token (0.00s) --- PASS: TestAgent_Members_ACLFilter/root_token (0.00s) === CONT TestAgent_Reload_ACLDeny -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:01.913Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:01.970Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:01.976 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4f7f4b40-b4db-e2c7-c81b-8a9bffbaa47d Address:127.0.0.1:13474}]" -43:01.976 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13474 [Follower]" leader= -43:01.977 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4f7f4b40-b4db-e2c7-c81b-8a9bffbaa47d.dc1 127.0.0.1 -43:01.978 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4f7f4b40-b4db-e2c7-c81b-8a9bffbaa47d 127.0.0.1 -2022-09-15T21:43:01.978Z [INFO] agent.router: Initializing LAN area manager -43:01.978 [INFO] TestAgent.server: Adding LAN server: server="Node-4f7f4b40-b4db-e2c7-c81b-8a9bffbaa47d (Addr: tcp/127.0.0.1:13474) (DC: dc1)" -43:01.978 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4f7f4b40-b4db-e2c7-c81b-8a9bffbaa47d.dc1 area=wan -43:01.978 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13483 network=udp -43:01.978 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13483 network=tcp -43:01.979 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13470 network=tcp -43:01.979 [INFO] TestAgent: started state syncer -43:01.979 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13475 network=tcp -43:02.022 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:02.023 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13474 [Candidate]" term=2 -43:02.023 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:02.023 [DEBUG] TestAgent.server.raft: vote granted: from=4f7f4b40-b4db-e2c7-c81b-8a9bffbaa47d term=2 tally=1 -43:02.023 [INFO] TestAgent.server.raft: election won: tally=1 -43:02.023 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13474 [Leader]" -43:02.023 [INFO] TestAgent.server: cluster leadership acquired -43:02.023 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13474 -43:02.025 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:02.026 [INFO] TestAgent.server: New leader elected: payload=Node-4f7f4b40-b4db-e2c7-c81b-8a9bffbaa47d -43:02.028 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:02.028 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:02.028 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:02.028 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:02.028 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:02.028 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4f7f4b40-b4db-e2c7-c81b-8a9bffbaa47d -43:02.028 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4f7f4b40-b4db-e2c7-c81b-8a9bffbaa47d -43:02.032 [INFO] TestAgent.server: federation state anti-entropy synced -43:02.253 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:02.254 [INFO] TestAgent: Synced node info -43:02.254 [DEBUG] TestAgent: Node info in sync -43:02.284 [INFO] TestAgent: Requesting shutdown -43:02.284 [INFO] TestAgent.server: shutting down server -43:02.284 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:02.284 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:02.284 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:02.284 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:02.284 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:02.285 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:02.285 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:02.285 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:02.285 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:02.285 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:02.285 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:02.285Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:02.285Z [INFO] agent.router.manager: shutting down -43:02.285 [INFO] TestAgent: consul server down -43:02.285 [INFO] TestAgent: shutdown complete -43:02.285 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13483 network=tcp -43:02.285 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13483 network=udp -43:02.285 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13470 network=tcp -43:02.785 [INFO] TestAgent: Waiting for endpoints to shut down -43:02.785 [INFO] TestAgent: Endpoints down ---- PASS: TestParseConsistency_Invalid (0.95s) +2021-08-13T15:34:27.832Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:27.848Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:27.851 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6d7bcf89-7ff5-8dea-d234-1384cf1098fd Address:127.0.0.1:25065}]" +34:27.851 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25065 [Follower]" leader= +34:27.851 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6d7bcf89-7ff5-8dea-d234-1384cf1098fd.dc1 127.0.0.1 +34:27.852 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6d7bcf89-7ff5-8dea-d234-1384cf1098fd 127.0.0.1 +2021-08-13T15:34:27.852Z [INFO] agent.router: Initializing LAN area manager +34:27.853 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25081 network=udp +34:27.853 [INFO] TestAgent.server: Adding LAN server: server="Node-6d7bcf89-7ff5-8dea-d234-1384cf1098fd (Addr: tcp/127.0.0.1:25065) (DC: dc1)" +34:27.853 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6d7bcf89-7ff5-8dea-d234-1384cf1098fd.dc1 area=wan +34:27.853 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25081 network=tcp +34:27.854 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25082 network=tcp +34:27.854 [INFO] TestAgent: started state syncer +34:27.854 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25066 network=tcp +34:27.909 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:27.909 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25065 [Candidate]" term=2 +34:27.909 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:27.909 [DEBUG] TestAgent.server.raft: vote granted: from=6d7bcf89-7ff5-8dea-d234-1384cf1098fd term=2 tally=1 +34:27.909 [INFO] TestAgent.server.raft: election won: tally=1 +34:27.909 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25065 [Leader]" +34:27.909 [INFO] TestAgent.server: cluster leadership acquired +34:27.910 [INFO] TestAgent.server: New leader elected: payload=Node-6d7bcf89-7ff5-8dea-d234-1384cf1098fd +34:27.910 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25065 +34:27.911 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:27.913 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:27.913 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:27.913 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:27.913 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:27.913 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:27.913 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6d7bcf89-7ff5-8dea-d234-1384cf1098fd +34:27.913 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6d7bcf89-7ff5-8dea-d234-1384cf1098fd +34:27.915 [INFO] TestAgent.server: federation state anti-entropy synced +34:27.998 [INFO] TestAgent: Requesting shutdown +34:27.998 [INFO] TestAgent.server: shutting down server +34:27.998 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:27.998 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:27.998 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:27.998 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:27.998 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:27.998 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:27.999 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:27.999 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:27.999 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:27.999 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:27.999 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:27.999 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:27.999Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:27.999Z [INFO] agent.router.manager: shutting down +34:27.999 [INFO] TestAgent: consul server down +34:27.999 [INFO] TestAgent: shutdown complete +34:27.999 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25081 network=tcp +34:27.999 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25081 network=udp +34:27.999 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25082 network=tcp +34:28.499 [INFO] TestAgent: Waiting for endpoints to shut down +34:28.499 [INFO] TestAgent: Endpoints down +=== RUN TestAgent_InmemKeyrings/server_with_keys +[WARN] freeport: 10 out of 45 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestAgent_Reload_ACLDeny/no_token +=== RUN TestAgent_Reload_ACLDeny/read-only_token +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:28.401Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:28.418Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:28.421 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:143b6058-f071-0150-1b44-f41558e9099d Address:127.0.0.1:25044}]" +34:28.421 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25044 [Follower]" leader= +34:28.422 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-143b6058-f071-0150-1b44-f41558e9099d.dc1 127.0.0.1 +34:28.423 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-143b6058-f071-0150-1b44-f41558e9099d 127.0.0.1 +2021-08-13T15:34:28.423Z [INFO] agent.router: Initializing LAN area manager +34:28.423 [INFO] TestAgent.server: Adding LAN server: server="Node-143b6058-f071-0150-1b44-f41558e9099d (Addr: tcp/127.0.0.1:25044) (DC: dc1)" +34:28.423 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-143b6058-f071-0150-1b44-f41558e9099d.dc1 area=wan +34:28.423 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25112 network=udp +34:28.423 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25112 network=tcp +34:28.424 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25100 network=tcp +34:28.424 [INFO] TestAgent: started state syncer +34:28.424 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25045 network=tcp +34:28.482 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:28.482 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25044 [Candidate]" term=2 +34:28.482 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:28.482 [DEBUG] TestAgent.server.raft: vote granted: from=143b6058-f071-0150-1b44-f41558e9099d term=2 tally=1 +34:28.482 [INFO] TestAgent.server.raft: election won: tally=1 +34:28.482 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25044 [Leader]" +34:28.482 [INFO] TestAgent.server: cluster leadership acquired +34:28.483 [INFO] TestAgent.server: New leader elected: payload=Node-143b6058-f071-0150-1b44-f41558e9099d +34:28.483 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25044 +34:28.484 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:28.486 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:28.486 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:28.486 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:28.486 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:28.486 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:28.486 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-143b6058-f071-0150-1b44-f41558e9099d +34:28.486 [INFO] TestAgent.server: member joined, marking health alive: member=Node-143b6058-f071-0150-1b44-f41558e9099d +34:28.487 [INFO] TestAgent.server: federation state anti-entropy synced +34:28.550 [INFO] TestAgent: Requesting shutdown +34:28.550 [INFO] TestAgent.server: shutting down server +34:28.550 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:28.550 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:28.550 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:28.550 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:28.550 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:28.550 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:28.550 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:28.550 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:28.550 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:28.550 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:28.550 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:28.551 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:28.551Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:28.551Z [INFO] agent.router.manager: shutting down +34:28.551 [INFO] TestAgent: consul server down +34:28.551 [INFO] TestAgent: shutdown complete +34:28.551 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25112 network=tcp +34:28.551 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25112 network=udp +34:28.551 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25100 network=tcp +34:29.051 [INFO] TestAgent: Waiting for endpoints to shut down +34:29.051 [INFO] TestAgent: Endpoints down +=== RUN TestAgent_LoadKeyrings/server_with_keys +2021-08-13T15:34:28.326Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:28.345Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:28.348 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:675420ac-b607-09de-3d11-812de4f9438d Address:127.0.0.1:25109}]" +34:28.348 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25109 [Follower]" leader= +34:28.349 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-675420ac-b607-09de-3d11-812de4f9438d.dc1 127.0.0.1 +34:28.350 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-675420ac-b607-09de-3d11-812de4f9438d 127.0.0.1 +2021-08-13T15:34:28.350Z [INFO] agent.router: Initializing LAN area manager +34:28.350 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25067 network=udp +34:28.350 [INFO] TestAgent.server: Adding LAN server: server="Node-675420ac-b607-09de-3d11-812de4f9438d (Addr: tcp/127.0.0.1:25109) (DC: dc1)" +34:28.350 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-675420ac-b607-09de-3d11-812de4f9438d.dc1 area=wan +34:28.350 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25067 network=tcp +34:28.351 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25068 network=tcp +34:28.351 [INFO] TestAgent: started state syncer +34:28.351 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25111 network=tcp +34:28.411 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:28.411 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25109 [Candidate]" term=2 +34:28.412 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:28.412 [DEBUG] TestAgent.server.raft: vote granted: from=675420ac-b607-09de-3d11-812de4f9438d term=2 tally=1 +34:28.412 [INFO] TestAgent.server.raft: election won: tally=1 +34:28.412 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25109 [Leader]" +34:28.412 [INFO] TestAgent.server: cluster leadership acquired +34:28.412 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25109 +34:28.413 [INFO] TestAgent.server: New leader elected: payload=Node-675420ac-b607-09de-3d11-812de4f9438d +34:28.413 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:28.416 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:28.416 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:28.416 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:28.416 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:28.416 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:28.416 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-675420ac-b607-09de-3d11-812de4f9438d +34:28.416 [INFO] TestAgent.server: member joined, marking health alive: member=Node-675420ac-b607-09de-3d11-812de4f9438d +34:28.417 [INFO] TestAgent.server: federation state anti-entropy synced +34:28.656 [INFO] TestAgent: Requesting shutdown +34:28.656 [INFO] TestAgent.server: shutting down server +34:28.656 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:28.656 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:28.656 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:28.656 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:28.656 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:28.656 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:28.656 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:28.656 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:28.656 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:28.656 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:28.656 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:28.657 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:28.657Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:28.657Z [INFO] agent.router.manager: shutting down +34:28.657 [INFO] TestAgent: consul server down +34:28.657 [INFO] TestAgent: shutdown complete +34:28.657 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25067 network=tcp +34:28.657 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25067 network=udp +34:28.657 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25068 network=tcp +34:29.157 [INFO] TestAgent: Waiting for endpoints to shut down +34:29.157 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Members_WAN (0.85s) +=== CONT TestIntentionsSpecificDelete_good +2021-08-13T15:34:28.516Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:28.531Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:28.534 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:11bea7d0-12f2-c533-f590-e4bb32ca2172 Address:127.0.0.1:25093}]" +34:28.534 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25093 [Follower]" leader= +34:28.535 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-11bea7d0-12f2-c533-f590-e4bb32ca2172.dc1 127.0.0.1 +34:28.535 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-11bea7d0-12f2-c533-f590-e4bb32ca2172 127.0.0.1 +2021-08-13T15:34:28.535Z [INFO] agent.router: Initializing LAN area manager +34:28.536 [INFO] TestAgent.server: Adding LAN server: server="Node-11bea7d0-12f2-c533-f590-e4bb32ca2172 (Addr: tcp/127.0.0.1:25093) (DC: dc1)" +34:28.536 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-11bea7d0-12f2-c533-f590-e4bb32ca2172.dc1 area=wan +34:28.536 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25088 network=udp +34:28.536 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25088 network=tcp +34:28.537 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25089 network=tcp +34:28.537 [INFO] TestAgent: started state syncer +34:28.537 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25094 network=tcp +34:28.580 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:28.580 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25093 [Candidate]" term=2 +34:28.581 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:28.581 [DEBUG] TestAgent.server.raft: vote granted: from=11bea7d0-12f2-c533-f590-e4bb32ca2172 term=2 tally=1 +34:28.581 [INFO] TestAgent.server.raft: election won: tally=1 +34:28.581 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25093 [Leader]" +34:28.581 [INFO] TestAgent.server: cluster leadership acquired +34:28.581 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25093 +34:28.582 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:28.583 [INFO] TestAgent.server: New leader elected: payload=Node-11bea7d0-12f2-c533-f590-e4bb32ca2172 +34:28.585 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:28.585 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:28.585 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:28.585 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:28.585 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:28.585 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-11bea7d0-12f2-c533-f590-e4bb32ca2172 +34:28.585 [INFO] TestAgent.server: member joined, marking health alive: member=Node-11bea7d0-12f2-c533-f590-e4bb32ca2172 +34:28.586 [INFO] TestAgent.server: federation state anti-entropy synced +34:28.711 [INFO] TestAgent: Requesting shutdown +34:28.711 [INFO] TestAgent.server: shutting down server +34:28.711 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:28.711 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:28.711 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:28.711 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:28.711 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:28.712 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:28.712 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:28.712 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:28.712 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:28.712 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:28.712 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:28.712 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:28.712Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:28.712Z [INFO] agent.router.manager: shutting down +34:28.712 [INFO] TestAgent: consul server down +34:28.712 [INFO] TestAgent: shutdown complete +34:28.712 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25088 network=tcp +34:28.712 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25088 network=udp +34:28.712 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25089 network=tcp +34:29.215 [INFO] TestAgent: Waiting for endpoints to shut down +34:29.215 [INFO] TestAgent: Endpoints down +=== RUN TestAgent_InmemKeyrings/client_with_keys +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:28.508Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:28.508Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:28.525Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:28.525Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:28.527 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bd2b69a3-130f-6d9e-1c27-1b075306b408 Address:127.0.0.1:25086}]" +34:28.528 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25086 [Follower]" leader= +34:28.528 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bd2b69a3-130f-6d9e-1c27-1b075306b408.dc1 127.0.0.1 +34:28.529 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bd2b69a3-130f-6d9e-1c27-1b075306b408 127.0.0.1 +2021-08-13T15:34:28.529Z [INFO] agent.router: Initializing LAN area manager +34:28.529 [INFO] TestAgent.server: Adding LAN server: server="Node-bd2b69a3-130f-6d9e-1c27-1b075306b408 (Addr: tcp/127.0.0.1:25086) (DC: dc1)" +34:28.529 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bd2b69a3-130f-6d9e-1c27-1b075306b408.dc1 area=wan +34:28.530 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25046 network=udp +34:28.530 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25046 network=tcp +34:28.531 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25047 network=tcp +34:28.531 [INFO] TestAgent: started state syncer +34:28.531 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25087 network=tcp +34:28.568 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:28.568 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25086 [Candidate]" term=2 +34:28.568 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:28.568 [DEBUG] TestAgent.server.raft: vote granted: from=bd2b69a3-130f-6d9e-1c27-1b075306b408 term=2 tally=1 +34:28.568 [INFO] TestAgent.server.raft: election won: tally=1 +34:28.568 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25086 [Leader]" +34:28.568 [INFO] TestAgent.server: cluster leadership acquired +34:28.568 [INFO] TestAgent.server: New leader elected: payload=Node-bd2b69a3-130f-6d9e-1c27-1b075306b408 +34:28.568 [INFO] TestAgent.server: initializing acls +34:28.569 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:28.569 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:28.570 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:28.570 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:28.570 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:28.570 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:28.570 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-bd2b69a3-130f-6d9e-1c27-1b075306b408 +34:28.571 [INFO] TestAgent.server: Updating LAN server: server="Node-bd2b69a3-130f-6d9e-1c27-1b075306b408 (Addr: tcp/127.0.0.1:25086) (DC: dc1)" +34:28.571 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-bd2b69a3-130f-6d9e-1c27-1b075306b408.dc1 +34:28.571 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-bd2b69a3-130f-6d9e-1c27-1b075306b408.dc1 area=wan +34:28.572 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:28.574 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:28.574 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:28.574 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:28.574 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:28.574 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:28.574 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bd2b69a3-130f-6d9e-1c27-1b075306b408 +34:28.574 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bd2b69a3-130f-6d9e-1c27-1b075306b408 +34:28.575 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bd2b69a3-130f-6d9e-1c27-1b075306b408 +34:28.575 [INFO] TestAgent.server: federation state anti-entropy synced +34:28.627 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:28.628 [INFO] TestAgent: Synced node info +34:28.942 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-bd2b69a3-130f-6d9e-1c27-1b075306b408 +34:28.942 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-bd2b69a3-130f-6d9e-1c27-1b075306b408 +34:28.944 [INFO] TestAgent: Requesting shutdown +34:28.944 [INFO] TestAgent.server: shutting down server +34:28.944 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:28.944 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:28.944 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:28.944 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:28.944 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:28.944 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:28.944 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:28.944 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:28.944 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:28.944 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:28.944 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:28.944 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:28.944 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:28.944 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:28.944 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:28.945Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:28.945Z [INFO] agent.router.manager: shutting down +34:28.945 [INFO] TestAgent: consul server down +34:28.945 [INFO] TestAgent: shutdown complete +34:28.945 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25046 network=tcp +34:28.945 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25046 network=udp +34:28.945 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25047 network=tcp +34:29.445 [INFO] TestAgent: Waiting for endpoints to shut down +34:29.445 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Reload_ACLDeny (0.95s) + --- PASS: TestAgent_Reload_ACLDeny/no_token (0.00s) + --- PASS: TestAgent_Reload_ACLDeny/read-only_token (0.00s) === CONT TestAgent_Reload -2022-09-15T21:43:02.281Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:02.303Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:02.306 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1aa755e6-e1bb-da1d-07cb-08af7d035d24 Address:127.0.0.1:13495}]" -43:02.307 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13495 [Follower]" leader= -43:02.308 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1aa755e6-e1bb-da1d-07cb-08af7d035d24.dc1 127.0.0.1 -43:02.308 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1aa755e6-e1bb-da1d-07cb-08af7d035d24 127.0.0.1 -2022-09-15T21:43:02.308Z [INFO] agent.router: Initializing LAN area manager -43:02.309 [INFO] TestAgent.server: Adding LAN server: server="Node-1aa755e6-e1bb-da1d-07cb-08af7d035d24 (Addr: tcp/127.0.0.1:13495) (DC: dc1)" -43:02.309 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1aa755e6-e1bb-da1d-07cb-08af7d035d24.dc1 area=wan -43:02.309 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13455 network=udp -43:02.309 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13455 network=tcp -43:02.310 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13477 network=tcp -43:02.310 [INFO] TestAgent: started state syncer -43:02.310 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13496 network=tcp -43:02.358 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:02.358 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13495 [Candidate]" term=2 -43:02.359 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:02.359 [DEBUG] TestAgent.server.raft: vote granted: from=1aa755e6-e1bb-da1d-07cb-08af7d035d24 term=2 tally=1 -43:02.359 [INFO] TestAgent.server.raft: election won: tally=1 -43:02.359 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13495 [Leader]" -43:02.359 [INFO] TestAgent.server: cluster leadership acquired -43:02.359 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13495 -43:02.360 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:02.362 [INFO] TestAgent.server: New leader elected: payload=Node-1aa755e6-e1bb-da1d-07cb-08af7d035d24 -43:02.363 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:02.363 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:02.363 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:02.363 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:02.363 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:02.363 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1aa755e6-e1bb-da1d-07cb-08af7d035d24 -43:02.363 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1aa755e6-e1bb-da1d-07cb-08af7d035d24 -43:02.364 [INFO] TestAgent.server: federation state anti-entropy synced -43:02.370 [INFO] TestAgent: Requesting shutdown -43:02.370 [INFO] TestAgent.server: shutting down server -43:02.370 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:02.370 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:02.370 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:02.370 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:02.370 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:02.370 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:02.370 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:02.370 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:02.370 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:02.370 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:02.370 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:02.370 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:02.371Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:02.371Z [INFO] agent.router.manager: shutting down -43:02.371 [INFO] TestAgent: consul server down -43:02.371 [INFO] TestAgent: shutdown complete -43:02.371 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13455 network=tcp -43:02.371 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13455 network=udp -43:02.371 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13477 network=tcp -43:02.874 [INFO] TestAgent: Waiting for endpoints to shut down -43:02.874 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Members_WAN (0.61s) +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +34:29.254 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-3d4013a3-bd4e-ec08-311f-111efebdb422 127.0.0.1 +2021-08-13T15:34:29.254Z [INFO] agent.router: Initializing LAN area manager +34:29.254 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25120 network=udp +34:29.255 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25120 network=tcp +34:29.256 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25128 network=tcp +34:29.256 [INFO] TestAgent: started state syncer +34:29.257 [INFO] TestAgent: Requesting shutdown +34:29.257 [INFO] TestAgent.client: shutting down client +34:29.257 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave +34:29.257 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25133 network=tcp +34:29.257 [ERROR] TestAgent: gRPC server failed: error="grpc: the server has been stopped" +2021-08-13T15:34:29.257Z [WARN] agent.router.manager: No servers available +34:29.257 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" +34:29.257 [INFO] TestAgent: consul client down +34:29.257 [INFO] TestAgent: shutdown complete +34:29.257 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25120 network=tcp +34:29.257 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25120 network=udp +34:29.257 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25128 network=tcp +34:29.757 [INFO] TestAgent: Waiting for endpoints to shut down +34:29.758 [INFO] TestAgent: Endpoints down +=== RUN TestAgent_InmemKeyrings/ignore_files +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:29.068Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:29.092Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:29.095 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5f4ad19c-e532-6fa4-b414-7381fbee305c Address:127.0.0.1:25104}]" +34:29.095 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25104 [Follower]" leader= +34:29.096 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5f4ad19c-e532-6fa4-b414-7381fbee305c.dc1 127.0.0.1 +34:29.097 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5f4ad19c-e532-6fa4-b414-7381fbee305c 127.0.0.1 +2021-08-13T15:34:29.097Z [INFO] agent.router: Initializing LAN area manager +34:29.097 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25095 network=udp +34:29.097 [INFO] TestAgent.server: Adding LAN server: server="Node-5f4ad19c-e532-6fa4-b414-7381fbee305c (Addr: tcp/127.0.0.1:25104) (DC: dc1)" +34:29.097 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5f4ad19c-e532-6fa4-b414-7381fbee305c.dc1 area=wan +34:29.097 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25095 network=tcp +34:29.098 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25096 network=tcp +34:29.098 [INFO] TestAgent: started state syncer +34:29.099 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25105 network=tcp +34:29.139 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:29.139 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25104 [Candidate]" term=2 +34:29.139 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:29.139 [DEBUG] TestAgent.server.raft: vote granted: from=5f4ad19c-e532-6fa4-b414-7381fbee305c term=2 tally=1 +34:29.139 [INFO] TestAgent.server.raft: election won: tally=1 +34:29.139 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25104 [Leader]" +34:29.139 [INFO] TestAgent.server: cluster leadership acquired +34:29.140 [INFO] TestAgent.server: New leader elected: payload=Node-5f4ad19c-e532-6fa4-b414-7381fbee305c +34:29.140 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25104 +34:29.141 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:29.143 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:29.144 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:29.144 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:29.144 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:29.144 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:29.144 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5f4ad19c-e532-6fa4-b414-7381fbee305c +34:29.144 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5f4ad19c-e532-6fa4-b414-7381fbee305c +34:29.144 [INFO] TestAgent.server: federation state anti-entropy synced +34:29.321 [INFO] TestAgent: Requesting shutdown +34:29.321 [INFO] TestAgent.server: shutting down server +34:29.321 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:29.321 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:29.321 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:29.321 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:29.321 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:29.321 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:29.321 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:29.321 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:29.321 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:29.321 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:29.321 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:29.321 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:29.322Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:29.322Z [INFO] agent.router.manager: shutting down +34:29.322 [INFO] TestAgent: consul server down +34:29.322 [INFO] TestAgent: shutdown complete +34:29.322 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25095 network=tcp +34:29.322 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25095 network=udp +34:29.322 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25096 network=tcp +34:29.822 [INFO] TestAgent: Waiting for endpoints to shut down +34:29.822 [INFO] TestAgent: Endpoints down +=== RUN TestAgent_LoadKeyrings/client_with_keys +2021-08-13T15:34:29.196Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:29.212Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:29.215 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8fcd5dfe-dc9c-1114-4683-4e5a27678354 Address:127.0.0.1:25118}]" +34:29.216 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8fcd5dfe-dc9c-1114-4683-4e5a27678354.dc1 127.0.0.1 +34:29.217 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8fcd5dfe-dc9c-1114-4683-4e5a27678354 127.0.0.1 +2021-08-13T15:34:29.217Z [INFO] agent.router: Initializing LAN area manager +34:29.217 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25106 network=udp +34:29.217 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25118 [Follower]" leader= +34:29.218 [INFO] TestAgent.server: Adding LAN server: server="Node-8fcd5dfe-dc9c-1114-4683-4e5a27678354 (Addr: tcp/127.0.0.1:25118) (DC: dc1)" +34:29.218 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8fcd5dfe-dc9c-1114-4683-4e5a27678354.dc1 area=wan +34:29.218 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25106 network=tcp +34:29.219 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25121 network=tcp +34:29.219 [INFO] TestAgent: started state syncer +34:29.219 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25119 network=tcp +34:29.277 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:29.277 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25118 [Candidate]" term=2 +34:29.277 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:29.277 [DEBUG] TestAgent.server.raft: vote granted: from=8fcd5dfe-dc9c-1114-4683-4e5a27678354 term=2 tally=1 +34:29.277 [INFO] TestAgent.server.raft: election won: tally=1 +34:29.277 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25118 [Leader]" +34:29.278 [INFO] TestAgent.server: cluster leadership acquired +34:29.278 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25118 +34:29.278 [INFO] TestAgent.server: New leader elected: payload=Node-8fcd5dfe-dc9c-1114-4683-4e5a27678354 +34:29.279 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:29.282 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:29.282 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:29.282 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:29.282 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:29.282 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:29.282 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8fcd5dfe-dc9c-1114-4683-4e5a27678354 +34:29.282 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8fcd5dfe-dc9c-1114-4683-4e5a27678354 +34:29.283 [INFO] TestAgent.server: federation state anti-entropy synced +34:29.402 [INFO] TestAgent: Requesting shutdown +34:29.402 [INFO] TestAgent.server: shutting down server +34:29.402 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:29.402 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:29.402 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:29.402 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:29.402 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:29.402 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:29.402 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:29.402 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:29.402 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:29.402 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:29.402 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:29.402 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:29.403Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:29.403Z [INFO] agent.router.manager: shutting down +34:29.403 [INFO] TestAgent: consul server down +34:29.403 [INFO] TestAgent: shutdown complete +34:29.403 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25106 network=tcp +34:29.403 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25106 network=udp +34:29.403 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25121 network=tcp +34:29.903 [INFO] TestAgent: Waiting for endpoints to shut down +34:29.903 [INFO] TestAgent: Endpoints down +--- PASS: TestIntentionsSpecificDelete_good (0.75s) +=== CONT TestIntentionsSpecificUpdate_good +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +34:24.936 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-3c1ba7c4-0abe-f77f-9eda-afce48669d8e 127.0.0.1 +2021-08-13T15:34:24.936Z [INFO] agent.router: Initializing LAN area manager +34:24.936 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24965 network=tcp +34:24.936 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24965 network=udp +34:24.937 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24966 network=tcp +34:24.937 [INFO] TestAgent: started state syncer +34:24.937 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24985 network=tcp +2021-08-13T15:34:24.937Z [WARN] agent.router.manager: No servers available +34:24.937 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" +34:24.938 [DEBUG] TestAgent.client.memberlist.lan: memberlist: Stream connection from=127.0.0.1:49728 +34:24.939 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-e3a3bad3-7d05-3c07-c2ad-30b6ef5e0dd7 127.0.0.1 +34:24.939 [INFO] TestAgent.client: adding server: server="Node-e3a3bad3-7d05-3c07-c2ad-30b6ef5e0dd7 (Addr: tcp/127.0.0.1:24963) (DC: dc1)" +34:24.939 [INFO] TestAgent.client: New leader elected: payload=Node-e3a3bad3-7d05-3c07-c2ad-30b6ef5e0dd7 +34:24.940 [INFO] TestAgent.client: client starting leave +34:25.076 [DEBUG] TestAgent.client.serf.lan: serf: messageUserEventType: consul:new-leader +34:25.076 [DEBUG] TestAgent.client.serf.lan: serf: messageJoinType: Node-e3a3bad3-7d05-3c07-c2ad-30b6ef5e0dd7 +34:25.275 [DEBUG] TestAgent.client.serf.lan: serf: messageUserEventType: consul:new-leader +34:25.275 [DEBUG] TestAgent.client.serf.lan: serf: messageJoinType: Node-e3a3bad3-7d05-3c07-c2ad-30b6ef5e0dd7 +34:25.275 [DEBUG] TestAgent.client.serf.lan: serf: messageLeaveType: Node-3c1ba7c4-0abe-f77f-9eda-afce48669d8e +34:25.479 [DEBUG] TestAgent.client.serf.lan: serf: messageUserEventType: consul:new-leader +34:25.479 [DEBUG] TestAgent.client.serf.lan: serf: messageJoinType: Node-e3a3bad3-7d05-3c07-c2ad-30b6ef5e0dd7 +34:25.479 [DEBUG] TestAgent.client.serf.lan: serf: messageLeaveType: Node-3c1ba7c4-0abe-f77f-9eda-afce48669d8e +34:25.675 [INFO] TestAgent.client.serf.lan: serf: EventMemberLeave: Node-3c1ba7c4-0abe-f77f-9eda-afce48669d8e 127.0.0.1 +34:25.675 [DEBUG] TestAgent.client.serf.lan: serf: messageLeaveType: Node-3c1ba7c4-0abe-f77f-9eda-afce48669d8e +34:25.675 [DEBUG] TestAgent.client.serf.lan: serf: messageUserEventType: consul:new-leader +34:25.675 [DEBUG] TestAgent.client.serf.lan: serf: messageJoinType: Node-e3a3bad3-7d05-3c07-c2ad-30b6ef5e0dd7 +34:25.676 [DEBUG] TestAgent.client.serf.lan: serf: messageLeaveType: Node-3c1ba7c4-0abe-f77f-9eda-afce48669d8e +34:26.536 [INFO] TestAgent: Synced node info +34:27.412 [DEBUG] TestAgent: Node info in sync +34:27.412 [DEBUG] TestAgent: Node info in sync +34:29.137 [INFO] TestAgent: Requesting shutdown +34:29.137 [INFO] TestAgent.client: shutting down client +34:29.137 [INFO] TestAgent: consul client down +34:29.137 [INFO] TestAgent: shutdown complete +34:29.163 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24965 network=tcp +34:29.163 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24965 network=udp +34:29.163 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24966 network=tcp +34:29.663 [INFO] TestAgent: Waiting for endpoints to shut down +34:29.663 [INFO] TestAgent: Endpoints down +2021-08-13T15:34:24.655Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:24.670Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:24.673 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e3a3bad3-7d05-3c07-c2ad-30b6ef5e0dd7 Address:127.0.0.1:24963}]" +34:24.673 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24963 [Follower]" leader= +34:24.674 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e3a3bad3-7d05-3c07-c2ad-30b6ef5e0dd7.dc1 127.0.0.1 +34:24.674 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e3a3bad3-7d05-3c07-c2ad-30b6ef5e0dd7 127.0.0.1 +2021-08-13T15:34:24.674Z [INFO] agent.router: Initializing LAN area manager +34:24.675 [INFO] TestAgent.server: Adding LAN server: server="Node-e3a3bad3-7d05-3c07-c2ad-30b6ef5e0dd7 (Addr: tcp/127.0.0.1:24963) (DC: dc1)" +34:24.675 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e3a3bad3-7d05-3c07-c2ad-30b6ef5e0dd7.dc1 area=wan +34:24.675 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24972 network=udp +34:24.675 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24972 network=tcp +34:24.676 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24973 network=tcp +34:24.676 [INFO] TestAgent: started state syncer +34:24.676 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24964 network=tcp +34:24.739 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:24.739 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24963 [Candidate]" term=2 +34:24.739 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:24.739 [DEBUG] TestAgent.server.raft: vote granted: from=e3a3bad3-7d05-3c07-c2ad-30b6ef5e0dd7 term=2 tally=1 +34:24.739 [INFO] TestAgent.server.raft: election won: tally=1 +34:24.739 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24963 [Leader]" +34:24.739 [INFO] TestAgent.server: cluster leadership acquired +34:24.739 [INFO] TestAgent.server: New leader elected: payload=Node-e3a3bad3-7d05-3c07-c2ad-30b6ef5e0dd7 +34:24.740 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24963 +34:24.741 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:24.743 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:24.743 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:24.743 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:24.743 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:24.743 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:24.743 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e3a3bad3-7d05-3c07-c2ad-30b6ef5e0dd7 +34:24.743 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e3a3bad3-7d05-3c07-c2ad-30b6ef5e0dd7 +34:24.744 [INFO] TestAgent.server: federation state anti-entropy synced +34:24.938 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:24975] +34:24.938 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:24975 +34:24.940 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3c1ba7c4-0abe-f77f-9eda-afce48669d8e 127.0.0.1 +34:24.940 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 +34:24.940 [DEBUG] TestAgent: systemd notify failed: error="No socket" +34:24.940 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3c1ba7c4-0abe-f77f-9eda-afce48669d8e +34:25.020 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:25.020 [INFO] TestAgent: Synced node info +34:25.020 [DEBUG] TestAgent: Node info in sync +34:25.137 [DEBUG] TestAgent.server.serf.lan: serf: messageJoinType: Node-e3a3bad3-7d05-3c07-c2ad-30b6ef5e0dd7 +34:25.137 [DEBUG] TestAgent.server.serf.lan: serf: messageLeaveType: Node-3c1ba7c4-0abe-f77f-9eda-afce48669d8e +34:25.342 [DEBUG] TestAgent.server.serf.lan: serf: messageJoinType: Node-e3a3bad3-7d05-3c07-c2ad-30b6ef5e0dd7 +34:25.342 [DEBUG] TestAgent.server.serf.lan: serf: messageLeaveType: Node-3c1ba7c4-0abe-f77f-9eda-afce48669d8e +34:25.537 [DEBUG] TestAgent.server.serf.lan: serf: messageJoinType: Node-e3a3bad3-7d05-3c07-c2ad-30b6ef5e0dd7 +34:25.537 [DEBUG] TestAgent.server.serf.lan: serf: messageLeaveType: Node-3c1ba7c4-0abe-f77f-9eda-afce48669d8e +34:25.557 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:25.557 [DEBUG] TestAgent: Node info in sync +34:25.676 [DEBUG] TestAgent.server.serf.lan: serf: messageJoinType: Node-e3a3bad3-7d05-3c07-c2ad-30b6ef5e0dd7 +34:25.676 [DEBUG] TestAgent.server.serf.lan: serf: messageLeaveType: Node-3c1ba7c4-0abe-f77f-9eda-afce48669d8e +34:25.736 [INFO] TestAgent.server.serf.lan: serf: EventMemberLeave: Node-3c1ba7c4-0abe-f77f-9eda-afce48669d8e 127.0.0.1 +34:25.736 [INFO] TestAgent.server: deregistering member: member=Node-3c1ba7c4-0abe-f77f-9eda-afce48669d8e reason=left +34:29.663 [INFO] TestAgent: Requesting shutdown +34:29.663 [INFO] TestAgent.server: shutting down server +34:29.663 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:29.663 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:29.663 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:29.663 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:29.663 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:29.663 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:29.663 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:29.663 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:29.663 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:29.663 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:29.663 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:29.664Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:29.664Z [INFO] agent.router.manager: shutting down +34:29.664 [INFO] TestAgent: consul server down +34:29.664 [INFO] TestAgent: shutdown complete +34:29.664 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24972 network=tcp +34:29.664 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24972 network=udp +34:29.664 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24973 network=tcp +34:30.164 [INFO] TestAgent: Waiting for endpoints to shut down +34:30.164 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Leave (5.53s) === CONT TestAgent_Metrics_ACLDeny -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:29.841Z [WARN] TestAgent_Reload: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:29.463Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:29.480Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:29.483 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a3c21670-94bc-6c6d-36c6-0957383381dc Address:127.0.0.1:25125}]" +34:29.483 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25125 [Follower]" leader= +34:29.484 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a3c21670-94bc-6c6d-36c6-0957383381dc.dc1 127.0.0.1 +34:29.484 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a3c21670-94bc-6c6d-36c6-0957383381dc 127.0.0.1 +2021-08-13T15:34:29.485Z [INFO] agent.router: Initializing LAN area manager +34:29.485 [INFO] TestAgent.server: Adding LAN server: server="Node-a3c21670-94bc-6c6d-36c6-0957383381dc (Addr: tcp/127.0.0.1:25125) (DC: dc1)" +34:29.485 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a3c21670-94bc-6c6d-36c6-0957383381dc.dc1 area=wan +34:29.485 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25134 network=tcp +34:29.486 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25134 network=udp +34:29.486 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25135 network=tcp +34:29.487 [WARN] TestAgent: The 'handler' field in watches has been deprecated and replaced with the 'args' field. See https://www.consul.io/docs/agent/watches.html +34:29.487 [INFO] TestAgent: started state syncer +34:29.487 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25126 network=tcp +34:29.550 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:29.550 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25125 [Candidate]" term=2 +34:29.550 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:29.550 [DEBUG] TestAgent.server.raft: vote granted: from=a3c21670-94bc-6c6d-36c6-0957383381dc term=2 tally=1 +34:29.550 [INFO] TestAgent.server.raft: election won: tally=1 +34:29.550 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25125 [Leader]" +34:29.551 [INFO] TestAgent.server: cluster leadership acquired +34:29.551 [INFO] TestAgent.server: New leader elected: payload=Node-a3c21670-94bc-6c6d-36c6-0957383381dc +34:29.551 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25125 +34:29.552 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:29.555 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:29.555 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:29.555 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:29.555 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:29.555 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:29.555 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a3c21670-94bc-6c6d-36c6-0957383381dc +34:29.555 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a3c21670-94bc-6c6d-36c6-0957383381dc +34:29.557 [INFO] TestAgent.server: federation state anti-entropy synced +34:29.629 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test?dc=dc1 from=127.0.0.1:33948 latency=141.029507ms +34:29.633 [DEBUG] TestAgent: watch handler output: watch_handler=true output= +34:29.677 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:29.678 [INFO] TestAgent: Synced node info +34:29.678 [INFO] TestAgent: Synced service: service=redis +34:29.841 [INFO] TestAgent: Requesting shutdown +34:29.841 [INFO] TestAgent.server: shutting down server +34:29.841 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:29.842 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:29.842 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:29.842Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:29.842Z [INFO] agent.router.manager: shutting down +34:29.842 [INFO] TestAgent: consul server down +34:29.842 [INFO] TestAgent: shutdown complete +34:29.842 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25134 network=tcp +34:29.842 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25134 network=udp +34:29.842 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25135 network=tcp +34:30.342 [INFO] TestAgent: Waiting for endpoints to shut down +34:30.342 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Reload (0.90s) +=== CONT TestAgent_Self_ACLDeny === RUN TestAgent_Metrics_ACLDeny/no_token === RUN TestAgent_Metrics_ACLDeny/agent_master_token === RUN TestAgent_Metrics_ACLDeny/read-only_token -=== RUN TestAgent_Reload_ACLDeny/no_token -=== RUN TestAgent_Reload_ACLDeny/read-only_token -2022-09-15T21:43:02.353Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:02.370Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:02.373 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:59fd0d94-9fda-cd5c-ac55-d2a48baa5c11 Address:127.0.0.1:13502}]" -43:02.373 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13502 [Follower]" leader= -43:02.375 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-59fd0d94-9fda-cd5c-ac55-d2a48baa5c11.dc1 127.0.0.1 -43:02.376 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-59fd0d94-9fda-cd5c-ac55-d2a48baa5c11 127.0.0.1 -2022-09-15T21:43:02.376Z [INFO] agent.router: Initializing LAN area manager -43:02.376 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13497 network=udp -43:02.377 [INFO] TestAgent.server: Adding LAN server: server="Node-59fd0d94-9fda-cd5c-ac55-d2a48baa5c11 (Addr: tcp/127.0.0.1:13502) (DC: dc1)" -43:02.377 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-59fd0d94-9fda-cd5c-ac55-d2a48baa5c11.dc1 area=wan -43:02.377 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13497 network=tcp -43:02.380 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13505 network=tcp -43:02.380 [INFO] TestAgent: started state syncer -43:02.380 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13503 network=tcp -43:02.416 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:02.416 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13502 [Candidate]" term=2 -43:02.416 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:02.416 [DEBUG] TestAgent.server.raft: vote granted: from=59fd0d94-9fda-cd5c-ac55-d2a48baa5c11 term=2 tally=1 -43:02.416 [INFO] TestAgent.server.raft: election won: tally=1 -43:02.416 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13502 [Leader]" -43:02.416 [INFO] TestAgent.server: cluster leadership acquired -43:02.417 [INFO] TestAgent.server: New leader elected: payload=Node-59fd0d94-9fda-cd5c-ac55-d2a48baa5c11 -43:02.417 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13502 -43:02.418 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:02.421 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:02.421 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:02.421 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:02.421 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:02.421 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:02.421 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-59fd0d94-9fda-cd5c-ac55-d2a48baa5c11 -43:02.421 [INFO] TestAgent.server: member joined, marking health alive: member=Node-59fd0d94-9fda-cd5c-ac55-d2a48baa5c11 -43:02.422 [INFO] TestAgent.server: federation state anti-entropy synced -43:02.701 [INFO] TestAgent: Requesting shutdown -43:02.701 [INFO] TestAgent.server: shutting down server -43:02.701 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:02.701 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:02.701 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:02.701 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:02.701 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:02.701 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:02.701 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:02.701 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:02.701 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:02.701 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:02.701 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:02.702 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:02.702Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:02.702Z [INFO] agent.router.manager: shutting down -43:02.702 [INFO] TestAgent: consul server down -43:02.702 [INFO] TestAgent: shutdown complete -43:02.702 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13497 network=tcp -43:02.702 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13497 network=udp -43:02.702 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13505 network=tcp -43:03.202 [INFO] TestAgent: Waiting for endpoints to shut down -43:03.202 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Members (0.87s) -=== CONT TestAgent_Self_ACLDeny -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! +34:29.859 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-c5f4863a-82c1-ca2e-e22c-95f5538bc561 127.0.0.1 +2021-08-13T15:34:29.860Z [INFO] agent.router: Initializing LAN area manager +34:29.860 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25141 network=tcp +34:29.860 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25141 network=udp +34:29.861 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25142 network=tcp +34:29.861 [INFO] TestAgent: started state syncer +34:29.862 [INFO] TestAgent: Requesting shutdown +34:29.862 [INFO] TestAgent.client: shutting down client +34:29.862 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave +34:29.862 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25147 network=tcp +34:29.862 [INFO] TestAgent: consul client down +34:29.862 [INFO] TestAgent: shutdown complete +34:29.862 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25141 network=tcp +34:29.862 [ERROR] TestAgent: gRPC server failed: error="grpc: the server has been stopped" +34:29.862 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25141 network=udp +34:29.862 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25142 network=tcp +2021-08-13T15:34:29.862Z [WARN] agent.router.manager: No servers available +34:29.862 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" +34:30.364 [INFO] TestAgent: Waiting for endpoints to shut down +34:30.364 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_LoadKeyrings (1.98s) + --- PASS: TestAgent_LoadKeyrings/no_keys (0.67s) + --- PASS: TestAgent_LoadKeyrings/server_with_keys (0.77s) + --- PASS: TestAgent_LoadKeyrings/client_with_keys (0.54s) +=== CONT TestIntentionsSpecificGet_invalidId +2021-08-13T15:34:29.775Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:29.792Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:29.794 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:340f6f82-6cac-eea8-442e-db8679296c3c Address:127.0.0.1:25139}]" +34:29.794 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25139 [Follower]" leader= +34:29.795 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-340f6f82-6cac-eea8-442e-db8679296c3c.dc1 127.0.0.1 +34:29.796 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-340f6f82-6cac-eea8-442e-db8679296c3c 127.0.0.1 +2021-08-13T15:34:29.796Z [INFO] agent.router: Initializing LAN area manager +34:29.796 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25127 network=udp +34:29.797 [INFO] TestAgent.server: Adding LAN server: server="Node-340f6f82-6cac-eea8-442e-db8679296c3c (Addr: tcp/127.0.0.1:25139) (DC: dc1)" +34:29.797 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-340f6f82-6cac-eea8-442e-db8679296c3c.dc1 area=wan +34:29.797 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25127 network=tcp +34:29.798 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25114 network=tcp +34:29.798 [INFO] TestAgent: started state syncer +34:29.798 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25140 network=tcp +34:29.842 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:29.842 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25139 [Candidate]" term=2 +34:29.843 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:29.843 [DEBUG] TestAgent.server.raft: vote granted: from=340f6f82-6cac-eea8-442e-db8679296c3c term=2 tally=1 +34:29.843 [INFO] TestAgent.server.raft: election won: tally=1 +34:29.843 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25139 [Leader]" +34:29.843 [INFO] TestAgent.server: cluster leadership acquired +34:29.843 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25139 +34:29.844 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:29.845 [INFO] TestAgent.server: New leader elected: payload=Node-340f6f82-6cac-eea8-442e-db8679296c3c +34:29.847 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:29.847 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:29.847 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:29.847 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:29.847 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:29.847 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-340f6f82-6cac-eea8-442e-db8679296c3c +34:29.847 [INFO] TestAgent.server: member joined, marking health alive: member=Node-340f6f82-6cac-eea8-442e-db8679296c3c +34:29.848 [INFO] TestAgent.server: federation state anti-entropy synced +34:29.956 [INFO] TestAgent: Requesting shutdown +34:29.957 [INFO] TestAgent.server: shutting down server +34:29.957 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:29.957 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:29.957 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:29.957 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:29.957 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:29.957 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:29.957 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:29.957 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:29.957 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:29.957 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +34:29.957 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:29.957 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2021-08-13T15:34:29.957Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:29.957Z [INFO] agent.router.manager: shutting down +34:29.957 [INFO] TestAgent: consul server down +34:29.957 [INFO] TestAgent: shutdown complete +34:29.957 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25127 network=tcp +34:29.957 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25127 network=udp +34:29.957 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25114 network=tcp +34:30.457 [INFO] TestAgent: Waiting for endpoints to shut down +34:30.457 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_InmemKeyrings (2.64s) + --- PASS: TestAgent_InmemKeyrings/no_keys (0.68s) + --- PASS: TestAgent_InmemKeyrings/server_with_keys (0.72s) + --- PASS: TestAgent_InmemKeyrings/client_with_keys (0.54s) + --- PASS: TestAgent_InmemKeyrings/ignore_files (0.70s) +=== CONT TestAgent_Self +=== RUN TestAgent_Self/normal === RUN TestAgent_Self_ACLDeny/no_token === RUN TestAgent_Self_ACLDeny/agent_master_token === RUN TestAgent_Self_ACLDeny/read-only_token -[WARN] freeport: 4 out of 4 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:02.898Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:02.898Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:02.917Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:02.917Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:02.921 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8f355275-c9f1-07c6-906a-ab4da6727e72 Address:127.0.0.1:13530}]" -43:02.922 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13530 [Follower]" leader= -43:02.922 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8f355275-c9f1-07c6-906a-ab4da6727e72.dc1 127.0.0.1 -43:02.923 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8f355275-c9f1-07c6-906a-ab4da6727e72 127.0.0.1 -2022-09-15T21:43:02.923Z [INFO] agent.router: Initializing LAN area manager -43:02.923 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13539 network=udp -43:02.923 [INFO] TestAgent.server: Adding LAN server: server="Node-8f355275-c9f1-07c6-906a-ab4da6727e72 (Addr: tcp/127.0.0.1:13530) (DC: dc1)" -43:02.923 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8f355275-c9f1-07c6-906a-ab4da6727e72.dc1 area=wan -43:02.924 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13539 network=tcp -43:02.925 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13526 network=tcp -43:02.925 [INFO] TestAgent: started state syncer -43:02.925 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13531 network=tcp -43:02.966 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:02.966 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13530 [Candidate]" term=2 -43:02.966 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:02.966 [DEBUG] TestAgent.server.raft: vote granted: from=8f355275-c9f1-07c6-906a-ab4da6727e72 term=2 tally=1 -43:02.966 [INFO] TestAgent.server.raft: election won: tally=1 -43:02.966 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13530 [Leader]" -43:02.966 [INFO] TestAgent.server: cluster leadership acquired -43:02.967 [INFO] TestAgent.server: initializing acls -43:02.967 [INFO] TestAgent.server: Created ACL 'global-management' policy -43:02.967 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -43:02.967 [INFO] TestAgent.server: New leader elected: payload=Node-8f355275-c9f1-07c6-906a-ab4da6727e72 -43:02.968 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -43:02.968 [INFO] TestAgent.server: Created ACL anonymous token from configuration -43:02.968 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -43:02.968 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -43:02.969 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-8f355275-c9f1-07c6-906a-ab4da6727e72 -43:02.969 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-8f355275-c9f1-07c6-906a-ab4da6727e72.dc1 -43:02.970 [INFO] TestAgent.server: Updating LAN server: server="Node-8f355275-c9f1-07c6-906a-ab4da6727e72 (Addr: tcp/127.0.0.1:13530) (DC: dc1)" -43:02.970 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-8f355275-c9f1-07c6-906a-ab4da6727e72.dc1 area=wan -43:02.970 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:02.973 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:02.973 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:02.973 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:02.973 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:02.973 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:02.973 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8f355275-c9f1-07c6-906a-ab4da6727e72 -43:02.973 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8f355275-c9f1-07c6-906a-ab4da6727e72 -43:02.976 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8f355275-c9f1-07c6-906a-ab4da6727e72 -43:02.977 [INFO] TestAgent.server: federation state anti-entropy synced -43:03.052 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-8f355275-c9f1-07c6-906a-ab4da6727e72 -43:03.052 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-8f355275-c9f1-07c6-906a-ab4da6727e72 -43:03.055 [INFO] TestAgent: Requesting shutdown -43:03.055 [INFO] TestAgent.server: shutting down server -43:03.055 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -43:03.055 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -43:03.055 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:03.055 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:03.055 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:03.055 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:03.055 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:03.055 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -43:03.055 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:03.055 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:03.055 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:03.055 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -43:03.055 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:03.055 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:03.055 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:03.055 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:03.055Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:03.055Z [INFO] agent.router.manager: shutting down -43:03.056 [INFO] TestAgent: consul server down -43:03.056 [INFO] TestAgent: shutdown complete -43:03.056 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13539 network=tcp -43:03.056 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13539 network=udp -43:03.056 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13526 network=tcp -43:03.556 [INFO] TestAgent: Waiting for endpoints to shut down -43:03.556 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Metrics_ACLDeny (0.68s) +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:29.925Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:29.961Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:29.965 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d7ce238b-2b69-726c-aa93-679e06980134 Address:127.0.0.1:25160}]" +34:29.965 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25160 [Follower]" leader= +34:29.966 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d7ce238b-2b69-726c-aa93-679e06980134.dc1 127.0.0.1 +34:29.967 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d7ce238b-2b69-726c-aa93-679e06980134 127.0.0.1 +2021-08-13T15:34:29.967Z [INFO] agent.router: Initializing LAN area manager +34:29.967 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d7ce238b-2b69-726c-aa93-679e06980134.dc1 area=wan +34:29.967 [INFO] TestAgent.server: Adding LAN server: server="Node-d7ce238b-2b69-726c-aa93-679e06980134 (Addr: tcp/127.0.0.1:25160) (DC: dc1)" +34:29.967 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25148 network=tcp +34:29.967 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25148 network=udp +34:29.968 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25149 network=tcp +34:29.968 [INFO] TestAgent: started state syncer +34:29.968 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25161 network=tcp +34:30.005 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:30.005 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25160 [Candidate]" term=2 +34:30.005 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:30.005 [DEBUG] TestAgent.server.raft: vote granted: from=d7ce238b-2b69-726c-aa93-679e06980134 term=2 tally=1 +34:30.005 [INFO] TestAgent.server.raft: election won: tally=1 +34:30.005 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25160 [Leader]" +34:30.005 [INFO] TestAgent.server: cluster leadership acquired +34:30.006 [INFO] TestAgent.server: New leader elected: payload=Node-d7ce238b-2b69-726c-aa93-679e06980134 +34:30.006 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25160 +34:30.007 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:30.010 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:30.010 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:30.010 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:30.010 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:30.010 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:30.010 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d7ce238b-2b69-726c-aa93-679e06980134 +34:30.010 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d7ce238b-2b69-726c-aa93-679e06980134 +34:30.011 [INFO] TestAgent.server: federation state anti-entropy synced +34:30.141 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:30.142 [INFO] TestAgent: Synced node info +34:30.204 [INFO] TestAgent: Requesting shutdown +34:30.204 [INFO] TestAgent.server: shutting down server +34:30.204 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:30.204 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:30.204 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:30.204 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:30.204 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:30.204 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:30.204 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:30.204 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:30.204 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:30.204 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:30.204 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:30.204Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:30.204Z [INFO] agent.router.manager: shutting down +34:30.205 [INFO] TestAgent: consul server down +34:30.205 [INFO] TestAgent: shutdown complete +34:30.205 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25148 network=tcp +34:30.205 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25148 network=udp +34:30.205 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25149 network=tcp +34:30.705 [INFO] TestAgent: Waiting for endpoints to shut down +34:30.705 [INFO] TestAgent: Endpoints down +--- PASS: TestIntentionsSpecificUpdate_good (0.80s) +=== CONT TestIntentionsSpecificGet_good +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:24.512Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:24.554Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:24.557 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f582df29-6ed2-e9f7-7ee3-6450c25ce8a6 Address:127.0.0.1:24942}]" +34:24.557 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24942 [Follower]" leader= +34:24.558 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f582df29-6ed2-e9f7-7ee3-6450c25ce8a6.dc1 127.0.0.1 +34:24.558 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f582df29-6ed2-e9f7-7ee3-6450c25ce8a6 127.0.0.1 +2021-08-13T15:34:24.558Z [INFO] agent.router: Initializing LAN area manager +34:24.559 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24958 network=udp +34:24.559 [INFO] TestAgent.server: Adding LAN server: server="Node-f582df29-6ed2-e9f7-7ee3-6450c25ce8a6 (Addr: tcp/127.0.0.1:24942) (DC: dc1)" +34:24.559 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f582df29-6ed2-e9f7-7ee3-6450c25ce8a6.dc1 area=wan +34:24.559 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24958 network=tcp +34:24.560 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24959 network=tcp +34:24.560 [INFO] TestAgent: started state syncer +34:24.560 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24936 network=tcp +34:24.625 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:24.625 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24942 [Candidate]" term=2 +34:24.625 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:24.625 [DEBUG] TestAgent.server.raft: vote granted: from=f582df29-6ed2-e9f7-7ee3-6450c25ce8a6 term=2 tally=1 +34:24.625 [INFO] TestAgent.server.raft: election won: tally=1 +34:24.625 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24942 [Leader]" +34:24.626 [INFO] TestAgent.server: cluster leadership acquired +34:24.626 [INFO] TestAgent.server: New leader elected: payload=Node-f582df29-6ed2-e9f7-7ee3-6450c25ce8a6 +34:24.626 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24942 +34:24.627 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:24.629 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:24.629 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:24.629 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:24.630 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:24.630 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:24.630 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f582df29-6ed2-e9f7-7ee3-6450c25ce8a6 +34:24.630 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f582df29-6ed2-e9f7-7ee3-6450c25ce8a6 +34:24.631 [INFO] TestAgent.server: federation state anti-entropy synced +34:24.925 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:50350 +34:24.926 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ed8f033c-39b8-118f-efbb-e41a4102bd2e 127.0.0.1 +34:24.927 [INFO] TestAgent: Requesting shutdown +34:24.927 [INFO] TestAgent.server: shutting down server +34:24.927 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:24.927 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:24.927 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:24.927 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:24.927 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:24.927 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:24.927 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:24.927 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:24.927 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:24.927 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:24.927Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:24.927Z [INFO] agent.router.manager: shutting down +34:24.927 [INFO] TestAgent: consul server down +34:24.927 [INFO] TestAgent: shutdown complete +34:24.927 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24958 network=tcp +34:24.928 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24958 network=udp +34:24.928 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24959 network=tcp +34:24.928 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:24.928 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:25.428 [INFO] TestAgent: Waiting for endpoints to shut down +34:25.428 [INFO] TestAgent: Endpoints down +2021-08-13T15:34:24.306Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:24.322Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:24.324 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ed8f033c-39b8-118f-efbb-e41a4102bd2e Address:127.0.0.1:24949}]" +34:24.324 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24949 [Follower]" leader= +34:24.325 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ed8f033c-39b8-118f-efbb-e41a4102bd2e.dc1 127.0.0.1 +34:24.326 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ed8f033c-39b8-118f-efbb-e41a4102bd2e 127.0.0.1 +2021-08-13T15:34:24.326Z [INFO] agent.router: Initializing LAN area manager +34:24.326 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ed8f033c-39b8-118f-efbb-e41a4102bd2e.dc1 area=wan +34:24.326 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24930 network=udp +34:24.326 [INFO] TestAgent.server: Adding LAN server: server="Node-ed8f033c-39b8-118f-efbb-e41a4102bd2e (Addr: tcp/127.0.0.1:24949) (DC: dc1)" +34:24.326 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24930 network=tcp +34:24.327 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24931 network=tcp +34:24.327 [INFO] TestAgent: started state syncer +34:24.327 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24922 network=tcp +34:24.386 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:24.386 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24949 [Candidate]" term=2 +34:24.386 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:24.386 [DEBUG] TestAgent.server.raft: vote granted: from=ed8f033c-39b8-118f-efbb-e41a4102bd2e term=2 tally=1 +34:24.386 [INFO] TestAgent.server.raft: election won: tally=1 +34:24.386 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24949 [Leader]" +34:24.387 [INFO] TestAgent.server: cluster leadership acquired +34:24.387 [INFO] TestAgent.server: New leader elected: payload=Node-ed8f033c-39b8-118f-efbb-e41a4102bd2e +34:24.387 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24949 +34:24.388 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:24.391 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:24.391 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:24.391 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:24.391 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:24.391 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:24.391 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ed8f033c-39b8-118f-efbb-e41a4102bd2e +34:24.391 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ed8f033c-39b8-118f-efbb-e41a4102bd2e +34:24.391 [INFO] TestAgent.server: federation state anti-entropy synced +34:24.778 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:24.779 [INFO] TestAgent: Synced node info +34:24.779 [DEBUG] TestAgent: Node info in sync +34:24.925 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:24940] +34:24.925 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:24940 +34:24.927 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f582df29-6ed2-e9f7-7ee3-6450c25ce8a6 127.0.0.1 +34:24.927 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 +34:24.927 [DEBUG] TestAgent: systemd notify failed: error="No socket" +34:24.927 [INFO] TestAgent.server: Adding LAN server: server="Node-f582df29-6ed2-e9f7-7ee3-6450c25ce8a6 (Addr: tcp/127.0.0.1:24942) (DC: dc1)" +34:24.927 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-f582df29-6ed2-e9f7-7ee3-6450c25ce8a6 other=Node-ed8f033c-39b8-118f-efbb-e41a4102bd2e +34:24.927 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f582df29-6ed2-e9f7-7ee3-6450c25ce8a6 +34:24.928 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:24941: connect: connection refused +34:24.928 [DEBUG] TestAgent.server: Failed to flood-join server at address: server=Node-f582df29-6ed2-e9f7-7ee3-6450c25ce8a6.dc1 address=127.0.0.1:24941 error="1 error occurred: + * Failed to join 127.0.0.1: dial tcp 127.0.0.1:24941: connect: connection refused + +" +34:25.827 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Failed ping: Node-f582df29-6ed2-e9f7-7ee3-6450c25ce8a6 (timeout reached) +34:26.285 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:26.285 [DEBUG] TestAgent: Node info in sync +34:26.326 [INFO] TestAgent.server.memberlist.lan: memberlist: Suspect Node-f582df29-6ed2-e9f7-7ee3-6450c25ce8a6 has failed, no acks received +34:26.388 [WARN] TestAgent: error getting server health from server: server=Node-f582df29-6ed2-e9f7-7ee3-6450c25ce8a6 error="rpc error getting client: failed to get conn: dial tcp 127.0.0.1:0->127.0.0.1:24942: connect: connection refused" +34:27.391 [WARN] TestAgent: error getting server health from server: server=Node-f582df29-6ed2-e9f7-7ee3-6450c25ce8a6 error="context deadline exceeded" +34:27.827 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Failed ping: Node-f582df29-6ed2-e9f7-7ee3-6450c25ce8a6 (timeout reached) +34:28.388 [WARN] TestAgent: error getting server health from server: server=Node-f582df29-6ed2-e9f7-7ee3-6450c25ce8a6 error="rpc error getting client: failed to get conn: dial tcp 127.0.0.1:0->127.0.0.1:24942: connect: connection refused" +34:29.327 [INFO] TestAgent.server.memberlist.lan: memberlist: Suspect Node-f582df29-6ed2-e9f7-7ee3-6450c25ce8a6 has failed, no acks received +34:29.388 [WARN] TestAgent: error getting server health from server: server=Node-f582df29-6ed2-e9f7-7ee3-6450c25ce8a6 error="context deadline exceeded" +34:29.831 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Failed ping: Node-f582df29-6ed2-e9f7-7ee3-6450c25ce8a6 (timeout reached) +34:30.326 [INFO] TestAgent.server.memberlist.lan: memberlist: Marking Node-f582df29-6ed2-e9f7-7ee3-6450c25ce8a6 as failed, suspect timeout reached (0 peer confirmations) +34:30.326 [INFO] TestAgent.server.serf.lan: serf: EventMemberFailed: Node-f582df29-6ed2-e9f7-7ee3-6450c25ce8a6 127.0.0.1 +34:30.327 [INFO] TestAgent.server: Removing LAN server: server="Node-f582df29-6ed2-e9f7-7ee3-6450c25ce8a6 (Addr: tcp/127.0.0.1:24942) (DC: dc1)" +34:30.327 [INFO] TestAgent.server: member failed, marking health critical: member=Node-f582df29-6ed2-e9f7-7ee3-6450c25ce8a6 +34:30.347 [INFO] TestAgent: Force leaving node: node=Node-f582df29-6ed2-e9f7-7ee3-6450c25ce8a6 +34:30.347 [INFO] TestAgent.server.serf.lan: serf: EventMemberLeave (forced): Node-f582df29-6ed2-e9f7-7ee3-6450c25ce8a6 127.0.0.1 +34:30.347 [INFO] TestAgent: Requesting shutdown +34:30.347 [INFO] TestAgent.server: shutting down server +34:30.347 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:30.347 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:30.347 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:30.347 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:30.347 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:30.348 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:30.348 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:30.348 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:30.348 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:30.348 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:30.348 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:30.348Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:30.348Z [INFO] agent.router.manager: shutting down +34:30.348 [INFO] TestAgent: consul server down +34:30.348 [INFO] TestAgent: shutdown complete +34:30.348 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24930 network=tcp +34:30.348 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24930 network=udp +34:30.348 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24931 network=tcp +34:30.849 [INFO] TestAgent: Waiting for endpoints to shut down +34:30.849 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_ForceLeave (6.56s) +=== CONT TestAgent_Checks_ACLFilter +2021-08-13T15:34:30.183Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:30.183Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:30.200Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:30.200Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:30.202 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d14c95cc-30ef-0774-03ed-6040b413b988 Address:127.0.0.1:25153}]" +34:30.203 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d14c95cc-30ef-0774-03ed-6040b413b988.dc1 127.0.0.1 +34:30.204 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d14c95cc-30ef-0774-03ed-6040b413b988 127.0.0.1 +2021-08-13T15:34:30.204Z [INFO] agent.router: Initializing LAN area manager +34:30.204 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25162 network=udp +34:30.204 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25153 [Follower]" leader= +34:30.205 [INFO] TestAgent.server: Adding LAN server: server="Node-d14c95cc-30ef-0774-03ed-6040b413b988 (Addr: tcp/127.0.0.1:25153) (DC: dc1)" +34:30.205 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d14c95cc-30ef-0774-03ed-6040b413b988.dc1 area=wan +34:30.205 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25162 network=tcp +34:30.206 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25163 network=tcp +34:30.206 [INFO] TestAgent: started state syncer +34:30.206 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25154 network=tcp +34:30.255 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:30.270 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:30.270 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25153 [Candidate]" term=2 +34:30.270 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:30.270 [DEBUG] TestAgent.server.raft: vote granted: from=d14c95cc-30ef-0774-03ed-6040b413b988 term=2 tally=1 +34:30.270 [INFO] TestAgent.server.raft: election won: tally=1 +34:30.270 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25153 [Leader]" +34:30.270 [INFO] TestAgent.server: cluster leadership acquired +34:30.271 [INFO] TestAgent.server: initializing acls +34:30.271 [INFO] TestAgent.server: New leader elected: payload=Node-d14c95cc-30ef-0774-03ed-6040b413b988 +34:30.271 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:30.271 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:30.272 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:30.272 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:30.272 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:30.272 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:30.273 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-d14c95cc-30ef-0774-03ed-6040b413b988 +34:30.273 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-d14c95cc-30ef-0774-03ed-6040b413b988.dc1 +34:30.273 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-d14c95cc-30ef-0774-03ed-6040b413b988.dc1 area=wan +34:30.273 [INFO] TestAgent.server: Updating LAN server: server="Node-d14c95cc-30ef-0774-03ed-6040b413b988 (Addr: tcp/127.0.0.1:25153) (DC: dc1)" +34:30.274 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:30.276 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:30.276 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:30.276 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:30.276 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:30.276 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:30.276 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d14c95cc-30ef-0774-03ed-6040b413b988 +34:30.276 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d14c95cc-30ef-0774-03ed-6040b413b988 +34:30.278 [INFO] TestAgent.server: federation state anti-entropy synced +34:30.278 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d14c95cc-30ef-0774-03ed-6040b413b988 +34:30.361 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-d14c95cc-30ef-0774-03ed-6040b413b988 +34:30.361 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-d14c95cc-30ef-0774-03ed-6040b413b988 +34:30.363 [INFO] TestAgent: Requesting shutdown +34:30.363 [INFO] TestAgent.server: shutting down server +34:30.363 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:30.363 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:30.363 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:30.363 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:30.363 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:30.363 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:30.363 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:30.363 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:30.363 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:30.363 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:30.363 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:30.363 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:30.363 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:30.363 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:30.363 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:30.363 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:30.364Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:30.364Z [INFO] agent.router.manager: shutting down +34:30.364 [INFO] TestAgent: consul server down +34:30.364 [INFO] TestAgent: shutdown complete +34:30.364 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25162 network=tcp +34:30.364 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25162 network=udp +34:30.364 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25163 network=tcp +34:30.864 [INFO] TestAgent: Waiting for endpoints to shut down +34:30.864 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Metrics_ACLDeny (0.70s) --- PASS: TestAgent_Metrics_ACLDeny/no_token (0.00s) --- PASS: TestAgent_Metrics_ACLDeny/agent_master_token (0.00s) --- PASS: TestAgent_Metrics_ACLDeny/read-only_token (0.00s) -=== CONT TestAgent_Self -=== RUN TestAgent_Self/normal -2022-09-15T21:43:02.646Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:02.646Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:02.665Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:02.665Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:02.668 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:150da930-3405-bf32-eee7-6bf07d25db51 Address:127.0.0.1:13509}]" -43:02.669 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-150da930-3405-bf32-eee7-6bf07d25db51.dc1 127.0.0.1 -43:02.670 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13509 [Follower]" leader= -43:02.670 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-150da930-3405-bf32-eee7-6bf07d25db51 127.0.0.1 -2022-09-15T21:43:02.670Z [INFO] agent.router: Initializing LAN area manager -43:02.671 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13504 network=udp -43:02.671 [INFO] TestAgent.server: Adding LAN server: server="Node-150da930-3405-bf32-eee7-6bf07d25db51 (Addr: tcp/127.0.0.1:13509) (DC: dc1)" -43:02.671 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-150da930-3405-bf32-eee7-6bf07d25db51.dc1 area=wan -43:02.671 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13504 network=tcp -43:02.672 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13491 network=tcp -43:02.672 [INFO] TestAgent: started state syncer -43:02.672 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13510 network=tcp -43:02.705 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:02.705 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13509 [Candidate]" term=2 -43:02.706 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:02.706 [DEBUG] TestAgent.server.raft: vote granted: from=150da930-3405-bf32-eee7-6bf07d25db51 term=2 tally=1 -43:02.706 [INFO] TestAgent.server.raft: election won: tally=1 -43:02.706 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13509 [Leader]" -43:02.706 [INFO] TestAgent.server: cluster leadership acquired -43:02.706 [INFO] TestAgent.server: New leader elected: payload=Node-150da930-3405-bf32-eee7-6bf07d25db51 -43:02.706 [INFO] TestAgent.server: initializing acls -43:02.707 [INFO] TestAgent.server: Created ACL 'global-management' policy -43:02.707 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -43:02.707 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -43:02.708 [INFO] TestAgent.server: Created ACL anonymous token from configuration -43:02.708 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -43:02.708 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -43:02.708 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-150da930-3405-bf32-eee7-6bf07d25db51 -43:02.708 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-150da930-3405-bf32-eee7-6bf07d25db51.dc1 -43:02.708 [INFO] TestAgent.server: Updating LAN server: server="Node-150da930-3405-bf32-eee7-6bf07d25db51 (Addr: tcp/127.0.0.1:13509) (DC: dc1)" -43:02.709 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-150da930-3405-bf32-eee7-6bf07d25db51.dc1 area=wan -43:02.710 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:02.712 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:02.712 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:02.712 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:02.712 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:02.712 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:02.712 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-150da930-3405-bf32-eee7-6bf07d25db51 -43:02.712 [INFO] TestAgent.server: member joined, marking health alive: member=Node-150da930-3405-bf32-eee7-6bf07d25db51 -43:02.713 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-150da930-3405-bf32-eee7-6bf07d25db51 -43:02.714 [INFO] TestAgent.server: federation state anti-entropy synced -43:03.077 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-150da930-3405-bf32-eee7-6bf07d25db51 -43:03.077 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-150da930-3405-bf32-eee7-6bf07d25db51 -43:03.079 [INFO] TestAgent: Requesting shutdown -43:03.079 [INFO] TestAgent.server: shutting down server -43:03.080 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -43:03.080 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -43:03.080 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:03.080 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:03.080 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -43:03.080 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:03.080 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:03.080 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:03.080 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:03.080 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:03.080 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:03.080 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -43:03.080 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:03.080 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:03.080 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:03.080 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:03.080Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:03.080Z [INFO] agent.router.manager: shutting down -43:03.080 [INFO] TestAgent: consul server down -43:03.080 [INFO] TestAgent: shutdown complete -43:03.080 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13504 network=tcp -43:03.081 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13504 network=udp -43:03.081 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13491 network=tcp -43:03.581 [INFO] TestAgent: Waiting for endpoints to shut down -43:03.581 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Reload_ACLDeny (0.97s) - --- PASS: TestAgent_Reload_ACLDeny/no_token (0.00s) - --- PASS: TestAgent_Reload_ACLDeny/read-only_token (0.00s) -=== CONT TestAgent_Checks_ACLFilter -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:03.289Z [WARN] TestAgent_Reload: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:02.868Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:02.903Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:02.906 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:58166ad7-cbe1-9248-2e38-a35cfd669617 Address:127.0.0.1:13537}]" -43:02.906 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13537 [Follower]" leader= -43:02.908 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-58166ad7-cbe1-9248-2e38-a35cfd669617.dc1 127.0.0.1 -43:02.908 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-58166ad7-cbe1-9248-2e38-a35cfd669617 127.0.0.1 -2022-09-15T21:43:02.909Z [INFO] agent.router: Initializing LAN area manager -43:02.909 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13511 network=udp -43:02.910 [INFO] TestAgent.server: Adding LAN server: server="Node-58166ad7-cbe1-9248-2e38-a35cfd669617 (Addr: tcp/127.0.0.1:13537) (DC: dc1)" -43:02.910 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-58166ad7-cbe1-9248-2e38-a35cfd669617.dc1 area=wan -43:02.910 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13511 network=tcp -43:02.911 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13512 network=tcp -43:02.912 [WARN] TestAgent: The 'handler' field in watches has been deprecated and replaced with the 'args' field. See https://www.consul.io/docs/agent/watches.html -43:02.912 [INFO] TestAgent: started state syncer -43:02.912 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13538 network=tcp -43:02.967 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:02.967 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13537 [Candidate]" term=2 -43:02.967 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:02.967 [DEBUG] TestAgent.server.raft: vote granted: from=58166ad7-cbe1-9248-2e38-a35cfd669617 term=2 tally=1 -43:02.967 [INFO] TestAgent.server.raft: election won: tally=1 -43:02.967 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13537 [Leader]" -43:02.968 [INFO] TestAgent.server: cluster leadership acquired -43:02.968 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13537 -43:02.969 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:02.970 [INFO] TestAgent.server: New leader elected: payload=Node-58166ad7-cbe1-9248-2e38-a35cfd669617 -43:02.973 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:02.973 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:02.973 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:02.973 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:02.973 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:02.973 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-58166ad7-cbe1-9248-2e38-a35cfd669617 -43:02.973 [INFO] TestAgent.server: member joined, marking health alive: member=Node-58166ad7-cbe1-9248-2e38-a35cfd669617 -43:02.975 [INFO] TestAgent.server: federation state anti-entropy synced -43:03.050 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:03.051 [INFO] TestAgent: Synced node info -43:03.052 [INFO] TestAgent: Synced service: service=redis -43:03.052 [DEBUG] TestAgent: Node info in sync -43:03.052 [DEBUG] TestAgent: Service in sync: service=redis -43:03.289 [INFO] TestAgent: Requesting shutdown -43:03.298 [INFO] TestAgent.server: shutting down server -43:03.298 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:03.298 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:03.298 [ERROR] TestAgent.http: Request error: method=GET url=/v1/kv/test?dc=dc1 from=127.0.0.1:42016 error="No cluster leader" -43:03.298 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:03.299Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:03.299Z [INFO] agent.router.manager: shutting down -43:03.299 [INFO] TestAgent: consul server down -43:03.299 [INFO] TestAgent: shutdown complete -43:03.299 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13511 network=tcp -43:03.299 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13511 network=udp -43:03.299 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13512 network=tcp -43:03.799 [INFO] TestAgent: Waiting for endpoints to shut down -43:03.799 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Reload (1.01s) -=== CONT TestAgent_HealthServicesACLEnforcement -=== RUN TestAgent_Checks_ACLFilter/no_token -=== RUN TestAgent_Checks_ACLFilter/root_token -2022-09-15T21:43:03.223Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:03.223Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:03.241Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:03.241Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:03.245 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3ac309da-ebe4-20a4-63ba-eda56654b84e Address:127.0.0.1:13551}]" -43:03.246 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13551 [Follower]" leader= -43:03.246 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3ac309da-ebe4-20a4-63ba-eda56654b84e.dc1 127.0.0.1 -43:03.247 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3ac309da-ebe4-20a4-63ba-eda56654b84e 127.0.0.1 -2022-09-15T21:43:03.247Z [INFO] agent.router: Initializing LAN area manager -43:03.247 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13532 network=udp -43:03.247 [INFO] TestAgent.server: Adding LAN server: server="Node-3ac309da-ebe4-20a4-63ba-eda56654b84e (Addr: tcp/127.0.0.1:13551) (DC: dc1)" -43:03.247 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3ac309da-ebe4-20a4-63ba-eda56654b84e.dc1 area=wan -43:03.248 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13532 network=tcp -43:03.248 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13540 network=tcp -43:03.249 [INFO] TestAgent: started state syncer -43:03.249 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13552 network=tcp -43:03.286 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:03.286 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13551 [Candidate]" term=2 -43:03.286 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:03.286 [DEBUG] TestAgent.server.raft: vote granted: from=3ac309da-ebe4-20a4-63ba-eda56654b84e term=2 tally=1 -43:03.286 [INFO] TestAgent.server.raft: election won: tally=1 -43:03.286 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13551 [Leader]" -43:03.286 [INFO] TestAgent.server: cluster leadership acquired -43:03.286 [INFO] TestAgent.server: initializing acls -43:03.287 [INFO] TestAgent.server: New leader elected: payload=Node-3ac309da-ebe4-20a4-63ba-eda56654b84e -43:03.287 [INFO] TestAgent.server: Created ACL 'global-management' policy -43:03.287 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -43:03.288 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -43:03.288 [INFO] TestAgent.server: Created ACL anonymous token from configuration -43:03.288 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -43:03.288 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -43:03.288 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-3ac309da-ebe4-20a4-63ba-eda56654b84e -43:03.289 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-3ac309da-ebe4-20a4-63ba-eda56654b84e.dc1 -43:03.290 [INFO] TestAgent.server: Updating LAN server: server="Node-3ac309da-ebe4-20a4-63ba-eda56654b84e (Addr: tcp/127.0.0.1:13551) (DC: dc1)" -43:03.290 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-3ac309da-ebe4-20a4-63ba-eda56654b84e.dc1 area=wan -43:03.297 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:03.300 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:03.300 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:03.300 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:03.300 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:03.300 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:03.300 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3ac309da-ebe4-20a4-63ba-eda56654b84e -43:03.300 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3ac309da-ebe4-20a4-63ba-eda56654b84e -43:03.301 [INFO] TestAgent.server: federation state anti-entropy synced -43:03.301 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3ac309da-ebe4-20a4-63ba-eda56654b84e -43:03.334 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:03.335 [INFO] TestAgent: Synced node info -43:03.428 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-3ac309da-ebe4-20a4-63ba-eda56654b84e -43:03.428 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-3ac309da-ebe4-20a4-63ba-eda56654b84e -43:03.437 [INFO] TestAgent: Requesting shutdown -43:03.437 [INFO] TestAgent.server: shutting down server -43:03.437 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -43:03.437 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -43:03.437 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:03.437 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:03.437 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:03.437 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:03.437 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:03.437 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -43:03.437 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:03.437 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:03.438 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:03.438 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:03.438 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:03.438 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:03.438Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:03.438Z [INFO] agent.router.manager: shutting down -43:03.438 [INFO] TestAgent: consul server down -43:03.438 [INFO] TestAgent: shutdown complete -43:03.438 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13532 network=tcp -43:03.438 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13532 network=udp -43:03.438 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13540 network=tcp -43:03.440 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -43:03.939 [INFO] TestAgent: Waiting for endpoints to shut down -43:03.939 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Self_ACLDeny (0.74s) +=== CONT TestIntentionsCreate_noBody +2021-08-13T15:34:30.360Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:30.361Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:30.377Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:30.377Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:30.380 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:30aecd08-aa05-b263-71b2-6688ecca6def Address:127.0.0.1:25174}]" +34:30.381 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-30aecd08-aa05-b263-71b2-6688ecca6def.dc1 127.0.0.1 +34:30.382 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-30aecd08-aa05-b263-71b2-6688ecca6def 127.0.0.1 +2021-08-13T15:34:30.382Z [INFO] agent.router: Initializing LAN area manager +34:30.382 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25155 network=udp +34:30.382 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25174 [Follower]" leader= +34:30.383 [INFO] TestAgent.server: Adding LAN server: server="Node-30aecd08-aa05-b263-71b2-6688ecca6def (Addr: tcp/127.0.0.1:25174) (DC: dc1)" +34:30.383 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-30aecd08-aa05-b263-71b2-6688ecca6def.dc1 area=wan +34:30.383 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25155 network=tcp +34:30.384 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25156 network=tcp +34:30.384 [INFO] TestAgent: started state syncer +34:30.384 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25175 network=tcp +34:30.433 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:30.435 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:30.435 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25174 [Candidate]" term=2 +34:30.436 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:30.436 [DEBUG] TestAgent.server.raft: vote granted: from=30aecd08-aa05-b263-71b2-6688ecca6def term=2 tally=1 +34:30.436 [INFO] TestAgent.server.raft: election won: tally=1 +34:30.436 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25174 [Leader]" +34:30.436 [INFO] TestAgent.server: cluster leadership acquired +34:30.436 [INFO] TestAgent.server: initializing acls +34:30.437 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:30.437 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:30.437 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:30.438 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:30.438 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:30.438 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:30.438 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-30aecd08-aa05-b263-71b2-6688ecca6def +34:30.438 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-30aecd08-aa05-b263-71b2-6688ecca6def.dc1 +34:30.439 [INFO] TestAgent.server: New leader elected: payload=Node-30aecd08-aa05-b263-71b2-6688ecca6def +34:30.439 [INFO] TestAgent.server: Updating LAN server: server="Node-30aecd08-aa05-b263-71b2-6688ecca6def (Addr: tcp/127.0.0.1:25174) (DC: dc1)" +34:30.439 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-30aecd08-aa05-b263-71b2-6688ecca6def.dc1 area=wan +34:30.440 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:30.442 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:30.442 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:30.442 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:30.442 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:30.442 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:30.442 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-30aecd08-aa05-b263-71b2-6688ecca6def +34:30.442 [INFO] TestAgent.server: member joined, marking health alive: member=Node-30aecd08-aa05-b263-71b2-6688ecca6def +34:30.443 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-30aecd08-aa05-b263-71b2-6688ecca6def +34:30.443 [INFO] TestAgent.server: federation state anti-entropy synced +34:30.506 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-30aecd08-aa05-b263-71b2-6688ecca6def +34:30.507 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-30aecd08-aa05-b263-71b2-6688ecca6def +34:30.510 [INFO] TestAgent: Requesting shutdown +34:30.510 [INFO] TestAgent.server: shutting down server +34:30.510 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:30.510 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:30.510 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:30.510 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:30.510 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:30.510 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:30.510 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:30.510 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:30.510 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:30.510 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:30.510 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:30.510 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:30.510 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:30.510 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:30.510 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:30.510 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:30.510Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:30.510Z [INFO] agent.router.manager: shutting down +34:30.510 [INFO] TestAgent: consul server down +34:30.510 [INFO] TestAgent: shutdown complete +34:30.510 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25155 network=tcp +34:30.510 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25155 network=udp +34:30.510 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25156 network=tcp +34:31.011 [INFO] TestAgent: Waiting for endpoints to shut down +34:31.011 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Self_ACLDeny (0.67s) --- PASS: TestAgent_Self_ACLDeny/no_token (0.00s) --- PASS: TestAgent_Self_ACLDeny/agent_master_token (0.00s) --- PASS: TestAgent_Self_ACLDeny/read-only_token (0.00s) -=== CONT TestAgent_HealthServiceByName -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== CONT TestAgent_HealthServicesACLEnforcement +[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestAgent_Checks_ACLFilter/no_token +=== RUN TestAgent_Checks_ACLFilter/root_token +2021-08-13T15:34:30.475Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:30.491Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:30.494 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f581ebd4-9073-6e17-af64-85f21cf0bea4 Address:127.0.0.1:25188}]" +34:30.494 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25188 [Follower]" leader= +34:30.494 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f581ebd4-9073-6e17-af64-85f21cf0bea4.dc1 127.0.0.1 +34:30.495 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f581ebd4-9073-6e17-af64-85f21cf0bea4 127.0.0.1 +2021-08-13T15:34:30.495Z [INFO] agent.router: Initializing LAN area manager +34:30.496 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25169 network=udp +34:30.496 [INFO] TestAgent.server: Adding LAN server: server="Node-f581ebd4-9073-6e17-af64-85f21cf0bea4 (Addr: tcp/127.0.0.1:25188) (DC: dc1)" +34:30.496 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f581ebd4-9073-6e17-af64-85f21cf0bea4.dc1 area=wan +34:30.496 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25169 network=tcp +34:30.497 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25170 network=tcp +34:30.497 [INFO] TestAgent: started state syncer +34:30.497 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25189 network=tcp +34:30.555 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:30.555 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25188 [Candidate]" term=2 +34:30.555 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:30.555 [DEBUG] TestAgent.server.raft: vote granted: from=f581ebd4-9073-6e17-af64-85f21cf0bea4 term=2 tally=1 +34:30.555 [INFO] TestAgent.server.raft: election won: tally=1 +34:30.555 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25188 [Leader]" +34:30.556 [INFO] TestAgent.server: cluster leadership acquired +34:30.556 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25188 +34:30.557 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:30.558 [INFO] TestAgent.server: New leader elected: payload=Node-f581ebd4-9073-6e17-af64-85f21cf0bea4 +34:30.559 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:30.559 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:30.559 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:30.559 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:30.559 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:30.560 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f581ebd4-9073-6e17-af64-85f21cf0bea4 +34:30.560 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f581ebd4-9073-6e17-af64-85f21cf0bea4 +34:30.560 [INFO] TestAgent.server: federation state anti-entropy synced +34:30.717 [INFO] TestAgent: Requesting shutdown +34:30.717 [INFO] TestAgent.server: shutting down server +34:30.717 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:30.717 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:30.717 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:30.717 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:30.717 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:30.718 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:30.718 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:30.718 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:30.718 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:30.718 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:30.718 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:30.718 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:30.718Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:30.718Z [INFO] agent.router.manager: shutting down +34:30.718 [INFO] TestAgent: consul server down +34:30.718 [INFO] TestAgent: shutdown complete +34:30.718 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25169 network=tcp +34:30.718 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25169 network=udp +34:30.718 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25170 network=tcp +34:31.218 [INFO] TestAgent: Waiting for endpoints to shut down +34:31.218 [INFO] TestAgent: Endpoints down +=== RUN TestAgent_Self/no_grpc +2021-08-13T15:34:30.380Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:30.396Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:30.399 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b24f6085-9be3-0680-85e6-4eb7dfb3e6e6 Address:127.0.0.1:25167}]" +34:30.399 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25167 [Follower]" leader= +34:30.400 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b24f6085-9be3-0680-85e6-4eb7dfb3e6e6.dc1 127.0.0.1 +34:30.401 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b24f6085-9be3-0680-85e6-4eb7dfb3e6e6 127.0.0.1 +2021-08-13T15:34:30.401Z [INFO] agent.router: Initializing LAN area manager +34:30.401 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25176 network=udp +34:30.401 [INFO] TestAgent.server: Adding LAN server: server="Node-b24f6085-9be3-0680-85e6-4eb7dfb3e6e6 (Addr: tcp/127.0.0.1:25167) (DC: dc1)" +34:30.401 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b24f6085-9be3-0680-85e6-4eb7dfb3e6e6.dc1 area=wan +34:30.402 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25176 network=tcp +34:30.402 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25177 network=tcp +34:30.403 [INFO] TestAgent: started state syncer +34:30.403 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25168 network=tcp +34:30.457 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:30.457 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25167 [Candidate]" term=2 +34:30.457 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:30.457 [DEBUG] TestAgent.server.raft: vote granted: from=b24f6085-9be3-0680-85e6-4eb7dfb3e6e6 term=2 tally=1 +34:30.457 [INFO] TestAgent.server.raft: election won: tally=1 +34:30.457 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25167 [Leader]" +34:30.457 [INFO] TestAgent.server: cluster leadership acquired +34:30.458 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25167 +34:30.459 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:30.460 [INFO] TestAgent.server: New leader elected: payload=Node-b24f6085-9be3-0680-85e6-4eb7dfb3e6e6 +34:30.461 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:30.461 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:30.461 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:30.461 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:30.461 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:30.461 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b24f6085-9be3-0680-85e6-4eb7dfb3e6e6 +34:30.461 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b24f6085-9be3-0680-85e6-4eb7dfb3e6e6 +34:30.462 [INFO] TestAgent.server: federation state anti-entropy synced +34:30.735 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:30.736 [INFO] TestAgent: Synced node info +34:30.736 [DEBUG] TestAgent: Node info in sync +34:30.738 [INFO] TestAgent: Requesting shutdown +34:30.739 [INFO] TestAgent.server: shutting down server +34:30.739 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:30.739 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:30.739 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:30.739 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:30.739 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:30.739 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:30.739 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:30.739 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:30.739 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:30.739 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:30.739 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:30.739Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:30.739Z [INFO] agent.router.manager: shutting down +34:30.739 [INFO] TestAgent: consul server down +34:30.739 [INFO] TestAgent: shutdown complete +34:30.739 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25176 network=tcp +34:30.739 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25176 network=udp +34:30.739 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25177 network=tcp +34:31.239 [INFO] TestAgent: Waiting for endpoints to shut down +34:31.239 [INFO] TestAgent: Endpoints down +--- PASS: TestIntentionsSpecificGet_invalidId (0.88s) +=== CONT TestIntentionsCreate_good +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! === RUN TestAgent_HealthServicesACLEnforcement/no-token-health-by-id === RUN TestAgent_HealthServicesACLEnforcement/no-token-health-by-name === RUN TestAgent_HealthServicesACLEnforcement/root-token-health-by-id === RUN TestAgent_HealthServicesACLEnforcement/root-token-health-by-name +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:30.721Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:30.736Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:30.739 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:28dbd537-6a80-d72b-0167-a9fec3961271 Address:127.0.0.1:25195}]" +34:30.739 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25195 [Follower]" leader= +34:30.740 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-28dbd537-6a80-d72b-0167-a9fec3961271.dc1 127.0.0.1 +34:30.741 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-28dbd537-6a80-d72b-0167-a9fec3961271 127.0.0.1 +2021-08-13T15:34:30.741Z [INFO] agent.router: Initializing LAN area manager +34:30.741 [INFO] TestAgent.server: Adding LAN server: server="Node-28dbd537-6a80-d72b-0167-a9fec3961271 (Addr: tcp/127.0.0.1:25195) (DC: dc1)" +34:30.741 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-28dbd537-6a80-d72b-0167-a9fec3961271.dc1 area=wan +34:30.741 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25190 network=udp +34:30.741 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25190 network=tcp +34:30.742 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25191 network=tcp +34:30.742 [INFO] TestAgent: started state syncer +34:30.742 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25196 network=tcp +34:30.782 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:30.782 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25195 [Candidate]" term=2 +34:30.782 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:30.782 [DEBUG] TestAgent.server.raft: vote granted: from=28dbd537-6a80-d72b-0167-a9fec3961271 term=2 tally=1 +34:30.782 [INFO] TestAgent.server.raft: election won: tally=1 +34:30.782 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25195 [Leader]" +34:30.782 [INFO] TestAgent.server: cluster leadership acquired +34:30.782 [INFO] TestAgent.server: New leader elected: payload=Node-28dbd537-6a80-d72b-0167-a9fec3961271 +34:30.783 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25195 +34:30.784 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:30.786 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:30.786 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:30.786 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:30.786 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:30.786 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:30.786 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-28dbd537-6a80-d72b-0167-a9fec3961271 +34:30.786 [INFO] TestAgent.server: member joined, marking health alive: member=Node-28dbd537-6a80-d72b-0167-a9fec3961271 +34:30.787 [INFO] TestAgent.server: federation state anti-entropy synced +34:31.050 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:31.050 [INFO] TestAgent: Synced node info +34:31.050 [DEBUG] TestAgent: Node info in sync +34:31.065 [INFO] TestAgent: Requesting shutdown +34:31.066 [INFO] TestAgent.server: shutting down server +34:31.066 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:31.066 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:31.066 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:31.066 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:31.066 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:31.066 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:31.066 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:31.066 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:31.066 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:31.066 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:31.066 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:31.066Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:31.066Z [INFO] agent.router.manager: shutting down +34:31.066 [INFO] TestAgent: consul server down +34:31.066 [INFO] TestAgent: shutdown complete +34:31.066 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25190 network=tcp +34:31.066 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25190 network=udp +34:31.066 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25191 network=tcp +34:31.566 [INFO] TestAgent: Waiting for endpoints to shut down +34:31.566 [INFO] TestAgent: Endpoints down +--- PASS: TestIntentionsSpecificGet_good (0.86s) +=== CONT TestAgent_HealthServiceByName +2021-08-13T15:34:30.866Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:30.866Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:30.919Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:30.919Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:30.922 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:508943c0-05da-7e0e-b8d9-c656cbc693e4 Address:127.0.0.1:25181}]" +34:30.923 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25181 [Follower]" leader= +34:30.923 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-508943c0-05da-7e0e-b8d9-c656cbc693e4.dc1 127.0.0.1 +34:30.924 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-508943c0-05da-7e0e-b8d9-c656cbc693e4 127.0.0.1 +2021-08-13T15:34:30.924Z [INFO] agent.router: Initializing LAN area manager +34:30.924 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25197 network=udp +34:30.924 [INFO] TestAgent.server: Adding LAN server: server="Node-508943c0-05da-7e0e-b8d9-c656cbc693e4 (Addr: tcp/127.0.0.1:25181) (DC: dc1)" +34:30.925 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-508943c0-05da-7e0e-b8d9-c656cbc693e4.dc1 area=wan +34:30.925 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25197 network=tcp +34:30.926 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25198 network=tcp +34:30.926 [INFO] TestAgent: started state syncer +34:30.926 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25182 network=tcp +34:30.975 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:30.975 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:30.975 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25181 [Candidate]" term=2 +34:30.976 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:30.976 [DEBUG] TestAgent.server.raft: vote granted: from=508943c0-05da-7e0e-b8d9-c656cbc693e4 term=2 tally=1 +34:30.976 [INFO] TestAgent.server.raft: election won: tally=1 +34:30.976 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25181 [Leader]" +34:30.976 [INFO] TestAgent.server: cluster leadership acquired +34:30.976 [INFO] TestAgent.server: New leader elected: payload=Node-508943c0-05da-7e0e-b8d9-c656cbc693e4 +34:30.976 [INFO] TestAgent.server: initializing acls +34:30.977 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:30.977 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:30.977 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:30.978 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:30.978 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:30.978 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:30.978 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-508943c0-05da-7e0e-b8d9-c656cbc693e4 +34:30.978 [INFO] TestAgent.server: Updating LAN server: server="Node-508943c0-05da-7e0e-b8d9-c656cbc693e4 (Addr: tcp/127.0.0.1:25181) (DC: dc1)" +34:30.978 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-508943c0-05da-7e0e-b8d9-c656cbc693e4.dc1 +34:30.978 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-508943c0-05da-7e0e-b8d9-c656cbc693e4.dc1 area=wan +34:30.979 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:30.982 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:30.982 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:30.982 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:30.982 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:30.982 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:30.982 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-508943c0-05da-7e0e-b8d9-c656cbc693e4 +34:30.982 [INFO] TestAgent.server: member joined, marking health alive: member=Node-508943c0-05da-7e0e-b8d9-c656cbc693e4 +34:30.983 [INFO] TestAgent.server: federation state anti-entropy synced +34:30.983 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-508943c0-05da-7e0e-b8d9-c656cbc693e4 +34:31.123 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-508943c0-05da-7e0e-b8d9-c656cbc693e4 +34:31.123 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-508943c0-05da-7e0e-b8d9-c656cbc693e4 +34:31.123 [DEBUG] TestAgent: dropping check from result due to ACLs: check=mysql +34:31.124 [INFO] TestAgent: Requesting shutdown +34:31.124 [INFO] TestAgent.server: shutting down server +34:31.124 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:31.124 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:31.124 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:31.124 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:31.124 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:31.124 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:31.124 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:31.124 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:31.124 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:31.124 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:31.124 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:31.124 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:31.124 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:31.124 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:31.124 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:31.124 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:31.125Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:31.125Z [INFO] agent.router.manager: shutting down +34:31.125 [INFO] TestAgent: consul server down +34:31.125 [INFO] TestAgent: shutdown complete +34:31.125 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25197 network=tcp +34:31.125 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25197 network=udp +34:31.125 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25198 network=tcp +34:31.625 [INFO] TestAgent: Waiting for endpoints to shut down +34:31.625 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Checks_ACLFilter (0.78s) + --- PASS: TestAgent_Checks_ACLFilter/no_token (0.00s) + --- PASS: TestAgent_Checks_ACLFilter/root_token (0.00s) +=== CONT TestIntentionsCheck_noDestination +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! === RUN TestAgent_HealthServiceByName/passing_checks === RUN TestAgent_HealthServiceByName/passing_checks/format=text === RUN TestAgent_HealthServiceByName/passing_checks/format=json @@ -44941,223 +39873,261 @@ === RUN TestAgent_HealthServiceByName/maintenance_check_on_node === RUN TestAgent_HealthServiceByName/maintenance_check_on_node/format=text === RUN TestAgent_HealthServiceByName/maintenance_check_on_node/format=json -[WARN] freeport: 4 out of 4 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:03.580Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:03.602Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:03.605 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:13bec3ee-88cb-3cb6-94ac-2eeaba426780 Address:127.0.0.1:13523}]" -43:03.606 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13523 [Follower]" leader= -43:03.606 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-13bec3ee-88cb-3cb6-94ac-2eeaba426780.dc1 127.0.0.1 -43:03.607 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-13bec3ee-88cb-3cb6-94ac-2eeaba426780 127.0.0.1 -2022-09-15T21:43:03.607Z [INFO] agent.router: Initializing LAN area manager -43:03.607 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13553 network=udp -43:03.608 [INFO] TestAgent.server: Adding LAN server: server="Node-13bec3ee-88cb-3cb6-94ac-2eeaba426780 (Addr: tcp/127.0.0.1:13523) (DC: dc1)" -43:03.608 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-13bec3ee-88cb-3cb6-94ac-2eeaba426780.dc1 area=wan -43:03.608 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13553 network=tcp -43:03.609 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13556 network=tcp -43:03.609 [INFO] TestAgent: started state syncer -43:03.609 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13524 network=tcp -43:03.673 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:03.673 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13523 [Candidate]" term=2 -43:03.674 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:03.674 [DEBUG] TestAgent.server.raft: vote granted: from=13bec3ee-88cb-3cb6-94ac-2eeaba426780 term=2 tally=1 -43:03.674 [INFO] TestAgent.server.raft: election won: tally=1 -43:03.674 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13523 [Leader]" -43:03.674 [INFO] TestAgent.server: cluster leadership acquired -43:03.674 [INFO] TestAgent.server: New leader elected: payload=Node-13bec3ee-88cb-3cb6-94ac-2eeaba426780 -43:03.674 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13523 -43:03.676 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:03.678 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:03.678 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:03.678 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:03.678 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:03.678 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:03.678 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-13bec3ee-88cb-3cb6-94ac-2eeaba426780 -43:03.678 [INFO] TestAgent.server: member joined, marking health alive: member=Node-13bec3ee-88cb-3cb6-94ac-2eeaba426780 -43:03.679 [INFO] TestAgent.server: federation state anti-entropy synced -43:03.877 [INFO] TestAgent: Requesting shutdown -43:03.877 [INFO] TestAgent.server: shutting down server -43:03.877 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:03.877 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:03.877 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:03.877 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:03.877 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:03.877 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:03.877 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:03.877 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:03.877 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:03.877 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:03.877 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:03.877 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:03.877Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:03.877Z [INFO] agent.router.manager: shutting down -43:03.877 [INFO] TestAgent: consul server down -43:03.877 [INFO] TestAgent: shutdown complete -43:03.877 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13553 network=tcp -43:03.878 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13553 network=udp -43:03.878 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13556 network=tcp -43:04.378 [INFO] TestAgent: Waiting for endpoints to shut down -43:04.378 [INFO] TestAgent: Endpoints down -=== RUN TestAgent_Self/no_grpc -2022-09-15T21:43:03.609Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:03.609Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:03.628Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:03.628Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:03.631 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1a66db2c-dba7-cb65-4282-b227bd4e203b Address:127.0.0.1:13571}]" -43:03.632 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1a66db2c-dba7-cb65-4282-b227bd4e203b.dc1 127.0.0.1 -43:03.632 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13571 [Follower]" leader= -43:03.633 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1a66db2c-dba7-cb65-4282-b227bd4e203b 127.0.0.1 -2022-09-15T21:43:03.633Z [INFO] agent.router: Initializing LAN area manager -43:03.633 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13525 network=udp -43:03.633 [INFO] TestAgent.server: Adding LAN server: server="Node-1a66db2c-dba7-cb65-4282-b227bd4e203b (Addr: tcp/127.0.0.1:13571) (DC: dc1)" -43:03.633 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1a66db2c-dba7-cb65-4282-b227bd4e203b.dc1 area=wan -43:03.634 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13525 network=tcp -43:03.635 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13533 network=tcp -43:03.635 [INFO] TestAgent: started state syncer -43:03.635 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13572 network=tcp -43:03.686 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -43:03.698 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:03.698 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13571 [Candidate]" term=2 -43:03.698 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:03.698 [DEBUG] TestAgent.server.raft: vote granted: from=1a66db2c-dba7-cb65-4282-b227bd4e203b term=2 tally=1 -43:03.698 [INFO] TestAgent.server.raft: election won: tally=1 -43:03.698 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13571 [Leader]" -43:03.698 [INFO] TestAgent.server: cluster leadership acquired -43:03.699 [INFO] TestAgent.server: initializing acls -43:03.699 [INFO] TestAgent.server: New leader elected: payload=Node-1a66db2c-dba7-cb65-4282-b227bd4e203b -43:03.699 [INFO] TestAgent.server: Created ACL 'global-management' policy -43:03.699 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -43:03.700 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -43:03.700 [INFO] TestAgent.server: Created ACL anonymous token from configuration -43:03.700 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -43:03.700 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -43:03.701 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-1a66db2c-dba7-cb65-4282-b227bd4e203b -43:03.701 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-1a66db2c-dba7-cb65-4282-b227bd4e203b.dc1 -43:03.702 [INFO] TestAgent.server: Updating LAN server: server="Node-1a66db2c-dba7-cb65-4282-b227bd4e203b (Addr: tcp/127.0.0.1:13571) (DC: dc1)" -43:03.702 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-1a66db2c-dba7-cb65-4282-b227bd4e203b.dc1 area=wan -43:03.702 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:03.705 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:03.705 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:03.705 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:03.705 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:03.705 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:03.705 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1a66db2c-dba7-cb65-4282-b227bd4e203b -43:03.705 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1a66db2c-dba7-cb65-4282-b227bd4e203b -43:03.707 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1a66db2c-dba7-cb65-4282-b227bd4e203b -43:03.709 [INFO] TestAgent.server: federation state anti-entropy synced -43:03.739 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:03.740 [INFO] TestAgent: Synced node info -43:03.740 [DEBUG] TestAgent: Node info in sync -43:03.907 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-1a66db2c-dba7-cb65-4282-b227bd4e203b -43:03.907 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-1a66db2c-dba7-cb65-4282-b227bd4e203b -43:03.907 [DEBUG] TestAgent: dropping check from result due to ACLs: check=mysql -43:03.907 [INFO] TestAgent: Requesting shutdown -43:03.907 [INFO] TestAgent.server: shutting down server -43:03.908 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:03.908 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:03.908 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:03.908 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:03.908 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -43:03.908 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -43:03.908 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:03.908 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:03.908 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:03.908 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:03.908 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:03.908 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -43:03.908 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:03.908 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -43:03.908 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -2022-09-15T21:43:03.908Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:03.908Z [INFO] agent.router.manager: shutting down -43:03.908 [INFO] TestAgent: consul server down -43:03.908 [INFO] TestAgent: shutdown complete -43:03.908 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13525 network=tcp -43:03.908 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13525 network=udp -43:03.908 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13533 network=tcp -43:04.409 [INFO] TestAgent: Waiting for endpoints to shut down -43:04.409 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Checks_ACLFilter (0.83s) - --- PASS: TestAgent_Checks_ACLFilter/no_token (0.00s) - --- PASS: TestAgent_Checks_ACLFilter/root_token (0.00s) +2021-08-13T15:34:30.915Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:30.938Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:30.941 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:667830d0-cf71-63fe-2946-8dcff52b4b35 Address:127.0.0.1:25202}]" +34:30.941 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25202 [Follower]" leader= +34:30.942 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-667830d0-cf71-63fe-2946-8dcff52b4b35.dc1 127.0.0.1 +34:30.943 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-667830d0-cf71-63fe-2946-8dcff52b4b35 127.0.0.1 +2021-08-13T15:34:30.943Z [INFO] agent.router: Initializing LAN area manager +34:30.944 [INFO] TestAgent.server: Adding LAN server: server="Node-667830d0-cf71-63fe-2946-8dcff52b4b35 (Addr: tcp/127.0.0.1:25202) (DC: dc1)" +34:30.944 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-667830d0-cf71-63fe-2946-8dcff52b4b35.dc1 area=wan +34:30.944 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25183 network=tcp +34:30.944 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25183 network=udp +34:30.945 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25184 network=tcp +34:30.945 [INFO] TestAgent: started state syncer +34:30.945 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25203 network=tcp +34:30.999 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:30.999 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25202 [Candidate]" term=2 +34:30.999 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:30.999 [DEBUG] TestAgent.server.raft: vote granted: from=667830d0-cf71-63fe-2946-8dcff52b4b35 term=2 tally=1 +34:30.999 [INFO] TestAgent.server.raft: election won: tally=1 +34:30.999 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25202 [Leader]" +34:30.999 [INFO] TestAgent.server: cluster leadership acquired +34:30.999 [INFO] TestAgent.server: New leader elected: payload=Node-667830d0-cf71-63fe-2946-8dcff52b4b35 +34:30.999 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25202 +34:31.001 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:31.003 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:31.003 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:31.003 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:31.003 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:31.003 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:31.003 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-667830d0-cf71-63fe-2946-8dcff52b4b35 +34:31.003 [INFO] TestAgent.server: member joined, marking health alive: member=Node-667830d0-cf71-63fe-2946-8dcff52b4b35 +34:31.004 [INFO] TestAgent.server: federation state anti-entropy synced +34:31.154 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:31.155 [INFO] TestAgent: Synced node info +34:31.372 [INFO] TestAgent: Requesting shutdown +34:31.372 [INFO] TestAgent.server: shutting down server +34:31.372 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:31.372 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:31.372 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:31.372 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:31.372 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:31.372 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:31.372 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:31.372 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:31.372 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:31.372 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:31.372 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:31.373Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:31.373Z [INFO] agent.router.manager: shutting down +34:31.373 [INFO] TestAgent: consul server down +34:31.373 [INFO] TestAgent: shutdown complete +34:31.373 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25183 network=tcp +34:31.373 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25183 network=udp +34:31.373 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25184 network=tcp +34:31.873 [INFO] TestAgent: Waiting for endpoints to shut down +34:31.873 [INFO] TestAgent: Endpoints down +--- PASS: TestIntentionsCreate_noBody (1.01s) === CONT TestAgent_HealthServiceByID -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:03.825Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:03.844Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:03.847 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b003be86-2a29-5b4d-0c81-13812110fbdf Address:127.0.0.1:13516}]" -43:03.847 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13516 [Follower]" leader= -43:03.848 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b003be86-2a29-5b4d-0c81-13812110fbdf.dc1 127.0.0.1 -43:03.849 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b003be86-2a29-5b4d-0c81-13812110fbdf 127.0.0.1 -2022-09-15T21:43:03.849Z [INFO] agent.router: Initializing LAN area manager -43:03.849 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13573 network=udp -43:03.849 [INFO] TestAgent.server: Adding LAN server: server="Node-b003be86-2a29-5b4d-0c81-13812110fbdf (Addr: tcp/127.0.0.1:13516) (DC: dc1)" -43:03.849 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b003be86-2a29-5b4d-0c81-13812110fbdf.dc1 area=wan -43:03.850 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13573 network=tcp -43:03.851 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13574 network=tcp -43:03.851 [INFO] TestAgent: started state syncer -43:03.851 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13517 network=tcp -43:03.896 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:03.896 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13516 [Candidate]" term=2 -43:03.897 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:03.897 [DEBUG] TestAgent.server.raft: vote granted: from=b003be86-2a29-5b4d-0c81-13812110fbdf term=2 tally=1 -43:03.897 [INFO] TestAgent.server.raft: election won: tally=1 -43:03.897 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13516 [Leader]" -43:03.897 [INFO] TestAgent.server: cluster leadership acquired -43:03.897 [INFO] TestAgent.server: initializing acls -43:03.898 [INFO] TestAgent.server: Created ACL 'global-management' policy -43:03.898 [INFO] TestAgent.server: New leader elected: payload=Node-b003be86-2a29-5b4d-0c81-13812110fbdf -43:03.898 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -43:03.899 [INFO] TestAgent.server: Created ACL anonymous token from configuration -43:03.899 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -43:03.899 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -43:03.899 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-b003be86-2a29-5b4d-0c81-13812110fbdf -43:03.899 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-b003be86-2a29-5b4d-0c81-13812110fbdf.dc1 -43:03.899 [INFO] TestAgent.server: Updating LAN server: server="Node-b003be86-2a29-5b4d-0c81-13812110fbdf (Addr: tcp/127.0.0.1:13516) (DC: dc1)" -43:03.900 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-b003be86-2a29-5b4d-0c81-13812110fbdf.dc1 area=wan -43:03.900 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:03.903 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:03.903 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:03.903 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:03.903 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:03.903 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:03.903 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b003be86-2a29-5b4d-0c81-13812110fbdf -43:03.903 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b003be86-2a29-5b4d-0c81-13812110fbdf -43:03.904 [INFO] TestAgent.server: federation state anti-entropy synced -43:03.904 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b003be86-2a29-5b4d-0c81-13812110fbdf -43:03.965 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:03.965 [INFO] TestAgent: Synced node info -43:04.127 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-b003be86-2a29-5b4d-0c81-13812110fbdf -43:04.128 [INFO] TestAgent: Requesting shutdown -43:04.128 [INFO] TestAgent.server: shutting down server -43:04.128 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -43:04.128 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -43:04.128 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:04.128 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:04.128 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:04.128 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:04.128 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:04.128 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:04.128 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:04.128 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -43:04.128 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -43:04.128 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:04.128 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:04.128 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:04.128 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:04.129Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:04.129Z [INFO] agent.router.manager: shutting down -43:04.129 [INFO] TestAgent: consul server down -43:04.129 [INFO] TestAgent: shutdown complete -43:04.129 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13573 network=tcp -43:04.129 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13573 network=udp -43:04.129 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13574 network=tcp -43:04.629 [INFO] TestAgent: Waiting for endpoints to shut down -43:04.629 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_HealthServicesACLEnforcement (0.83s) +2021-08-13T15:34:31.236Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:31.252Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:31.255 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ac4c0b79-59ba-75db-799b-c2acea0e7d34 Address:127.0.0.1:25209}]" +34:31.255 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25209 [Follower]" leader= +34:31.256 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ac4c0b79-59ba-75db-799b-c2acea0e7d34.dc1 127.0.0.1 +34:31.257 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ac4c0b79-59ba-75db-799b-c2acea0e7d34 127.0.0.1 +2021-08-13T15:34:31.257Z [INFO] agent.router: Initializing LAN area manager +34:31.257 [INFO] TestAgent.server: Adding LAN server: server="Node-ac4c0b79-59ba-75db-799b-c2acea0e7d34 (Addr: tcp/127.0.0.1:25209) (DC: dc1)" +34:31.257 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ac4c0b79-59ba-75db-799b-c2acea0e7d34.dc1 area=wan +34:31.257 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25218 network=udp +34:31.257 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25218 network=tcp +34:31.258 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25219 network=tcp +34:31.258 [INFO] TestAgent: started state syncer +34:31.321 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:31.321 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25209 [Candidate]" term=2 +34:31.321 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:31.321 [DEBUG] TestAgent.server.raft: vote granted: from=ac4c0b79-59ba-75db-799b-c2acea0e7d34 term=2 tally=1 +34:31.321 [INFO] TestAgent.server.raft: election won: tally=1 +34:31.321 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25209 [Leader]" +34:31.321 [INFO] TestAgent.server: cluster leadership acquired +34:31.322 [INFO] TestAgent.server: New leader elected: payload=Node-ac4c0b79-59ba-75db-799b-c2acea0e7d34 +34:31.322 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25209 +34:31.323 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:31.325 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:31.326 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:31.326 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:31.326 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:31.326 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:31.326 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ac4c0b79-59ba-75db-799b-c2acea0e7d34 +34:31.326 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ac4c0b79-59ba-75db-799b-c2acea0e7d34 +34:31.327 [INFO] TestAgent.server: federation state anti-entropy synced +34:31.382 [INFO] TestAgent: Requesting shutdown +34:31.382 [INFO] TestAgent.server: shutting down server +34:31.382 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:31.382 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:31.382 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:31.382 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:31.382 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:31.382 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:31.382 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:31.382 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:31.382 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:31.382 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:31.382 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:31.382 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:31.383Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:31.383Z [INFO] agent.router.manager: shutting down +34:31.383 [INFO] TestAgent: consul server down +34:31.383 [INFO] TestAgent: shutdown complete +34:31.383 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25218 network=tcp +34:31.383 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25218 network=udp +34:31.383 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25219 network=tcp +34:31.887 [INFO] TestAgent: Waiting for endpoints to shut down +34:31.887 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Self (1.43s) + --- PASS: TestAgent_Self/normal (0.76s) + --- PASS: TestAgent_Self/no_grpc (0.67s) +=== CONT TestIntentionsCheck_noSource +2021-08-13T15:34:31.029Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:31.045Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:31.048 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8b8368f3-8dda-c02a-05aa-e44a14eaa048 Address:127.0.0.1:25216}]" +34:31.048 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25216 [Follower]" leader= +34:31.049 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8b8368f3-8dda-c02a-05aa-e44a14eaa048.dc1 127.0.0.1 +34:31.050 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8b8368f3-8dda-c02a-05aa-e44a14eaa048 127.0.0.1 +2021-08-13T15:34:31.050Z [INFO] agent.router: Initializing LAN area manager +34:31.050 [INFO] TestAgent.server: Adding LAN server: server="Node-8b8368f3-8dda-c02a-05aa-e44a14eaa048 (Addr: tcp/127.0.0.1:25216) (DC: dc1)" +34:31.050 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8b8368f3-8dda-c02a-05aa-e44a14eaa048.dc1 area=wan +34:31.050 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25204 network=udp +34:31.050 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25204 network=tcp +34:31.051 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25205 network=tcp +34:31.051 [INFO] TestAgent: started state syncer +34:31.051 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25217 network=tcp +34:31.100 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:31.105 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:31.105 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25216 [Candidate]" term=2 +34:31.105 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:31.105 [DEBUG] TestAgent.server.raft: vote granted: from=8b8368f3-8dda-c02a-05aa-e44a14eaa048 term=2 tally=1 +34:31.105 [INFO] TestAgent.server.raft: election won: tally=1 +34:31.105 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25216 [Leader]" +34:31.105 [INFO] TestAgent.server: cluster leadership acquired +34:31.105 [INFO] TestAgent.server: New leader elected: payload=Node-8b8368f3-8dda-c02a-05aa-e44a14eaa048 +34:31.105 [INFO] TestAgent.server: initializing acls +34:31.106 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:31.107 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:31.107 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:31.107 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:31.107 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:31.107 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-8b8368f3-8dda-c02a-05aa-e44a14eaa048 +34:31.108 [INFO] TestAgent.server: Updating LAN server: server="Node-8b8368f3-8dda-c02a-05aa-e44a14eaa048 (Addr: tcp/127.0.0.1:25216) (DC: dc1)" +34:31.108 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-8b8368f3-8dda-c02a-05aa-e44a14eaa048.dc1 +34:31.108 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-8b8368f3-8dda-c02a-05aa-e44a14eaa048.dc1 area=wan +34:31.109 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:31.111 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:31.111 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:31.111 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:31.111 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:31.111 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:31.111 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8b8368f3-8dda-c02a-05aa-e44a14eaa048 +34:31.111 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8b8368f3-8dda-c02a-05aa-e44a14eaa048 +34:31.112 [INFO] TestAgent.server: federation state anti-entropy synced +34:31.112 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8b8368f3-8dda-c02a-05aa-e44a14eaa048 +34:31.437 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:31.438 [INFO] TestAgent: Synced node info +34:31.438 [DEBUG] TestAgent: Node info in sync +34:31.501 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-8b8368f3-8dda-c02a-05aa-e44a14eaa048 +34:31.502 [INFO] TestAgent: Requesting shutdown +34:31.502 [INFO] TestAgent.server: shutting down server +34:31.502 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:31.502 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:31.502 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:31.502 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:31.502 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:31.502 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:31.502 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:31.502 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:31.502 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:31.502 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:31.502 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:31.502 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:31.502 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:31.502 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:31.502 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:31.502Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:31.502Z [INFO] agent.router.manager: shutting down +34:31.502 [INFO] TestAgent: consul server down +34:31.502 [INFO] TestAgent: shutdown complete +34:31.502 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25204 network=tcp +34:31.503 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25204 network=udp +34:31.503 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25205 network=tcp +34:32.003 [INFO] TestAgent: Waiting for endpoints to shut down +34:32.003 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_HealthServicesACLEnforcement (0.99s) --- PASS: TestAgent_HealthServicesACLEnforcement/no-token-health-by-id (0.00s) --- PASS: TestAgent_HealthServicesACLEnforcement/no-token-health-by-name (0.00s) --- PASS: TestAgent_HealthServicesACLEnforcement/root-token-health-by-id (0.00s) --- PASS: TestAgent_HealthServicesACLEnforcement/root-token-health-by-name (0.00s) === CONT TestAgent_ChecksWithFilter +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:31.255Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:31.271Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:31.274 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:562045a5-eb56-0d0b-311d-1924609f04e8 Address:127.0.0.1:25223}]" +34:31.274 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25223 [Follower]" leader= +34:31.274 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-562045a5-eb56-0d0b-311d-1924609f04e8.dc1 127.0.0.1 +34:31.275 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-562045a5-eb56-0d0b-311d-1924609f04e8 127.0.0.1 +2021-08-13T15:34:31.275Z [INFO] agent.router: Initializing LAN area manager +34:31.275 [INFO] TestAgent.server: Adding LAN server: server="Node-562045a5-eb56-0d0b-311d-1924609f04e8 (Addr: tcp/127.0.0.1:25223) (DC: dc1)" +34:31.276 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-562045a5-eb56-0d0b-311d-1924609f04e8.dc1 area=wan +34:31.276 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25211 network=udp +34:31.276 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25211 network=tcp +34:31.277 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25212 network=tcp +34:31.277 [INFO] TestAgent: started state syncer +34:31.277 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25224 network=tcp +34:31.329 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:31.329 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25223 [Candidate]" term=2 +34:31.329 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:31.329 [DEBUG] TestAgent.server.raft: vote granted: from=562045a5-eb56-0d0b-311d-1924609f04e8 term=2 tally=1 +34:31.329 [INFO] TestAgent.server.raft: election won: tally=1 +34:31.329 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25223 [Leader]" +34:31.329 [INFO] TestAgent.server: cluster leadership acquired +34:31.329 [INFO] TestAgent.server: New leader elected: payload=Node-562045a5-eb56-0d0b-311d-1924609f04e8 +34:31.329 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25223 +34:31.330 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:31.333 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:31.333 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:31.333 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:31.333 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:31.333 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:31.333 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-562045a5-eb56-0d0b-311d-1924609f04e8 +34:31.333 [INFO] TestAgent.server: member joined, marking health alive: member=Node-562045a5-eb56-0d0b-311d-1924609f04e8 +34:31.334 [INFO] TestAgent.server: federation state anti-entropy synced +34:31.648 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:31.648 [INFO] TestAgent: Synced node info +34:31.648 [DEBUG] TestAgent: Node info in sync +34:31.656 [INFO] TestAgent: Requesting shutdown +34:31.656 [INFO] TestAgent.server: shutting down server +34:31.656 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:31.656 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:31.656 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:31.656 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:31.656 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:31.656 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:31.656 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:31.656 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:31.656 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:31.656 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:31.656 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:31.657Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:31.657Z [INFO] agent.router.manager: shutting down +34:31.657 [INFO] TestAgent: consul server down +34:31.657 [INFO] TestAgent: shutdown complete +34:31.657 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25211 network=tcp +34:31.657 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25211 network=udp +34:31.657 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25212 network=tcp +34:32.157 [INFO] TestAgent: Waiting for endpoints to shut down +34:32.157 [INFO] TestAgent: Endpoints down +=== CONT TestIntentionsCheck_basic +--- PASS: TestIntentionsCreate_good (0.92s) === RUN TestAgent_HealthServiceByID/passing_checks === RUN TestAgent_HealthServiceByID/passing_checks/format=text === RUN TestAgent_HealthServiceByID/passing_checks/format=json @@ -45176,61 +40146,64 @@ === RUN TestAgent_HealthServiceByID/maintenance_check_on_node === RUN TestAgent_HealthServiceByID/maintenance_check_on_node/format=text === RUN TestAgent_HealthServiceByID/maintenance_check_on_node/format=json -2022-09-15T21:43:03.959Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:03.977Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:03.980 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5ac11726-9f8f-2789-fd29-63948654fdfc Address:127.0.0.1:13578}]" -43:03.980 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13578 [Follower]" leader= -43:03.981 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5ac11726-9f8f-2789-fd29-63948654fdfc.dc1 127.0.0.1 -43:03.984 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5ac11726-9f8f-2789-fd29-63948654fdfc 127.0.0.1 -2022-09-15T21:43:03.984Z [INFO] agent.router: Initializing LAN area manager -43:03.984 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5ac11726-9f8f-2789-fd29-63948654fdfc.dc1 area=wan -43:03.984 [INFO] TestAgent.server: Adding LAN server: server="Node-5ac11726-9f8f-2789-fd29-63948654fdfc (Addr: tcp/127.0.0.1:13578) (DC: dc1)" -43:03.984 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13518 network=udp -43:03.985 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13518 network=tcp -43:03.985 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13519 network=tcp -43:03.986 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13579 network=tcp -43:03.986 [INFO] TestAgent: started state syncer -43:04.018 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:04.018 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13578 [Candidate]" term=2 -43:04.018 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:04.018 [DEBUG] TestAgent.server.raft: vote granted: from=5ac11726-9f8f-2789-fd29-63948654fdfc term=2 tally=1 -43:04.018 [INFO] TestAgent.server.raft: election won: tally=1 -43:04.018 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13578 [Leader]" -43:04.018 [INFO] TestAgent.server: cluster leadership acquired -43:04.019 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13578 -43:04.019 [INFO] TestAgent.server: New leader elected: payload=Node-5ac11726-9f8f-2789-fd29-63948654fdfc -43:04.020 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:04.023 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:04.023 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:04.023 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:04.023 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:04.023 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:04.023 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5ac11726-9f8f-2789-fd29-63948654fdfc -43:04.023 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5ac11726-9f8f-2789-fd29-63948654fdfc -43:04.024 [INFO] TestAgent.server: federation state anti-entropy synced -43:04.168 [INFO] TestAgent: Requesting shutdown -43:04.168 [INFO] TestAgent.server: shutting down server -43:04.168 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:04.168 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:04.168 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:04.168 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:04.168 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:04.168 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:04.168 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:04.168 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:04.168 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:04.168 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:04.168 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:04.168 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:04.168Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:04.169Z [INFO] agent.router.manager: shutting down -43:04.169 [INFO] TestAgent: consul server down -43:04.169 [INFO] TestAgent: shutdown complete -43:04.169 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13518 network=tcp -43:04.169 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13518 network=udp -43:04.169 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13519 network=tcp -43:04.670 [INFO] TestAgent: Waiting for endpoints to shut down -43:04.670 [INFO] TestAgent: Endpoints down +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:31.583Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:31.599Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:31.602 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e9cbfa1c-4947-78e4-f252-89c3c1ec2b78 Address:127.0.0.1:25251}]" +34:31.602 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25251 [Follower]" leader= +34:31.603 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e9cbfa1c-4947-78e4-f252-89c3c1ec2b78.dc1 127.0.0.1 +34:31.604 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e9cbfa1c-4947-78e4-f252-89c3c1ec2b78 127.0.0.1 +2021-08-13T15:34:31.604Z [INFO] agent.router: Initializing LAN area manager +34:31.604 [INFO] TestAgent.server: Adding LAN server: server="Node-e9cbfa1c-4947-78e4-f252-89c3c1ec2b78 (Addr: tcp/127.0.0.1:25251) (DC: dc1)" +34:31.604 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e9cbfa1c-4947-78e4-f252-89c3c1ec2b78.dc1 area=wan +34:31.605 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25225 network=udp +34:31.605 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25225 network=tcp +34:31.606 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25226 network=tcp +34:31.606 [INFO] TestAgent: started state syncer +34:31.606 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25252 network=tcp +34:31.641 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:31.642 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25251 [Candidate]" term=2 +34:31.642 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:31.642 [DEBUG] TestAgent.server.raft: vote granted: from=e9cbfa1c-4947-78e4-f252-89c3c1ec2b78 term=2 tally=1 +34:31.642 [INFO] TestAgent.server.raft: election won: tally=1 +34:31.642 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25251 [Leader]" +34:31.642 [INFO] TestAgent.server: cluster leadership acquired +34:31.642 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25251 +34:31.644 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:31.646 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:31.646 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:31.646 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:31.646 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:31.646 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:31.646 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e9cbfa1c-4947-78e4-f252-89c3c1ec2b78 +34:31.646 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e9cbfa1c-4947-78e4-f252-89c3c1ec2b78 +34:31.647 [INFO] TestAgent.server: New leader elected: payload=Node-e9cbfa1c-4947-78e4-f252-89c3c1ec2b78 +34:31.649 [INFO] TestAgent.server: federation state anti-entropy synced +34:31.733 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:31.734 [INFO] TestAgent: Synced node info +34:31.734 [DEBUG] TestAgent: Node info in sync +34:31.790 [INFO] TestAgent: Requesting shutdown +34:31.791 [INFO] TestAgent.server: shutting down server +34:31.791 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:31.791 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:31.791 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:31.791 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:31.791 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:31.791 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:31.791 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:31.791 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:31.791 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:31.791 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:31.791 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:31.791Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:31.791Z [INFO] agent.router.manager: shutting down +34:31.791 [INFO] TestAgent: consul server down +34:31.791 [INFO] TestAgent: shutdown complete +34:31.791 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25225 network=tcp +34:31.791 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25225 network=udp +34:31.791 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25226 network=tcp +34:32.292 [INFO] TestAgent: Waiting for endpoints to shut down +34:32.292 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_HealthServiceByName (0.73s) --- PASS: TestAgent_HealthServiceByName/passing_checks (0.00s) --- PASS: TestAgent_HealthServiceByName/passing_checks/format=text (0.00s) @@ -45251,240 +40224,120 @@ --- PASS: TestAgent_HealthServiceByName/maintenance_check_on_node/format=text (0.00s) --- PASS: TestAgent_HealthServiceByName/maintenance_check_on_node/format=json (0.00s) === CONT TestAgent_Checks -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 4 out of 4 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:04.400Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:04.428Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:04.431 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:25c0544a-80c8-eba6-a56d-dcf8cff547a0 Address:127.0.0.1:13564}]" -43:04.431 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13564 [Follower]" leader= -43:04.432 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-25c0544a-80c8-eba6-a56d-dcf8cff547a0.dc1 127.0.0.1 -43:04.433 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-25c0544a-80c8-eba6-a56d-dcf8cff547a0 127.0.0.1 -2022-09-15T21:43:04.433Z [INFO] agent.router: Initializing LAN area manager -43:04.433 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13580 network=udp -43:04.433 [INFO] TestAgent.server: Adding LAN server: server="Node-25c0544a-80c8-eba6-a56d-dcf8cff547a0 (Addr: tcp/127.0.0.1:13564) (DC: dc1)" -43:04.433 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-25c0544a-80c8-eba6-a56d-dcf8cff547a0.dc1 area=wan -43:04.434 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13580 network=tcp -43:04.435 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13581 network=tcp -43:04.435 [INFO] TestAgent: started state syncer -43:04.473 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:04.473 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13564 [Candidate]" term=2 -43:04.473 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:04.473 [DEBUG] TestAgent.server.raft: vote granted: from=25c0544a-80c8-eba6-a56d-dcf8cff547a0 term=2 tally=1 -43:04.473 [INFO] TestAgent.server.raft: election won: tally=1 -43:04.473 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13564 [Leader]" -43:04.474 [INFO] TestAgent.server: cluster leadership acquired -43:04.474 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13564 -43:04.475 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:04.478 [INFO] TestAgent.server: New leader elected: payload=Node-25c0544a-80c8-eba6-a56d-dcf8cff547a0 -43:04.479 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:04.479 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:04.479 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:04.479 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:04.479 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:04.479 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-25c0544a-80c8-eba6-a56d-dcf8cff547a0 -43:04.479 [INFO] TestAgent.server: member joined, marking health alive: member=Node-25c0544a-80c8-eba6-a56d-dcf8cff547a0 -43:04.480 [INFO] TestAgent.server: federation state anti-entropy synced -43:04.526 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:04.526 [INFO] TestAgent: Synced node info -43:04.527 [DEBUG] TestAgent: Node info in sync -43:04.553 [INFO] TestAgent: Requesting shutdown -43:04.553 [INFO] TestAgent.server: shutting down server -43:04.553 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:04.553 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:04.553 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:04.553 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:04.553 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:04.553 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:04.553 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:04.553 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:04.553 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:04.553 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:04.554 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:04.554Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:04.554Z [INFO] agent.router.manager: shutting down -43:04.554 [INFO] TestAgent: consul server down -43:04.554 [INFO] TestAgent: shutdown complete -43:04.554 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13580 network=tcp -43:04.554 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13580 network=udp -43:04.554 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13581 network=tcp -43:05.055 [INFO] TestAgent: Waiting for endpoints to shut down -43:05.055 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Self (1.50s) - --- PASS: TestAgent_Self/normal (0.82s) - --- PASS: TestAgent_Self/no_grpc (0.68s) -=== CONT TestAgent_Services_ACLFilter -42:59.880 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-e4cb355d-101f-c3c0-c5d3-c94e70061261 127.0.0.1 -2022-09-15T21:42:59.880Z [INFO] agent.router: Initializing LAN area manager -42:59.880 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13416 network=udp -42:59.880 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13416 network=tcp -42:59.881 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13417 network=tcp -42:59.881 [INFO] TestAgent: started state syncer -42:59.882 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13415 network=tcp -2022-09-15T21:42:59.882Z [WARN] agent.router.manager: No servers available -42:59.882 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" -42:59.883 [DEBUG] TestAgent.client.memberlist.lan: memberlist: Stream connection from=127.0.0.1:52390 -42:59.884 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-7d816596-84ce-ff51-472d-361952af0ced 127.0.0.1 -42:59.884 [INFO] TestAgent.client: client starting leave -42:59.884 [INFO] TestAgent.client: adding server: server="Node-7d816596-84ce-ff51-472d-361952af0ced (Addr: tcp/127.0.0.1:13386) (DC: dc1)" -42:59.884 [INFO] TestAgent.client: New leader elected: payload=Node-7d816596-84ce-ff51-472d-361952af0ced -43:00.037 [DEBUG] TestAgent.client.serf.lan: serf: messageUserEventType: consul:new-leader -43:00.037 [DEBUG] TestAgent.client.serf.lan: serf: messageJoinType: Node-7d816596-84ce-ff51-472d-361952af0ced -43:00.242 [DEBUG] TestAgent.client.serf.lan: serf: messageUserEventType: consul:new-leader -43:00.242 [DEBUG] TestAgent.client.serf.lan: serf: messageJoinType: Node-7d816596-84ce-ff51-472d-361952af0ced -43:00.242 [DEBUG] TestAgent.client.serf.lan: serf: messageLeaveType: Node-e4cb355d-101f-c3c0-c5d3-c94e70061261 -43:00.438 [DEBUG] TestAgent.client.serf.lan: serf: messageUserEventType: consul:new-leader -43:00.438 [DEBUG] TestAgent.client.serf.lan: serf: messageJoinType: Node-7d816596-84ce-ff51-472d-361952af0ced -43:00.438 [DEBUG] TestAgent.client.serf.lan: serf: messageLeaveType: Node-e4cb355d-101f-c3c0-c5d3-c94e70061261 -43:00.438 [DEBUG] TestAgent.client.serf.lan: serf: messageUserEventType: consul:new-leader -43:00.438 [DEBUG] TestAgent.client.serf.lan: serf: messageJoinType: Node-7d816596-84ce-ff51-472d-361952af0ced -43:00.438 [DEBUG] TestAgent.client.serf.lan: serf: messageLeaveType: Node-e4cb355d-101f-c3c0-c5d3-c94e70061261 -43:00.480 [INFO] TestAgent.client.serf.lan: serf: EventMemberLeave: Node-e4cb355d-101f-c3c0-c5d3-c94e70061261 127.0.0.1 -43:00.638 [DEBUG] TestAgent.client.serf.lan: serf: messageLeaveType: Node-e4cb355d-101f-c3c0-c5d3-c94e70061261 -43:01.452 [INFO] TestAgent: Synced node info -43:02.278 [DEBUG] TestAgent: Node info in sync -43:02.278 [DEBUG] TestAgent: Node info in sync -43:04.087 [INFO] TestAgent: Requesting shutdown -43:04.087 [INFO] TestAgent.client: shutting down client -43:04.087 [INFO] TestAgent: consul client down -43:04.087 [INFO] TestAgent: shutdown complete -43:04.087 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13416 network=tcp -43:04.087 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13416 network=udp -43:04.087 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13417 network=tcp -43:04.588 [INFO] TestAgent: Waiting for endpoints to shut down -43:04.588 [INFO] TestAgent: Endpoints down -2022-09-15T21:42:59.410Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:59.431Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:59.434 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7d816596-84ce-ff51-472d-361952af0ced Address:127.0.0.1:13386}]" -42:59.434 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13386 [Follower]" leader= -42:59.435 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7d816596-84ce-ff51-472d-361952af0ced.dc1 127.0.0.1 -42:59.436 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7d816596-84ce-ff51-472d-361952af0ced 127.0.0.1 -2022-09-15T21:42:59.436Z [INFO] agent.router: Initializing LAN area manager -42:59.436 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7d816596-84ce-ff51-472d-361952af0ced.dc1 area=wan -42:59.436 [INFO] TestAgent.server: Adding LAN server: server="Node-7d816596-84ce-ff51-472d-361952af0ced (Addr: tcp/127.0.0.1:13386) (DC: dc1)" -42:59.436 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13388 network=tcp -42:59.436 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13388 network=udp -42:59.437 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13389 network=tcp -42:59.437 [INFO] TestAgent: started state syncer -42:59.437 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13387 network=tcp -42:59.478 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:59.478 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13386 [Candidate]" term=2 -42:59.479 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:59.479 [DEBUG] TestAgent.server.raft: vote granted: from=7d816596-84ce-ff51-472d-361952af0ced term=2 tally=1 -42:59.479 [INFO] TestAgent.server.raft: election won: tally=1 -42:59.479 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13386 [Leader]" -42:59.479 [INFO] TestAgent.server: cluster leadership acquired -42:59.479 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13386 -42:59.480 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:59.481 [INFO] TestAgent.server: New leader elected: payload=Node-7d816596-84ce-ff51-472d-361952af0ced -42:59.486 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:59.486 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:59.486 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:59.486 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:59.486 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:59.486 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7d816596-84ce-ff51-472d-361952af0ced -42:59.486 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7d816596-84ce-ff51-472d-361952af0ced -42:59.488 [INFO] TestAgent.server: federation state anti-entropy synced -42:59.686 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:59.687 [INFO] TestAgent: Synced node info -42:59.687 [DEBUG] TestAgent: Node info in sync -42:59.882 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:13419] -42:59.883 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:13419 -42:59.884 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e4cb355d-101f-c3c0-c5d3-c94e70061261 127.0.0.1 -42:59.884 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 -42:59.884 [DEBUG] TestAgent: systemd notify failed: error="No socket" -42:59.884 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e4cb355d-101f-c3c0-c5d3-c94e70061261 -43:00.085 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:00.085 [DEBUG] TestAgent: Node info in sync -43:00.086 [DEBUG] TestAgent.server.serf.lan: serf: messageJoinType: Node-7d816596-84ce-ff51-472d-361952af0ced -43:00.086 [DEBUG] TestAgent.server.serf.lan: serf: messageLeaveType: Node-e4cb355d-101f-c3c0-c5d3-c94e70061261 -43:00.285 [DEBUG] TestAgent.server.serf.lan: serf: messageJoinType: Node-7d816596-84ce-ff51-472d-361952af0ced -43:00.285 [DEBUG] TestAgent.server.serf.lan: serf: messageLeaveType: Node-e4cb355d-101f-c3c0-c5d3-c94e70061261 -43:00.438 [DEBUG] TestAgent.server.serf.lan: serf: messageJoinType: Node-7d816596-84ce-ff51-472d-361952af0ced -43:00.438 [DEBUG] TestAgent.server.serf.lan: serf: messageLeaveType: Node-e4cb355d-101f-c3c0-c5d3-c94e70061261 -43:00.481 [DEBUG] TestAgent.server.serf.lan: serf: messageJoinType: Node-7d816596-84ce-ff51-472d-361952af0ced -43:00.481 [DEBUG] TestAgent.server.serf.lan: serf: messageLeaveType: Node-e4cb355d-101f-c3c0-c5d3-c94e70061261 -43:00.680 [INFO] TestAgent.server.serf.lan: serf: EventMemberLeave: Node-e4cb355d-101f-c3c0-c5d3-c94e70061261 127.0.0.1 -43:00.681 [INFO] TestAgent.server: deregistering member: member=Node-e4cb355d-101f-c3c0-c5d3-c94e70061261 reason=left -43:04.588 [INFO] TestAgent: Requesting shutdown -43:04.588 [INFO] TestAgent.server: shutting down server -43:04.588 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:04.588 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:04.588 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:04.588 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:04.588 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:04.588 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:04.588 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:04.588 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:04.588 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:04.588 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:04.588 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:04.589Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:04.589Z [INFO] agent.router.manager: shutting down -43:04.589 [INFO] TestAgent: consul server down -43:04.589 [INFO] TestAgent: shutdown complete -43:04.589 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13388 network=tcp -43:04.589 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13388 network=udp -43:04.589 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13389 network=tcp -43:05.094 [INFO] TestAgent: Waiting for endpoints to shut down -43:05.094 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Leave (5.71s) -=== CONT TestAgent_Services_TerminatingGateway -2022-09-15T21:43:04.436Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:04.453Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:04.456 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:835e1ed2-1e7a-15c4-7682-db2815008dc0 Address:127.0.0.1:13613}]" -43:04.458 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-835e1ed2-1e7a-15c4-7682-db2815008dc0.dc1 127.0.0.1 -43:04.458 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13613 [Follower]" leader= -43:04.458 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-835e1ed2-1e7a-15c4-7682-db2815008dc0 127.0.0.1 -2022-09-15T21:43:04.459Z [INFO] agent.router: Initializing LAN area manager -43:04.459 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13566 network=udp -43:04.459 [INFO] TestAgent.server: Adding LAN server: server="Node-835e1ed2-1e7a-15c4-7682-db2815008dc0 (Addr: tcp/127.0.0.1:13613) (DC: dc1)" -43:04.459 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-835e1ed2-1e7a-15c4-7682-db2815008dc0.dc1 area=wan -43:04.459 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13566 network=tcp -43:04.460 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13567 network=tcp -43:04.460 [INFO] TestAgent: started state syncer -43:04.461 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13614 network=tcp -43:04.515 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:04.515 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13613 [Candidate]" term=2 -43:04.515 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:04.516 [DEBUG] TestAgent.server.raft: vote granted: from=835e1ed2-1e7a-15c4-7682-db2815008dc0 term=2 tally=1 -43:04.516 [INFO] TestAgent.server.raft: election won: tally=1 -43:04.516 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13613 [Leader]" -43:04.516 [INFO] TestAgent.server: cluster leadership acquired -43:04.516 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13613 -43:04.517 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:04.518 [INFO] TestAgent.server: New leader elected: payload=Node-835e1ed2-1e7a-15c4-7682-db2815008dc0 -43:04.520 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:04.520 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:04.520 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:04.520 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:04.520 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:04.521 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-835e1ed2-1e7a-15c4-7682-db2815008dc0 -43:04.521 [INFO] TestAgent.server: member joined, marking health alive: member=Node-835e1ed2-1e7a-15c4-7682-db2815008dc0 -43:04.522 [INFO] TestAgent.server: federation state anti-entropy synced -43:04.639 [INFO] TestAgent: Requesting shutdown -43:04.639 [INFO] TestAgent.server: shutting down server -43:04.640 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:04.640 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:04.640 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:04.640 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:04.640 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:04.640 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:04.640 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:04.640 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:04.640 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:04.640 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:04.640 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:04.640 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:04.640Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:04.640Z [INFO] agent.router.manager: shutting down -43:04.640 [INFO] TestAgent: consul server down -43:04.640 [INFO] TestAgent: shutdown complete -43:04.640 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13566 network=tcp -43:04.641 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13566 network=udp -43:04.641 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13567 network=tcp -43:05.142 [INFO] TestAgent: Waiting for endpoints to shut down -43:05.142 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_HealthServiceByID (0.73s) +2021-08-13T15:34:31.642Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:31.658Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:31.660 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a2439058-9708-70c5-48a6-05f98ac230a9 Address:127.0.0.1:25237}]" +34:31.660 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25237 [Follower]" leader= +34:31.661 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a2439058-9708-70c5-48a6-05f98ac230a9.dc1 127.0.0.1 +34:31.662 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a2439058-9708-70c5-48a6-05f98ac230a9 127.0.0.1 +2021-08-13T15:34:31.662Z [INFO] agent.router: Initializing LAN area manager +34:31.663 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25253 network=udp +34:31.663 [INFO] TestAgent.server: Adding LAN server: server="Node-a2439058-9708-70c5-48a6-05f98ac230a9 (Addr: tcp/127.0.0.1:25237) (DC: dc1)" +34:31.663 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a2439058-9708-70c5-48a6-05f98ac230a9.dc1 area=wan +34:31.663 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25253 network=tcp +34:31.664 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25254 network=tcp +34:31.664 [INFO] TestAgent: started state syncer +34:31.664 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25238 network=tcp +34:31.705 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:31.705 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25237 [Candidate]" term=2 +34:31.705 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:31.705 [DEBUG] TestAgent.server.raft: vote granted: from=a2439058-9708-70c5-48a6-05f98ac230a9 term=2 tally=1 +34:31.705 [INFO] TestAgent.server.raft: election won: tally=1 +34:31.705 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25237 [Leader]" +34:31.705 [INFO] TestAgent.server: cluster leadership acquired +34:31.706 [INFO] TestAgent.server: New leader elected: payload=Node-a2439058-9708-70c5-48a6-05f98ac230a9 +34:31.706 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25237 +34:31.707 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:31.709 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:31.709 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:31.709 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:31.709 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:31.709 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:31.709 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a2439058-9708-70c5-48a6-05f98ac230a9 +34:31.709 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a2439058-9708-70c5-48a6-05f98ac230a9 +34:31.710 [INFO] TestAgent.server: federation state anti-entropy synced +34:31.957 [INFO] TestAgent: Requesting shutdown +34:31.957 [INFO] TestAgent.server: shutting down server +34:31.957 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:31.957 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:31.957 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:31.957 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:31.957 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:31.957 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:31.957 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:31.957 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:31.957 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:31.957 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:31.957 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:31.957 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:31.958Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:31.958Z [INFO] agent.router.manager: shutting down +34:31.958 [INFO] TestAgent: consul server down +34:31.958 [INFO] TestAgent: shutdown complete +34:31.958 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25253 network=tcp +34:31.958 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25253 network=udp +34:31.958 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25254 network=tcp +34:32.458 [INFO] TestAgent: Waiting for endpoints to shut down +34:32.458 [INFO] TestAgent: Endpoints down +--- PASS: TestIntentionsCheck_noDestination (0.83s) +=== CONT TestIntentionsMatch_noName +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:31.921Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:31.937Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:31.940 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4e8d3b2a-22cc-7d85-2620-7b704e37da2d Address:127.0.0.1:25244}]" +34:31.940 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25244 [Follower]" leader= +34:31.941 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4e8d3b2a-22cc-7d85-2620-7b704e37da2d.dc1 127.0.0.1 +34:31.942 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4e8d3b2a-22cc-7d85-2620-7b704e37da2d 127.0.0.1 +2021-08-13T15:34:31.942Z [INFO] agent.router: Initializing LAN area manager +34:31.942 [INFO] TestAgent.server: Adding LAN server: server="Node-4e8d3b2a-22cc-7d85-2620-7b704e37da2d (Addr: tcp/127.0.0.1:25244) (DC: dc1)" +34:31.942 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4e8d3b2a-22cc-7d85-2620-7b704e37da2d.dc1 area=wan +34:31.943 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25239 network=udp +34:31.943 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25239 network=tcp +34:31.944 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25240 network=tcp +34:31.944 [INFO] TestAgent: started state syncer +34:31.944 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25245 network=tcp +34:31.990 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:31.990 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25244 [Candidate]" term=2 +34:31.991 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:31.991 [DEBUG] TestAgent.server.raft: vote granted: from=4e8d3b2a-22cc-7d85-2620-7b704e37da2d term=2 tally=1 +34:31.991 [INFO] TestAgent.server.raft: election won: tally=1 +34:31.991 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25244 [Leader]" +34:31.991 [INFO] TestAgent.server: cluster leadership acquired +34:31.991 [INFO] TestAgent.server: New leader elected: payload=Node-4e8d3b2a-22cc-7d85-2620-7b704e37da2d +34:31.991 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25244 +34:31.992 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:31.995 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:31.995 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:31.995 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:31.995 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:31.995 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:31.995 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4e8d3b2a-22cc-7d85-2620-7b704e37da2d +34:31.995 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4e8d3b2a-22cc-7d85-2620-7b704e37da2d +34:31.996 [INFO] TestAgent.server: federation state anti-entropy synced +34:32.191 [INFO] TestAgent: Requesting shutdown +34:32.191 [INFO] TestAgent.server: shutting down server +34:32.191 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:32.191 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:32.191 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:32.191 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:32.191 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:32.191 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:32.191 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:32.191 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:32.191 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:32.191 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:32.191 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:32.191 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:32.192Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:32.192Z [INFO] agent.router.manager: shutting down +34:32.192 [INFO] TestAgent: consul server down +34:32.192 [INFO] TestAgent: shutdown complete +34:32.192 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25239 network=tcp +34:32.192 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25239 network=udp +34:32.192 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25240 network=tcp +34:32.692 [INFO] TestAgent: Waiting for endpoints to shut down +34:32.692 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_HealthServiceByID (0.82s) --- PASS: TestAgent_HealthServiceByID/passing_checks (0.00s) --- PASS: TestAgent_HealthServiceByID/passing_checks/format=text (0.00s) --- PASS: TestAgent_HealthServiceByID/passing_checks/format=json (0.00s) @@ -45503,770 +40356,1573 @@ --- PASS: TestAgent_HealthServiceByID/maintenance_check_on_node (0.00s) --- PASS: TestAgent_HealthServiceByID/maintenance_check_on_node/format=text (0.00s) --- PASS: TestAgent_HealthServiceByID/maintenance_check_on_node/format=json (0.00s) +=== CONT TestAgent_Services_ACLFilter +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:32.174Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:32.190Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:32.193 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:165b729d-d749-55f3-eb39-9fb811e02eb2 Address:127.0.0.1:25279}]" +34:32.193 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25279 [Follower]" leader= +34:32.194 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-165b729d-d749-55f3-eb39-9fb811e02eb2.dc1 127.0.0.1 +34:32.195 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-165b729d-d749-55f3-eb39-9fb811e02eb2 127.0.0.1 +2021-08-13T15:34:32.195Z [INFO] agent.router: Initializing LAN area manager +34:32.196 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25274 network=udp +34:32.196 [INFO] TestAgent.server: Adding LAN server: server="Node-165b729d-d749-55f3-eb39-9fb811e02eb2 (Addr: tcp/127.0.0.1:25279) (DC: dc1)" +34:32.196 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-165b729d-d749-55f3-eb39-9fb811e02eb2.dc1 area=wan +34:32.196 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25274 network=tcp +34:32.197 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25275 network=tcp +34:32.197 [INFO] TestAgent: started state syncer +34:32.197 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25280 network=tcp +34:32.259 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:32.259 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25279 [Candidate]" term=2 +34:32.259 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:32.259 [DEBUG] TestAgent.server.raft: vote granted: from=165b729d-d749-55f3-eb39-9fb811e02eb2 term=2 tally=1 +34:32.259 [INFO] TestAgent.server.raft: election won: tally=1 +34:32.259 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25279 [Leader]" +34:32.259 [INFO] TestAgent.server: cluster leadership acquired +34:32.260 [INFO] TestAgent.server: New leader elected: payload=Node-165b729d-d749-55f3-eb39-9fb811e02eb2 +34:32.260 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25279 +34:32.261 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:32.263 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:32.263 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:32.263 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:32.263 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:32.263 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:32.263 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-165b729d-d749-55f3-eb39-9fb811e02eb2 +34:32.263 [INFO] TestAgent.server: member joined, marking health alive: member=Node-165b729d-d749-55f3-eb39-9fb811e02eb2 +34:32.264 [INFO] TestAgent.server: federation state anti-entropy synced +34:32.386 [INFO] TestAgent: Requesting shutdown +34:32.386 [INFO] TestAgent.server: shutting down server +34:32.386 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:32.386 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:32.386 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:32.387 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:32.387 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:32.387 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:32.387 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:32.387 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:32.387 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:32.387 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:32.387 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:32.387 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:32.387Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:32.387Z [INFO] agent.router.manager: shutting down +34:32.387 [INFO] TestAgent: consul server down +34:32.387 [INFO] TestAgent: shutdown complete +34:32.387 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25274 network=tcp +34:32.387 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25274 network=udp +34:32.387 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25275 network=tcp +34:32.887 [INFO] TestAgent: Waiting for endpoints to shut down +34:32.887 [INFO] TestAgent: Endpoints down +--- PASS: TestIntentionsCheck_basic (0.73s) +=== CONT TestIntentionsMatch_byInvalid +2021-08-13T15:34:31.926Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:31.943Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:31.946 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:804d1fe3-004d-3b91-8e8d-6185c565f11a Address:127.0.0.1:25265}]" +34:31.946 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25265 [Follower]" leader= +34:31.946 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-804d1fe3-004d-3b91-8e8d-6185c565f11a.dc1 127.0.0.1 +34:31.947 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-804d1fe3-004d-3b91-8e8d-6185c565f11a 127.0.0.1 +2021-08-13T15:34:31.947Z [INFO] agent.router: Initializing LAN area manager +34:31.948 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25246 network=udp +34:31.948 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-804d1fe3-004d-3b91-8e8d-6185c565f11a.dc1 area=wan +34:31.948 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25246 network=tcp +34:31.949 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25247 network=tcp +34:31.949 [INFO] TestAgent: started state syncer +34:31.949 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25266 network=tcp +34:31.951 [INFO] TestAgent.server: Adding LAN server: server="Node-804d1fe3-004d-3b91-8e8d-6185c565f11a (Addr: tcp/127.0.0.1:25265) (DC: dc1)" +34:32.001 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:32.001 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25265 [Candidate]" term=2 +34:32.001 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:32.002 [DEBUG] TestAgent.server.raft: vote granted: from=804d1fe3-004d-3b91-8e8d-6185c565f11a term=2 tally=1 +34:32.002 [INFO] TestAgent.server.raft: election won: tally=1 +34:32.002 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25265 [Leader]" +34:32.002 [INFO] TestAgent.server: cluster leadership acquired +34:32.002 [INFO] TestAgent.server: New leader elected: payload=Node-804d1fe3-004d-3b91-8e8d-6185c565f11a +34:32.002 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25265 +34:32.003 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:32.006 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:32.006 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:32.006 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:32.006 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:32.006 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:32.006 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-804d1fe3-004d-3b91-8e8d-6185c565f11a +34:32.006 [INFO] TestAgent.server: member joined, marking health alive: member=Node-804d1fe3-004d-3b91-8e8d-6185c565f11a +34:32.007 [INFO] TestAgent.server: federation state anti-entropy synced +34:32.122 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:32.123 [INFO] TestAgent: Synced node info +34:32.123 [DEBUG] TestAgent: Node info in sync +34:32.427 [INFO] TestAgent: Requesting shutdown +34:32.427 [INFO] TestAgent.server: shutting down server +34:32.427 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:32.427 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:32.427 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:32.427 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:32.427 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:32.427 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:32.427 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:32.427 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:32.427 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:32.427 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:32.427 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:32.427Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:32.427Z [INFO] agent.router.manager: shutting down +34:32.427 [INFO] TestAgent: consul server down +34:32.427 [INFO] TestAgent: shutdown complete +34:32.427 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25246 network=tcp +34:32.427 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25246 network=udp +34:32.427 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25247 network=tcp +34:32.928 [INFO] TestAgent: Waiting for endpoints to shut down +34:32.928 [INFO] TestAgent: Endpoints down +--- PASS: TestIntentionsCheck_noSource (1.04s) +=== CONT TestAgent_Services_TerminatingGateway +2021-08-13T15:34:32.020Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:32.036Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:32.039 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4fb42849-3719-a3cb-5d89-6465567a670f Address:127.0.0.1:25272}]" +34:32.039 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25272 [Follower]" leader= +34:32.040 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4fb42849-3719-a3cb-5d89-6465567a670f.dc1 127.0.0.1 +34:32.041 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4fb42849-3719-a3cb-5d89-6465567a670f 127.0.0.1 +2021-08-13T15:34:32.041Z [INFO] agent.router: Initializing LAN area manager +34:32.041 [INFO] TestAgent.server: Adding LAN server: server="Node-4fb42849-3719-a3cb-5d89-6465567a670f (Addr: tcp/127.0.0.1:25272) (DC: dc1)" +34:32.041 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4fb42849-3719-a3cb-5d89-6465567a670f.dc1 area=wan +34:32.041 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25267 network=tcp +34:32.041 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25267 network=udp +34:32.042 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25268 network=tcp +34:32.042 [INFO] TestAgent: started state syncer +34:32.042 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25273 network=tcp +34:32.108 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:32.108 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25272 [Candidate]" term=2 +34:32.108 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:32.108 [DEBUG] TestAgent.server.raft: vote granted: from=4fb42849-3719-a3cb-5d89-6465567a670f term=2 tally=1 +34:32.108 [INFO] TestAgent.server.raft: election won: tally=1 +34:32.108 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25272 [Leader]" +34:32.109 [INFO] TestAgent.server: cluster leadership acquired +34:32.109 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25272 +34:32.109 [INFO] TestAgent.server: New leader elected: payload=Node-4fb42849-3719-a3cb-5d89-6465567a670f +34:32.110 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:32.113 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:32.113 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:32.113 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:32.113 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:32.113 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:32.113 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4fb42849-3719-a3cb-5d89-6465567a670f +34:32.113 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4fb42849-3719-a3cb-5d89-6465567a670f +34:32.114 [INFO] TestAgent.server: federation state anti-entropy synced +34:32.414 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:32.415 [INFO] TestAgent: Synced node info +34:32.415 [DEBUG] TestAgent: Node info in sync +34:32.433 [INFO] TestAgent: Requesting shutdown +34:32.433 [INFO] TestAgent.server: shutting down server +34:32.433 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:32.433 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:32.433 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:32.433 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:32.433 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:32.433 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:32.433 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:32.433 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:32.433 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:32.433 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:32.434 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:32.434Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:32.434Z [INFO] agent.router.manager: shutting down +34:32.434 [INFO] TestAgent: consul server down +34:32.434 [INFO] TestAgent: shutdown complete +34:32.434 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25267 network=tcp +34:32.434 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25267 network=udp +34:32.434 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25268 network=tcp +34:32.934 [INFO] TestAgent: Waiting for endpoints to shut down +34:32.934 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_ChecksWithFilter (0.93s) +=== CONT TestIntentionsMatch_noBy +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:32.475Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:32.490Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:32.493 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fa903985-857d-5e38-9394-ffaaf0e636ab Address:127.0.0.1:25300}]" +34:32.493 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25300 [Follower]" leader= +34:32.494 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fa903985-857d-5e38-9394-ffaaf0e636ab.dc1 127.0.0.1 +34:32.495 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fa903985-857d-5e38-9394-ffaaf0e636ab 127.0.0.1 +2021-08-13T15:34:32.495Z [INFO] agent.router: Initializing LAN area manager +34:32.495 [INFO] TestAgent.server: Adding LAN server: server="Node-fa903985-857d-5e38-9394-ffaaf0e636ab (Addr: tcp/127.0.0.1:25300) (DC: dc1)" +34:32.495 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fa903985-857d-5e38-9394-ffaaf0e636ab.dc1 area=wan +34:32.495 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25288 network=tcp +34:32.495 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25288 network=udp +34:32.496 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25289 network=tcp +34:32.496 [INFO] TestAgent: started state syncer +34:32.496 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25301 network=tcp +34:32.545 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:32.545 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25300 [Candidate]" term=2 +34:32.545 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:32.545 [DEBUG] TestAgent.server.raft: vote granted: from=fa903985-857d-5e38-9394-ffaaf0e636ab term=2 tally=1 +34:32.545 [INFO] TestAgent.server.raft: election won: tally=1 +34:32.545 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25300 [Leader]" +34:32.545 [INFO] TestAgent.server: cluster leadership acquired +34:32.546 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25300 +34:32.547 [INFO] TestAgent.server: New leader elected: payload=Node-fa903985-857d-5e38-9394-ffaaf0e636ab +34:32.547 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:32.549 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:32.549 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:32.549 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:32.549 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:32.550 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:32.550 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fa903985-857d-5e38-9394-ffaaf0e636ab +34:32.550 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fa903985-857d-5e38-9394-ffaaf0e636ab +34:32.551 [INFO] TestAgent.server: federation state anti-entropy synced +34:32.648 [INFO] TestAgent: Requesting shutdown +34:32.648 [INFO] TestAgent.server: shutting down server +34:32.649 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:32.649 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:32.649 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:32.649 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:32.649 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:32.649 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:32.649 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:32.649 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:32.649 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:32.649 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:32.649 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:32.649Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:32.649Z [INFO] agent.router.manager: shutting down +34:32.649 [INFO] TestAgent: consul server down +34:32.649 [INFO] TestAgent: shutdown complete +34:32.649 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25288 network=tcp +34:32.649 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25288 network=udp +34:32.649 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25289 network=tcp +34:33.149 [INFO] TestAgent: Waiting for endpoints to shut down +34:33.149 [INFO] TestAgent: Endpoints down +--- PASS: TestIntentionsMatch_noName (0.69s) === CONT TestAgent_Services_MeshGateway +2021-08-13T15:34:32.308Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:32.323Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:32.326 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1f1b1426-8f18-e41d-4e7a-16c394762d9b Address:127.0.0.1:25286}]" +34:32.326 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25286 [Follower]" leader= +34:32.327 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1f1b1426-8f18-e41d-4e7a-16c394762d9b.dc1 127.0.0.1 +34:32.328 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1f1b1426-8f18-e41d-4e7a-16c394762d9b 127.0.0.1 +2021-08-13T15:34:32.328Z [INFO] agent.router: Initializing LAN area manager +34:32.328 [INFO] TestAgent.server: Adding LAN server: server="Node-1f1b1426-8f18-e41d-4e7a-16c394762d9b (Addr: tcp/127.0.0.1:25286) (DC: dc1)" +34:32.328 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1f1b1426-8f18-e41d-4e7a-16c394762d9b.dc1 area=wan +34:32.328 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25281 network=tcp +34:32.328 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25281 network=udp +34:32.329 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25282 network=tcp +34:32.329 [INFO] TestAgent: started state syncer +34:32.329 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25287 network=tcp +34:32.372 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:32.372 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25286 [Candidate]" term=2 +34:32.372 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:32.372 [DEBUG] TestAgent.server.raft: vote granted: from=1f1b1426-8f18-e41d-4e7a-16c394762d9b term=2 tally=1 +34:32.373 [INFO] TestAgent.server.raft: election won: tally=1 +34:32.373 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25286 [Leader]" +34:32.373 [INFO] TestAgent.server: cluster leadership acquired +34:32.373 [INFO] TestAgent.server: New leader elected: payload=Node-1f1b1426-8f18-e41d-4e7a-16c394762d9b +34:32.373 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25286 +34:32.374 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:32.377 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:32.377 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:32.377 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:32.377 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:32.377 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:32.377 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1f1b1426-8f18-e41d-4e7a-16c394762d9b +34:32.377 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1f1b1426-8f18-e41d-4e7a-16c394762d9b +34:32.378 [INFO] TestAgent.server: federation state anti-entropy synced +34:32.480 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:32.480 [INFO] TestAgent: Synced node info +34:32.652 [INFO] TestAgent: Requesting shutdown +34:32.652 [INFO] TestAgent.server: shutting down server +34:32.652 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:32.652 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:32.652 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:32.652 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:32.652 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:32.652 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:32.652 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:32.652 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:32.652 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:32.652 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:32.652 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:32.652Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:32.652Z [INFO] agent.router.manager: shutting down +34:32.652 [INFO] TestAgent: consul server down +34:32.652 [INFO] TestAgent: shutdown complete +34:32.652 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25281 network=tcp +34:32.652 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25281 network=udp +34:32.652 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25282 network=tcp +34:33.153 [INFO] TestAgent: Waiting for endpoints to shut down +34:33.153 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Checks (0.86s) +=== CONT TestIntentionsMatch_basic === RUN TestAgent_Services_ACLFilter/no_token === RUN TestAgent_Services_ACLFilter/root_token -[WARN] freeport: 4 out of 32 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:04.723Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:04.744Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:04.747 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:18fc4404-f9c6-9e5a-894d-6abd04775489 Address:127.0.0.1:13627}]" -43:04.747 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13627 [Follower]" leader= -43:04.748 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-18fc4404-f9c6-9e5a-894d-6abd04775489.dc1 127.0.0.1 -43:04.748 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-18fc4404-f9c6-9e5a-894d-6abd04775489 127.0.0.1 -2022-09-15T21:43:04.748Z [INFO] agent.router: Initializing LAN area manager -43:04.749 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-18fc4404-f9c6-9e5a-894d-6abd04775489.dc1 area=wan -43:04.749 [INFO] TestAgent.server: Adding LAN server: server="Node-18fc4404-f9c6-9e5a-894d-6abd04775489 (Addr: tcp/127.0.0.1:13627) (DC: dc1)" -43:04.749 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13601 network=udp -43:04.749 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13601 network=tcp -43:04.750 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13602 network=tcp -43:04.751 [INFO] TestAgent: started state syncer -43:04.751 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13628 network=tcp -43:04.806 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:04.806 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13627 [Candidate]" term=2 -43:04.806 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:04.806 [DEBUG] TestAgent.server.raft: vote granted: from=18fc4404-f9c6-9e5a-894d-6abd04775489 term=2 tally=1 -43:04.806 [INFO] TestAgent.server.raft: election won: tally=1 -43:04.806 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13627 [Leader]" -43:04.806 [INFO] TestAgent.server: cluster leadership acquired -43:04.807 [INFO] TestAgent.server: New leader elected: payload=Node-18fc4404-f9c6-9e5a-894d-6abd04775489 -43:04.807 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13627 -43:04.808 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:04.811 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:04.811 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:04.811 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:04.811 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:04.811 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:04.811 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-18fc4404-f9c6-9e5a-894d-6abd04775489 -43:04.811 [INFO] TestAgent.server: member joined, marking health alive: member=Node-18fc4404-f9c6-9e5a-894d-6abd04775489 -43:04.812 [INFO] TestAgent.server: federation state anti-entropy synced -43:04.887 [INFO] TestAgent: Requesting shutdown -43:04.887 [INFO] TestAgent.server: shutting down server -43:04.887 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:04.887 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:04.887 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:04.887 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:04.887 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:04.887 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:04.887 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:04.887 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:04.887 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:04.887 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:04.887 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:04.887 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:04.887Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:04.887Z [INFO] agent.router.manager: shutting down -43:04.887 [INFO] TestAgent: consul server down -43:04.887 [INFO] TestAgent: shutdown complete -43:04.887 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13601 network=tcp -43:04.888 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13601 network=udp -43:04.888 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13602 network=tcp -43:05.388 [INFO] TestAgent: Waiting for endpoints to shut down -43:05.388 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Checks (0.72s) +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:32.944Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:32.959Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:32.962 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e3e2108d-320a-75f9-9854-8368862aa4df Address:127.0.0.1:24301}]" +34:32.962 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24301 [Follower]" leader= +34:32.962 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e3e2108d-320a-75f9-9854-8368862aa4df.dc1 127.0.0.1 +34:32.963 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e3e2108d-320a-75f9-9854-8368862aa4df 127.0.0.1 +2021-08-13T15:34:32.963Z [INFO] agent.router: Initializing LAN area manager +34:32.964 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25309 network=udp +34:32.964 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e3e2108d-320a-75f9-9854-8368862aa4df.dc1 area=wan +34:32.964 [INFO] TestAgent.server: Adding LAN server: server="Node-e3e2108d-320a-75f9-9854-8368862aa4df (Addr: tcp/127.0.0.1:24301) (DC: dc1)" +34:32.964 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25309 network=tcp +34:32.965 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25310 network=tcp +34:32.965 [INFO] TestAgent: started state syncer +34:32.965 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24302 network=tcp +34:33.014 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:33.014 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24301 [Candidate]" term=2 +34:33.014 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:33.014 [DEBUG] TestAgent.server.raft: vote granted: from=e3e2108d-320a-75f9-9854-8368862aa4df term=2 tally=1 +34:33.014 [INFO] TestAgent.server.raft: election won: tally=1 +34:33.014 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24301 [Leader]" +34:33.014 [INFO] TestAgent.server: cluster leadership acquired +34:33.015 [INFO] TestAgent.server: New leader elected: payload=Node-e3e2108d-320a-75f9-9854-8368862aa4df +34:33.015 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24301 +34:33.016 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:33.018 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:33.018 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:33.018 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:33.018 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:33.018 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:33.018 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e3e2108d-320a-75f9-9854-8368862aa4df +34:33.018 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e3e2108d-320a-75f9-9854-8368862aa4df +34:33.019 [INFO] TestAgent.server: federation state anti-entropy synced +34:33.030 [INFO] TestAgent: Requesting shutdown +34:33.030 [INFO] TestAgent.server: shutting down server +34:33.030 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:33.030 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:33.030 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:33.030 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:33.030 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:33.030 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:33.030 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:33.030 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:33.030 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:33.030 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:33.030 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:33.030 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:33.030Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:33.030Z [INFO] agent.router.manager: shutting down +34:33.031 [INFO] TestAgent: consul server down +34:33.031 [INFO] TestAgent: shutdown complete +34:33.031 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25309 network=tcp +34:33.031 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25309 network=udp +34:33.031 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25310 network=tcp +34:33.531 [INFO] TestAgent: Waiting for endpoints to shut down +34:33.531 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Services_TerminatingGateway (0.60s) === CONT TestAgent_Services_Sidecar -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:04.663Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:04.722Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:04.728 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ca961b6b-28ad-05a9-c5fc-0deed5768643 Address:127.0.0.1:13599}]" -43:04.729 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13599 [Follower]" leader= -43:04.729 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ca961b6b-28ad-05a9-c5fc-0deed5768643.dc1 127.0.0.1 -43:04.730 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ca961b6b-28ad-05a9-c5fc-0deed5768643 127.0.0.1 -2022-09-15T21:43:04.730Z [INFO] agent.router: Initializing LAN area manager -43:04.731 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13615 network=udp -43:04.731 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ca961b6b-28ad-05a9-c5fc-0deed5768643.dc1 area=wan -43:04.731 [INFO] TestAgent.server: Adding LAN server: server="Node-ca961b6b-28ad-05a9-c5fc-0deed5768643 (Addr: tcp/127.0.0.1:13599) (DC: dc1)" -43:04.731 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13615 network=tcp -43:04.732 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13616 network=tcp -43:04.733 [INFO] TestAgent: started state syncer -43:04.733 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13600 network=tcp -43:04.768 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:04.768 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13599 [Candidate]" term=2 -43:04.768 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:04.768 [DEBUG] TestAgent.server.raft: vote granted: from=ca961b6b-28ad-05a9-c5fc-0deed5768643 term=2 tally=1 -43:04.768 [INFO] TestAgent.server.raft: election won: tally=1 -43:04.768 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13599 [Leader]" -43:04.768 [INFO] TestAgent.server: cluster leadership acquired -43:04.769 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13599 -43:04.769 [INFO] TestAgent.server: New leader elected: payload=Node-ca961b6b-28ad-05a9-c5fc-0deed5768643 -43:04.770 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:04.773 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:04.773 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:04.773 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:04.773 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:04.773 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:04.773 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ca961b6b-28ad-05a9-c5fc-0deed5768643 -43:04.773 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ca961b6b-28ad-05a9-c5fc-0deed5768643 -43:04.774 [INFO] TestAgent.server: federation state anti-entropy synced -43:04.914 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:04.915 [INFO] TestAgent: Synced node info -43:05.168 [INFO] TestAgent: Requesting shutdown -43:05.168 [INFO] TestAgent.server: shutting down server -43:05.168 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:05.168 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:05.168 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:05.168 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:05.168 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:05.168 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:05.168 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:05.168 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:05.168 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:05.168 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:05.168 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:05.169Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:05.169Z [INFO] agent.router.manager: shutting down -43:05.169 [INFO] TestAgent: consul server down -43:05.169 [INFO] TestAgent: shutdown complete -43:05.169 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13615 network=tcp -43:05.169 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13615 network=udp -43:05.169 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13616 network=tcp -43:05.669 [INFO] TestAgent: Waiting for endpoints to shut down -43:05.669 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_ChecksWithFilter (1.04s) -=== CONT TestAgent_Services_ExternalConnectProxy -2022-09-15T21:43:05.085Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:05.085Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:05.124Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:05.124Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:05.132 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b22f183c-16bc-26ca-9dd2-956e00b63543 Address:127.0.0.1:13544}]" -43:05.133 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b22f183c-16bc-26ca-9dd2-956e00b63543.dc1 127.0.0.1 -43:05.134 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13544 [Follower]" leader= -43:05.138 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b22f183c-16bc-26ca-9dd2-956e00b63543 127.0.0.1 -2022-09-15T21:43:05.138Z [INFO] agent.router: Initializing LAN area manager -43:05.138 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b22f183c-16bc-26ca-9dd2-956e00b63543.dc1 area=wan -43:05.138 [INFO] TestAgent.server: Adding LAN server: server="Node-b22f183c-16bc-26ca-9dd2-956e00b63543 (Addr: tcp/127.0.0.1:13544) (DC: dc1)" -43:05.138 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13629 network=udp -43:05.139 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13629 network=tcp -43:05.139 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13630 network=tcp -43:05.139 [INFO] TestAgent: started state syncer -43:05.140 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13545 network=tcp -43:05.182 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:05.182 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13544 [Candidate]" term=2 -43:05.182 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:05.182 [DEBUG] TestAgent.server.raft: vote granted: from=b22f183c-16bc-26ca-9dd2-956e00b63543 term=2 tally=1 -43:05.182 [INFO] TestAgent.server.raft: election won: tally=1 -43:05.182 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13544 [Leader]" -43:05.182 [INFO] TestAgent.server: cluster leadership acquired -43:05.182 [INFO] TestAgent.server: initializing acls -43:05.183 [INFO] TestAgent.server: Created ACL 'global-management' policy -43:05.183 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -43:05.184 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -43:05.184 [INFO] TestAgent.server: Created ACL anonymous token from configuration -43:05.184 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -43:05.184 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -43:05.184 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-b22f183c-16bc-26ca-9dd2-956e00b63543 -43:05.185 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-b22f183c-16bc-26ca-9dd2-956e00b63543.dc1 -43:05.186 [INFO] TestAgent.server: New leader elected: payload=Node-b22f183c-16bc-26ca-9dd2-956e00b63543 -43:05.186 [INFO] TestAgent.server: Updating LAN server: server="Node-b22f183c-16bc-26ca-9dd2-956e00b63543 (Addr: tcp/127.0.0.1:13544) (DC: dc1)" -43:05.186 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-b22f183c-16bc-26ca-9dd2-956e00b63543.dc1 area=wan -43:05.186 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:05.189 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:05.189 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:05.189 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:05.189 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:05.189 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:05.189 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b22f183c-16bc-26ca-9dd2-956e00b63543 -43:05.189 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b22f183c-16bc-26ca-9dd2-956e00b63543 -43:05.190 [INFO] TestAgent.server: federation state anti-entropy synced -43:05.190 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b22f183c-16bc-26ca-9dd2-956e00b63543 -43:05.229 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-b22f183c-16bc-26ca-9dd2-956e00b63543 -43:05.230 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-b22f183c-16bc-26ca-9dd2-956e00b63543 -43:05.230 [DEBUG] TestAgent: dropping service from result due to ACLs: service=mysql -43:05.231 [INFO] TestAgent: Requesting shutdown -43:05.231 [INFO] TestAgent.server: shutting down server -43:05.231 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -43:05.231 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -43:05.231 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:05.231 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:05.231 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:05.231 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:05.231 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:05.231 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:05.231 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:05.231 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:05.231 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -43:05.231 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -43:05.231 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:05.231 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:05.231 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:05.231 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:05.232Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:05.232Z [INFO] agent.router.manager: shutting down -43:05.232 [INFO] TestAgent: consul server down -43:05.232 [INFO] TestAgent: shutdown complete -43:05.232 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13629 network=tcp -43:05.232 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13629 network=udp -43:05.232 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13630 network=tcp -43:05.732 [INFO] TestAgent: Waiting for endpoints to shut down -43:05.732 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Services_ACLFilter (0.68s) +2021-08-13T15:34:32.950Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:32.965Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:32.967 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a52e033d-c69f-e6ce-0259-91a12b468b11 Address:127.0.0.1:24308}]" +34:32.967 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24308 [Follower]" leader= +34:32.968 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a52e033d-c69f-e6ce-0259-91a12b468b11.dc1 127.0.0.1 +34:32.969 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a52e033d-c69f-e6ce-0259-91a12b468b11 127.0.0.1 +2021-08-13T15:34:32.969Z [INFO] agent.router: Initializing LAN area manager +34:32.969 [INFO] TestAgent.server: Adding LAN server: server="Node-a52e033d-c69f-e6ce-0259-91a12b468b11 (Addr: tcp/127.0.0.1:24308) (DC: dc1)" +34:32.969 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a52e033d-c69f-e6ce-0259-91a12b468b11.dc1 area=wan +34:32.969 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24303 network=tcp +34:32.969 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24303 network=udp +34:32.970 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24304 network=tcp +34:32.970 [INFO] TestAgent: started state syncer +34:32.970 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24309 network=tcp +34:33.035 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:33.035 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24308 [Candidate]" term=2 +34:33.036 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:33.036 [DEBUG] TestAgent.server.raft: vote granted: from=a52e033d-c69f-e6ce-0259-91a12b468b11 term=2 tally=1 +34:33.036 [INFO] TestAgent.server.raft: election won: tally=1 +34:33.036 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24308 [Leader]" +34:33.036 [INFO] TestAgent.server: cluster leadership acquired +34:33.036 [INFO] TestAgent.server: New leader elected: payload=Node-a52e033d-c69f-e6ce-0259-91a12b468b11 +34:33.036 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24308 +34:33.037 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:33.040 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:33.040 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:33.040 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:33.040 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:33.040 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:33.040 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a52e033d-c69f-e6ce-0259-91a12b468b11 +34:33.040 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a52e033d-c69f-e6ce-0259-91a12b468b11 +34:33.040 [INFO] TestAgent.server: federation state anti-entropy synced +34:33.126 [INFO] TestAgent: Requesting shutdown +34:33.126 [INFO] TestAgent.server: shutting down server +34:33.127 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:33.127 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:33.127 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:33.127 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:33.127 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:33.127 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:33.127 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:33.127 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:33.127 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:33.127 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:33.127 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:33.127 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:33.127Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:33.127Z [INFO] agent.router.manager: shutting down +34:33.127 [INFO] TestAgent: consul server down +34:33.127 [INFO] TestAgent: shutdown complete +34:33.127 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24303 network=tcp +34:33.127 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24303 network=udp +34:33.127 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24304 network=tcp +34:33.627 [INFO] TestAgent: Waiting for endpoints to shut down +34:33.627 [INFO] TestAgent: Endpoints down +--- PASS: TestIntentionsMatch_noBy (0.69s) +=== CONT TestIntentionsList_values +2021-08-13T15:34:32.709Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:32.709Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:32.737Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:32.737Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:32.749 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7cd96bc3-e4f8-becc-2207-2987c849a1a5 Address:127.0.0.1:25293}]" +34:32.750 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7cd96bc3-e4f8-becc-2207-2987c849a1a5.dc1 127.0.0.1 +34:32.751 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25293 [Follower]" leader= +34:32.756 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7cd96bc3-e4f8-becc-2207-2987c849a1a5 127.0.0.1 +2021-08-13T15:34:32.756Z [INFO] agent.router: Initializing LAN area manager +34:32.757 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25302 network=udp +34:32.757 [INFO] TestAgent.server: Adding LAN server: server="Node-7cd96bc3-e4f8-becc-2207-2987c849a1a5 (Addr: tcp/127.0.0.1:25293) (DC: dc1)" +34:32.757 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7cd96bc3-e4f8-becc-2207-2987c849a1a5.dc1 area=wan +34:32.757 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25302 network=tcp +34:32.759 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25303 network=tcp +34:32.759 [INFO] TestAgent: started state syncer +34:32.760 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25294 network=tcp +34:32.801 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:32.801 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25293 [Candidate]" term=2 +34:32.801 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:32.801 [DEBUG] TestAgent.server.raft: vote granted: from=7cd96bc3-e4f8-becc-2207-2987c849a1a5 term=2 tally=1 +34:32.801 [INFO] TestAgent.server.raft: election won: tally=1 +34:32.801 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25293 [Leader]" +34:32.802 [INFO] TestAgent.server: cluster leadership acquired +34:32.802 [INFO] TestAgent.server: initializing acls +34:32.802 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:32.802 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:32.803 [INFO] TestAgent.server: New leader elected: payload=Node-7cd96bc3-e4f8-becc-2207-2987c849a1a5 +34:32.803 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:32.803 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:32.804 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:32.804 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:32.804 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-7cd96bc3-e4f8-becc-2207-2987c849a1a5 +34:32.804 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-7cd96bc3-e4f8-becc-2207-2987c849a1a5.dc1 +34:32.805 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:32.807 [INFO] TestAgent.server: Updating LAN server: server="Node-7cd96bc3-e4f8-becc-2207-2987c849a1a5 (Addr: tcp/127.0.0.1:25293) (DC: dc1)" +34:32.807 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-7cd96bc3-e4f8-becc-2207-2987c849a1a5.dc1 area=wan +34:32.816 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:32.816 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:32.816 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:32.816 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:32.816 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:32.816 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7cd96bc3-e4f8-becc-2207-2987c849a1a5 +34:32.816 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7cd96bc3-e4f8-becc-2207-2987c849a1a5 +34:32.817 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7cd96bc3-e4f8-becc-2207-2987c849a1a5 +34:32.817 [INFO] TestAgent.server: federation state anti-entropy synced +34:32.832 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:32.833 [INFO] TestAgent: Synced node info +34:32.833 [DEBUG] TestAgent: Node info in sync +34:32.958 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:32.958 [DEBUG] TestAgent: Node info in sync +34:33.157 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-7cd96bc3-e4f8-becc-2207-2987c849a1a5 +34:33.158 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-7cd96bc3-e4f8-becc-2207-2987c849a1a5 +34:33.158 [DEBUG] TestAgent: dropping service from result due to ACLs: service=mysql +34:33.158 [INFO] TestAgent: Requesting shutdown +34:33.158 [INFO] TestAgent.server: shutting down server +34:33.158 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:33.158 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:33.158 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:33.158 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:33.158 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:33.158 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:33.158 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:33.158 [DEBUG] TestAgent: Node info in sync +34:33.158 [WARN] TestAgent: Service registration blocked by ACLs: service=mysql accessorID=00000000-0000-0000-0000-000000000002 +34:33.158 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:33.158 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:33.159 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:33.159 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:33.159 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:33.159 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:33.159 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:33.159 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:33.159Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:33.159Z [INFO] agent.router.manager: shutting down +34:33.159 [INFO] TestAgent: consul server down +34:33.159 [INFO] TestAgent: shutdown complete +34:33.159 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25302 network=tcp +34:33.159 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25302 network=udp +34:33.159 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25303 network=tcp +34:33.659 [INFO] TestAgent: Waiting for endpoints to shut down +34:33.659 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Services_ACLFilter (0.97s) --- PASS: TestAgent_Services_ACLFilter/no_token (0.00s) --- PASS: TestAgent_Services_ACLFilter/root_token (0.00s) +=== CONT TestAgent_Services_ExternalConnectProxy +2021-08-13T15:34:32.904Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:32.919Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:32.922 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0ed7f0ca-9619-6dca-7df8-d23638c006c4 Address:127.0.0.1:25307}]" +34:32.922 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25307 [Follower]" leader= +34:32.923 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0ed7f0ca-9619-6dca-7df8-d23638c006c4.dc1 127.0.0.1 +34:32.924 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0ed7f0ca-9619-6dca-7df8-d23638c006c4 127.0.0.1 +2021-08-13T15:34:32.924Z [INFO] agent.router: Initializing LAN area manager +34:32.924 [INFO] TestAgent.server: Adding LAN server: server="Node-0ed7f0ca-9619-6dca-7df8-d23638c006c4 (Addr: tcp/127.0.0.1:25307) (DC: dc1)" +34:32.924 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0ed7f0ca-9619-6dca-7df8-d23638c006c4.dc1 area=wan +34:32.924 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25295 network=tcp +34:32.924 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25295 network=udp +34:32.925 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25296 network=tcp +34:32.925 [INFO] TestAgent: started state syncer +34:32.925 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25308 network=tcp +34:32.972 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:32.972 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25307 [Candidate]" term=2 +34:32.972 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:32.972 [DEBUG] TestAgent.server.raft: vote granted: from=0ed7f0ca-9619-6dca-7df8-d23638c006c4 term=2 tally=1 +34:32.972 [INFO] TestAgent.server.raft: election won: tally=1 +34:32.972 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25307 [Leader]" +34:32.972 [INFO] TestAgent.server: cluster leadership acquired +34:32.972 [INFO] TestAgent.server: New leader elected: payload=Node-0ed7f0ca-9619-6dca-7df8-d23638c006c4 +34:32.972 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25307 +34:32.973 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:32.976 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:32.976 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:32.976 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:32.976 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:32.976 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:32.976 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0ed7f0ca-9619-6dca-7df8-d23638c006c4 +34:32.976 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0ed7f0ca-9619-6dca-7df8-d23638c006c4 +34:32.976 [INFO] TestAgent.server: federation state anti-entropy synced +34:33.063 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:33.063 [INFO] TestAgent: Synced node info +34:33.063 [DEBUG] TestAgent: Node info in sync +34:33.172 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:33.172 [DEBUG] TestAgent: Node info in sync +34:33.230 [INFO] TestAgent: Requesting shutdown +34:33.230 [INFO] TestAgent.server: shutting down server +34:33.230 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:33.230 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:33.230 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:33.230 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:33.230 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:33.230 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:33.230 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:33.230 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:33.230 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:33.230 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:33.230 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:33.230Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:33.230Z [INFO] agent.router.manager: shutting down +34:33.230 [INFO] TestAgent: consul server down +34:33.230 [INFO] TestAgent: shutdown complete +34:33.230 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25295 network=tcp +34:33.230 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25295 network=udp +34:33.230 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25296 network=tcp +34:33.731 [INFO] TestAgent: Waiting for endpoints to shut down +34:33.731 [INFO] TestAgent: Endpoints down +--- PASS: TestIntentionsMatch_byInvalid (0.84s) +=== CONT TestIntentionsList_empty +[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:33.168Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:33.182Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:33.185 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:34913ed6-d1c1-8f76-f7cd-3e6c72d8fbc5 Address:127.0.0.1:24336}]" +34:33.185 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24336 [Follower]" leader= +34:33.185 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-34913ed6-d1c1-8f76-f7cd-3e6c72d8fbc5.dc1 127.0.0.1 +34:33.186 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-34913ed6-d1c1-8f76-f7cd-3e6c72d8fbc5 127.0.0.1 +2021-08-13T15:34:33.186Z [INFO] agent.router: Initializing LAN area manager +34:33.186 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24366 network=udp +34:33.187 [INFO] TestAgent.server: Adding LAN server: server="Node-34913ed6-d1c1-8f76-f7cd-3e6c72d8fbc5 (Addr: tcp/127.0.0.1:24336) (DC: dc1)" +34:33.187 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-34913ed6-d1c1-8f76-f7cd-3e6c72d8fbc5.dc1 area=wan +34:33.187 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24366 network=tcp +34:33.188 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24353 network=tcp +34:33.188 [INFO] TestAgent: started state syncer +34:33.188 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24337 network=tcp +34:33.234 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:33.234 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24336 [Candidate]" term=2 +34:33.234 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:33.234 [DEBUG] TestAgent.server.raft: vote granted: from=34913ed6-d1c1-8f76-f7cd-3e6c72d8fbc5 term=2 tally=1 +34:33.234 [INFO] TestAgent.server.raft: election won: tally=1 +34:33.234 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24336 [Leader]" +34:33.234 [INFO] TestAgent.server: cluster leadership acquired +34:33.234 [INFO] TestAgent.server: New leader elected: payload=Node-34913ed6-d1c1-8f76-f7cd-3e6c72d8fbc5 +34:33.234 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24336 +34:33.236 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:33.238 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:33.238 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:33.238 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:33.238 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:33.238 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:33.238 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-34913ed6-d1c1-8f76-f7cd-3e6c72d8fbc5 +34:33.238 [INFO] TestAgent.server: member joined, marking health alive: member=Node-34913ed6-d1c1-8f76-f7cd-3e6c72d8fbc5 +34:33.239 [INFO] TestAgent.server: federation state anti-entropy synced +34:33.280 [INFO] TestAgent: Requesting shutdown +34:33.281 [INFO] TestAgent.server: shutting down server +34:33.281 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:33.281 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:33.281 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:33.281 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:33.281 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:33.281 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:33.281 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:33.281 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:33.281 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:33.281 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:33.281 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:33.281 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:33.281Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:33.281Z [INFO] agent.router.manager: shutting down +34:33.281 [INFO] TestAgent: consul server down +34:33.281 [INFO] TestAgent: shutdown complete +34:33.281 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24366 network=tcp +34:33.281 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24366 network=udp +34:33.281 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24353 network=tcp +34:33.781 [INFO] TestAgent: Waiting for endpoints to shut down +34:33.781 [INFO] TestAgent: Endpoints down +--- PASS: TestIntentionsMatch_basic (0.63s) === CONT TestAgent_ServicesFiltered -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:59.743Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:59.822Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:59.829 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2be61511-b992-1de3-2f78-75f4839f1e67 Address:127.0.0.1:13379}]" -42:59.830 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13379 [Follower]" leader= -42:59.831 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2be61511-b992-1de3-2f78-75f4839f1e67.dc1 127.0.0.1 -42:59.832 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2be61511-b992-1de3-2f78-75f4839f1e67 127.0.0.1 -2022-09-15T21:42:59.832Z [INFO] agent.router: Initializing LAN area manager -42:59.832 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13395 network=udp -42:59.833 [INFO] TestAgent.server: Adding LAN server: server="Node-2be61511-b992-1de3-2f78-75f4839f1e67 (Addr: tcp/127.0.0.1:13379) (DC: dc1)" -42:59.833 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2be61511-b992-1de3-2f78-75f4839f1e67.dc1 area=wan -42:59.833 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13395 network=tcp -42:59.835 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13396 network=tcp -42:59.835 [INFO] TestAgent: started state syncer -42:59.835 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13380 network=tcp -42:59.890 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:59.890 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13379 [Candidate]" term=2 -42:59.890 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:59.890 [DEBUG] TestAgent.server.raft: vote granted: from=2be61511-b992-1de3-2f78-75f4839f1e67 term=2 tally=1 -42:59.890 [INFO] TestAgent.server.raft: election won: tally=1 -42:59.890 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13379 [Leader]" -42:59.890 [INFO] TestAgent.server: cluster leadership acquired -42:59.890 [INFO] TestAgent.server: New leader elected: payload=Node-2be61511-b992-1de3-2f78-75f4839f1e67 -42:59.892 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13379 -42:59.894 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:59.896 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:59.896 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:59.896 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:59.896 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:59.896 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:59.897 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2be61511-b992-1de3-2f78-75f4839f1e67 -42:59.897 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2be61511-b992-1de3-2f78-75f4839f1e67 -42:59.898 [INFO] TestAgent.server: federation state anti-entropy synced -43:00.102 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:00.103 [INFO] TestAgent: Synced node info -43:00.170 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:60462 -43:00.170 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ad6b391b-f8b2-ca17-b3c2-cf7c43beda2d 127.0.0.1 -43:00.171 [INFO] TestAgent.server: Adding LAN server: server="Node-ad6b391b-f8b2-ca17-b3c2-cf7c43beda2d (Addr: tcp/127.0.0.1:13344) (DC: dc1)" -43:00.171 [INFO] TestAgent.server: New leader elected: payload=Node-ad6b391b-f8b2-ca17-b3c2-cf7c43beda2d -43:00.171 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-ad6b391b-f8b2-ca17-b3c2-cf7c43beda2d other=Node-2be61511-b992-1de3-2f78-75f4839f1e67 -43:00.171 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ad6b391b-f8b2-ca17-b3c2-cf7c43beda2d -43:00.174 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: Node-ad6b391b-f8b2-ca17-b3c2-cf7c43beda2d.dc1 127.0.0.1:13378 -43:00.175 [INFO] TestAgent: Requesting shutdown -43:00.175 [INFO] TestAgent.server: shutting down server -43:00.175 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:00.175 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:00.175 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:00.175 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:00.175 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:00.176 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ad6b391b-f8b2-ca17-b3c2-cf7c43beda2d.dc1 127.0.0.1 -43:00.176 [DEBUG] TestAgent.server: Successfully performed flood-join for server at address: server=Node-ad6b391b-f8b2-ca17-b3c2-cf7c43beda2d.dc1 address=127.0.0.1:13378 -43:00.176 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ad6b391b-f8b2-ca17-b3c2-cf7c43beda2d.dc1 area=wan -43:00.176 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:00.176 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:00.176 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:00.176 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:00.176 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:00.176 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:00.176Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:00.176Z [INFO] agent.router.manager: shutting down -43:00.176 [INFO] TestAgent: consul server down -43:00.176 [INFO] TestAgent: shutdown complete -43:00.176 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13395 network=tcp -43:00.176 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13395 network=udp -43:00.176 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13396 network=tcp -43:00.677 [INFO] TestAgent: Waiting for endpoints to shut down -43:00.677 [INFO] TestAgent: Endpoints down -2022-09-15T21:42:59.295Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:59.336Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:59.340 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ad6b391b-f8b2-ca17-b3c2-cf7c43beda2d Address:127.0.0.1:13344}]" -42:59.341 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ad6b391b-f8b2-ca17-b3c2-cf7c43beda2d.dc1 127.0.0.1 -42:59.342 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13344 [Follower]" leader= -42:59.342 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ad6b391b-f8b2-ca17-b3c2-cf7c43beda2d 127.0.0.1 -2022-09-15T21:42:59.342Z [INFO] agent.router: Initializing LAN area manager -42:59.342 [INFO] TestAgent.server: Adding LAN server: server="Node-ad6b391b-f8b2-ca17-b3c2-cf7c43beda2d (Addr: tcp/127.0.0.1:13344) (DC: dc1)" -42:59.345 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ad6b391b-f8b2-ca17-b3c2-cf7c43beda2d.dc1 area=wan -42:59.345 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13374 network=udp -42:59.345 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13374 network=tcp -42:59.346 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13375 network=tcp -42:59.346 [INFO] TestAgent: started state syncer -42:59.346 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13345 network=tcp -42:59.394 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:59.394 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13344 [Candidate]" term=2 -42:59.394 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:59.395 [DEBUG] TestAgent.server.raft: vote granted: from=ad6b391b-f8b2-ca17-b3c2-cf7c43beda2d term=2 tally=1 -42:59.395 [INFO] TestAgent.server.raft: election won: tally=1 -42:59.395 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13344 [Leader]" -42:59.395 [INFO] TestAgent.server: cluster leadership acquired -42:59.395 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13344 -42:59.396 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:59.398 [INFO] TestAgent.server: New leader elected: payload=Node-ad6b391b-f8b2-ca17-b3c2-cf7c43beda2d -42:59.403 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:59.403 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:59.403 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:59.403 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:59.403 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:59.403 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ad6b391b-f8b2-ca17-b3c2-cf7c43beda2d -42:59.403 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ad6b391b-f8b2-ca17-b3c2-cf7c43beda2d -42:59.404 [INFO] TestAgent.server: federation state anti-entropy synced -42:59.783 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:59.784 [INFO] TestAgent: Synced node info -42:59.784 [DEBUG] TestAgent: Node info in sync -42:59.854 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:59.854 [DEBUG] TestAgent: Node info in sync -43:00.169 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:13398] -43:00.169 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:13398 -43:00.174 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:41018 -43:00.175 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2be61511-b992-1de3-2f78-75f4839f1e67 127.0.0.1 -43:00.175 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 -43:00.175 [DEBUG] TestAgent: systemd notify failed: error="No socket" -43:00.175 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2be61511-b992-1de3-2f78-75f4839f1e67.dc1 127.0.0.1 -43:00.175 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2be61511-b992-1de3-2f78-75f4839f1e67.dc1 area=wan -43:00.175 [INFO] TestAgent.server: Adding LAN server: server="Node-2be61511-b992-1de3-2f78-75f4839f1e67 (Addr: tcp/127.0.0.1:13379) (DC: dc1)" -43:00.175 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-2be61511-b992-1de3-2f78-75f4839f1e67 other=Node-ad6b391b-f8b2-ca17-b3c2-cf7c43beda2d -43:00.175 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2be61511-b992-1de3-2f78-75f4839f1e67 -43:00.842 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Failed ping: Node-2be61511-b992-1de3-2f78-75f4839f1e67 (timeout reached) -43:01.342 [INFO] TestAgent.server.memberlist.lan: memberlist: Suspect Node-2be61511-b992-1de3-2f78-75f4839f1e67 has failed, no acks received -43:01.398 [WARN] TestAgent: error getting server health from server: server=Node-2be61511-b992-1de3-2f78-75f4839f1e67 error="rpc error getting client: failed to get conn: dial tcp 127.0.0.1:0->127.0.0.1:13379: connect: connection refused" -43:02.398 [WARN] TestAgent: error getting server health from server: server=Node-2be61511-b992-1de3-2f78-75f4839f1e67 error="context deadline exceeded" -43:02.843 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Failed ping: Node-2be61511-b992-1de3-2f78-75f4839f1e67 (timeout reached) -43:03.399 [WARN] TestAgent: error getting server health from server: server=Node-2be61511-b992-1de3-2f78-75f4839f1e67 error="rpc error getting client: failed to get conn: dial tcp 127.0.0.1:0->127.0.0.1:13379: connect: connection refused" -43:04.342 [INFO] TestAgent.server.memberlist.lan: memberlist: Suspect Node-2be61511-b992-1de3-2f78-75f4839f1e67 has failed, no acks received -43:04.399 [WARN] TestAgent: error getting server health from server: server=Node-2be61511-b992-1de3-2f78-75f4839f1e67 error="context deadline exceeded" -43:04.842 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Failed ping: Node-2be61511-b992-1de3-2f78-75f4839f1e67 (timeout reached) -43:05.343 [INFO] TestAgent.server.memberlist.lan: memberlist: Marking Node-2be61511-b992-1de3-2f78-75f4839f1e67 as failed, suspect timeout reached (0 peer confirmations) -43:05.343 [INFO] TestAgent.server.serf.lan: serf: EventMemberFailed: Node-2be61511-b992-1de3-2f78-75f4839f1e67 127.0.0.1 -43:05.344 [INFO] TestAgent.server: Removing LAN server: server="Node-2be61511-b992-1de3-2f78-75f4839f1e67 (Addr: tcp/127.0.0.1:13379) (DC: dc1)" -43:05.344 [INFO] TestAgent.server: member failed, marking health critical: member=Node-2be61511-b992-1de3-2f78-75f4839f1e67 -43:05.402 [INFO] TestAgent: Force leaving node: node=Node-2be61511-b992-1de3-2f78-75f4839f1e67 -43:05.402 [INFO] TestAgent.server.serf.lan: serf: EventMemberLeave (forced): Node-2be61511-b992-1de3-2f78-75f4839f1e67 127.0.0.1 -43:05.402 [INFO] TestAgent: Requesting shutdown -43:05.402 [INFO] TestAgent.server: shutting down server -43:05.402 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:05.402 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:05.402 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:05.402 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:05.402 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:05.402 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:05.402 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:05.402 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:05.402 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:05.402 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:05.403 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:05.403Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:05.403Z [INFO] agent.router.manager: shutting down -43:05.404 [INFO] TestAgent: consul server down -43:05.404 [INFO] TestAgent: shutdown complete -43:05.404 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13374 network=tcp -43:05.404 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13374 network=udp -43:05.404 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13375 network=tcp -43:05.404 [WARN] TestAgent: error getting server health from server: server=Node-ad6b391b-f8b2-ca17-b3c2-cf7c43beda2d error="rpc error making call: EOF" -43:05.910 [INFO] TestAgent: Waiting for endpoints to shut down -43:05.910 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_ForceLeave (6.64s) +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:33.166Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:33.181Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:33.183 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:272a8495-ed48-e069-ccd1-506af9119d44 Address:127.0.0.1:24350}]" +34:33.183 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24350 [Follower]" leader= +34:33.184 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-272a8495-ed48-e069-ccd1-506af9119d44.dc1 127.0.0.1 +34:33.185 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-272a8495-ed48-e069-ccd1-506af9119d44 127.0.0.1 +2021-08-13T15:34:33.185Z [INFO] agent.router: Initializing LAN area manager +34:33.185 [INFO] TestAgent.server: Adding LAN server: server="Node-272a8495-ed48-e069-ccd1-506af9119d44 (Addr: tcp/127.0.0.1:24350) (DC: dc1)" +34:33.185 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-272a8495-ed48-e069-ccd1-506af9119d44.dc1 area=wan +34:33.185 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24310 network=udp +34:33.185 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24310 network=tcp +34:33.186 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24339 network=tcp +34:33.186 [INFO] TestAgent: started state syncer +34:33.187 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24365 network=tcp +34:33.243 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:33.243 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24350 [Candidate]" term=2 +34:33.243 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:33.243 [DEBUG] TestAgent.server.raft: vote granted: from=272a8495-ed48-e069-ccd1-506af9119d44 term=2 tally=1 +34:33.243 [INFO] TestAgent.server.raft: election won: tally=1 +34:33.243 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24350 [Leader]" +34:33.243 [INFO] TestAgent.server: cluster leadership acquired +34:33.243 [INFO] TestAgent.server: New leader elected: payload=Node-272a8495-ed48-e069-ccd1-506af9119d44 +34:33.243 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24350 +34:33.244 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:33.247 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:33.247 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:33.247 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:33.247 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:33.247 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:33.247 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-272a8495-ed48-e069-ccd1-506af9119d44 +34:33.247 [INFO] TestAgent.server: member joined, marking health alive: member=Node-272a8495-ed48-e069-ccd1-506af9119d44 +34:33.248 [INFO] TestAgent.server: federation state anti-entropy synced +34:33.335 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:33.335 [INFO] TestAgent: Synced node info +34:33.583 [INFO] TestAgent: Requesting shutdown +34:33.583 [INFO] TestAgent.server: shutting down server +34:33.584 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:33.584 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:33.584 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:33.584 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:33.584 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:33.584 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:33.584 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:33.584 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:33.584 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:33.584 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:33.584 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:33.584Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:33.584Z [INFO] agent.router.manager: shutting down +34:33.585 [INFO] TestAgent: consul server down +34:33.585 [INFO] TestAgent: shutdown complete +34:33.585 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24310 network=tcp +34:33.585 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24310 network=udp +34:33.585 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24339 network=tcp +34:34.085 [INFO] TestAgent: Waiting for endpoints to shut down +34:34.085 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Services_MeshGateway (0.94s) +=== CONT TestRPC_HTTPSMaxConnsPerClient +=== RUN TestRPC_HTTPSMaxConnsPerClient/HTTP +2021-08-13T15:34:33.547Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:33.586Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:33.590 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:19e4febd-52fb-640c-259a-726636939775 Address:127.0.0.1:24357}]" +34:33.590 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24357 [Follower]" leader= +34:33.591 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-19e4febd-52fb-640c-259a-726636939775.dc1 127.0.0.1 +34:33.592 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-19e4febd-52fb-640c-259a-726636939775 127.0.0.1 +2021-08-13T15:34:33.592Z [INFO] agent.router: Initializing LAN area manager +34:33.592 [INFO] TestAgent.server: Adding LAN server: server="Node-19e4febd-52fb-640c-259a-726636939775 (Addr: tcp/127.0.0.1:24357) (DC: dc1)" +34:33.592 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-19e4febd-52fb-640c-259a-726636939775.dc1 area=wan +34:33.593 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24338 network=udp +34:33.593 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24338 network=tcp +34:33.593 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24346 network=tcp +34:33.594 [INFO] TestAgent: started state syncer +34:33.594 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24344 network=tcp +34:33.631 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:33.631 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24357 [Candidate]" term=2 +34:33.632 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:33.632 [DEBUG] TestAgent.server.raft: vote granted: from=19e4febd-52fb-640c-259a-726636939775 term=2 tally=1 +34:33.632 [INFO] TestAgent.server.raft: election won: tally=1 +34:33.632 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24357 [Leader]" +34:33.632 [INFO] TestAgent.server: cluster leadership acquired +34:33.632 [INFO] TestAgent.server: New leader elected: payload=Node-19e4febd-52fb-640c-259a-726636939775 +34:33.632 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24357 +34:33.633 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:33.636 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:33.636 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:33.636 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:33.636 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:33.636 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:33.636 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-19e4febd-52fb-640c-259a-726636939775 +34:33.636 [INFO] TestAgent.server: member joined, marking health alive: member=Node-19e4febd-52fb-640c-259a-726636939775 +34:33.637 [INFO] TestAgent: Requesting shutdown +34:33.637 [INFO] TestAgent.server: shutting down server +34:33.637 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:33.637 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:33.637 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:33.637 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:33.637 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:33.637 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:33.637 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:33.637 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:33.637 [INFO] TestAgent.server: federation state anti-entropy synced +34:33.637 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:33.637 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:33.637 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:33.638Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:33.638Z [INFO] agent.router.manager: shutting down +34:33.638 [INFO] TestAgent: consul server down +34:33.638 [INFO] TestAgent: shutdown complete +34:33.638 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24338 network=tcp +34:33.638 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24338 network=udp +34:33.638 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24346 network=tcp +34:34.138 [INFO] TestAgent: Waiting for endpoints to shut down +34:34.138 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Services_Sidecar (0.61s) === CONT TestAgent_Services -2022-09-15T21:43:05.127Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:05.161Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:05.169 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e3f3c1d0-39d9-526b-23b5-8fda2a0c9635 Address:127.0.0.1:13641}]" -43:05.169 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13641 [Follower]" leader= -43:05.174 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e3f3c1d0-39d9-526b-23b5-8fda2a0c9635.dc1 127.0.0.1 -43:05.175 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e3f3c1d0-39d9-526b-23b5-8fda2a0c9635 127.0.0.1 -2022-09-15T21:43:05.175Z [INFO] agent.router: Initializing LAN area manager -43:05.175 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13546 network=udp -43:05.175 [INFO] TestAgent.server: Adding LAN server: server="Node-e3f3c1d0-39d9-526b-23b5-8fda2a0c9635 (Addr: tcp/127.0.0.1:13641) (DC: dc1)" -43:05.175 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e3f3c1d0-39d9-526b-23b5-8fda2a0c9635.dc1 area=wan -43:05.176 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13546 network=tcp -43:05.176 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13547 network=tcp -43:05.177 [INFO] TestAgent: started state syncer -43:05.177 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13642 network=tcp -43:05.207 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:05.207 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13641 [Candidate]" term=2 -43:05.207 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:05.207 [DEBUG] TestAgent.server.raft: vote granted: from=e3f3c1d0-39d9-526b-23b5-8fda2a0c9635 term=2 tally=1 -43:05.207 [INFO] TestAgent.server.raft: election won: tally=1 -43:05.207 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13641 [Leader]" -43:05.207 [INFO] TestAgent.server: cluster leadership acquired -43:05.208 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13641 -43:05.209 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:05.209 [INFO] TestAgent.server: New leader elected: payload=Node-e3f3c1d0-39d9-526b-23b5-8fda2a0c9635 -43:05.212 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:05.212 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:05.212 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:05.212 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:05.212 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:05.212 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e3f3c1d0-39d9-526b-23b5-8fda2a0c9635 -43:05.212 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e3f3c1d0-39d9-526b-23b5-8fda2a0c9635 -43:05.212 [INFO] TestAgent.server: federation state anti-entropy synced -43:05.370 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:05.371 [INFO] TestAgent: Synced node info -43:05.450 [INFO] TestAgent: Requesting shutdown -43:05.450 [INFO] TestAgent.server: shutting down server -43:05.450 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:05.450 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:05.450 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:05.450 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:05.450 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:05.451 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:05.451 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:05.451 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:05.451 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:05.451 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:05.451 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:05.451Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:05.451Z [INFO] agent.router.manager: shutting down -43:05.452 [INFO] TestAgent: consul server down -43:05.452 [INFO] TestAgent: shutdown complete -43:05.452 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13546 network=tcp -43:05.452 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13546 network=udp -43:05.452 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13547 network=tcp -43:05.954 [INFO] TestAgent: Waiting for endpoints to shut down -43:05.954 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Services_TerminatingGateway (0.86s) +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:33.644Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:33.660Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:33.663 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c8eb7c25-76f7-1f69-014d-06a2d2249217 Address:127.0.0.1:24315}]" +34:33.663 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24315 [Follower]" leader= +34:33.664 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c8eb7c25-76f7-1f69-014d-06a2d2249217.dc1 127.0.0.1 +34:33.665 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c8eb7c25-76f7-1f69-014d-06a2d2249217 127.0.0.1 +2021-08-13T15:34:33.665Z [INFO] agent.router: Initializing LAN area manager +34:33.665 [INFO] TestAgent.server: Adding LAN server: server="Node-c8eb7c25-76f7-1f69-014d-06a2d2249217 (Addr: tcp/127.0.0.1:24315) (DC: dc1)" +34:33.665 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c8eb7c25-76f7-1f69-014d-06a2d2249217.dc1 area=wan +34:33.665 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24345 network=udp +34:33.665 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24345 network=tcp +34:33.666 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24311 network=tcp +34:33.666 [INFO] TestAgent: started state syncer +34:33.666 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24316 network=tcp +34:33.719 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:33.719 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24315 [Candidate]" term=2 +34:33.719 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:33.719 [DEBUG] TestAgent.server.raft: vote granted: from=c8eb7c25-76f7-1f69-014d-06a2d2249217 term=2 tally=1 +34:33.719 [INFO] TestAgent.server.raft: election won: tally=1 +34:33.719 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24315 [Leader]" +34:33.719 [INFO] TestAgent.server: cluster leadership acquired +34:33.719 [INFO] TestAgent.server: New leader elected: payload=Node-c8eb7c25-76f7-1f69-014d-06a2d2249217 +34:33.719 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24315 +34:33.721 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:33.723 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:33.723 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:33.723 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:33.723 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:33.723 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:33.723 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c8eb7c25-76f7-1f69-014d-06a2d2249217 +34:33.723 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c8eb7c25-76f7-1f69-014d-06a2d2249217 +34:33.724 [INFO] TestAgent.server: federation state anti-entropy synced +34:33.889 [INFO] TestAgent: Requesting shutdown +34:33.889 [INFO] TestAgent.server: shutting down server +34:33.889 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:33.889 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:33.889 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:33.889 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:33.889 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:33.889 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:33.889 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:33.889 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:33.889 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:33.889 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:33.889 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:33.889 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:33.889Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:33.889Z [INFO] agent.router.manager: shutting down +34:33.889 [INFO] TestAgent: consul server down +34:33.889 [INFO] TestAgent: shutdown complete +34:33.889 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24345 network=tcp +34:33.890 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24345 network=udp +34:33.890 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24311 network=tcp +34:34.390 [INFO] TestAgent: Waiting for endpoints to shut down +34:34.390 [INFO] TestAgent: Endpoints down +--- PASS: TestIntentionsList_values (0.76s) +=== CONT TestHTTPServer_HandshakeTimeout +2021-08-13T15:34:33.799Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:33.814Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:33.816 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fd5e52ac-c484-9d3c-375e-2f8ef2664d7f Address:127.0.0.1:24329}]" +34:33.817 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24329 [Follower]" leader= +34:33.817 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fd5e52ac-c484-9d3c-375e-2f8ef2664d7f.dc1 127.0.0.1 +34:33.818 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fd5e52ac-c484-9d3c-375e-2f8ef2664d7f 127.0.0.1 +2021-08-13T15:34:33.818Z [INFO] agent.router: Initializing LAN area manager +34:33.818 [INFO] TestAgent.server: Adding LAN server: server="Node-fd5e52ac-c484-9d3c-375e-2f8ef2664d7f (Addr: tcp/127.0.0.1:24329) (DC: dc1)" +34:33.818 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fd5e52ac-c484-9d3c-375e-2f8ef2664d7f.dc1 area=wan +34:33.819 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24324 network=udp +34:33.819 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24324 network=tcp +34:33.819 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24367 network=tcp +34:33.820 [INFO] TestAgent: started state syncer +34:33.820 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24330 network=tcp +34:33.884 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:33.884 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24329 [Candidate]" term=2 +34:33.884 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:33.884 [DEBUG] TestAgent.server.raft: vote granted: from=fd5e52ac-c484-9d3c-375e-2f8ef2664d7f term=2 tally=1 +34:33.884 [INFO] TestAgent.server.raft: election won: tally=1 +34:33.884 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24329 [Leader]" +34:33.884 [INFO] TestAgent.server: cluster leadership acquired +34:33.885 [INFO] TestAgent.server: New leader elected: payload=Node-fd5e52ac-c484-9d3c-375e-2f8ef2664d7f +34:33.885 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24329 +34:33.886 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:33.888 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:33.888 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:33.888 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:33.888 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:33.888 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:33.888 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fd5e52ac-c484-9d3c-375e-2f8ef2664d7f +34:33.888 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fd5e52ac-c484-9d3c-375e-2f8ef2664d7f +34:33.889 [INFO] TestAgent.server: federation state anti-entropy synced +34:33.936 [INFO] TestAgent: Requesting shutdown +34:33.936 [INFO] TestAgent.server: shutting down server +34:33.936 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:33.936 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:33.936 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:33.936 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:33.936 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:33.936 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:33.936 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:33.936 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:33.936 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:33.936 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:33.936 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:33.936 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:33.937Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:33.937Z [INFO] agent.router.manager: shutting down +34:33.937 [INFO] TestAgent: consul server down +34:33.937 [INFO] TestAgent: shutdown complete +34:33.937 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24324 network=tcp +34:33.937 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24324 network=udp +34:33.937 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24367 network=tcp +34:34.437 [INFO] TestAgent: Waiting for endpoints to shut down +34:34.437 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_ServicesFiltered (0.66s) === CONT TestACL_ResolveIdentity -2022-09-15T21:43:05.172Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:05.198Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:05.201 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:696797de-cfe0-b2cf-14f3-5b4361ec416f Address:127.0.0.1:13585}]" -43:05.201 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13585 [Follower]" leader= -43:05.202 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-696797de-cfe0-b2cf-14f3-5b4361ec416f.dc1 127.0.0.1 -43:05.203 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-696797de-cfe0-b2cf-14f3-5b4361ec416f 127.0.0.1 -2022-09-15T21:43:05.203Z [INFO] agent.router: Initializing LAN area manager -43:05.203 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13643 network=udp -43:05.204 [INFO] TestAgent.server: Adding LAN server: server="Node-696797de-cfe0-b2cf-14f3-5b4361ec416f (Addr: tcp/127.0.0.1:13585) (DC: dc1)" -43:05.204 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-696797de-cfe0-b2cf-14f3-5b4361ec416f.dc1 area=wan -43:05.204 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13643 network=tcp -43:05.205 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13644 network=tcp -43:05.205 [INFO] TestAgent: started state syncer -43:05.205 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13586 network=tcp -43:05.264 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:05.264 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13585 [Candidate]" term=2 -43:05.264 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:05.264 [DEBUG] TestAgent.server.raft: vote granted: from=696797de-cfe0-b2cf-14f3-5b4361ec416f term=2 tally=1 -43:05.264 [INFO] TestAgent.server.raft: election won: tally=1 -43:05.264 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13585 [Leader]" -43:05.264 [INFO] TestAgent.server: cluster leadership acquired -43:05.265 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13585 -43:05.266 [INFO] TestAgent.server: New leader elected: payload=Node-696797de-cfe0-b2cf-14f3-5b4361ec416f -43:05.266 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:05.269 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:05.269 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:05.269 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:05.269 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:05.269 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:05.269 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-696797de-cfe0-b2cf-14f3-5b4361ec416f -43:05.269 [INFO] TestAgent.server: member joined, marking health alive: member=Node-696797de-cfe0-b2cf-14f3-5b4361ec416f -43:05.271 [INFO] TestAgent.server: federation state anti-entropy synced -43:05.353 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:05.354 [INFO] TestAgent: Synced node info -43:05.484 [INFO] TestAgent: Requesting shutdown -43:05.484 [INFO] TestAgent.server: shutting down server -43:05.484 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:05.484 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:05.484 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:05.484 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:05.484 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:05.484 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:05.484 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:05.485 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:05.485 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:05.485 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:05.485 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:05.485Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:05.485Z [INFO] agent.router.manager: shutting down -43:05.485 [INFO] TestAgent: consul server down -43:05.485 [INFO] TestAgent: shutdown complete -43:05.485 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13643 network=tcp -43:05.485 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13643 network=udp -43:05.485 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13644 network=tcp -43:05.986 [INFO] TestAgent: Waiting for endpoints to shut down -43:05.986 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Services_MeshGateway (0.84s) -=== CONT TestACLEndpoint_LoginLogout_jwt -2022-09-15T21:43:06.003Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:06.003Z [WARN] agent: bootstrap = true: do not enable unless necessary ---- PASS: TestACL_ResolveIdentity (0.05s) +2021-08-13T15:34:34.451Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:34.451Z [WARN] agent: bootstrap = true: do not enable unless necessary +--- PASS: TestACL_ResolveIdentity (0.01s) +=== CONT TestEnableWebUI +2021-08-13T15:34:33.676Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:33.691Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:33.694 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8e8b4d5c-25cf-3ea7-cb5b-c6a744855610 Address:127.0.0.1:24371}]" +34:33.694 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24371 [Follower]" leader= +34:33.695 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8e8b4d5c-25cf-3ea7-cb5b-c6a744855610.dc1 127.0.0.1 +34:33.695 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8e8b4d5c-25cf-3ea7-cb5b-c6a744855610 127.0.0.1 +2021-08-13T15:34:33.695Z [INFO] agent.router: Initializing LAN area manager +34:33.696 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8e8b4d5c-25cf-3ea7-cb5b-c6a744855610.dc1 area=wan +34:33.696 [INFO] TestAgent.server: Adding LAN server: server="Node-8e8b4d5c-25cf-3ea7-cb5b-c6a744855610 (Addr: tcp/127.0.0.1:24371) (DC: dc1)" +34:33.696 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24317 network=tcp +34:33.696 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24317 network=udp +34:33.697 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24318 network=tcp +34:33.697 [INFO] TestAgent: started state syncer +34:33.697 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24358 network=tcp +34:33.747 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:33.747 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24371 [Candidate]" term=2 +34:33.747 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:33.747 [DEBUG] TestAgent.server.raft: vote granted: from=8e8b4d5c-25cf-3ea7-cb5b-c6a744855610 term=2 tally=1 +34:33.747 [INFO] TestAgent.server.raft: election won: tally=1 +34:33.747 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24371 [Leader]" +34:33.747 [INFO] TestAgent.server: cluster leadership acquired +34:33.748 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24371 +34:33.748 [INFO] TestAgent.server: New leader elected: payload=Node-8e8b4d5c-25cf-3ea7-cb5b-c6a744855610 +34:33.749 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:33.751 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:33.751 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:33.751 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:33.751 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:33.751 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:33.751 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8e8b4d5c-25cf-3ea7-cb5b-c6a744855610 +34:33.751 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8e8b4d5c-25cf-3ea7-cb5b-c6a744855610 +34:33.752 [INFO] TestAgent.server: federation state anti-entropy synced +34:33.981 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:33.982 [INFO] TestAgent: Synced node info +34:33.988 [INFO] TestAgent: Requesting shutdown +34:33.988 [INFO] TestAgent.server: shutting down server +34:33.988 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:33.988 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:33.988 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:33.988 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:33.988 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:33.988 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:33.988 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:33.988 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:33.988 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:33.988 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:33.989 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:33.989Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:33.989Z [INFO] agent.router.manager: shutting down +34:33.989 [INFO] TestAgent: consul server down +34:33.989 [INFO] TestAgent: shutdown complete +34:33.989 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24317 network=tcp +34:33.989 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24317 network=udp +34:33.989 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24318 network=tcp +34:34.489 [INFO] TestAgent: Waiting for endpoints to shut down +34:34.489 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Services_ExternalConnectProxy (0.83s) +=== CONT TestACL_filterChecks +2021-08-13T15:34:34.503Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:34.503Z [WARN] agent: bootstrap = true: do not enable unless necessary +34:34.504 [DEBUG] TestACL_filterChecks: dropping check from result due to ACLs: check=my-node +34:34.504 [DEBUG] TestACL_filterChecks: dropping check from result due to ACLs: check=my-other +34:34.504 [DEBUG] TestACL_filterChecks: dropping check from result due to ACLs: check=my-other +34:34.504 [DEBUG] TestACL_filterChecks: dropping check from result due to ACLs: check=my-service +--- PASS: TestACL_filterChecks (0.01s) +=== CONT TestACLResolution +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021/08/13 15:34:34 http: TLS handshake error from 127.0.0.1:54866: read tcp 127.0.0.1:24375->127.0.0.1:54866: i/o timeout +2021-08-13T15:34:33.747Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:33.762Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:33.765 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:26945d59-4352-5911-5d90-f7435a9d50c3 Address:127.0.0.1:24322}]" +34:33.765 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24322 [Follower]" leader= +34:33.766 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-26945d59-4352-5911-5d90-f7435a9d50c3.dc1 127.0.0.1 +34:33.767 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-26945d59-4352-5911-5d90-f7435a9d50c3 127.0.0.1 +2021-08-13T15:34:33.767Z [INFO] agent.router: Initializing LAN area manager +34:33.767 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24359 network=udp +34:33.767 [INFO] TestAgent.server: Adding LAN server: server="Node-26945d59-4352-5911-5d90-f7435a9d50c3 (Addr: tcp/127.0.0.1:24322) (DC: dc1)" +34:33.767 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-26945d59-4352-5911-5d90-f7435a9d50c3.dc1 area=wan +34:33.767 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24359 network=tcp +34:33.768 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24360 network=tcp +34:33.768 [INFO] TestAgent: started state syncer +34:33.768 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24323 network=tcp +34:33.822 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:33.822 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24322 [Candidate]" term=2 +34:33.823 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:33.823 [DEBUG] TestAgent.server.raft: vote granted: from=26945d59-4352-5911-5d90-f7435a9d50c3 term=2 tally=1 +34:33.823 [INFO] TestAgent.server.raft: election won: tally=1 +34:33.823 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24322 [Leader]" +34:33.823 [INFO] TestAgent.server: cluster leadership acquired +34:33.823 [INFO] TestAgent.server: New leader elected: payload=Node-26945d59-4352-5911-5d90-f7435a9d50c3 +34:33.823 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24322 +34:33.824 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:33.827 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:33.827 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:33.827 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:33.827 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:33.827 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:33.827 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-26945d59-4352-5911-5d90-f7435a9d50c3 +34:33.827 [INFO] TestAgent.server: member joined, marking health alive: member=Node-26945d59-4352-5911-5d90-f7435a9d50c3 +34:33.827 [INFO] TestAgent.server: federation state anti-entropy synced +34:34.021 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:34.021 [INFO] TestAgent: Synced node info +34:34.021 [DEBUG] TestAgent: Node info in sync +34:34.153 [INFO] TestAgent: Requesting shutdown +34:34.153 [INFO] TestAgent.server: shutting down server +34:34.153 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:34.153 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:34.153 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:34.153 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:34.153 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:34.153 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:34.153 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:34.153 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:34.153 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:34.153 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:34.153 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:34.154Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:34.154Z [INFO] agent.router.manager: shutting down +34:34.154 [INFO] TestAgent: consul server down +34:34.154 [INFO] TestAgent: shutdown complete +34:34.154 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24359 network=tcp +34:34.154 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24359 network=udp +34:34.154 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24360 network=tcp +34:34.654 [INFO] TestAgent: Waiting for endpoints to shut down +34:34.654 [INFO] TestAgent: Endpoints down +--- PASS: TestIntentionsList_empty (0.92s) === CONT TestACL_filterServices -[WARN] freeport: 4 out of 32 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:06.036Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:06.036Z [WARN] agent: bootstrap = true: do not enable unless necessary -43:06.036 [DEBUG] TestACL_filterServices: dropping service from result due to ACLs: service=my-other ---- PASS: TestACL_filterServices (0.03s) +2021-08-13T15:34:34.669Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:34.669Z [WARN] agent: bootstrap = true: do not enable unless necessary +34:34.669 [DEBUG] TestACL_filterServices: dropping service from result due to ACLs: service=my-other +--- PASS: TestACL_filterServices (0.01s) +=== CONT TestParseConsistency_Invalid +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:34.155Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:34.188Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:34.196 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4783a1e0-fa28-ca73-71f7-71a8822e489e Address:127.0.0.1:24385}]" +34:34.196 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24385 [Follower]" leader= +34:34.201 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4783a1e0-fa28-ca73-71f7-71a8822e489e.dc1 127.0.0.1 +34:34.202 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4783a1e0-fa28-ca73-71f7-71a8822e489e 127.0.0.1 +2021-08-13T15:34:34.202Z [INFO] agent.router: Initializing LAN area manager +34:34.203 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24380 network=udp +34:34.203 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4783a1e0-fa28-ca73-71f7-71a8822e489e.dc1 area=wan +34:34.203 [INFO] TestAgent.server: Adding LAN server: server="Node-4783a1e0-fa28-ca73-71f7-71a8822e489e (Addr: tcp/127.0.0.1:24385) (DC: dc1)" +34:34.203 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24380 network=tcp +34:34.204 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24325 network=tcp +34:34.204 [INFO] TestAgent: started state syncer +34:34.204 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24372 network=tcp +34:34.257 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:34.257 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24385 [Candidate]" term=2 +34:34.257 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:34.257 [DEBUG] TestAgent.server.raft: vote granted: from=4783a1e0-fa28-ca73-71f7-71a8822e489e term=2 tally=1 +34:34.257 [INFO] TestAgent.server.raft: election won: tally=1 +34:34.257 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24385 [Leader]" +34:34.257 [INFO] TestAgent.server: cluster leadership acquired +34:34.257 [INFO] TestAgent.server: New leader elected: payload=Node-4783a1e0-fa28-ca73-71f7-71a8822e489e +34:34.257 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24385 +34:34.259 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:34.262 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:34.262 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:34.262 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:34.262 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:34.262 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:34.262 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4783a1e0-fa28-ca73-71f7-71a8822e489e +34:34.262 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4783a1e0-fa28-ca73-71f7-71a8822e489e +34:34.263 [INFO] TestAgent.server: federation state anti-entropy synced +34:34.294 [INFO] TestAgent: Requesting shutdown +34:34.294 [INFO] TestAgent.server: shutting down server +34:34.294 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:34.294 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:34.294 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:34.294 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:34.294 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:34.294 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:34.294 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:34.294 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:34.294 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:34.294 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:34.294 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:34.294 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:34.295Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:34.295Z [INFO] agent.router.manager: shutting down +34:34.295 [INFO] TestAgent: consul server down +34:34.295 [INFO] TestAgent: shutdown complete +34:34.295 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24380 network=tcp +34:34.295 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24380 network=udp +34:34.295 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24325 network=tcp +34:34.795 [INFO] TestAgent: Waiting for endpoints to shut down +34:34.795 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Services (0.66s) === CONT TestACL_filterMembers -2022-09-15T21:43:06.053Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:06.053Z [WARN] agent: bootstrap = true: do not enable unless necessary -43:06.053 [DEBUG] TestACL_filterMembers: dropping node from result due to ACLs: node=Nope accessorID=9df2d1a4-2d07-414e-8ead-6053f56ed2eb ---- PASS: TestACL_filterMembers (0.02s) +2021-08-13T15:34:34.810Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:34.810Z [WARN] agent: bootstrap = true: do not enable unless necessary +34:34.810 [DEBUG] TestACL_filterMembers: dropping node from result due to ACLs: node=Nope accessorID=9df2d1a4-2d07-414e-8ead-6053f56ed2eb +--- PASS: TestACL_filterMembers (0.01s) === CONT TestACL_vetCheckUpdate -2022-09-15T21:43:06.078Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:06.078Z [WARN] agent: bootstrap = true: do not enable unless necessary ---- PASS: TestACL_vetCheckUpdate (0.03s) +2021-08-13T15:34:34.868Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:34.868Z [WARN] agent: bootstrap = true: do not enable unless necessary +--- PASS: TestACL_vetCheckUpdate (0.06s) === CONT TestACL_vetCheckRegister -2022-09-15T21:43:06.103Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:06.103Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:34.892Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:34.892Z [WARN] agent: bootstrap = true: do not enable unless necessary --- PASS: TestACL_vetCheckRegister (0.02s) +=== CONT TestParseWait_InvalidIndex +--- PASS: TestParseWait_InvalidIndex (0.00s) === CONT TestACL_vetServiceUpdate -2022-09-15T21:43:06.131Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:06.131Z [WARN] agent: bootstrap = true: do not enable unless necessary ---- PASS: TestACL_vetServiceUpdate (0.03s) +2021-08-13T15:34:34.907Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:34.907Z [WARN] agent: bootstrap = true: do not enable unless necessary +--- PASS: TestACL_vetServiceUpdate (0.01s) +=== CONT TestParseWait_InvalidTime +--- PASS: TestParseWait_InvalidTime (0.00s) === CONT TestACL_vetServiceRegister -2022-09-15T21:43:06.149Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:06.149Z [WARN] agent: bootstrap = true: do not enable unless necessary ---- PASS: TestACL_vetServiceRegister (0.02s) +2021-08-13T15:34:34.922Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:34.922Z [WARN] agent: bootstrap = true: do not enable unless necessary +--- PASS: TestACL_vetServiceRegister (0.01s) +=== CONT TestPProfHandlers_ACLs +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:34.102Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:34.118Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:34.120 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5067d8ea-0d3d-a479-f9f8-eb692c6038ff Address:127.0.0.1:24364}]" +34:34.120 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24364 [Follower]" leader= +34:34.121 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5067d8ea-0d3d-a479-f9f8-eb692c6038ff.dc1 127.0.0.1 +34:34.122 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5067d8ea-0d3d-a479-f9f8-eb692c6038ff 127.0.0.1 +2021-08-13T15:34:34.122Z [INFO] agent.router: Initializing LAN area manager +34:34.122 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5067d8ea-0d3d-a479-f9f8-eb692c6038ff.dc1 area=wan +34:34.122 [INFO] TestAgent.server: Adding LAN server: server="Node-5067d8ea-0d3d-a479-f9f8-eb692c6038ff (Addr: tcp/127.0.0.1:24364) (DC: dc1)" +34:34.122 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24331 network=tcp +34:34.122 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24331 network=udp +34:34.123 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24381 network=tcp +34:34.123 [INFO] TestAgent: started state syncer +34:34.123 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24379 network=tcp +34:34.163 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:34.163 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24364 [Candidate]" term=2 +34:34.165 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:34.165 [DEBUG] TestAgent.server.raft: vote granted: from=5067d8ea-0d3d-a479-f9f8-eb692c6038ff term=2 tally=1 +34:34.165 [INFO] TestAgent.server.raft: election won: tally=1 +34:34.165 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24364 [Leader]" +34:34.166 [INFO] TestAgent.server: cluster leadership acquired +34:34.166 [INFO] TestAgent.server: New leader elected: payload=Node-5067d8ea-0d3d-a479-f9f8-eb692c6038ff +34:34.195 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24364 +34:34.201 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:34.204 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:34.204 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:34.204 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:34.204 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:34.204 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:34.204 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5067d8ea-0d3d-a479-f9f8-eb692c6038ff +34:34.204 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5067d8ea-0d3d-a479-f9f8-eb692c6038ff +34:34.205 [INFO] TestAgent.server: federation state anti-entropy synced +34:34.213 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:34.214 [INFO] TestAgent: Synced node info +34:34.214 [DEBUG] TestAgent: Node info in sync +34:34.416 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:34.416 [DEBUG] TestAgent: Node info in sync +34:34.560 [INFO] TestAgent: Requesting shutdown +34:34.560 [INFO] TestAgent.server: shutting down server +34:34.560 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:34.560 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:34.561 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:34.561Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:34.561Z [INFO] agent.router.manager: shutting down +34:34.561 [INFO] TestAgent: consul server down +34:34.561 [INFO] TestAgent: shutdown complete +34:34.561 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24331 network=tcp +34:34.561 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24331 network=udp +34:34.561 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24381 network=tcp +34:35.061 [INFO] TestAgent: Waiting for endpoints to shut down +34:35.061 [INFO] TestAgent: Endpoints down +=== RUN TestRPC_HTTPSMaxConnsPerClient/HTTPS +2021-08-13T15:34:34.468Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:34.485Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:34.487 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4050641e-c05e-876a-241e-b1166e56461a Address:127.0.0.1:24392}]" +34:34.487 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24392 [Follower]" leader= +34:34.488 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4050641e-c05e-876a-241e-b1166e56461a.dc1 127.0.0.1 +34:34.489 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4050641e-c05e-876a-241e-b1166e56461a 127.0.0.1 +2021-08-13T15:34:34.489Z [INFO] agent.router: Initializing LAN area manager +34:34.489 [INFO] TestAgent.server: Adding LAN server: server="Node-4050641e-c05e-876a-241e-b1166e56461a (Addr: tcp/127.0.0.1:24392) (DC: dc1)" +34:34.489 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4050641e-c05e-876a-241e-b1166e56461a.dc1 area=wan +34:34.489 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24387 network=tcp +34:34.490 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24387 network=udp +34:34.491 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24388 network=tcp +34:34.491 [INFO] TestAgent: started state syncer +34:34.491 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24393 network=tcp +34:34.547 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:34.547 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24392 [Candidate]" term=2 +34:34.548 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:34.548 [DEBUG] TestAgent.server.raft: vote granted: from=4050641e-c05e-876a-241e-b1166e56461a term=2 tally=1 +34:34.548 [INFO] TestAgent.server.raft: election won: tally=1 +34:34.548 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24392 [Leader]" +34:34.548 [INFO] TestAgent.server: cluster leadership acquired +34:34.548 [INFO] TestAgent.server: New leader elected: payload=Node-4050641e-c05e-876a-241e-b1166e56461a +34:34.548 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24392 +34:34.549 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:34.552 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:34.552 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:34.552 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:34.552 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:34.552 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:34.552 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4050641e-c05e-876a-241e-b1166e56461a +34:34.552 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4050641e-c05e-876a-241e-b1166e56461a +34:34.553 [INFO] TestAgent.server: federation state anti-entropy synced +34:34.617 [INFO] TestAgent: Requesting shutdown +34:34.618 [INFO] TestAgent.server: shutting down server +34:34.618 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:34.618 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:34.618 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:34.618 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:34.618 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:34.618 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:34.618 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:34.618 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:34.618 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:34.618 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:34.618 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:34.618 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:34.618Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:34.618Z [INFO] agent.router.manager: shutting down +34:34.618 [INFO] TestAgent: consul server down +34:34.618 [INFO] TestAgent: shutdown complete +34:34.618 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24387 network=tcp +34:34.618 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24387 network=udp +34:34.619 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24388 network=tcp +34:35.119 [INFO] TestAgent: Waiting for endpoints to shut down +34:35.119 [INFO] TestAgent: Endpoints down +--- PASS: TestEnableWebUI (0.67s) === CONT TestACL_RootAuthorizersDenied -2022-09-15T21:43:06.175Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:06.175Z [WARN] agent: bootstrap = true: do not enable unless necessary ---- PASS: TestACL_RootAuthorizersDenied (0.02s) +2021-08-13T15:34:35.134Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:35.134Z [WARN] agent: bootstrap = true: do not enable unless necessary +--- PASS: TestACL_RootAuthorizersDenied (0.01s) +=== CONT TestPProfHandlers_DisableDebugNoACLs +2021-08-13T15:34:34.408Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:34.433Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:34.436 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fdad4989-5907-91c8-5cf7-85d6022f8909 Address:127.0.0.1:24378}]" +34:34.436 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24378 [Follower]" leader= +34:34.437 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fdad4989-5907-91c8-5cf7-85d6022f8909.dc1 127.0.0.1 +34:34.438 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fdad4989-5907-91c8-5cf7-85d6022f8909 127.0.0.1 +2021-08-13T15:34:34.438Z [INFO] agent.router: Initializing LAN area manager +34:34.438 [INFO] TestAgent.server: Adding LAN server: server="Node-fdad4989-5907-91c8-5cf7-85d6022f8909 (Addr: tcp/127.0.0.1:24378) (DC: dc1)" +34:34.438 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fdad4989-5907-91c8-5cf7-85d6022f8909.dc1 area=wan +34:34.438 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24373 network=udp +34:34.439 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24373 network=tcp +34:34.439 [INFO] TestAgent: Started HTTPS server: address=127.0.0.1:24375 network=tcp +34:34.440 [INFO] TestAgent: started state syncer +34:34.440 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24386 network=tcp +34:34.475 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:34.475 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24378 [Candidate]" term=2 +34:34.475 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:34.475 [DEBUG] TestAgent.server.raft: vote granted: from=fdad4989-5907-91c8-5cf7-85d6022f8909 term=2 tally=1 +34:34.475 [INFO] TestAgent.server.raft: election won: tally=1 +34:34.475 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24378 [Leader]" +34:34.475 [INFO] TestAgent.server: cluster leadership acquired +34:34.476 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24378 +34:34.477 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:34.478 [INFO] TestAgent.server: New leader elected: payload=Node-fdad4989-5907-91c8-5cf7-85d6022f8909 +34:34.481 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:34.481 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:34.481 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:34.481 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:34.481 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:34.481 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fdad4989-5907-91c8-5cf7-85d6022f8909 +34:34.481 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fdad4989-5907-91c8-5cf7-85d6022f8909 +34:34.482 [INFO] TestAgent.server: federation state anti-entropy synced +34:34.686 [INFO] TestAgent: Requesting shutdown +34:34.686 [INFO] TestAgent.server: shutting down server +34:34.686 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:34.687 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:34.687 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:34.687 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:34.687 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:34.687 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:34.687 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:34.687 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:34.687 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:34.687 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:34.687 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:34.687 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:34.687Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:34.687Z [INFO] agent.router.manager: shutting down +34:34.687 [INFO] TestAgent: consul server down +34:34.687 [INFO] TestAgent: shutdown complete +34:34.687 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24373 network=tcp +34:34.687 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24373 network=udp +34:34.687 [INFO] TestAgent: Stopping server: protocol=HTTPS address=127.0.0.1:24375 network=tcp +34:35.188 [INFO] TestAgent: Waiting for endpoints to shut down +34:35.188 [INFO] TestAgent: Endpoints down +--- PASS: TestHTTPServer_HandshakeTimeout (0.80s) === CONT TestACL_AgentMasterToken -2022-09-15T21:43:06.191Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:06.191Z [WARN] agent: bootstrap = true: do not enable unless necessary ---- PASS: TestACL_AgentMasterToken (0.02s) +2021-08-13T15:34:35.203Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:35.203Z [WARN] agent: bootstrap = true: do not enable unless necessary +--- PASS: TestACL_AgentMasterToken (0.01s) +=== CONT TestPProfHandlers_EnableDebug +2021/08/13 15:34:35 http: TLS handshake error from 127.0.0.1:57664: read tcp 127.0.0.1:25227->127.0.0.1:57664: use of closed network connection +2021/08/13 15:34:35 http: TLS handshake error from 127.0.0.1:57666: EOF +2021/08/13 15:34:35 http: TLS handshake error from 127.0.0.1:57660: EOF +2021/08/13 15:34:35 http: TLS handshake error from 127.0.0.1:57662: EOF +=== RUN TestPProfHandlers_ACLs/case_0_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:200,_token:"master",_endpoint:"/debug/pprof/heap",_nilResponse:false}) +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:34.522Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:34.537Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:34.540 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:003cef11-06e0-efd0-65da-9230da5cfdd2 Address:127.0.0.1:24399}]" +34:34.540 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24399 [Follower]" leader= +34:34.541 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-003cef11-06e0-efd0-65da-9230da5cfdd2.dc1 127.0.0.1 +34:34.541 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-003cef11-06e0-efd0-65da-9230da5cfdd2 127.0.0.1 +2021-08-13T15:34:34.541Z [INFO] agent.router: Initializing LAN area manager +34:34.541 [INFO] TestAgent.server: Adding LAN server: server="Node-003cef11-06e0-efd0-65da-9230da5cfdd2 (Addr: tcp/127.0.0.1:24399) (DC: dc1)" +34:34.542 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-003cef11-06e0-efd0-65da-9230da5cfdd2.dc1 area=wan +34:34.542 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24394 network=udp +34:34.542 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24394 network=tcp +34:34.543 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24395 network=tcp +34:34.543 [INFO] TestAgent: started state syncer +34:34.543 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24400 network=tcp +34:34.596 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:34.596 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24399 [Candidate]" term=2 +34:34.596 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:34.596 [DEBUG] TestAgent.server.raft: vote granted: from=003cef11-06e0-efd0-65da-9230da5cfdd2 term=2 tally=1 +34:34.596 [INFO] TestAgent.server.raft: election won: tally=1 +34:34.596 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24399 [Leader]" +34:34.596 [INFO] TestAgent.server: cluster leadership acquired +34:34.596 [INFO] TestAgent.server: New leader elected: payload=Node-003cef11-06e0-efd0-65da-9230da5cfdd2 +34:34.596 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24399 +34:34.598 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:34.600 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:34.600 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:34.600 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:34.600 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:34.600 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:34.600 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-003cef11-06e0-efd0-65da-9230da5cfdd2 +34:34.600 [INFO] TestAgent.server: member joined, marking health alive: member=Node-003cef11-06e0-efd0-65da-9230da5cfdd2 +34:34.602 [INFO] TestAgent.server: federation state anti-entropy synced +34:34.804 [INFO] TestAgent: Requesting shutdown +34:34.804 [INFO] TestAgent.server: shutting down server +34:34.804 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:34.804 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:34.804 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:34.804 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:34.804 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:34.804 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:34.804 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:34.805 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:34.805 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:34.805 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:34.805 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:34.805 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:34.805Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:34.805Z [INFO] agent.router.manager: shutting down +34:34.805 [INFO] TestAgent: consul server down +34:34.805 [INFO] TestAgent: shutdown complete +34:34.805 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24394 network=tcp +34:34.805 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24394 network=udp +34:34.805 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24395 network=tcp +34:35.311 [INFO] TestAgent: Waiting for endpoints to shut down +34:35.311 [INFO] TestAgent: Endpoints down +--- PASS: TestACLResolution (0.81s) === CONT TestACL_Version8EnabledByDefault -2022-09-15T21:43:06.208Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:06.208Z [WARN] agent: bootstrap = true: do not enable unless necessary ---- PASS: TestACL_Version8EnabledByDefault (0.02s) +2021-08-13T15:34:34.685Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:34.702Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:34.705 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a7110db6-733b-8852-01b4-e0c824e89bf9 Address:127.0.0.1:24427}]" +34:34.705 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a7110db6-733b-8852-01b4-e0c824e89bf9.dc1 127.0.0.1 +34:34.706 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a7110db6-733b-8852-01b4-e0c824e89bf9 127.0.0.1 +2021-08-13T15:34:34.706Z [INFO] agent.router: Initializing LAN area manager +34:34.707 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24401 network=udp +34:34.707 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24427 [Follower]" leader= +34:34.707 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a7110db6-733b-8852-01b4-e0c824e89bf9.dc1 area=wan +34:34.707 [INFO] TestAgent.server: Adding LAN server: server="Node-a7110db6-733b-8852-01b4-e0c824e89bf9 (Addr: tcp/127.0.0.1:24427) (DC: dc1)" +34:34.707 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24401 network=tcp +34:34.708 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24409 network=tcp +34:34.708 [INFO] TestAgent: started state syncer +34:34.708 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24414 network=tcp +34:34.751 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:34.751 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24427 [Candidate]" term=2 +34:34.751 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:34.751 [DEBUG] TestAgent.server.raft: vote granted: from=a7110db6-733b-8852-01b4-e0c824e89bf9 term=2 tally=1 +34:34.751 [INFO] TestAgent.server.raft: election won: tally=1 +34:34.751 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24427 [Leader]" +34:34.751 [INFO] TestAgent.server: cluster leadership acquired +34:34.752 [INFO] TestAgent.server: New leader elected: payload=Node-a7110db6-733b-8852-01b4-e0c824e89bf9 +34:34.752 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24427 +34:34.753 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:34.756 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:34.756 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:34.756 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:34.756 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:34.756 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:34.756 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a7110db6-733b-8852-01b4-e0c824e89bf9 +34:34.756 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a7110db6-733b-8852-01b4-e0c824e89bf9 +34:34.757 [INFO] TestAgent.server: federation state anti-entropy synced +34:34.835 [INFO] TestAgent: Requesting shutdown +34:34.835 [INFO] TestAgent.server: shutting down server +34:34.835 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:34.835 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:34.835 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:34.835 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:34.835 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:34.835 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:34.835 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:34.836 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:34.836 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:34.836 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:34.836 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:34.836 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:34.836Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:34.836Z [INFO] agent.router.manager: shutting down +34:34.836 [INFO] TestAgent: consul server down +34:34.836 [INFO] TestAgent: shutdown complete +34:34.836 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24401 network=tcp +34:34.836 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24401 network=udp +34:34.836 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24409 network=tcp +34:35.341 [INFO] TestAgent: Waiting for endpoints to shut down +34:35.341 [INFO] TestAgent: Endpoints down +--- PASS: TestParseConsistency_Invalid (0.67s) +=== CONT TestParseWait +--- PASS: TestParseWait (0.00s) === CONT TestACL_Authorize -[WARN] freeport: 4 out of 4 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestACLEndpoint_LoginLogout_jwt/success_-_jwt_jwks -=== RUN TestACLEndpoint_LoginLogout_jwt/success_-_jwt_jwks/invalid_bearer_token -=== RUN TestACLEndpoint_LoginLogout_jwt/success_-_jwt_jwks/valid_bearer_token_no_bindings -=== RUN TestACLEndpoint_LoginLogout_jwt/success_-_jwt_jwks/valid_bearer_token_1_service_binding -=== RUN TestACLEndpoint_LoginLogout_jwt/success_-_jwt_oidc_discovery -=== RUN TestACLEndpoint_LoginLogout_jwt/success_-_jwt_oidc_discovery/invalid_bearer_token -2022-09-15T21:43:05.442Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:05.472Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:05.478 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c83833b5-8905-5d7e-cf39-15be4325f20a Address:127.0.0.1:13606}]" -43:05.479 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c83833b5-8905-5d7e-cf39-15be4325f20a.dc1 127.0.0.1 -43:05.480 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c83833b5-8905-5d7e-cf39-15be4325f20a 127.0.0.1 -2022-09-15T21:43:05.480Z [INFO] agent.router: Initializing LAN area manager -43:05.480 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13587 network=udp -43:05.481 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13606 [Follower]" leader= -43:05.481 [INFO] TestAgent.server: Adding LAN server: server="Node-c83833b5-8905-5d7e-cf39-15be4325f20a (Addr: tcp/127.0.0.1:13606) (DC: dc1)" -43:05.481 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c83833b5-8905-5d7e-cf39-15be4325f20a.dc1 area=wan -43:05.481 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13587 network=tcp -43:05.483 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13588 network=tcp -43:05.483 [INFO] TestAgent: started state syncer -43:05.483 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13607 network=tcp -43:05.524 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:05.524 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13606 [Candidate]" term=2 -43:05.525 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:05.525 [DEBUG] TestAgent.server.raft: vote granted: from=c83833b5-8905-5d7e-cf39-15be4325f20a term=2 tally=1 -43:05.525 [INFO] TestAgent.server.raft: election won: tally=1 -43:05.525 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13606 [Leader]" -43:05.525 [INFO] TestAgent.server: cluster leadership acquired -43:05.525 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13606 -43:05.525 [INFO] TestAgent.server: New leader elected: payload=Node-c83833b5-8905-5d7e-cf39-15be4325f20a -43:05.527 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:05.529 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:05.529 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:05.529 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:05.529 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:05.529 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:05.529 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c83833b5-8905-5d7e-cf39-15be4325f20a -43:05.529 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c83833b5-8905-5d7e-cf39-15be4325f20a -43:05.531 [INFO] TestAgent.server: federation state anti-entropy synced -43:05.862 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:05.871 [INFO] TestAgent: Synced node info -43:05.871 [DEBUG] TestAgent: Node info in sync -43:05.894 [INFO] TestAgent: Requesting shutdown -43:05.894 [INFO] TestAgent.server: shutting down server -43:05.895 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:05.895 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:05.895 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:05.895 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:05.895 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:05.895 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:05.895 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:05.895 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:05.895 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:05.895 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:05.896 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:05.896Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:05.896Z [INFO] agent.router.manager: shutting down -43:05.896 [INFO] TestAgent: consul server down -43:05.896 [INFO] TestAgent: shutdown complete -43:05.896 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13587 network=tcp -43:05.897 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13587 network=udp -43:05.897 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13588 network=tcp -43:06.398 [INFO] TestAgent: Waiting for endpoints to shut down -43:06.398 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Services_Sidecar (1.01s) -=== CONT TestKVSEndpoint_DELETE_CAS -=== RUN TestACLEndpoint_LoginLogout_jwt/success_-_jwt_oidc_discovery/valid_bearer_token_no_bindings -=== RUN TestACLEndpoint_LoginLogout_jwt/success_-_jwt_oidc_discovery/valid_bearer_token_1_service_binding -=== RUN TestACLEndpoint_LoginLogout_jwt/success_-_jwt_static_keys -=== RUN TestACLEndpoint_LoginLogout_jwt/success_-_jwt_static_keys/invalid_bearer_token -=== RUN TestACLEndpoint_LoginLogout_jwt/success_-_jwt_static_keys/valid_bearer_token_no_bindings -=== RUN TestACLEndpoint_LoginLogout_jwt/success_-_jwt_static_keys/valid_bearer_token_1_service_binding -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:05.696Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:05.717Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:05.720 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3acbad5a-9222-4592-e982-03ede62e4190 Address:127.0.0.1:13662}]" -43:05.721 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13662 [Follower]" leader= -43:05.722 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3acbad5a-9222-4592-e982-03ede62e4190.dc1 127.0.0.1 -43:05.723 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3acbad5a-9222-4592-e982-03ede62e4190 127.0.0.1 -2022-09-15T21:43:05.723Z [INFO] agent.router: Initializing LAN area manager -43:05.723 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13608 network=udp -43:05.724 [INFO] TestAgent.server: Adding LAN server: server="Node-3acbad5a-9222-4592-e982-03ede62e4190 (Addr: tcp/127.0.0.1:13662) (DC: dc1)" -43:05.724 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3acbad5a-9222-4592-e982-03ede62e4190.dc1 area=wan -43:05.724 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13608 network=tcp -43:05.725 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13609 network=tcp -43:05.725 [INFO] TestAgent: started state syncer -43:05.725 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13663 network=tcp -43:05.781 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:05.781 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13662 [Candidate]" term=2 -43:05.781 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:05.781 [DEBUG] TestAgent.server.raft: vote granted: from=3acbad5a-9222-4592-e982-03ede62e4190 term=2 tally=1 -43:05.781 [INFO] TestAgent.server.raft: election won: tally=1 -43:05.781 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13662 [Leader]" -43:05.782 [INFO] TestAgent.server: cluster leadership acquired -43:05.782 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13662 -43:05.783 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:05.785 [INFO] TestAgent.server: New leader elected: payload=Node-3acbad5a-9222-4592-e982-03ede62e4190 -43:05.790 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:05.790 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:05.790 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:05.790 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:05.790 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:05.790 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3acbad5a-9222-4592-e982-03ede62e4190 -43:05.790 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3acbad5a-9222-4592-e982-03ede62e4190 -43:05.792 [INFO] TestAgent.server: federation state anti-entropy synced -43:05.926 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:05.927 [INFO] TestAgent: Synced node info -43:06.038 [INFO] TestAgent: Requesting shutdown -43:06.038 [INFO] TestAgent.server: shutting down server -43:06.038 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:06.038 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:06.038 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:06.038 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:06.038 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:06.039 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:06.039 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:06.039 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:06.039 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:06.039 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:06.039 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:06.040Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:06.040Z [INFO] agent.router.manager: shutting down -43:06.040 [INFO] TestAgent: consul server down -43:06.040 [INFO] TestAgent: shutdown complete -43:06.040 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13608 network=tcp -43:06.040 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13608 network=udp -43:06.040 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13609 network=tcp -43:06.542 [INFO] TestAgent: Waiting for endpoints to shut down -43:06.542 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Services_ExternalConnectProxy (0.87s) -=== CONT TestOperator_KeyringList -2022-09-15T21:43:05.973Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:06.018Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:06.022 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8f953f35-e2de-8381-aaf2-7cd3914aff4a Address:127.0.0.1:13683}]" -43:06.023 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8f953f35-e2de-8381-aaf2-7cd3914aff4a.dc1 127.0.0.1 -43:06.024 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8f953f35-e2de-8381-aaf2-7cd3914aff4a 127.0.0.1 -2022-09-15T21:43:06.024Z [INFO] agent.router: Initializing LAN area manager -43:06.025 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13683 [Follower]" leader= -43:06.026 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13650 network=tcp -43:06.026 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13650 network=udp -43:06.026 [INFO] TestAgent.server: Adding LAN server: server="Node-8f953f35-e2de-8381-aaf2-7cd3914aff4a (Addr: tcp/127.0.0.1:13683) (DC: dc1)" -43:06.026 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8f953f35-e2de-8381-aaf2-7cd3914aff4a.dc1 area=wan -43:06.027 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13651 network=tcp -43:06.027 [INFO] TestAgent: started state syncer -43:06.027 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13684 network=tcp -43:06.090 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:06.090 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13683 [Candidate]" term=2 -43:06.090 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:06.090 [DEBUG] TestAgent.server.raft: vote granted: from=8f953f35-e2de-8381-aaf2-7cd3914aff4a term=2 tally=1 -43:06.090 [INFO] TestAgent.server.raft: election won: tally=1 -43:06.090 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13683 [Leader]" -43:06.090 [INFO] TestAgent.server: cluster leadership acquired -43:06.090 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13683 -43:06.092 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:06.094 [INFO] TestAgent.server: New leader elected: payload=Node-8f953f35-e2de-8381-aaf2-7cd3914aff4a -43:06.095 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:06.095 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:06.095 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:06.095 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:06.095 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:06.095 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8f953f35-e2de-8381-aaf2-7cd3914aff4a -43:06.095 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8f953f35-e2de-8381-aaf2-7cd3914aff4a -43:06.097 [INFO] TestAgent.server: federation state anti-entropy synced -43:06.138 [INFO] TestAgent: Requesting shutdown -43:06.138 [INFO] TestAgent.server: shutting down server -43:06.138 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:06.138 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:06.138 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:06.138 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:06.138 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:06.139 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:06.139 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:06.139 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:06.139 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:06.139 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:06.139 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:06.139 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:06.139Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:06.139Z [INFO] agent.router.manager: shutting down -43:06.139 [INFO] TestAgent: consul server down -43:06.139 [INFO] TestAgent: shutdown complete -43:06.139 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13650 network=tcp -43:06.139 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13650 network=udp -43:06.139 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13651 network=tcp -43:06.640 [INFO] TestAgent: Waiting for endpoints to shut down -43:06.640 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Services (0.73s) -=== CONT TestOperator_KeyringInstall -2022-09-15T21:43:05.768Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:05.787Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:05.791 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5958a359-1c83-7ed5-d98d-8a2dbd5824d7 Address:127.0.0.1:13648}]" -43:05.792 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13648 [Follower]" leader= -43:05.792 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5958a359-1c83-7ed5-d98d-8a2dbd5824d7.dc1 127.0.0.1 -43:05.793 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5958a359-1c83-7ed5-d98d-8a2dbd5824d7 127.0.0.1 -2022-09-15T21:43:05.793Z [INFO] agent.router: Initializing LAN area manager -43:05.794 [INFO] TestAgent.server: Adding LAN server: server="Node-5958a359-1c83-7ed5-d98d-8a2dbd5824d7 (Addr: tcp/127.0.0.1:13648) (DC: dc1)" -43:05.794 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5958a359-1c83-7ed5-d98d-8a2dbd5824d7.dc1 area=wan -43:05.795 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13664 network=tcp -43:05.795 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13664 network=udp -43:05.796 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13665 network=tcp -43:05.796 [INFO] TestAgent: started state syncer -43:05.796 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13649 network=tcp -43:05.845 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:05.845 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13648 [Candidate]" term=2 -43:05.846 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:05.846 [DEBUG] TestAgent.server.raft: vote granted: from=5958a359-1c83-7ed5-d98d-8a2dbd5824d7 term=2 tally=1 -43:05.846 [INFO] TestAgent.server.raft: election won: tally=1 -43:05.846 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13648 [Leader]" -43:05.846 [INFO] TestAgent.server: cluster leadership acquired -43:05.846 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13648 -43:05.847 [INFO] TestAgent.server: New leader elected: payload=Node-5958a359-1c83-7ed5-d98d-8a2dbd5824d7 -43:05.848 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:05.873 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:05.873 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:05.873 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:05.873 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:05.873 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:05.873 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5958a359-1c83-7ed5-d98d-8a2dbd5824d7 -43:05.874 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5958a359-1c83-7ed5-d98d-8a2dbd5824d7 -43:05.875 [INFO] TestAgent.server: federation state anti-entropy synced -43:05.919 [INFO] TestAgent: Synced node info -43:05.919 [DEBUG] TestAgent: Node info in sync -43:06.235 [INFO] TestAgent: Requesting shutdown -43:06.235 [INFO] TestAgent.server: shutting down server -43:06.235 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:06.235 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:06.235 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:06.235 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:06.235 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:06.236 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:06.236 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:06.236 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:06.236 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:06.236 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:06.236 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:06.236Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:06.236Z [INFO] agent.router.manager: shutting down -43:06.236 [INFO] TestAgent: consul server down -43:06.236 [INFO] TestAgent: shutdown complete -43:06.236 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13664 network=tcp -43:06.236 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13664 network=udp -43:06.236 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13665 network=tcp -43:06.737 [INFO] TestAgent: Waiting for endpoints to shut down -43:06.737 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_ServicesFiltered (1.00s) -=== CONT TestOperator_RaftPeer -=== RUN TestOperator_RaftPeer/#00 -[WARN] freeport: 4 out of 25 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:35.371Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:35.371Z [WARN] agent: bootstrap = true: do not enable unless necessary +--- PASS: TestACL_Version8EnabledByDefault (0.06s) +=== CONT TestParseSource +=== RUN TestPProfHandlers_ACLs/case_1_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:403,_token:"agent",_endpoint:"/debug/pprof/heap",_nilResponse:true}) +=== RUN TestPProfHandlers_ACLs/case_2_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:403,_token:"agent",_endpoint:"/debug/pprof/",_nilResponse:true}) +=== RUN TestPProfHandlers_ACLs/case_3_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:403,_token:"",_endpoint:"/debug/pprof/",_nilResponse:true}) +=== RUN TestPProfHandlers_ACLs/case_4_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:200,_token:"master",_endpoint:"/debug/pprof/heap",_nilResponse:false}) +=== RUN TestPProfHandlers_ACLs/case_5_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:403,_token:"towel",_endpoint:"/debug/pprof/heap",_nilResponse:true}) +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:35.080Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:35.098Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:35.102 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:dcf98e7e-62f4-63e6-7891-9b93f9e5ed21 Address:127.0.0.1:25230}]" +34:35.102 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25230 [Follower]" leader= +34:35.103 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-dcf98e7e-62f4-63e6-7891-9b93f9e5ed21.dc1 127.0.0.1 +34:35.104 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-dcf98e7e-62f4-63e6-7891-9b93f9e5ed21 127.0.0.1 +2021-08-13T15:34:35.104Z [INFO] agent.router: Initializing LAN area manager +34:35.104 [INFO] TestAgent.server: Adding LAN server: server="Node-dcf98e7e-62f4-63e6-7891-9b93f9e5ed21 (Addr: tcp/127.0.0.1:25230) (DC: dc1)" +34:35.104 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-dcf98e7e-62f4-63e6-7891-9b93f9e5ed21.dc1 area=wan +34:35.104 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25260 network=udp +34:35.104 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25260 network=tcp +34:35.105 [INFO] TestAgent: Started HTTPS server: address=127.0.0.1:25227 network=tcp +34:35.105 [INFO] TestAgent: started state syncer +34:35.106 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25231 network=tcp +34:35.170 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:35.170 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25230 [Candidate]" term=2 +34:35.171 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:35.171 [DEBUG] TestAgent.server.raft: vote granted: from=dcf98e7e-62f4-63e6-7891-9b93f9e5ed21 term=2 tally=1 +34:35.171 [INFO] TestAgent.server.raft: election won: tally=1 +34:35.171 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25230 [Leader]" +34:35.171 [INFO] TestAgent.server: cluster leadership acquired +34:35.171 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25230 +34:35.172 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:35.173 [INFO] TestAgent.server: New leader elected: payload=Node-dcf98e7e-62f4-63e6-7891-9b93f9e5ed21 +34:35.175 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:35.175 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:35.175 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:35.175 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:35.175 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:35.175 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-dcf98e7e-62f4-63e6-7891-9b93f9e5ed21 +34:35.175 [INFO] TestAgent.server: member joined, marking health alive: member=Node-dcf98e7e-62f4-63e6-7891-9b93f9e5ed21 +34:35.176 [INFO] TestAgent.server: federation state anti-entropy synced +34:35.207 [INFO] TestAgent: Requesting shutdown +34:35.207 [INFO] TestAgent.server: shutting down server +34:35.207 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:35.207 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:35.207 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +34:35.207 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +2021-08-13T15:34:35.208Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:35.208Z [INFO] agent.router.manager: shutting down +34:35.208 [INFO] TestAgent: consul server down +34:35.208 [INFO] TestAgent: shutdown complete +34:35.208 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25260 network=tcp +34:35.208 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25260 network=udp +34:35.208 [INFO] TestAgent: Stopping server: protocol=HTTPS address=127.0.0.1:25227 network=tcp +34:35.711 [INFO] TestAgent: Waiting for endpoints to shut down +34:35.711 [INFO] TestAgent: Endpoints down +--- PASS: TestRPC_HTTPSMaxConnsPerClient (1.63s) + --- PASS: TestRPC_HTTPSMaxConnsPerClient/HTTP (0.98s) + --- PASS: TestRPC_HTTPSMaxConnsPerClient/HTTPS (0.65s) +=== CONT TestACLEndpoint_LoginLogout_jwt +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:35.153Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:35.170Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:35.173 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3d8eb80a-e411-0952-8c25-5577c58a9eea Address:127.0.0.1:24406}]" +34:35.174 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3d8eb80a-e411-0952-8c25-5577c58a9eea.dc1 127.0.0.1 +34:35.175 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3d8eb80a-e411-0952-8c25-5577c58a9eea 127.0.0.1 +2021-08-13T15:34:35.175Z [INFO] agent.router: Initializing LAN area manager +34:35.176 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25232 network=udp +34:35.176 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24406 [Follower]" leader= +34:35.176 [INFO] TestAgent.server: Adding LAN server: server="Node-3d8eb80a-e411-0952-8c25-5577c58a9eea (Addr: tcp/127.0.0.1:24406) (DC: dc1)" +34:35.176 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3d8eb80a-e411-0952-8c25-5577c58a9eea.dc1 area=wan +34:35.176 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25232 network=tcp +34:35.177 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25233 network=tcp +34:35.177 [INFO] TestAgent: started state syncer +34:35.178 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24407 network=tcp +34:35.234 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:35.234 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24406 [Candidate]" term=2 +34:35.234 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:35.234 [DEBUG] TestAgent.server.raft: vote granted: from=3d8eb80a-e411-0952-8c25-5577c58a9eea term=2 tally=1 +34:35.234 [INFO] TestAgent.server.raft: election won: tally=1 +34:35.234 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24406 [Leader]" +34:35.235 [INFO] TestAgent.server: cluster leadership acquired +34:35.235 [INFO] TestAgent.server: New leader elected: payload=Node-3d8eb80a-e411-0952-8c25-5577c58a9eea +34:35.235 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24406 +34:35.236 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:35.239 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:35.239 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:35.239 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:35.239 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:35.239 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:35.239 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3d8eb80a-e411-0952-8c25-5577c58a9eea +34:35.239 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3d8eb80a-e411-0952-8c25-5577c58a9eea +34:35.240 [INFO] TestAgent.server: federation state anti-entropy synced +34:35.307 [INFO] TestAgent: Requesting shutdown +34:35.307 [INFO] TestAgent.server: shutting down server +34:35.315 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:35.315 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:35.315 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:35.315 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:35.315 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:35.315 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:35.315 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:35.316 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:35.316 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:35.316 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:35.316 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:35.316 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:35.317Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:35.317Z [INFO] agent.router.manager: shutting down +34:35.317 [INFO] TestAgent: consul server down +34:35.317 [INFO] TestAgent: shutdown complete +34:35.317 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25232 network=tcp +34:35.317 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25232 network=udp +34:35.317 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25233 network=tcp +34:35.818 [INFO] TestAgent: Waiting for endpoints to shut down +34:35.818 [INFO] TestAgent: Endpoints down +--- PASS: TestPProfHandlers_DisableDebugNoACLs (0.68s) +=== CONT TestPrettyPrintBare +2021-08-13T15:34:34.940Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:34.940Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:34.956Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:34.956Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:34.959 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6b445de2-54e8-9506-9077-5e7890727b8b Address:127.0.0.1:25258}]" +34:34.959 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25258 [Follower]" leader= +34:34.960 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6b445de2-54e8-9506-9077-5e7890727b8b.dc1 127.0.0.1 +34:34.961 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6b445de2-54e8-9506-9077-5e7890727b8b 127.0.0.1 +2021-08-13T15:34:34.961Z [INFO] agent.router: Initializing LAN area manager +34:34.961 [INFO] TestAgent.server: Adding LAN server: server="Node-6b445de2-54e8-9506-9077-5e7890727b8b (Addr: tcp/127.0.0.1:25258) (DC: dc1)" +34:34.961 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6b445de2-54e8-9506-9077-5e7890727b8b.dc1 area=wan +34:34.961 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24415 network=udp +34:34.961 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24415 network=tcp +34:34.962 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24430 network=tcp +34:34.962 [INFO] TestAgent: started state syncer +34:34.963 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25259 network=tcp +34:35.011 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:35.019 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:35.019 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25258 [Candidate]" term=2 +34:35.020 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:35.020 [DEBUG] TestAgent.server.raft: vote granted: from=6b445de2-54e8-9506-9077-5e7890727b8b term=2 tally=1 +34:35.020 [INFO] TestAgent.server.raft: election won: tally=1 +34:35.020 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25258 [Leader]" +34:35.020 [INFO] TestAgent.server: cluster leadership acquired +34:35.020 [INFO] TestAgent.server: initializing acls +34:35.021 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:35.021 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:35.021 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:35.022 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:35.022 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:35.022 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:35.022 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-6b445de2-54e8-9506-9077-5e7890727b8b +34:35.022 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-6b445de2-54e8-9506-9077-5e7890727b8b.dc1 +34:35.023 [INFO] TestAgent.server: New leader elected: payload=Node-6b445de2-54e8-9506-9077-5e7890727b8b +34:35.023 [INFO] TestAgent.server: Updating LAN server: server="Node-6b445de2-54e8-9506-9077-5e7890727b8b (Addr: tcp/127.0.0.1:25258) (DC: dc1)" +34:35.023 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-6b445de2-54e8-9506-9077-5e7890727b8b.dc1 area=wan +34:35.024 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:35.026 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:35.026 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:35.026 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:35.026 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:35.026 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:35.026 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6b445de2-54e8-9506-9077-5e7890727b8b +34:35.026 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6b445de2-54e8-9506-9077-5e7890727b8b +34:35.028 [INFO] TestAgent.server: federation state anti-entropy synced +34:35.028 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6b445de2-54e8-9506-9077-5e7890727b8b +34:35.156 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="ACL not found" +34:35.267 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-6b445de2-54e8-9506-9077-5e7890727b8b +34:35.267 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-6b445de2-54e8-9506-9077-5e7890727b8b +34:35.422 [INFO] TestAgent: Requesting shutdown +34:35.423 [INFO] TestAgent.server: shutting down server +34:35.423 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:35.423 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:35.423 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:35.423 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:35.423 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:35.423 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:35.423 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:35.423 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:35.423 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:35.423 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:35.423 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:35.423 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:35.423 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:35.423 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +34:35.423 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +2021-08-13T15:34:35.423Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:35.423Z [INFO] agent.router.manager: shutting down +34:35.423 [INFO] TestAgent: consul server down +34:35.423 [INFO] TestAgent: shutdown complete +34:35.423 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24415 network=tcp +34:35.424 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24415 network=udp +34:35.424 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24430 network=tcp +34:35.924 [INFO] TestAgent: Waiting for endpoints to shut down +34:35.924 [INFO] TestAgent: Endpoints down +--- PASS: TestPProfHandlers_ACLs (1.00s) + --- PASS: TestPProfHandlers_ACLs/case_0_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:200,_token:"master",_endpoint:"/debug/pprof/heap",_nilResponse:false}) (0.12s) + --- PASS: TestPProfHandlers_ACLs/case_1_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:403,_token:"agent",_endpoint:"/debug/pprof/heap",_nilResponse:true}) (0.00s) + --- PASS: TestPProfHandlers_ACLs/case_2_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:403,_token:"agent",_endpoint:"/debug/pprof/",_nilResponse:true}) (0.00s) + --- PASS: TestPProfHandlers_ACLs/case_3_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:403,_token:"",_endpoint:"/debug/pprof/",_nilResponse:true}) (0.00s) + --- PASS: TestPProfHandlers_ACLs/case_4_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:200,_token:"master",_endpoint:"/debug/pprof/heap",_nilResponse:false}) (0.04s) + --- PASS: TestPProfHandlers_ACLs/case_5_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:403,_token:"towel",_endpoint:"/debug/pprof/heap",_nilResponse:true}) (0.00s) +=== CONT TestACL_LoginProcedure_HTTP === RUN TestACL_Authorize/master-token === RUN TestACL_Authorize/master-token/dc1 === RUN TestACL_Authorize/master-token/dc2 +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! === RUN TestACL_Authorize/custom-token === RUN TestACL_Authorize/custom-token/dc1 === RUN TestACL_Authorize/custom-token/dc2 @@ -46275,873 +41931,109 @@ === RUN TestACL_Authorize/acl-not-found === RUN TestACL_Authorize/local-token-in-secondary-dc === RUN TestACL_Authorize/local-token-wrong-dc -[WARN] freeport: 4 out of 4 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:06.023Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:06.060Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:06.068 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:75f71e67-0bfc-7745-e294-86fe3cc8da1c Address:127.0.0.1:13655}]" -43:06.069 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-75f71e67-0bfc-7745-e294-86fe3cc8da1c.dc1 127.0.0.1 -43:06.070 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13655 [Follower]" leader= -43:06.074 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-75f71e67-0bfc-7745-e294-86fe3cc8da1c 127.0.0.1 -2022-09-15T21:43:06.074Z [INFO] agent.router: Initializing LAN area manager -43:06.075 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13685 network=udp -43:06.075 [INFO] TestAgent.server: Adding LAN server: server="Node-75f71e67-0bfc-7745-e294-86fe3cc8da1c (Addr: tcp/127.0.0.1:13655) (DC: dc1)" -43:06.075 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-75f71e67-0bfc-7745-e294-86fe3cc8da1c.dc1 area=wan -43:06.075 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13685 network=tcp -43:06.076 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13686 network=tcp -43:06.076 [INFO] TestAgent: started state syncer -43:06.077 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13656 network=tcp -43:06.106 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:06.106 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13655 [Candidate]" term=2 -43:06.106 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:06.106 [DEBUG] TestAgent.server.raft: vote granted: from=75f71e67-0bfc-7745-e294-86fe3cc8da1c term=2 tally=1 -43:06.106 [INFO] TestAgent.server.raft: election won: tally=1 -43:06.106 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13655 [Leader]" -43:06.106 [INFO] TestAgent.server: cluster leadership acquired -43:06.106 [INFO] TestAgent.server: initializing acls -43:06.107 [INFO] TestAgent.server: Created ACL 'global-management' policy -43:06.107 [INFO] TestAgent.server: New leader elected: payload=Node-75f71e67-0bfc-7745-e294-86fe3cc8da1c -43:06.107 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="ACL not found" -43:06.107 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -43:06.108 [INFO] TestAgent.server: Created ACL anonymous token from configuration -43:06.108 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -43:06.108 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -43:06.108 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-75f71e67-0bfc-7745-e294-86fe3cc8da1c -43:06.108 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-75f71e67-0bfc-7745-e294-86fe3cc8da1c.dc1 -43:06.109 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:06.110 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-75f71e67-0bfc-7745-e294-86fe3cc8da1c.dc1 area=wan -43:06.110 [INFO] TestAgent.server: Updating LAN server: server="Node-75f71e67-0bfc-7745-e294-86fe3cc8da1c (Addr: tcp/127.0.0.1:13655) (DC: dc1)" -43:06.113 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:06.113 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:06.113 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:06.113 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:06.113 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:06.113 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-75f71e67-0bfc-7745-e294-86fe3cc8da1c -43:06.113 [INFO] TestAgent.server: member joined, marking health alive: member=Node-75f71e67-0bfc-7745-e294-86fe3cc8da1c -43:06.118 [INFO] TestAgent.server: federation state anti-entropy synced -43:06.119 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-75f71e67-0bfc-7745-e294-86fe3cc8da1c -43:06.302 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-75f71e67-0bfc-7745-e294-86fe3cc8da1c -43:06.302 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-75f71e67-0bfc-7745-e294-86fe3cc8da1c -43:06.528 [INFO] TestAgent: Requesting shutdown -43:06.528 [INFO] TestAgent.server: shutting down server -43:06.528 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -43:06.528 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:06.528 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:06.529 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:06.529 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:06.529 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -43:06.529 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:06.529 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:06.529 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:06.529 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -43:06.529 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:06.529 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:06.529 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:06.529 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -43:06.529 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:06.529Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:06.529Z [INFO] agent.router.manager: shutting down -43:06.529 [INFO] TestAgent: consul server down -43:06.529 [INFO] TestAgent: shutdown complete -43:06.529 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13685 network=tcp -43:06.529 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13685 network=udp -43:06.529 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13686 network=tcp -43:07.030 [INFO] TestAgent: Waiting for endpoints to shut down -43:07.030 [INFO] TestAgent: Endpoints down ---- PASS: TestACLEndpoint_LoginLogout_jwt (1.04s) - --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_jwks (0.06s) - --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_jwks/invalid_bearer_token (0.01s) - --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_jwks/valid_bearer_token_no_bindings (0.02s) - --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_jwks/valid_bearer_token_1_service_binding (0.01s) - --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_oidc_discovery (0.10s) - --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_oidc_discovery/invalid_bearer_token (0.02s) - --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_oidc_discovery/valid_bearer_token_no_bindings (0.02s) - --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_oidc_discovery/valid_bearer_token_1_service_binding (0.01s) - --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_static_keys (0.06s) - --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_static_keys/invalid_bearer_token (0.00s) - --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_static_keys/valid_bearer_token_no_bindings (0.00s) - --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_static_keys/valid_bearer_token_1_service_binding (0.03s) -=== CONT TestOperator_RaftConfiguration -2022-09-15T21:43:06.528Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:06.566Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:06.569 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4a657d93-1e50-6620-79e3-d626e1941e3c Address:127.0.0.1:13698}]" -43:06.570 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13698 [Follower]" leader= -43:06.570 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4a657d93-1e50-6620-79e3-d626e1941e3c.dc1 127.0.0.1 -43:06.571 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4a657d93-1e50-6620-79e3-d626e1941e3c 127.0.0.1 -2022-09-15T21:43:06.571Z [INFO] agent.router: Initializing LAN area manager -43:06.572 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13672 network=udp -43:06.572 [INFO] TestAgent.server: Adding LAN server: server="Node-4a657d93-1e50-6620-79e3-d626e1941e3c (Addr: tcp/127.0.0.1:13698) (DC: dc1)" -43:06.572 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4a657d93-1e50-6620-79e3-d626e1941e3c.dc1 area=wan -43:06.572 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13672 network=tcp -43:06.573 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13652 network=tcp -43:06.573 [INFO] TestAgent: started state syncer -43:06.573 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13699 network=tcp -43:06.623 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:06.623 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13698 [Candidate]" term=2 -43:06.623 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:06.623 [DEBUG] TestAgent.server.raft: vote granted: from=4a657d93-1e50-6620-79e3-d626e1941e3c term=2 tally=1 -43:06.623 [INFO] TestAgent.server.raft: election won: tally=1 -43:06.623 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13698 [Leader]" -43:06.623 [INFO] TestAgent.server: cluster leadership acquired -43:06.624 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13698 -43:06.625 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:06.626 [INFO] TestAgent.server: New leader elected: payload=Node-4a657d93-1e50-6620-79e3-d626e1941e3c -43:06.629 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:06.629 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:06.629 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:06.629 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:06.629 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:06.629 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4a657d93-1e50-6620-79e3-d626e1941e3c -43:06.629 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4a657d93-1e50-6620-79e3-d626e1941e3c -43:06.634 [INFO] TestAgent.server: federation state anti-entropy synced -43:06.700 [INFO] TestAgent: Requesting shutdown -43:06.700 [INFO] TestAgent.server: shutting down server -43:06.700 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:06.700 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:06.700 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:06.700 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:06.700 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:06.700 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:06.700 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:06.700 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:06.700 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:06.700 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:06.700 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:06.701 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:06.701Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:06.701Z [INFO] agent.router.manager: shutting down -43:06.701 [INFO] TestAgent: consul server down -43:06.701 [INFO] TestAgent: shutdown complete -43:06.701 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13672 network=tcp -43:06.701 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13672 network=udp -43:06.701 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13652 network=tcp -43:07.202 [INFO] TestAgent: Waiting for endpoints to shut down -43:07.202 [INFO] TestAgent: Endpoints down ---- PASS: TestKVSEndpoint_DELETE_CAS (0.80s) -=== CONT TestKVSEndpoint_DELETE_ConflictingFlags -[WARN] freeport: 4 out of 19 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:06.759Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:06.783Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:06.793 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c02446c3-b957-a83f-0c27-680fda627978 Address:127.0.0.1:13719}]" -43:06.794 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13719 [Follower]" leader= -43:06.795 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c02446c3-b957-a83f-0c27-680fda627978.dc1 127.0.0.1 -43:06.796 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c02446c3-b957-a83f-0c27-680fda627978 127.0.0.1 -2022-09-15T21:43:06.796Z [INFO] agent.router: Initializing LAN area manager -43:06.796 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13693 network=udp -43:06.797 [INFO] TestAgent.server: Adding LAN server: server="Node-c02446c3-b957-a83f-0c27-680fda627978 (Addr: tcp/127.0.0.1:13719) (DC: dc1)" -43:06.797 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c02446c3-b957-a83f-0c27-680fda627978.dc1 area=wan -43:06.797 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13693 network=tcp -43:06.806 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13715 network=tcp -43:06.806 [INFO] TestAgent: started state syncer -43:06.807 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13720 network=tcp -43:06.859 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:06.859 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13719 [Candidate]" term=2 -43:06.859 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:06.859 [DEBUG] TestAgent.server.raft: vote granted: from=c02446c3-b957-a83f-0c27-680fda627978 term=2 tally=1 -43:06.859 [INFO] TestAgent.server.raft: election won: tally=1 -43:06.859 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13719 [Leader]" -43:06.859 [INFO] TestAgent.server: cluster leadership acquired -43:06.860 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13719 -43:06.861 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:06.862 [INFO] TestAgent.server: New leader elected: payload=Node-c02446c3-b957-a83f-0c27-680fda627978 -43:06.864 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:06.864 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:06.864 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:06.864 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:06.864 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:06.864 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c02446c3-b957-a83f-0c27-680fda627978 -43:06.864 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c02446c3-b957-a83f-0c27-680fda627978 -43:06.889 [INFO] TestAgent.server: federation state anti-entropy synced -43:06.903 [INFO] TestAgent: Requesting shutdown -43:06.903 [INFO] TestAgent.server: shutting down server -43:06.903 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:06.903 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:06.903 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:06.903 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:06.903 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:06.903 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:06.903 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:06.903 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:06.903 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:06.903 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:06.903 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:06.903 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:06.903Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:06.903Z [INFO] agent.router.manager: shutting down -43:06.904 [INFO] TestAgent: consul server down -43:06.904 [INFO] TestAgent: shutdown complete -43:06.904 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13693 network=tcp -43:06.904 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13693 network=udp -43:06.904 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13715 network=tcp -43:07.406 [INFO] TestAgent: Waiting for endpoints to shut down -43:07.406 [INFO] TestAgent: Endpoints down -2022-09-15T21:43:06.560Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:06.583Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:06.591 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a6bc6310-41b8-5cbb-803e-065f7a67ab07 Address:127.0.0.1:13740}]" -43:06.592 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a6bc6310-41b8-5cbb-803e-065f7a67ab07.dc1 127.0.0.1 -43:06.593 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a6bc6310-41b8-5cbb-803e-065f7a67ab07 127.0.0.1 -2022-09-15T21:43:06.593Z [INFO] agent.router: Initializing LAN area manager -43:06.593 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13700 network=udp -43:06.593 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13740 [Follower]" leader= -43:06.594 [INFO] TestAgent.server: Adding LAN server: server="Node-a6bc6310-41b8-5cbb-803e-065f7a67ab07 (Addr: tcp/127.0.0.1:13740) (DC: dc1)" -43:06.594 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a6bc6310-41b8-5cbb-803e-065f7a67ab07.dc1 area=wan -43:06.594 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13700 network=tcp -43:06.595 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13687 network=tcp -43:06.595 [INFO] TestAgent: started state syncer -43:06.596 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13741 network=tcp -43:06.649 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:06.649 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13740 [Candidate]" term=2 -43:06.649 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:06.649 [DEBUG] TestAgent.server.raft: vote granted: from=a6bc6310-41b8-5cbb-803e-065f7a67ab07 term=2 tally=1 -43:06.649 [INFO] TestAgent.server.raft: election won: tally=1 -43:06.649 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13740 [Leader]" -43:06.649 [INFO] TestAgent.server: cluster leadership acquired -43:06.649 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13740 -43:06.650 [INFO] TestAgent.server: New leader elected: payload=Node-a6bc6310-41b8-5cbb-803e-065f7a67ab07 -43:06.651 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:06.654 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:06.654 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:06.654 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:06.654 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:06.654 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:06.654 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a6bc6310-41b8-5cbb-803e-065f7a67ab07 -43:06.654 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a6bc6310-41b8-5cbb-803e-065f7a67ab07 -43:06.656 [INFO] TestAgent.server: federation state anti-entropy synced -43:06.720 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:06.721 [INFO] TestAgent: Synced node info -43:06.902 [INFO] TestAgent.server.serf.wan: serf: Received list-keys query -43:06.903 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-a6bc6310-41b8-5cbb-803e-065f7a67ab07.dc1 -43:06.903 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query -43:06.903 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-a6bc6310-41b8-5cbb-803e-065f7a67ab07 -43:06.903 [INFO] TestAgent: Requesting shutdown -43:06.903 [INFO] TestAgent.server: shutting down server -43:06.903 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:06.903 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:06.903 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:06.903 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:06.903 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:06.903 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:06.904 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:06.904 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:06.904 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:06.904 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:06.904 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:06.904Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:06.904Z [INFO] agent.router.manager: shutting down -43:06.904 [INFO] TestAgent: consul server down -43:06.904 [INFO] TestAgent: shutdown complete -43:06.904 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13700 network=tcp -43:06.904 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13700 network=udp -43:06.904 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13687 network=tcp -43:07.406 [INFO] TestAgent: Waiting for endpoints to shut down -43:07.406 [INFO] TestAgent: Endpoints down ---- PASS: TestOperator_KeyringList (0.86s) -=== CONT TestKVSEndpoint_PUT_ConflictingFlags -=== RUN TestOperator_RaftPeer/#01 -2022-09-15T21:43:06.661Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:06.680Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:06.684 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d3a7bcfb-760b-c37a-7aa5-472e6d4a7684 Address:127.0.0.1:13691}]" -43:06.685 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13691 [Follower]" leader= -43:06.685 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d3a7bcfb-760b-c37a-7aa5-472e6d4a7684.dc1 127.0.0.1 -43:06.686 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d3a7bcfb-760b-c37a-7aa5-472e6d4a7684 127.0.0.1 -2022-09-15T21:43:06.686Z [INFO] agent.router: Initializing LAN area manager -43:06.687 [INFO] TestAgent.server: Adding LAN server: server="Node-d3a7bcfb-760b-c37a-7aa5-472e6d4a7684 (Addr: tcp/127.0.0.1:13691) (DC: dc1)" -43:06.687 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d3a7bcfb-760b-c37a-7aa5-472e6d4a7684.dc1 area=wan -43:06.687 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13742 network=udp -43:06.687 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13742 network=tcp -43:06.688 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13729 network=tcp -43:06.688 [INFO] TestAgent: started state syncer -43:06.688 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13692 network=tcp -43:06.721 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:06.721 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13691 [Candidate]" term=2 -43:06.721 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:06.721 [DEBUG] TestAgent.server.raft: vote granted: from=d3a7bcfb-760b-c37a-7aa5-472e6d4a7684 term=2 tally=1 -43:06.721 [INFO] TestAgent.server.raft: election won: tally=1 -43:06.721 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13691 [Leader]" -43:06.721 [INFO] TestAgent.server: cluster leadership acquired -43:06.721 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13691 -43:06.722 [INFO] TestAgent.server: New leader elected: payload=Node-d3a7bcfb-760b-c37a-7aa5-472e6d4a7684 -43:06.723 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:06.725 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:06.725 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:06.725 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:06.725 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:06.725 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:06.725 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d3a7bcfb-760b-c37a-7aa5-472e6d4a7684 -43:06.725 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d3a7bcfb-760b-c37a-7aa5-472e6d4a7684 -43:06.727 [INFO] TestAgent.server: federation state anti-entropy synced -43:06.817 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:06.818 [INFO] TestAgent: Synced node info -43:06.978 [INFO] TestAgent.server.serf.wan: serf: Received install-key query -43:06.986 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-d3a7bcfb-760b-c37a-7aa5-472e6d4a7684.dc1 -43:06.986 [INFO] TestAgent.server.serf.lan: serf: Received install-key query -43:06.987 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-d3a7bcfb-760b-c37a-7aa5-472e6d4a7684 -43:06.987 [INFO] TestAgent.server.serf.wan: serf: Received list-keys query -43:06.987 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-d3a7bcfb-760b-c37a-7aa5-472e6d4a7684.dc1 -43:06.988 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query -43:06.988 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-d3a7bcfb-760b-c37a-7aa5-472e6d4a7684 -43:06.988 [INFO] TestAgent: Requesting shutdown -43:06.988 [INFO] TestAgent.server: shutting down server -43:06.988 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:06.988 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:06.988 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:06.988 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:06.988 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:06.988 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:06.988 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:06.988 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:06.988 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:06.988 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:06.988 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:06.989Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:06.989Z [INFO] agent.router.manager: shutting down -43:06.989 [INFO] TestAgent: consul server down -43:06.989 [INFO] TestAgent: shutdown complete -43:06.989 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13742 network=tcp -43:06.989 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13742 network=udp -43:06.989 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13729 network=tcp -43:07.489 [INFO] TestAgent: Waiting for endpoints to shut down -43:07.489 [INFO] TestAgent: Endpoints down ---- PASS: TestOperator_KeyringInstall (0.85s) -=== CONT TestKVSEndpoint_GET_Raw -[WARN] freeport: 4 out of 25 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 4 out of 4 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:06.507Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:06.538Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:06.541 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fd6c94df-3a48-5121-015a-4841cb658a61 Address:127.0.0.1:13670}]" -43:06.542 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13670 [Follower]" leader= -43:06.543 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fd6c94df-3a48-5121-015a-4841cb658a61.dc2 127.0.0.1 -43:06.544 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fd6c94df-3a48-5121-015a-4841cb658a61 127.0.0.1 -2022-09-15T21:43:06.544Z [INFO] agent.router: Initializing LAN area manager -43:06.544 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13707 network=udp -43:06.545 [INFO] TestAgent.server: Adding LAN server: server="Node-fd6c94df-3a48-5121-015a-4841cb658a61 (Addr: tcp/127.0.0.1:13670) (DC: dc2)" -43:06.545 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fd6c94df-3a48-5121-015a-4841cb658a61.dc2 area=wan -43:06.545 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13707 network=tcp -43:06.546 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13722 network=tcp -43:06.547 [INFO] TestAgent: started state syncer -43:06.547 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13671 network=tcp -43:06.586 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:06.586 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13670 [Candidate]" term=2 -43:06.586 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:06.586 [DEBUG] TestAgent.server.raft: vote granted: from=fd6c94df-3a48-5121-015a-4841cb658a61 term=2 tally=1 -43:06.586 [INFO] TestAgent.server.raft: election won: tally=1 -43:06.586 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13670 [Leader]" -43:06.586 [INFO] TestAgent.server: cluster leadership acquired -43:06.587 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs, servers in acl datacenter are not yet upgraded: ACLDatacenter=dc1 mode=1 found=false -43:06.587 [INFO] TestAgent.leader: started routine: routine="legacy ACL replication" -43:06.587 [INFO] TestAgent.server: started legacy ACL replication -43:06.587 [WARN] TestAgent.server.connect: primary datacenter is configured but unreachable - deferring initialization of the secondary datacenter CA -43:06.587 [INFO] TestAgent.leader: started routine: routine="config entry replication" -43:06.587 [INFO] TestAgent.leader: started routine: routine="federation state replication" -43:06.587 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:06.587 [INFO] TestAgent.leader: started routine: routine="secondary CA roots watch" -43:06.587 [INFO] TestAgent.leader: started routine: routine="intention replication" -43:06.587 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:06.587 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:06.588 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fd6c94df-3a48-5121-015a-4841cb658a61 -43:06.588 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fd6c94df-3a48-5121-015a-4841cb658a61 -43:06.589 [INFO] TestAgent.server: New leader elected: payload=Node-fd6c94df-3a48-5121-015a-4841cb658a61 -43:06.589 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ACL.List -43:06.589 [WARN] TestAgent.server.replication.acl.legacy: Legacy ACL replication error (will retry if still leader): error="failed to retrieve remote ACLs: No path to datacenter" -43:06.589 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ACL.List -43:06.589 [WARN] TestAgent.server.replication.acl.legacy: Legacy ACL replication error (will retry if still leader): error="failed to retrieve remote ACLs: No path to datacenter" -43:06.589 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ACL.List -43:06.589 [WARN] TestAgent.server.replication.acl.legacy: Legacy ACL replication error (will retry if still leader): error="failed to retrieve remote ACLs: No path to datacenter" -43:06.589 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ACL.List -43:06.589 [WARN] TestAgent.server.replication.acl.legacy: Legacy ACL replication error (will retry if still leader): error="failed to retrieve remote ACLs: No path to datacenter" -43:06.589 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ACL.List -43:06.589 [WARN] TestAgent.server.replication.acl.legacy: Legacy ACL replication error (will retry if still leader): error="failed to retrieve remote ACLs: No path to datacenter" -43:06.589 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ConfigEntry.ListAll -43:06.589 [DEBUG] TestAgent.server.connect: starting Connect intention replication from primary datacenter: primary=dc1 -43:06.589 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=Intention.List -43:06.589 [ERROR] TestAgent.server.connect: error replicating intentions: routine="intention replication" error="No path to datacenter" -43:06.589 [DEBUG] TestAgent.server: federation states are not enabled in the primary dc -43:06.589 [DEBUG] TestAgent.server.connect: starting Connect CA root replication from primary datacenter: primary=dc1 -43:06.589 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ConnectCA.Roots -43:06.589 [ERROR] TestAgent.server.connect: CA root replication failed, will retry: routine="secondary CA roots watch" error="Error retrieving the primary datacenter's roots: No path to datacenter" -43:06.589 [DEBUG] TestAgent.server: federation states are not enabled in the primary dc -43:06.589 [DEBUG] TestAgent.server: federation states are not enabled in the primary dc -43:06.589 [DEBUG] TestAgent.server: federation states are not enabled in the primary dc -43:06.589 [DEBUG] TestAgent.server: federation states are not enabled in the primary dc -43:06.589 [DEBUG] TestAgent.server: federation states are not enabled in the primary dc -43:06.595 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs, servers in acl datacenter are not yet upgraded: ACLDatacenter=dc1 mode=1 found=false -43:06.695 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs, servers in acl datacenter are not yet upgraded: ACLDatacenter=dc1 mode=1 found=false -43:06.878 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ACL.GetPolicy -43:06.878 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-fd6c94df-3a48-5121-015a-4841cb658a61 -43:06.878 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:13668] -43:06.879 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ACL.GetPolicy -43:06.879 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth -43:06.879 [WARN] TestAgent: Node info update blocked by ACLs: node=fd6c94df-3a48-5121-015a-4841cb658a61 accessorID=legacy-token -43:06.879 [DEBUG] TestAgent: Node info in sync -43:06.879 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:13668 -43:06.881 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d18dd042-4b23-10de-f6c7-acabeed577b2.dc1 127.0.0.1 -43:06.882 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d18dd042-4b23-10de-f6c7-acabeed577b2.dc1 area=wan -43:06.888 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 -43:06.889 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-fd6c94df-3a48-5121-015a-4841cb658a61 -43:06.904 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL replication" -43:06.904 [INFO] TestAgent.leader: started routine: routine="ACL policy replication" -43:06.904 [INFO] TestAgent.leader: started routine: routine="ACL role replication" -43:06.904 [INFO] TestAgent.leader: started routine: routine="ACL token replication" -43:06.904 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -43:06.904 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode -43:06.905 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-fd6c94df-3a48-5121-015a-4841cb658a61 -43:06.905 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-fd6c94df-3a48-5121-015a-4841cb658a61.dc2 -43:06.905 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-fd6c94df-3a48-5121-015a-4841cb658a61.dc2 area=wan -43:06.905 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL replication" -43:06.905 [INFO] TestAgent.server.replication.acl.policy: started ACL Policy replication -43:06.905 [INFO] TestAgent.server.replication.acl.role: started ACL Role replication -43:06.908 [INFO] TestAgent.server: Updating LAN server: server="Node-fd6c94df-3a48-5121-015a-4841cb658a61 (Addr: tcp/127.0.0.1:13670) (DC: dc2)" -43:06.908 [INFO] TestAgent.server.replication.acl.token: started ACL Token replication -43:06.908 [DEBUG] TestAgent.server.replication.acl.policy: finished fetching acls: amount=2 -43:06.908 [DEBUG] TestAgent.server.replication.acl.policy: acl replication: local=0 remote=2 -43:06.908 [DEBUG] TestAgent.server.replication.acl.policy: acl replication: deletions=0 updates=2 -43:06.908 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fd6c94df-3a48-5121-015a-4841cb658a61 -43:06.914 [DEBUG] TestAgent.server.replication.acl.policy: acl replication - downloaded updates: amount=2 -43:06.914 [DEBUG] TestAgent.server.replication.acl.policy: acl replication - performing updates -43:06.915 [DEBUG] TestAgent.server.replication.acl.policy: acl replication - upserted batch: number_upserted=2 batch_size=675 -43:06.915 [DEBUG] TestAgent.server.replication.acl.policy: acl replication - finished updates -43:06.915 [DEBUG] TestAgent.server.replication.acl.policy: ACL replication completed through remote index: index=16 -43:06.917 [DEBUG] TestAgent.server.replication.acl.role: finished fetching acls: amount=0 -43:06.917 [DEBUG] TestAgent.server.replication.acl.role: acl replication: local=0 remote=0 -43:06.917 [DEBUG] TestAgent.server.replication.acl.role: acl replication: deletions=0 updates=0 -43:06.917 [DEBUG] TestAgent.server.replication.acl.role: ACL replication completed through remote index: index=1 -43:06.922 [DEBUG] TestAgent.server.replication.acl.token: finished fetching acls: amount=3 -43:06.922 [DEBUG] TestAgent.server.replication.acl.token: acl replication: local=0 remote=3 -43:06.922 [DEBUG] TestAgent.server.replication.acl.token: acl replication: deletions=0 updates=3 -43:06.931 [DEBUG] TestAgent.server.replication.acl.token: acl replication - downloaded updates: amount=3 -43:06.931 [DEBUG] TestAgent.server.replication.acl.token: acl replication - performing updates -43:06.932 [DEBUG] TestAgent.server.replication.acl.token: acl replication - upserted batch: number_upserted=3 batch_size=442 -43:06.932 [DEBUG] TestAgent.server.replication.acl.token: acl replication - finished updates -43:06.932 [DEBUG] TestAgent.server.replication.acl.token: ACL replication completed through remote index: index=17 -43:06.958 [INFO] TestAgent: Requesting shutdown -43:06.958 [INFO] TestAgent.server: shutting down server -43:06.958 [DEBUG] TestAgent.leader: stopping routine: routine="intention replication" -43:06.958 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:06.958 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -43:06.958 [DEBUG] TestAgent.leader: stopping routine: routine="ACL policy replication" -43:06.958 [DEBUG] TestAgent.leader: stopping routine: routine="config entry replication" -43:06.958 [DEBUG] TestAgent.leader: stopping routine: routine="federation state replication" -43:06.958 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:06.958 [DEBUG] TestAgent.leader: stopping routine: routine="secondary CA roots watch" -43:06.958 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:06.958 [DEBUG] TestAgent.leader: stopping routine: routine="ACL role replication" -43:06.958 [DEBUG] TestAgent.leader: stopping routine: routine="ACL token replication" -43:06.958 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:06.959 [DEBUG] TestAgent.leader: stopped routine: routine="intention replication" -43:06.959 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:06.959 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -43:06.959 [INFO] TestAgent.server.replication.config_entry: stopped replication -43:06.959 [DEBUG] TestAgent.leader: stopped routine: routine="config entry replication" -43:06.959 [INFO] TestAgent.replication.federation_state: stopped replication -43:06.959 [DEBUG] TestAgent.leader: stopped routine: routine="federation state replication" -43:06.959 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:06.959 [DEBUG] TestAgent.leader: stopped routine: routine="secondary CA roots watch" -43:06.959 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:06.959 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:06.959Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:06.959Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:06.959Z [INFO] agent.router.manager: shutting down -43:06.959 [INFO] TestAgent: consul server down -43:06.959 [INFO] TestAgent: shutdown complete -43:06.959 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13707 network=tcp -43:06.959 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13707 network=udp -43:06.959 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13722 network=tcp -43:06.960 [ERROR] TestAgent.server.rpc: RPC failed to server in DC: server=127.0.0.1:13704 datacenter=dc1 method=ACL.TokenList error="rpc error making call: EOF" -43:06.960 [WARN] TestAgent.server.replication.acl.token: ACL replication error (will retry if still leader): error="failed to retrieve remote ACL tokens: rpc error making call: EOF" -43:06.960 [DEBUG] TestAgent.leader: stopped routine: routine="ACL token replication" -43:06.960 [ERROR] TestAgent.server.rpc: RPC failed to server in DC: server=127.0.0.1:13704 datacenter=dc1 method=ACL.RoleList error="rpc error making call: EOF" -43:06.960 [WARN] TestAgent.server.replication.acl.role: ACL replication error (will retry if still leader): error="failed to retrieve remote ACL roles: rpc error making call: EOF" -43:06.960 [DEBUG] TestAgent.leader: stopped routine: routine="ACL role replication" -43:06.960 [ERROR] TestAgent.server.rpc: RPC failed to server in DC: server=127.0.0.1:13704 datacenter=dc1 method=ACL.PolicyList error="rpc error making call: EOF" -43:06.960 [WARN] TestAgent.server.replication.acl.policy: ACL replication error (will retry if still leader): error="failed to retrieve remote ACL policies: rpc error making call: EOF" -43:06.960 [DEBUG] TestAgent.leader: stopped routine: routine="ACL policy replication" -43:07.466 [INFO] TestAgent: Waiting for endpoints to shut down -43:07.466 [INFO] TestAgent: Endpoints down -2022-09-15T21:43:06.228Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:06.251Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:06.254 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d18dd042-4b23-10de-f6c7-acabeed577b2 Address:127.0.0.1:13704}]" -43:06.255 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d18dd042-4b23-10de-f6c7-acabeed577b2.dc1 127.0.0.1 -43:06.256 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d18dd042-4b23-10de-f6c7-acabeed577b2 127.0.0.1 -2022-09-15T21:43:06.256Z [INFO] agent.router: Initializing LAN area manager -43:06.256 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13657 network=udp -43:06.256 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13704 [Follower]" leader= -43:06.257 [INFO] TestAgent.server: Adding LAN server: server="Node-d18dd042-4b23-10de-f6c7-acabeed577b2 (Addr: tcp/127.0.0.1:13704) (DC: dc1)" -43:06.257 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d18dd042-4b23-10de-f6c7-acabeed577b2.dc1 area=wan -43:06.257 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13657 network=tcp -43:06.258 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13658 network=tcp -43:06.258 [INFO] TestAgent: started state syncer -43:06.258 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13705 network=tcp -43:06.298 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:06.298 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13704 [Candidate]" term=2 -43:06.298 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:06.298 [DEBUG] TestAgent.server.raft: vote granted: from=d18dd042-4b23-10de-f6c7-acabeed577b2 term=2 tally=1 -43:06.298 [INFO] TestAgent.server.raft: election won: tally=1 -43:06.298 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13704 [Leader]" -43:06.298 [INFO] TestAgent.server: cluster leadership acquired -43:06.299 [INFO] TestAgent.server: initializing acls -43:06.299 [INFO] TestAgent.server: Created ACL 'global-management' policy -43:06.300 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -43:06.300 [INFO] TestAgent.server: Created ACL anonymous token from configuration -43:06.300 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -43:06.300 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -43:06.301 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-d18dd042-4b23-10de-f6c7-acabeed577b2 -43:06.301 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-d18dd042-4b23-10de-f6c7-acabeed577b2.dc1 -43:06.302 [INFO] TestAgent.server: New leader elected: payload=Node-d18dd042-4b23-10de-f6c7-acabeed577b2 -43:06.302 [INFO] TestAgent.server: Updating LAN server: server="Node-d18dd042-4b23-10de-f6c7-acabeed577b2 (Addr: tcp/127.0.0.1:13704) (DC: dc1)" -43:06.302 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-d18dd042-4b23-10de-f6c7-acabeed577b2.dc1 area=wan -43:06.314 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:06.317 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:06.317 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:06.317 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:06.317 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:06.317 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:06.317 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d18dd042-4b23-10de-f6c7-acabeed577b2 -43:06.317 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d18dd042-4b23-10de-f6c7-acabeed577b2 -43:06.327 [INFO] TestAgent: Synced node info -43:06.327 [DEBUG] TestAgent: Node info in sync -43:06.327 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d18dd042-4b23-10de-f6c7-acabeed577b2 -43:06.327 [INFO] TestAgent.server: federation state anti-entropy synced -43:06.370 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-d18dd042-4b23-10de-f6c7-acabeed577b2 -43:06.880 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:32788 -43:06.881 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fd6c94df-3a48-5121-015a-4841cb658a61.dc2 127.0.0.1 -43:06.881 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fd6c94df-3a48-5121-015a-4841cb658a61.dc2 area=wan -43:07.466 [INFO] TestAgent: Requesting shutdown -43:07.466 [INFO] TestAgent.server: shutting down server -43:07.466 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:07.466 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:07.466 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:07.466 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:07.466 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -43:07.466 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -43:07.466 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:07.466 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:07.466 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:07.466 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:07.466 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:07.466 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -43:07.466 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -43:07.466 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:07.466 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -2022-09-15T21:43:07.466Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:07.466Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:07.466Z [INFO] agent.router.manager: shutting down -43:07.467 [INFO] TestAgent: consul server down -43:07.467 [INFO] TestAgent: shutdown complete -43:07.467 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13657 network=tcp -43:07.467 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13657 network=udp -43:07.467 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13658 network=tcp -43:07.970 [INFO] TestAgent: Waiting for endpoints to shut down -43:07.970 [INFO] TestAgent: Endpoints down ---- PASS: TestACL_Authorize (1.76s) - --- PASS: TestACL_Authorize/master-token (0.00s) - --- PASS: TestACL_Authorize/master-token/dc1 (0.00s) - --- PASS: TestACL_Authorize/master-token/dc2 (0.00s) - --- PASS: TestACL_Authorize/custom-token (0.01s) - --- PASS: TestACL_Authorize/custom-token/dc1 (0.00s) - --- PASS: TestACL_Authorize/custom-token/dc2 (0.01s) - --- PASS: TestACL_Authorize/too-many-requests (0.00s) - --- PASS: TestACL_Authorize/decode-failure (0.00s) - --- PASS: TestACL_Authorize/acl-not-found (0.00s) - --- PASS: TestACL_Authorize/local-token-in-secondary-dc (0.00s) - --- PASS: TestACL_Authorize/local-token-wrong-dc (0.00s) -=== CONT TestKVSEndpoint_AcquireRelease -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:07.097Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:07.116Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:07.125 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7a29cfc3-4ddc-73a5-9aa0-bf7da792ae51 Address:127.0.0.1:13726}]" -43:07.126 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13726 [Follower]" leader= -43:07.128 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7a29cfc3-4ddc-73a5-9aa0-bf7da792ae51.dc1 127.0.0.1 -43:07.129 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7a29cfc3-4ddc-73a5-9aa0-bf7da792ae51 127.0.0.1 -2022-09-15T21:43:07.129Z [INFO] agent.router: Initializing LAN area manager -43:07.130 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13721 network=udp -43:07.130 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7a29cfc3-4ddc-73a5-9aa0-bf7da792ae51.dc1 area=wan -43:07.130 [INFO] TestAgent.server: Adding LAN server: server="Node-7a29cfc3-4ddc-73a5-9aa0-bf7da792ae51 (Addr: tcp/127.0.0.1:13726) (DC: dc1)" -43:07.130 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13721 network=tcp -43:07.131 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13701 network=tcp -43:07.131 [INFO] TestAgent: started state syncer -43:07.131 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13727 network=tcp -43:07.194 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:07.194 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13726 [Candidate]" term=2 -43:07.194 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:07.194 [DEBUG] TestAgent.server.raft: vote granted: from=7a29cfc3-4ddc-73a5-9aa0-bf7da792ae51 term=2 tally=1 -43:07.194 [INFO] TestAgent.server.raft: election won: tally=1 -43:07.194 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13726 [Leader]" -43:07.194 [INFO] TestAgent.server: cluster leadership acquired -43:07.195 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13726 -43:07.196 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:07.198 [INFO] TestAgent.server: New leader elected: payload=Node-7a29cfc3-4ddc-73a5-9aa0-bf7da792ae51 -43:07.199 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:07.199 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:07.199 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:07.199 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:07.199 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:07.199 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7a29cfc3-4ddc-73a5-9aa0-bf7da792ae51 -43:07.199 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7a29cfc3-4ddc-73a5-9aa0-bf7da792ae51 -43:07.201 [INFO] TestAgent.server: federation state anti-entropy synced -43:07.212 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:07.212 [INFO] TestAgent: Synced node info -43:07.212 [DEBUG] TestAgent: Node info in sync -43:07.570 [INFO] TestAgent: Requesting shutdown -43:07.570 [INFO] TestAgent.server: shutting down server -43:07.570 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:07.570 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:07.570 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:07.570 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:07.570 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:07.570 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:07.570 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:07.570 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:07.570 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:07.570 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:07.571 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:07.571Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:07.571Z [INFO] agent.router.manager: shutting down -43:07.571 [INFO] TestAgent: consul server down -43:07.571 [INFO] TestAgent: shutdown complete -43:07.571 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13721 network=tcp -43:07.571 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13721 network=udp -43:07.571 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13701 network=tcp -43:08.074 [INFO] TestAgent: Waiting for endpoints to shut down -43:08.074 [INFO] TestAgent: Endpoints down ---- PASS: TestOperator_RaftConfiguration (1.04s) -=== CONT TestKVSEndpoint_ListKeys -2022-09-15T21:43:07.441Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:07.512Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:07.519 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f3f4fae0-c199-6722-0bdc-a46c552500a3 Address:127.0.0.1:13712}]" -43:07.520 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f3f4fae0-c199-6722-0bdc-a46c552500a3.dc1 127.0.0.1 -43:07.521 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f3f4fae0-c199-6722-0bdc-a46c552500a3 127.0.0.1 -2022-09-15T21:43:07.521Z [INFO] agent.router: Initializing LAN area manager -43:07.521 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13756 network=udp -43:07.521 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13712 [Follower]" leader= -43:07.522 [INFO] TestAgent.server: Adding LAN server: server="Node-f3f4fae0-c199-6722-0bdc-a46c552500a3 (Addr: tcp/127.0.0.1:13712) (DC: dc1)" -43:07.522 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f3f4fae0-c199-6722-0bdc-a46c552500a3.dc1 area=wan -43:07.522 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13756 network=tcp -43:07.523 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13743 network=tcp -43:07.523 [INFO] TestAgent: started state syncer -43:07.523 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13713 network=tcp -43:07.589 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:07.589 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13712 [Candidate]" term=2 -43:07.589 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:07.589 [DEBUG] TestAgent.server.raft: vote granted: from=f3f4fae0-c199-6722-0bdc-a46c552500a3 term=2 tally=1 -43:07.589 [INFO] TestAgent.server.raft: election won: tally=1 -43:07.589 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13712 [Leader]" -43:07.589 [INFO] TestAgent.server: cluster leadership acquired -43:07.590 [INFO] TestAgent.server: New leader elected: payload=Node-f3f4fae0-c199-6722-0bdc-a46c552500a3 -43:07.590 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13712 -43:07.591 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:07.594 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:07.594 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:07.594 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:07.594 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:07.594 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:07.594 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f3f4fae0-c199-6722-0bdc-a46c552500a3 -43:07.594 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f3f4fae0-c199-6722-0bdc-a46c552500a3 -43:07.596 [INFO] TestAgent.server: federation state anti-entropy synced -43:07.734 [INFO] TestAgent: Requesting shutdown -43:07.734 [INFO] TestAgent.server: shutting down server -43:07.734 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:07.734 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:07.735 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:07.735 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:07.735 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:07.735 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:07.735 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:07.735 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:07.735 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:07.735 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:07.735 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:07.735 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:07.735Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:07.735Z [INFO] agent.router.manager: shutting down -43:07.735 [INFO] TestAgent: consul server down -43:07.735 [INFO] TestAgent: shutdown complete -43:07.735 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13756 network=tcp -43:07.735 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13756 network=udp -43:07.736 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13743 network=tcp -43:08.238 [INFO] TestAgent: Waiting for endpoints to shut down -43:08.238 [INFO] TestAgent: Endpoints down ---- PASS: TestOperator_RaftPeer (1.50s) - --- PASS: TestOperator_RaftPeer/#00 (0.67s) - --- PASS: TestOperator_RaftPeer/#01 (0.83s) -=== CONT TestKVSEndpoint_CAS -2022-09-15T21:43:07.236Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:07.325Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:07.329 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2a28c11a-520a-ce4c-fd6d-8c9e6fb7d56d Address:127.0.0.1:13733}]" -43:07.330 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13733 [Follower]" leader= -43:07.331 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2a28c11a-520a-ce4c-fd6d-8c9e6fb7d56d.dc1 127.0.0.1 -43:07.333 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2a28c11a-520a-ce4c-fd6d-8c9e6fb7d56d 127.0.0.1 -2022-09-15T21:43:07.333Z [INFO] agent.router: Initializing LAN area manager -43:07.334 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13728 network=udp -43:07.334 [INFO] TestAgent.server: Adding LAN server: server="Node-2a28c11a-520a-ce4c-fd6d-8c9e6fb7d56d (Addr: tcp/127.0.0.1:13733) (DC: dc1)" -43:07.334 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2a28c11a-520a-ce4c-fd6d-8c9e6fb7d56d.dc1 area=wan -43:07.335 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13728 network=tcp -43:07.336 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13708 network=tcp -43:07.336 [INFO] TestAgent: started state syncer -43:07.336 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13734 network=tcp -43:07.372 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:07.372 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13733 [Candidate]" term=2 -43:07.372 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:07.372 [DEBUG] TestAgent.server.raft: vote granted: from=2a28c11a-520a-ce4c-fd6d-8c9e6fb7d56d term=2 tally=1 -43:07.372 [INFO] TestAgent.server.raft: election won: tally=1 -43:07.372 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13733 [Leader]" -43:07.372 [INFO] TestAgent.server: cluster leadership acquired -43:07.372 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13733 -43:07.374 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:07.377 [INFO] TestAgent.server: New leader elected: payload=Node-2a28c11a-520a-ce4c-fd6d-8c9e6fb7d56d -43:07.377 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:07.377 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:07.377 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:07.377 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:07.377 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:07.377 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2a28c11a-520a-ce4c-fd6d-8c9e6fb7d56d -43:07.377 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2a28c11a-520a-ce4c-fd6d-8c9e6fb7d56d -43:07.379 [INFO] TestAgent.server: federation state anti-entropy synced -43:07.489 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:07.489 [INFO] TestAgent: Synced node info -43:07.489 [DEBUG] TestAgent: Node info in sync -43:07.751 [INFO] TestAgent: Requesting shutdown -43:07.751 [INFO] TestAgent.server: shutting down server -43:07.751 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:07.751 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:07.751 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:07.751 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:07.751 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:07.751 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:07.751 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:07.751 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:07.751 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:07.751 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:07.751 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:07.751Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:07.751Z [INFO] agent.router.manager: shutting down -43:07.752 [INFO] TestAgent: consul server down -43:07.752 [INFO] TestAgent: shutdown complete -43:07.752 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13728 network=tcp -43:07.752 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13728 network=udp -43:07.752 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13708 network=tcp -43:08.253 [INFO] TestAgent: Waiting for endpoints to shut down -43:08.253 [INFO] TestAgent: Endpoints down ---- PASS: TestKVSEndpoint_DELETE_ConflictingFlags (1.05s) -=== CONT TestACL_Legacy_List -[WARN] freeport: 4 out of 25 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:07.554Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:07.629Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:07.632 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5a12e8a7-dc65-1e44-97fd-59c82cf99e3b Address:127.0.0.1:13796}]" -43:07.633 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5a12e8a7-dc65-1e44-97fd-59c82cf99e3b.dc1 127.0.0.1 -43:07.634 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13796 [Follower]" leader= -43:07.634 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5a12e8a7-dc65-1e44-97fd-59c82cf99e3b 127.0.0.1 -2022-09-15T21:43:07.634Z [INFO] agent.router: Initializing LAN area manager -43:07.635 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13714 network=udp -43:07.635 [INFO] TestAgent.server: Adding LAN server: server="Node-5a12e8a7-dc65-1e44-97fd-59c82cf99e3b (Addr: tcp/127.0.0.1:13796) (DC: dc1)" -43:07.635 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5a12e8a7-dc65-1e44-97fd-59c82cf99e3b.dc1 area=wan -43:07.635 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13714 network=tcp -43:07.636 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13736 network=tcp -43:07.636 [INFO] TestAgent: started state syncer -43:07.636 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13797 network=tcp -43:07.670 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:07.670 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13796 [Candidate]" term=2 -43:07.670 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:07.670 [DEBUG] TestAgent.server.raft: vote granted: from=5a12e8a7-dc65-1e44-97fd-59c82cf99e3b term=2 tally=1 -43:07.670 [INFO] TestAgent.server.raft: election won: tally=1 -43:07.670 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13796 [Leader]" -43:07.671 [INFO] TestAgent.server: cluster leadership acquired -43:07.671 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13796 -43:07.672 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:07.674 [INFO] TestAgent.server: New leader elected: payload=Node-5a12e8a7-dc65-1e44-97fd-59c82cf99e3b -43:07.675 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:07.676 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:07.676 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:07.676 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:07.676 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:07.676 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5a12e8a7-dc65-1e44-97fd-59c82cf99e3b -43:07.676 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5a12e8a7-dc65-1e44-97fd-59c82cf99e3b -43:07.677 [INFO] TestAgent.server: federation state anti-entropy synced -43:07.737 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:07.738 [INFO] TestAgent: Synced node info -43:07.738 [DEBUG] TestAgent: Node info in sync -43:07.806 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:07.806 [DEBUG] TestAgent: Node info in sync -43:07.827 [INFO] TestAgent: Requesting shutdown -43:07.827 [INFO] TestAgent.server: shutting down server -43:07.827 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:07.827 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:07.827 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:07.827 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:07.827 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:07.827 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:07.827 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:07.827 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:07.827 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:07.827 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:07.827 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:07.827Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:07.827Z [INFO] agent.router.manager: shutting down -43:07.828 [INFO] TestAgent: consul server down -43:07.828 [INFO] TestAgent: shutdown complete -43:07.828 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13714 network=tcp -43:07.828 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13714 network=udp -43:07.828 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13736 network=tcp -43:08.328 [INFO] TestAgent: Waiting for endpoints to shut down -43:08.328 [INFO] TestAgent: Endpoints down ---- PASS: TestKVSEndpoint_GET_Raw (0.84s) -=== CONT TestACL_LoginProcedure_HTTP -2022-09-15T21:43:07.439Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:07.457Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:07.461 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:92293a77-191d-9179-31ed-4064da2f42bf Address:127.0.0.1:13754}]" -43:07.462 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13754 [Follower]" leader= -43:07.462 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-92293a77-191d-9179-31ed-4064da2f42bf.dc1 127.0.0.1 -43:07.463 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-92293a77-191d-9179-31ed-4064da2f42bf 127.0.0.1 -2022-09-15T21:43:07.463Z [INFO] agent.router: Initializing LAN area manager -43:07.463 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13735 network=udp -43:07.464 [INFO] TestAgent.server: Adding LAN server: server="Node-92293a77-191d-9179-31ed-4064da2f42bf (Addr: tcp/127.0.0.1:13754) (DC: dc1)" -43:07.464 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-92293a77-191d-9179-31ed-4064da2f42bf.dc1 area=wan -43:07.464 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13735 network=tcp -43:07.465 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13750 network=tcp -43:07.465 [INFO] TestAgent: started state syncer -43:07.465 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13755 network=tcp -43:07.498 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:07.498 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13754 [Candidate]" term=2 -43:07.498 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:07.498 [DEBUG] TestAgent.server.raft: vote granted: from=92293a77-191d-9179-31ed-4064da2f42bf term=2 tally=1 -43:07.498 [INFO] TestAgent.server.raft: election won: tally=1 -43:07.498 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13754 [Leader]" -43:07.498 [INFO] TestAgent.server: cluster leadership acquired -43:07.499 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13754 -43:07.500 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:07.502 [INFO] TestAgent.server: New leader elected: payload=Node-92293a77-191d-9179-31ed-4064da2f42bf -43:07.506 [INFO] TestAgent: Synced node info -43:07.507 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:07.507 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:07.507 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:07.507 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:07.507 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:07.508 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-92293a77-191d-9179-31ed-4064da2f42bf -43:07.508 [INFO] TestAgent.server: member joined, marking health alive: member=Node-92293a77-191d-9179-31ed-4064da2f42bf -43:07.509 [INFO] TestAgent.server: federation state anti-entropy synced -43:07.858 [INFO] TestAgent: Requesting shutdown -43:07.858 [INFO] TestAgent.server: shutting down server -43:07.859 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:07.859 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:07.859 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:07.859 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:07.859 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:07.859 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:07.859 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:07.859 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:07.859 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:07.859 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:07.859 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:07.859Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:07.859Z [INFO] agent.router.manager: shutting down -43:07.859 [INFO] TestAgent: consul server down -43:07.859 [INFO] TestAgent: shutdown complete -43:07.859 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13735 network=tcp -43:07.860 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13735 network=udp -43:07.860 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13750 network=tcp -43:08.362 [INFO] TestAgent: Waiting for endpoints to shut down -43:08.362 [INFO] TestAgent: Endpoints down ---- PASS: TestKVSEndpoint_PUT_ConflictingFlags (0.96s) -=== CONT TestACL_HTTP -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:24.490Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:24.490Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:24.557Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:24.557Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:24.561 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:aac17db4-f8e6-3051-e9fd-8dc10d087cf4 Address:127.0.0.1:24935}]" +34:24.561 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24935 [Follower]" leader= +34:24.562 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-aac17db4-f8e6-3051-e9fd-8dc10d087cf4.dc1 127.0.0.1 +34:24.563 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-aac17db4-f8e6-3051-e9fd-8dc10d087cf4 127.0.0.1 +2021-08-13T15:34:24.563Z [INFO] agent.router: Initializing LAN area manager +34:24.563 [INFO] TestAgent.server: Adding LAN server: server="Node-aac17db4-f8e6-3051-e9fd-8dc10d087cf4 (Addr: tcp/127.0.0.1:24935) (DC: dc1)" +34:24.563 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24951 network=udp +34:24.563 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-aac17db4-f8e6-3051-e9fd-8dc10d087cf4.dc1 area=wan +34:24.563 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24951 network=tcp +34:24.564 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24952 network=tcp +34:24.564 [INFO] TestAgent: started state syncer +34:24.564 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24957 network=tcp +34:24.613 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:24.629 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:24.629 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24935 [Candidate]" term=2 +34:24.629 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:24.629 [DEBUG] TestAgent.server.raft: vote granted: from=aac17db4-f8e6-3051-e9fd-8dc10d087cf4 term=2 tally=1 +34:24.629 [INFO] TestAgent.server.raft: election won: tally=1 +34:24.629 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24935 [Leader]" +34:24.629 [INFO] TestAgent.server: cluster leadership acquired +34:24.629 [INFO] TestAgent.server: initializing acls +34:24.630 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:24.630 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:24.630 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:24.631 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:24.631 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:24.631 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:24.631 [INFO] TestAgent.server: New leader elected: payload=Node-aac17db4-f8e6-3051-e9fd-8dc10d087cf4 +34:24.631 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-aac17db4-f8e6-3051-e9fd-8dc10d087cf4 +34:24.631 [INFO] TestAgent.server: Updating LAN server: server="Node-aac17db4-f8e6-3051-e9fd-8dc10d087cf4 (Addr: tcp/127.0.0.1:24935) (DC: dc1)" +34:24.631 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-aac17db4-f8e6-3051-e9fd-8dc10d087cf4.dc1 +34:24.632 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:24.634 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:24.634 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:24.634 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:24.634 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:24.634 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:24.634 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-aac17db4-f8e6-3051-e9fd-8dc10d087cf4 +34:24.634 [INFO] TestAgent.server: member joined, marking health alive: member=Node-aac17db4-f8e6-3051-e9fd-8dc10d087cf4 +34:24.635 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-aac17db4-f8e6-3051-e9fd-8dc10d087cf4.dc1 area=wan +34:24.636 [INFO] TestAgent.server: federation state anti-entropy synced +34:24.636 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-aac17db4-f8e6-3051-e9fd-8dc10d087cf4 +34:24.639 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-aac17db4-f8e6-3051-e9fd-8dc10d087cf4 +34:24.640 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-aac17db4-f8e6-3051-e9fd-8dc10d087cf4 +34:24.641 [INFO] TestAgent.server: server starting leave +34:24.641 [INFO] TestAgent.server.serf.wan: serf: EventMemberLeave: Node-aac17db4-f8e6-3051-e9fd-8dc10d087cf4.dc1 127.0.0.1 +34:24.642 [INFO] TestAgent.server: Handled event for server in area: event=member-leave server=Node-aac17db4-f8e6-3051-e9fd-8dc10d087cf4.dc1 area=wan +2021-08-13T15:34:24.642Z [INFO] agent.router.manager: shutting down +34:24.886 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:24.887 [INFO] TestAgent: Synced node info +34:24.887 [DEBUG] TestAgent: Node info in sync +34:25.066 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:25.066 [DEBUG] TestAgent: Node info in sync +34:27.642 [INFO] TestAgent.server.serf.lan: serf: EventMemberLeave: Node-aac17db4-f8e6-3051-e9fd-8dc10d087cf4 127.0.0.1 +34:27.642 [INFO] TestAgent.server: Removing LAN server: server="Node-aac17db4-f8e6-3051-e9fd-8dc10d087cf4 (Addr: tcp/127.0.0.1:24935) (DC: dc1)" +34:27.642 [WARN] TestAgent.server: deregistering self should be done by follower: name=Node-aac17db4-f8e6-3051-e9fd-8dc10d087cf4 +34:28.635 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" +34:30.635 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" +34:30.642 [INFO] TestAgent.server: Waiting to drain RPC traffic: drain_time=5s +34:32.635 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" +34:34.635 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" +34:34.635 [ERROR] TestAgent.server.autopilot: Error promoting servers: error="error getting server raft protocol versions: No servers found" +34:35.643 [INFO] TestAgent: Requesting shutdown +34:35.643 [INFO] TestAgent.server: shutting down server +34:35.643 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:35.643 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:35.643 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:35.643 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:35.643 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:35.643 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:35.643 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:35.643 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:35.643 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:35.643 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:35.643 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:35.643 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:35.643 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +2021-08-13T15:34:35.644Z [INFO] agent.router.manager: shutting down +34:35.644 [INFO] TestAgent: consul server down +34:35.644 [INFO] TestAgent: shutdown complete +34:35.644 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24951 network=tcp +34:35.644 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24951 network=udp +34:35.644 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24952 network=tcp +34:36.144 [INFO] TestAgent: Waiting for endpoints to shut down +34:36.144 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Leave_ACLDeny (11.67s) + --- PASS: TestAgent_Leave_ACLDeny/no_token (0.00s) + --- PASS: TestAgent_Leave_ACLDeny/read-only_token (0.00s) + --- PASS: TestAgent_Leave_ACLDeny/agent_master_token (11.00s) +=== CONT TestPrettyPrint +=== RUN TestACLEndpoint_LoginLogout_jwt/success_-_jwt_static_keys +=== RUN TestACLEndpoint_LoginLogout_jwt/success_-_jwt_static_keys/invalid_bearer_token +=== RUN TestACLEndpoint_LoginLogout_jwt/success_-_jwt_static_keys/valid_bearer_token_no_bindings +=== RUN TestACLEndpoint_LoginLogout_jwt/success_-_jwt_static_keys/valid_bearer_token_1_service_binding +=== RUN TestACLEndpoint_LoginLogout_jwt/success_-_jwt_jwks +=== RUN TestACLEndpoint_LoginLogout_jwt/success_-_jwt_jwks/invalid_bearer_token +=== RUN TestACLEndpoint_LoginLogout_jwt/success_-_jwt_jwks/valid_bearer_token_no_bindings +=== RUN TestACLEndpoint_LoginLogout_jwt/success_-_jwt_jwks/valid_bearer_token_1_service_binding +=== RUN TestACLEndpoint_LoginLogout_jwt/success_-_jwt_oidc_discovery === RUN TestACL_LoginProcedure_HTTP/AuthMethod === RUN TestACL_LoginProcedure_HTTP/AuthMethod/Create === RUN TestACL_LoginProcedure_HTTP/AuthMethod/Create_other @@ -47160,71 +42052,76 @@ === RUN TestACL_LoginProcedure_HTTP/BindingRule/Invalid_payload === RUN TestACL_LoginProcedure_HTTP/BindingRule/List === RUN TestACL_LoginProcedure_HTTP/BindingRule/Delete +=== RUN TestACLEndpoint_LoginLogout_jwt/success_-_jwt_oidc_discovery/invalid_bearer_token === RUN TestACL_LoginProcedure_HTTP/BindingRule/Read === RUN TestACL_LoginProcedure_HTTP/Login === RUN TestACL_LoginProcedure_HTTP/Login/Create_Token_1 === RUN TestACL_LoginProcedure_HTTP/Login/Create_Token_2 +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! === RUN TestACL_LoginProcedure_HTTP/Login/List_Tokens_by_(incorrect)_Method === RUN TestACL_LoginProcedure_HTTP/Login/List_Tokens_by_(correct)_Method === RUN TestACL_LoginProcedure_HTTP/Login/Logout === RUN TestACL_LoginProcedure_HTTP/Login/Token_is_gone_after_Logout -2022-09-15T21:43:08.015Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:08.033Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:08.036 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e57aa9de-186a-ab5d-924b-8a644095bd03 Address:127.0.0.1:13768}]" -43:08.037 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e57aa9de-186a-ab5d-924b-8a644095bd03.dc1 127.0.0.1 -43:08.038 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13768 [Follower]" leader= -43:08.041 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e57aa9de-186a-ab5d-924b-8a644095bd03 127.0.0.1 -2022-09-15T21:43:08.041Z [INFO] agent.router: Initializing LAN area manager -43:08.042 [INFO] TestAgent.server: Adding LAN server: server="Node-e57aa9de-186a-ab5d-924b-8a644095bd03 (Addr: tcp/127.0.0.1:13768) (DC: dc1)" -43:08.042 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e57aa9de-186a-ab5d-924b-8a644095bd03.dc1 area=wan -43:08.042 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13798 network=tcp -43:08.043 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13798 network=udp -43:08.044 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13799 network=tcp -43:08.044 [INFO] TestAgent: started state syncer -43:08.044 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13769 network=tcp -43:08.094 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:08.094 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13768 [Candidate]" term=2 -43:08.094 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:08.094 [DEBUG] TestAgent.server.raft: vote granted: from=e57aa9de-186a-ab5d-924b-8a644095bd03 term=2 tally=1 -43:08.094 [INFO] TestAgent.server.raft: election won: tally=1 -43:08.094 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13768 [Leader]" -43:08.094 [INFO] TestAgent.server: cluster leadership acquired -43:08.095 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13768 -43:08.096 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:08.098 [INFO] TestAgent.server: New leader elected: payload=Node-e57aa9de-186a-ab5d-924b-8a644095bd03 -43:08.100 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:08.101 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:08.101 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:08.101 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:08.101 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:08.101 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e57aa9de-186a-ab5d-924b-8a644095bd03 -43:08.101 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e57aa9de-186a-ab5d-924b-8a644095bd03 -43:08.102 [INFO] TestAgent.server: federation state anti-entropy synced -43:08.172 [INFO] TestAgent: Requesting shutdown -43:08.172 [INFO] TestAgent.server: shutting down server -43:08.172 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:08.172 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:08.172 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:08.172 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:08.172 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:08.172 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:08.172 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:08.172 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:08.172 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:08.172 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:08.172 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:08.173 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:08.173Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:08.173Z [INFO] agent.router.manager: shutting down -43:08.173 [INFO] TestAgent: consul server down -43:08.173 [INFO] TestAgent: shutdown complete -43:08.173 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13798 network=tcp -43:08.173 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13798 network=udp -43:08.173 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13799 network=tcp -43:08.673 [INFO] TestAgent: Waiting for endpoints to shut down -43:08.673 [INFO] TestAgent: Endpoints down ---- PASS: TestKVSEndpoint_AcquireRelease (0.70s) -=== CONT TestACL_Bootstrap +=== RUN TestACLEndpoint_LoginLogout_jwt/success_-_jwt_oidc_discovery/valid_bearer_token_no_bindings +=== RUN TestACLEndpoint_LoginLogout_jwt/success_-_jwt_oidc_discovery/valid_bearer_token_1_service_binding +2021-08-13T15:34:35.400Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:35.418Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:35.421 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e92c0507-1ff9-1e32-871c-58e2ef796855 Address:127.0.0.1:24462}]" +34:35.421 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e92c0507-1ff9-1e32-871c-58e2ef796855.dc1 127.0.0.1 +34:35.422 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e92c0507-1ff9-1e32-871c-58e2ef796855 127.0.0.1 +2021-08-13T15:34:35.422Z [INFO] agent.router: Initializing LAN area manager +34:35.423 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24436 network=udp +34:35.423 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24462 [Follower]" leader= +34:35.423 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e92c0507-1ff9-1e32-871c-58e2ef796855.dc1 area=wan +34:35.424 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24436 network=tcp +34:35.424 [INFO] TestAgent.server: Adding LAN server: server="Node-e92c0507-1ff9-1e32-871c-58e2ef796855 (Addr: tcp/127.0.0.1:24462) (DC: dc1)" +34:35.425 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24444 network=tcp +34:35.425 [INFO] TestAgent: started state syncer +34:35.425 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24463 network=tcp +34:35.476 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:35.476 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24462 [Candidate]" term=2 +34:35.476 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:35.476 [DEBUG] TestAgent.server.raft: vote granted: from=e92c0507-1ff9-1e32-871c-58e2ef796855 term=2 tally=1 +34:35.476 [INFO] TestAgent.server.raft: election won: tally=1 +34:35.476 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24462 [Leader]" +34:35.476 [INFO] TestAgent.server: cluster leadership acquired +34:35.477 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24462 +34:35.478 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:35.480 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:35.480 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:35.480 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:35.480 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:35.480 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:35.480 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e92c0507-1ff9-1e32-871c-58e2ef796855 +34:35.480 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e92c0507-1ff9-1e32-871c-58e2ef796855 +34:35.481 [INFO] TestAgent.server: New leader elected: payload=Node-e92c0507-1ff9-1e32-871c-58e2ef796855 +34:35.481 [INFO] TestAgent.server: federation state anti-entropy synced +34:35.567 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:35.567 [INFO] TestAgent: Synced node info +34:35.790 [INFO] TestAgent: Requesting shutdown +34:35.790 [INFO] TestAgent.server: shutting down server +34:35.790 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:35.790 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:35.790 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:35.790 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:35.790 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:35.790 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:35.790 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:35.790 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:35.790 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:35.790 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:35.790 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:35.791Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:35.791Z [INFO] agent.router.manager: shutting down +34:35.791 [INFO] TestAgent: consul server down +34:35.791 [INFO] TestAgent: shutdown complete +34:35.791 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24436 network=tcp +34:35.791 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24436 network=udp +34:35.791 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24444 network=tcp +34:36.293 [INFO] TestAgent: Waiting for endpoints to shut down +34:36.293 [INFO] TestAgent: Endpoints down +--- PASS: TestParseSource (0.92s) +=== CONT TestACL_HTTP === RUN TestACL_HTTP/Policy === RUN TestACL_HTTP/Policy/Create === RUN TestACL_HTTP/Policy/Minimal @@ -47251,63 +42148,6 @@ === RUN TestACL_HTTP/Role/Read === RUN TestACL_HTTP/Token === RUN TestACL_HTTP/Token/Create -2022-09-15T21:43:08.106Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:08.145Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:08.148 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:73ff5ae4-5982-baf1-ec4b-0b7f7faf5f66 Address:127.0.0.1:13761}]" -43:08.149 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-73ff5ae4-5982-baf1-ec4b-0b7f7faf5f66.dc1 127.0.0.1 -43:08.150 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13761 [Follower]" leader= -43:08.150 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-73ff5ae4-5982-baf1-ec4b-0b7f7faf5f66 127.0.0.1 -2022-09-15T21:43:08.150Z [INFO] agent.router: Initializing LAN area manager -43:08.151 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13770 network=udp -43:08.151 [INFO] TestAgent.server: Adding LAN server: server="Node-73ff5ae4-5982-baf1-ec4b-0b7f7faf5f66 (Addr: tcp/127.0.0.1:13761) (DC: dc1)" -43:08.151 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-73ff5ae4-5982-baf1-ec4b-0b7f7faf5f66.dc1 area=wan -43:08.151 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13770 network=tcp -43:08.152 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13757 network=tcp -43:08.152 [INFO] TestAgent: started state syncer -43:08.152 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13762 network=tcp -43:08.190 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:08.190 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13761 [Candidate]" term=2 -43:08.190 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:08.190 [DEBUG] TestAgent.server.raft: vote granted: from=73ff5ae4-5982-baf1-ec4b-0b7f7faf5f66 term=2 tally=1 -43:08.190 [INFO] TestAgent.server.raft: election won: tally=1 -43:08.190 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13761 [Leader]" -43:08.190 [INFO] TestAgent.server: cluster leadership acquired -43:08.191 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13761 -43:08.192 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:08.194 [INFO] TestAgent.server: New leader elected: payload=Node-73ff5ae4-5982-baf1-ec4b-0b7f7faf5f66 -43:08.207 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:08.207 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:08.207 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:08.207 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:08.207 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:08.207 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-73ff5ae4-5982-baf1-ec4b-0b7f7faf5f66 -43:08.207 [INFO] TestAgent.server: member joined, marking health alive: member=Node-73ff5ae4-5982-baf1-ec4b-0b7f7faf5f66 -43:08.209 [INFO] TestAgent.server: federation state anti-entropy synced -43:08.217 [INFO] TestAgent: Requesting shutdown -43:08.217 [INFO] TestAgent.server: shutting down server -43:08.217 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:08.217 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:08.217 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:08.217 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:08.217 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:08.217 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:08.217 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:08.218 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:08.218 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:08.218 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:08.218 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:08.218 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:08.218Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:08.218Z [INFO] agent.router.manager: shutting down -43:08.218 [INFO] TestAgent: consul server down -43:08.218 [INFO] TestAgent: shutdown complete -43:08.218 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13770 network=tcp -43:08.218 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13770 network=udp -43:08.218 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13757 network=tcp -43:08.720 [INFO] TestAgent: Waiting for endpoints to shut down -43:08.720 [INFO] TestAgent: Endpoints down ---- PASS: TestKVSEndpoint_ListKeys (0.66s) -=== CONT TestACL_Disabled_Response === RUN TestACL_HTTP/Token/Create_Local === RUN TestACL_HTTP/Token/Read === RUN TestACL_HTTP/Token/Self @@ -47321,123 +42161,159 @@ === RUN TestACL_HTTP/Token/Create_with_Accessor === RUN TestACL_HTTP/Token/Create_with_Secret === RUN TestACL_HTTP/Token/Create_with_Accessor_and_Secret -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! === RUN TestACL_HTTP/Token/Create_with_Accessor_Dup === RUN TestACL_HTTP/Token/Create_with_Secret_as_Accessor_Dup === RUN TestACL_HTTP/Token/Create_with_Secret_Dup === RUN TestACL_HTTP/Token/Create_with_Accessor_as_Secret_Dup === RUN TestACL_HTTP/Token/Create_with_Reserved_Accessor === RUN TestACL_HTTP/Token/Create_with_Reserved_Secret -=== RUN TestACL_Disabled_Response/ACLBootstrap -=== RUN TestACL_Disabled_Response/ACLReplicationStatus -=== RUN TestACL_Disabled_Response/AgentToken -=== RUN TestACL_Disabled_Response/ACLRulesTranslate -=== RUN TestACL_Disabled_Response/ACLRulesTranslateLegacyToken -=== RUN TestACL_Disabled_Response/ACLPolicyList -=== RUN TestACL_Disabled_Response/ACLPolicyCRUD -=== RUN TestACL_Disabled_Response/ACLPolicyCreate -=== RUN TestACL_Disabled_Response/ACLTokenList -=== RUN TestACL_Disabled_Response/ACLTokenCreate -=== RUN TestACL_Disabled_Response/ACLTokenSelf -=== RUN TestACL_Disabled_Response/ACLTokenCRUD -=== RUN TestACL_Disabled_Response/ACLRoleList -=== RUN TestACL_Disabled_Response/ACLRoleCreate -=== RUN TestACL_Disabled_Response/ACLRoleCRUD -=== RUN TestACL_Disabled_Response/ACLBindingRuleList -=== RUN TestACL_Disabled_Response/ACLBindingRuleCreate -=== RUN TestACL_Disabled_Response/ACLBindingRuleCRUD -=== RUN TestACL_Disabled_Response/ACLAuthMethodList -=== RUN TestACL_Disabled_Response/ACLAuthMethodCreate -=== RUN TestACL_Disabled_Response/ACLAuthMethodCRUD -=== RUN TestACL_Disabled_Response/ACLLogin -=== RUN TestACL_Disabled_Response/ACLLogout -=== RUN TestACL_Disabled_Response/ACLAuthorize -[WARN] freeport: 4 out of 4 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:08.460Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:08.460Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:08.489Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:08.489Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:08.508 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:df9c759c-8e63-21d1-3259-baf9c844b6b2 Address:127.0.0.1:13782}]" -43:08.509 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-df9c759c-8e63-21d1-3259-baf9c844b6b2.dc1 127.0.0.1 -43:08.510 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13782 [Follower]" leader= -43:08.510 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-df9c759c-8e63-21d1-3259-baf9c844b6b2 127.0.0.1 -2022-09-15T21:43:08.510Z [INFO] agent.router: Initializing LAN area manager -43:08.511 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13791 network=udp -43:08.511 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-df9c759c-8e63-21d1-3259-baf9c844b6b2.dc1 area=wan -43:08.511 [INFO] TestAgent.server: Adding LAN server: server="Node-df9c759c-8e63-21d1-3259-baf9c844b6b2 (Addr: tcp/127.0.0.1:13782) (DC: dc1)" -43:08.511 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13791 network=tcp -43:08.512 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13792 network=tcp -43:08.512 [INFO] TestAgent: started state syncer -43:08.512 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13783 network=tcp -43:08.554 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:08.554 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13782 [Candidate]" term=2 -43:08.554 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:08.554 [DEBUG] TestAgent.server.raft: vote granted: from=df9c759c-8e63-21d1-3259-baf9c844b6b2 term=2 tally=1 -43:08.554 [INFO] TestAgent.server.raft: election won: tally=1 -43:08.554 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13782 [Leader]" -43:08.554 [INFO] TestAgent.server: cluster leadership acquired -43:08.555 [INFO] TestAgent.server: initializing acls -43:08.555 [INFO] TestAgent.server: Created ACL 'global-management' policy -43:08.555 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -43:08.555 [INFO] TestAgent.server: New leader elected: payload=Node-df9c759c-8e63-21d1-3259-baf9c844b6b2 -43:08.556 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -43:08.556 [INFO] TestAgent.server: Created ACL anonymous token from configuration -43:08.556 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -43:08.556 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -43:08.557 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-df9c759c-8e63-21d1-3259-baf9c844b6b2 -43:08.557 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-df9c759c-8e63-21d1-3259-baf9c844b6b2.dc1 -43:08.558 [INFO] TestAgent.server: Updating LAN server: server="Node-df9c759c-8e63-21d1-3259-baf9c844b6b2 (Addr: tcp/127.0.0.1:13782) (DC: dc1)" -43:08.558 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-df9c759c-8e63-21d1-3259-baf9c844b6b2.dc1 area=wan -43:08.558 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:08.561 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:08.561 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:08.561 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:08.561 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:08.561 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:08.561 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-df9c759c-8e63-21d1-3259-baf9c844b6b2 -43:08.561 [INFO] TestAgent.server: member joined, marking health alive: member=Node-df9c759c-8e63-21d1-3259-baf9c844b6b2 -43:08.566 [INFO] TestAgent.server: federation state anti-entropy synced -43:08.567 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-df9c759c-8e63-21d1-3259-baf9c844b6b2 -43:08.594 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-df9c759c-8e63-21d1-3259-baf9c844b6b2 -43:08.594 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-df9c759c-8e63-21d1-3259-baf9c844b6b2 -43:08.624 [INFO] TestAgent: Requesting shutdown -43:08.624 [INFO] TestAgent.server: shutting down server -43:08.624 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -43:08.624 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:08.624 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:08.624 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:08.624 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:08.624 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -43:08.624 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:08.624 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:08.624 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:08.624 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:08.624 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -43:08.624 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:08.624 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:08.624 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:08.624 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -43:08.625 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:08.625Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:08.625Z [INFO] agent.router.manager: shutting down -43:08.625 [INFO] TestAgent: consul server down -43:08.625 [INFO] TestAgent: shutdown complete -43:08.625 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13791 network=tcp -43:08.625 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13791 network=udp -43:08.625 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13792 network=tcp -43:09.126 [INFO] TestAgent: Waiting for endpoints to shut down -43:09.126 [INFO] TestAgent: Endpoints down ---- PASS: TestACL_LoginProcedure_HTTP (0.80s) +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:35.866Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:35.882Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:35.884 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ae0d2900-8a86-766e-ff8a-51901258b5aa Address:127.0.0.1:24483}]" +34:35.884 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24483 [Follower]" leader= +34:35.885 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ae0d2900-8a86-766e-ff8a-51901258b5aa.dc1 127.0.0.1 +34:35.886 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ae0d2900-8a86-766e-ff8a-51901258b5aa 127.0.0.1 +2021-08-13T15:34:35.886Z [INFO] agent.router: Initializing LAN area manager +34:35.886 [INFO] TestAgent.server: Adding LAN server: server="Node-ae0d2900-8a86-766e-ff8a-51901258b5aa (Addr: tcp/127.0.0.1:24483) (DC: dc1)" +34:35.886 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ae0d2900-8a86-766e-ff8a-51901258b5aa.dc1 area=wan +34:35.886 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24457 network=tcp +34:35.886 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24457 network=udp +34:35.888 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24416 network=tcp +34:35.888 [INFO] TestAgent: started state syncer +34:35.888 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24491 network=tcp +34:35.919 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:35.920 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24483 [Candidate]" term=2 +34:35.920 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:35.920 [DEBUG] TestAgent.server.raft: vote granted: from=ae0d2900-8a86-766e-ff8a-51901258b5aa term=2 tally=1 +34:35.920 [INFO] TestAgent.server.raft: election won: tally=1 +34:35.920 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24483 [Leader]" +34:35.920 [INFO] TestAgent.server: cluster leadership acquired +34:35.920 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24483 +34:35.920 [INFO] TestAgent.server: New leader elected: payload=Node-ae0d2900-8a86-766e-ff8a-51901258b5aa +34:35.921 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:35.923 [INFO] TestAgent: Synced node info +34:35.923 [DEBUG] TestAgent: Node info in sync +34:35.924 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:35.924 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:35.924 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:35.924 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:35.924 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:35.924 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ae0d2900-8a86-766e-ff8a-51901258b5aa +34:35.924 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ae0d2900-8a86-766e-ff8a-51901258b5aa +34:35.925 [INFO] TestAgent.server: federation state anti-entropy synced +34:36.276 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/key?pretty from= latency=69.3µs +34:36.276 [INFO] TestAgent: Requesting shutdown +34:36.276 [INFO] TestAgent.server: shutting down server +34:36.276 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:36.276 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:36.276 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:36.276 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:36.276 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:36.276 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:36.276 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:36.276 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:36.276 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:36.276 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:36.277 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:36.277Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:36.277Z [INFO] agent.router.manager: shutting down +34:36.277 [INFO] TestAgent: consul server down +34:36.277 [INFO] TestAgent: shutdown complete +34:36.277 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24457 network=tcp +34:36.277 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24457 network=udp +34:36.277 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24416 network=tcp +34:36.777 [INFO] TestAgent: Waiting for endpoints to shut down +34:36.777 [INFO] TestAgent: Endpoints down +--- PASS: TestPrettyPrintBare (0.96s) +=== CONT TestHTTP_wrap_obfuscateLog +2021-08-13T15:34:35.941Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:35.941Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:35.957Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:35.957Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:35.960 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ef1867e9-d9b7-3b6b-c553-e7f1ac8bf53b Address:127.0.0.1:24476}]" +34:35.960 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24476 [Follower]" leader= +34:35.961 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ef1867e9-d9b7-3b6b-c553-e7f1ac8bf53b.dc1 127.0.0.1 +34:35.962 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ef1867e9-d9b7-3b6b-c553-e7f1ac8bf53b 127.0.0.1 +2021-08-13T15:34:35.962Z [INFO] agent.router: Initializing LAN area manager +34:35.962 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24492 network=udp +34:35.962 [INFO] TestAgent.server: Adding LAN server: server="Node-ef1867e9-d9b7-3b6b-c553-e7f1ac8bf53b (Addr: tcp/127.0.0.1:24476) (DC: dc1)" +34:35.963 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ef1867e9-d9b7-3b6b-c553-e7f1ac8bf53b.dc1 area=wan +34:35.963 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24492 network=tcp +34:35.964 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24472 network=tcp +34:35.964 [INFO] TestAgent: started state syncer +34:35.964 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24477 network=tcp +34:36.013 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:36.026 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:36.026 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24476 [Candidate]" term=2 +34:36.026 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:36.026 [DEBUG] TestAgent.server.raft: vote granted: from=ef1867e9-d9b7-3b6b-c553-e7f1ac8bf53b term=2 tally=1 +34:36.026 [INFO] TestAgent.server.raft: election won: tally=1 +34:36.026 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24476 [Leader]" +34:36.026 [INFO] TestAgent.server: cluster leadership acquired +34:36.027 [INFO] TestAgent.server: initializing acls +34:36.027 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:36.027 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:36.028 [INFO] TestAgent.server: New leader elected: payload=Node-ef1867e9-d9b7-3b6b-c553-e7f1ac8bf53b +34:36.028 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:36.028 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:36.028 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:36.028 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:36.028 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-ef1867e9-d9b7-3b6b-c553-e7f1ac8bf53b +34:36.029 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-ef1867e9-d9b7-3b6b-c553-e7f1ac8bf53b.dc1 +34:36.030 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:36.031 [INFO] TestAgent.server: Updating LAN server: server="Node-ef1867e9-d9b7-3b6b-c553-e7f1ac8bf53b (Addr: tcp/127.0.0.1:24476) (DC: dc1)" +34:36.031 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-ef1867e9-d9b7-3b6b-c553-e7f1ac8bf53b.dc1 area=wan +34:36.033 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:36.033 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:36.033 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:36.033 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:36.033 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:36.033 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ef1867e9-d9b7-3b6b-c553-e7f1ac8bf53b +34:36.033 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ef1867e9-d9b7-3b6b-c553-e7f1ac8bf53b +34:36.034 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ef1867e9-d9b7-3b6b-c553-e7f1ac8bf53b +34:36.034 [INFO] TestAgent.server: federation state anti-entropy synced +34:36.239 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-ef1867e9-d9b7-3b6b-c553-e7f1ac8bf53b +34:36.240 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-ef1867e9-d9b7-3b6b-c553-e7f1ac8bf53b +34:36.278 [INFO] TestAgent: Requesting shutdown +34:36.278 [INFO] TestAgent.server: shutting down server +34:36.279 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:36.279 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:36.279 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:36.279 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:36.279 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:36.279 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:36.279 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:36.279 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:36.279 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:36.279 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:36.279 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:36.279 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:36.279 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:36.279 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:36.279 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:36.279 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:36.279Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:36.279Z [INFO] agent.router.manager: shutting down +34:36.279 [INFO] TestAgent: consul server down +34:36.279 [INFO] TestAgent: shutdown complete +34:36.279 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24492 network=tcp +34:36.279 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24492 network=udp +34:36.279 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24472 network=tcp +34:36.780 [INFO] TestAgent: Waiting for endpoints to shut down +34:36.780 [INFO] TestAgent: Endpoints down +--- PASS: TestACL_LoginProcedure_HTTP (0.86s) --- PASS: TestACL_LoginProcedure_HTTP/AuthMethod (0.01s) --- PASS: TestACL_LoginProcedure_HTTP/AuthMethod/Create (0.00s) --- PASS: TestACL_LoginProcedure_HTTP/AuthMethod/Create_other (0.00s) --- PASS: TestACL_LoginProcedure_HTTP/AuthMethod/Update_Name_URL_Mismatch (0.00s) - --- PASS: TestACL_LoginProcedure_HTTP/AuthMethod/Update (0.00s) + --- PASS: TestACL_LoginProcedure_HTTP/AuthMethod/Update (0.01s) --- PASS: TestACL_LoginProcedure_HTTP/AuthMethod/Invalid_payload (0.00s) --- PASS: TestACL_LoginProcedure_HTTP/AuthMethod/List (0.00s) --- PASS: TestACL_LoginProcedure_HTTP/AuthMethod/Delete (0.00s) --- PASS: TestACL_LoginProcedure_HTTP/AuthMethod/Read (0.00s) - --- PASS: TestACL_LoginProcedure_HTTP/BindingRule (0.01s) + --- PASS: TestACL_LoginProcedure_HTTP/BindingRule (0.02s) --- PASS: TestACL_LoginProcedure_HTTP/BindingRule/Create (0.01s) --- PASS: TestACL_LoginProcedure_HTTP/BindingRule/Create_other (0.00s) --- PASS: TestACL_LoginProcedure_HTTP/BindingRule/BindingRule_CRUD_Missing_ID_in_URL (0.00s) @@ -47454,145 +42330,229 @@ --- PASS: TestACL_LoginProcedure_HTTP/Login/List_Tokens_by_(correct)_Method (0.00s) --- PASS: TestACL_LoginProcedure_HTTP/Login/Logout (0.00s) --- PASS: TestACL_LoginProcedure_HTTP/Login/Token_is_gone_after_Logout (0.00s) -=== CONT TestACLReplicationStatus -2022-09-15T21:43:08.256Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:08.273Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:08.276 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6fb3fe84-fc5c-7893-4eba-0dd19e5b05f5 Address:127.0.0.1:13775}]" -43:08.277 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6fb3fe84-fc5c-7893-4eba-0dd19e5b05f5.dc1 127.0.0.1 -43:08.278 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13775 [Follower]" leader= -43:08.280 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6fb3fe84-fc5c-7893-4eba-0dd19e5b05f5 127.0.0.1 -2022-09-15T21:43:08.280Z [INFO] agent.router: Initializing LAN area manager -43:08.280 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13763 network=udp -43:08.280 [INFO] TestAgent.server: Adding LAN server: server="Node-6fb3fe84-fc5c-7893-4eba-0dd19e5b05f5 (Addr: tcp/127.0.0.1:13775) (DC: dc1)" -43:08.280 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6fb3fe84-fc5c-7893-4eba-0dd19e5b05f5.dc1 area=wan -43:08.281 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13763 network=tcp -43:08.300 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13778 network=tcp -43:08.300 [INFO] TestAgent: started state syncer -43:08.301 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13776 network=tcp -43:08.326 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:08.326 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13775 [Candidate]" term=2 -43:08.326 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:08.326 [DEBUG] TestAgent.server.raft: vote granted: from=6fb3fe84-fc5c-7893-4eba-0dd19e5b05f5 term=2 tally=1 -43:08.326 [INFO] TestAgent.server.raft: election won: tally=1 -43:08.326 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13775 [Leader]" -43:08.327 [INFO] TestAgent.server: cluster leadership acquired -43:08.327 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13775 -43:08.331 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:08.334 [INFO] TestAgent.server: New leader elected: payload=Node-6fb3fe84-fc5c-7893-4eba-0dd19e5b05f5 -43:08.347 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:08.347 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:08.347 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:08.347 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:08.347 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:08.347 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6fb3fe84-fc5c-7893-4eba-0dd19e5b05f5 -43:08.347 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6fb3fe84-fc5c-7893-4eba-0dd19e5b05f5 -43:08.349 [INFO] TestAgent.server: federation state anti-entropy synced -43:08.647 [INFO] TestAgent: Requesting shutdown -43:08.647 [INFO] TestAgent.server: shutting down server -43:08.647 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:08.647 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:08.647 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:08.647 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:08.647 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:08.647 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:08.647 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:08.648 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:08.648 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:08.648 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:08.648 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:08.648 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:08.648Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:08.648Z [INFO] agent.router.manager: shutting down -43:08.648 [INFO] TestAgent: consul server down -43:08.648 [INFO] TestAgent: shutdown complete -43:08.648 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13763 network=tcp -43:08.648 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13763 network=udp -43:08.648 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13778 network=tcp -43:09.150 [INFO] TestAgent: Waiting for endpoints to shut down -43:09.150 [INFO] TestAgent: Endpoints down ---- PASS: TestKVSEndpoint_CAS (0.91s) -=== CONT TestPProfHandlers_ACLs -=== RUN TestACL_Bootstrap/bootstrap -=== RUN TestACL_Bootstrap/not_again -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:08.460Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:08.461Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:08.503Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:08.503Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:08.510 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:114e0622-e92e-4994-8ea6-fa82055f86ca Address:127.0.0.1:13810}]" -43:08.510 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13810 [Follower]" leader= -43:08.511 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-114e0622-e92e-4994-8ea6-fa82055f86ca.dc1 127.0.0.1 -43:08.512 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-114e0622-e92e-4994-8ea6-fa82055f86ca 127.0.0.1 -2022-09-15T21:43:08.512Z [INFO] agent.router: Initializing LAN area manager -43:08.512 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13784 network=udp -43:08.513 [INFO] TestAgent.server: Adding LAN server: server="Node-114e0622-e92e-4994-8ea6-fa82055f86ca (Addr: tcp/127.0.0.1:13810) (DC: dc1)" -43:08.513 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-114e0622-e92e-4994-8ea6-fa82055f86ca.dc1 area=wan -43:08.513 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13784 network=tcp -43:08.518 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13771 network=tcp -43:08.518 [INFO] TestAgent: started state syncer -43:08.518 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13811 network=tcp -43:08.563 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -43:08.573 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:08.573 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13810 [Candidate]" term=2 -43:08.574 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:08.574 [DEBUG] TestAgent.server.raft: vote granted: from=114e0622-e92e-4994-8ea6-fa82055f86ca term=2 tally=1 -43:08.574 [INFO] TestAgent.server.raft: election won: tally=1 -43:08.574 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13810 [Leader]" -43:08.574 [INFO] TestAgent.server: cluster leadership acquired -43:08.574 [INFO] TestAgent.server: initializing acls -43:08.575 [INFO] TestAgent.server: Created ACL 'global-management' policy -43:08.575 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -43:08.576 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -43:08.576 [INFO] TestAgent.server: Created ACL anonymous token from configuration -43:08.576 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -43:08.576 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -43:08.576 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-114e0622-e92e-4994-8ea6-fa82055f86ca -43:08.577 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-114e0622-e92e-4994-8ea6-fa82055f86ca.dc1 -43:08.578 [INFO] TestAgent.server: New leader elected: payload=Node-114e0622-e92e-4994-8ea6-fa82055f86ca -43:08.578 [INFO] TestAgent.server: Updating LAN server: server="Node-114e0622-e92e-4994-8ea6-fa82055f86ca (Addr: tcp/127.0.0.1:13810) (DC: dc1)" -43:08.578 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:08.581 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:08.581 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:08.581 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:08.581 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:08.581 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:08.581 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-114e0622-e92e-4994-8ea6-fa82055f86ca -43:08.581 [INFO] TestAgent.server: member joined, marking health alive: member=Node-114e0622-e92e-4994-8ea6-fa82055f86ca -43:08.582 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-114e0622-e92e-4994-8ea6-fa82055f86ca.dc1 area=wan -43:08.582 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-114e0622-e92e-4994-8ea6-fa82055f86ca -43:08.582 [INFO] TestAgent.server: federation state anti-entropy synced -43:08.680 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-114e0622-e92e-4994-8ea6-fa82055f86ca -43:08.680 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-114e0622-e92e-4994-8ea6-fa82055f86ca -43:08.714 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:08.735 [INFO] TestAgent: Synced node info -43:08.799 [INFO] TestAgent: Requesting shutdown -43:08.799 [INFO] TestAgent.server: shutting down server -43:08.800 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -43:08.800 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:08.800 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:08.800 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:08.800 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:08.800 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -43:08.800 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:08.800 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:08.800 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:08.800 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -43:08.800 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:08.800 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:08.800 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:08.800 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -43:08.800 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:08.800Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:08.800Z [INFO] agent.router.manager: shutting down -43:08.800 [INFO] TestAgent: consul server down -43:08.800 [INFO] TestAgent: shutdown complete -43:08.800 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13784 network=tcp -43:08.800 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13784 network=udp -43:08.800 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13771 network=tcp -43:09.303 [INFO] TestAgent: Waiting for endpoints to shut down -43:09.303 [INFO] TestAgent: Endpoints down ---- PASS: TestACL_HTTP (0.94s) - --- PASS: TestACL_HTTP/Policy (0.02s) +=== CONT TestACL_Bootstrap +2021-08-13T15:34:35.742Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:35.769Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:35.772 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fe58d785-44b0-b3d1-64f5-95c16d1fb5a7 Address:127.0.0.1:24455}]" +34:35.773 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fe58d785-44b0-b3d1-64f5-95c16d1fb5a7.dc1 127.0.0.1 +34:35.774 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fe58d785-44b0-b3d1-64f5-95c16d1fb5a7 127.0.0.1 +2021-08-13T15:34:35.774Z [INFO] agent.router: Initializing LAN area manager +34:35.774 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24455 [Follower]" leader= +34:35.775 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fe58d785-44b0-b3d1-64f5-95c16d1fb5a7.dc1 area=wan +34:35.775 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24429 network=tcp +34:35.775 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24429 network=udp +34:35.775 [INFO] TestAgent.server: Adding LAN server: server="Node-fe58d785-44b0-b3d1-64f5-95c16d1fb5a7 (Addr: tcp/127.0.0.1:24455) (DC: dc1)" +34:35.776 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24479 network=tcp +34:35.776 [INFO] TestAgent: started state syncer +34:35.776 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24456 network=tcp +34:35.827 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:35.839 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:35.839 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24455 [Candidate]" term=2 +34:35.839 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:35.840 [DEBUG] TestAgent.server.raft: vote granted: from=fe58d785-44b0-b3d1-64f5-95c16d1fb5a7 term=2 tally=1 +34:35.840 [INFO] TestAgent.server.raft: election won: tally=1 +34:35.840 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24455 [Leader]" +34:35.840 [INFO] TestAgent.server: cluster leadership acquired +34:35.840 [INFO] TestAgent.server: initializing acls +34:35.841 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:35.842 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:35.842 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:35.842 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:35.843 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:35.843 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-fe58d785-44b0-b3d1-64f5-95c16d1fb5a7 +34:35.843 [INFO] TestAgent.server: New leader elected: payload=Node-fe58d785-44b0-b3d1-64f5-95c16d1fb5a7 +34:35.843 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-fe58d785-44b0-b3d1-64f5-95c16d1fb5a7.dc1 +34:35.843 [INFO] TestAgent.server: Updating LAN server: server="Node-fe58d785-44b0-b3d1-64f5-95c16d1fb5a7 (Addr: tcp/127.0.0.1:24455) (DC: dc1)" +34:35.843 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-fe58d785-44b0-b3d1-64f5-95c16d1fb5a7.dc1 area=wan +34:35.845 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:35.854 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:35.854 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:35.854 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:35.854 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:35.854 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:35.854 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fe58d785-44b0-b3d1-64f5-95c16d1fb5a7 +34:35.854 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fe58d785-44b0-b3d1-64f5-95c16d1fb5a7 +34:35.856 [INFO] TestAgent.server: federation state anti-entropy synced +34:35.856 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fe58d785-44b0-b3d1-64f5-95c16d1fb5a7 +34:35.871 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:35.872 [INFO] TestAgent: Synced node info +34:36.152 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-fe58d785-44b0-b3d1-64f5-95c16d1fb5a7 +34:36.153 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-fe58d785-44b0-b3d1-64f5-95c16d1fb5a7 +34:36.292 [INFO] TestAgent: Requesting shutdown +34:36.292 [INFO] TestAgent.server: shutting down server +34:36.292 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:36.292 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:36.292 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:36.292 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:36.292 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:36.292 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:36.292 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:36.292 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:36.292 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:36.292 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:36.292 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:36.292 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:36.292 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:36.292 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:36.293 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:36.293Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:36.293Z [INFO] agent.router.manager: shutting down +34:36.293 [INFO] TestAgent: consul server down +34:36.293 [INFO] TestAgent: shutdown complete +34:36.293 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24429 network=tcp +34:36.293 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24429 network=udp +34:36.293 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24479 network=tcp +34:36.793 [INFO] TestAgent: Waiting for endpoints to shut down +34:36.793 [INFO] TestAgent: Endpoints down +--- PASS: TestACLEndpoint_LoginLogout_jwt (1.08s) + --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_static_keys (0.01s) + --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_static_keys/invalid_bearer_token (0.00s) + --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_static_keys/valid_bearer_token_no_bindings (0.00s) + --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_static_keys/valid_bearer_token_1_service_binding (0.00s) + --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_jwks (0.06s) + --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_jwks/invalid_bearer_token (0.02s) + --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_jwks/valid_bearer_token_no_bindings (0.02s) + --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_jwks/valid_bearer_token_1_service_binding (0.00s) + --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_oidc_discovery (0.06s) + --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_oidc_discovery/invalid_bearer_token (0.02s) + --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_oidc_discovery/valid_bearer_token_no_bindings (0.00s) + --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_oidc_discovery/valid_bearer_token_1_service_binding (0.00s) +=== CONT TestContentTypeIsJSON +2021-08-13T15:34:36.161Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:36.177Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:36.183 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f8b61fb7-7ba7-f192-a7b1-ff9ca2199a93 Address:127.0.0.1:24497}]" +34:36.185 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f8b61fb7-7ba7-f192-a7b1-ff9ca2199a93.dc1 127.0.0.1 +34:36.185 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f8b61fb7-7ba7-f192-a7b1-ff9ca2199a93 127.0.0.1 +2021-08-13T15:34:36.185Z [INFO] agent.router: Initializing LAN area manager +34:36.186 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24478 network=udp +34:36.186 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24497 [Follower]" leader= +34:36.186 [INFO] TestAgent.server: Adding LAN server: server="Node-f8b61fb7-7ba7-f192-a7b1-ff9ca2199a93 (Addr: tcp/127.0.0.1:24497) (DC: dc1)" +34:36.186 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f8b61fb7-7ba7-f192-a7b1-ff9ca2199a93.dc1 area=wan +34:36.187 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24478 network=tcp +34:36.188 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24493 network=tcp +34:36.188 [INFO] TestAgent: started state syncer +34:36.188 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24484 network=tcp +34:36.251 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:36.251 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24497 [Candidate]" term=2 +34:36.251 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:36.251 [DEBUG] TestAgent.server.raft: vote granted: from=f8b61fb7-7ba7-f192-a7b1-ff9ca2199a93 term=2 tally=1 +34:36.251 [INFO] TestAgent.server.raft: election won: tally=1 +34:36.251 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24497 [Leader]" +34:36.251 [INFO] TestAgent.server: cluster leadership acquired +34:36.252 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24497 +34:36.253 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:36.255 [INFO] TestAgent.server: New leader elected: payload=Node-f8b61fb7-7ba7-f192-a7b1-ff9ca2199a93 +34:36.259 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:36.259 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:36.259 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:36.259 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:36.259 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:36.260 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f8b61fb7-7ba7-f192-a7b1-ff9ca2199a93 +34:36.260 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f8b61fb7-7ba7-f192-a7b1-ff9ca2199a93 +34:36.261 [INFO] TestAgent.server: federation state anti-entropy synced +34:36.299 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/key?pretty=1 from= latency=51.921µs +34:36.299 [INFO] TestAgent: Requesting shutdown +34:36.299 [INFO] TestAgent.server: shutting down server +34:36.299 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:36.299 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:36.299 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:36.299 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:36.299 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:36.299 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:36.299 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:36.300 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:36.300 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:36.300 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:36.300 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:36.300 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:36.300Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:36.300Z [INFO] agent.router.manager: shutting down +34:36.300 [INFO] TestAgent: consul server down +34:36.300 [INFO] TestAgent: shutdown complete +34:36.300 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24478 network=tcp +34:36.300 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24478 network=udp +34:36.300 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24493 network=tcp +34:36.800 [INFO] TestAgent: Waiting for endpoints to shut down +34:36.801 [INFO] TestAgent: Endpoints down +--- PASS: TestPrettyPrint (0.68s) +=== CONT TestACL_Disabled_Response +2021-08-13T15:34:36.314Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:36.314Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:36.334Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:36.334Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:36.337 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0095ea1a-11dd-7242-9aee-1bc94e15cb22 Address:127.0.0.1:24470}]" +34:36.338 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24470 [Follower]" leader= +34:36.339 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0095ea1a-11dd-7242-9aee-1bc94e15cb22.dc1 127.0.0.1 +34:36.340 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0095ea1a-11dd-7242-9aee-1bc94e15cb22 127.0.0.1 +2021-08-13T15:34:36.340Z [INFO] agent.router: Initializing LAN area manager +34:36.340 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24465 network=udp +34:36.340 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0095ea1a-11dd-7242-9aee-1bc94e15cb22.dc1 area=wan +34:36.340 [INFO] TestAgent.server: Adding LAN server: server="Node-0095ea1a-11dd-7242-9aee-1bc94e15cb22 (Addr: tcp/127.0.0.1:24470) (DC: dc1)" +34:36.340 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24465 network=tcp +34:36.341 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24466 network=tcp +34:36.342 [INFO] TestAgent: started state syncer +34:36.342 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24471 network=tcp +34:36.390 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:36.392 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:36.392 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24470 [Candidate]" term=2 +34:36.393 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:36.393 [DEBUG] TestAgent.server.raft: vote granted: from=0095ea1a-11dd-7242-9aee-1bc94e15cb22 term=2 tally=1 +34:36.393 [INFO] TestAgent.server.raft: election won: tally=1 +34:36.393 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24470 [Leader]" +34:36.393 [INFO] TestAgent.server: cluster leadership acquired +34:36.393 [INFO] TestAgent.server: New leader elected: payload=Node-0095ea1a-11dd-7242-9aee-1bc94e15cb22 +34:36.393 [INFO] TestAgent.server: initializing acls +34:36.393 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:36.393 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:36.394 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:36.395 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:36.395 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:36.395 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:36.395 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-0095ea1a-11dd-7242-9aee-1bc94e15cb22 +34:36.395 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-0095ea1a-11dd-7242-9aee-1bc94e15cb22.dc1 +34:36.395 [INFO] TestAgent.server: Updating LAN server: server="Node-0095ea1a-11dd-7242-9aee-1bc94e15cb22 (Addr: tcp/127.0.0.1:24470) (DC: dc1)" +34:36.395 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-0095ea1a-11dd-7242-9aee-1bc94e15cb22.dc1 area=wan +34:36.396 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:36.398 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:36.398 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:36.398 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:36.398 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:36.398 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:36.398 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0095ea1a-11dd-7242-9aee-1bc94e15cb22 +34:36.398 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0095ea1a-11dd-7242-9aee-1bc94e15cb22 +34:36.400 [INFO] TestAgent.server: federation state anti-entropy synced +34:36.400 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0095ea1a-11dd-7242-9aee-1bc94e15cb22 +34:36.427 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-0095ea1a-11dd-7242-9aee-1bc94e15cb22 +34:36.427 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-0095ea1a-11dd-7242-9aee-1bc94e15cb22 +34:36.455 [INFO] TestAgent: Requesting shutdown +34:36.455 [INFO] TestAgent.server: shutting down server +34:36.455 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:36.455 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:36.455 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:36.455 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:36.455 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:36.455 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:36.455 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:36.455 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:36.455 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:36.455 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:36.455 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:36.455 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:36.455 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:36.455 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:36.455 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:36.456 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:36.456Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:36.456Z [INFO] agent.router.manager: shutting down +34:36.456 [INFO] TestAgent: consul server down +34:36.456 [INFO] TestAgent: shutdown complete +34:36.456 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24465 network=tcp +34:36.456 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24465 network=udp +34:36.456 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24466 network=tcp +34:36.956 [INFO] TestAgent: Waiting for endpoints to shut down +34:36.956 [INFO] TestAgent: Endpoints down +--- PASS: TestACL_HTTP (0.66s) + --- PASS: TestACL_HTTP/Policy (0.01s) --- PASS: TestACL_HTTP/Policy/Create (0.00s) --- PASS: TestACL_HTTP/Policy/Minimal (0.00s) --- PASS: TestACL_HTTP/Policy/Name_Chars (0.00s) @@ -47605,9 +42565,9 @@ --- PASS: TestACL_HTTP/Policy/List (0.00s) --- PASS: TestACL_HTTP/Policy/Read (0.00s) --- PASS: TestACL_HTTP/Policy/Read_Name (0.00s) - --- PASS: TestACL_HTTP/Role (0.00s) + --- PASS: TestACL_HTTP/Role (0.01s) --- PASS: TestACL_HTTP/Role/Create (0.00s) - --- PASS: TestACL_HTTP/Role/Name_Chars (0.00s) + --- PASS: TestACL_HTTP/Role/Name_Chars (0.01s) --- PASS: TestACL_HTTP/Role/Update_Name_ID_Mismatch (0.00s) --- PASS: TestACL_HTTP/Role/Role_CRUD_Missing_ID_in_URL (0.00s) --- PASS: TestACL_HTTP/Role/Update (0.00s) @@ -47616,8 +42576,8 @@ --- PASS: TestACL_HTTP/Role/Delete (0.00s) --- PASS: TestACL_HTTP/Role/List (0.00s) --- PASS: TestACL_HTTP/Role/Read (0.00s) - --- PASS: TestACL_HTTP/Token (0.09s) - --- PASS: TestACL_HTTP/Token/Create (0.03s) + --- PASS: TestACL_HTTP/Token (0.01s) + --- PASS: TestACL_HTTP/Token/Create (0.00s) --- PASS: TestACL_HTTP/Token/Create_Local (0.00s) --- PASS: TestACL_HTTP/Token/Read (0.00s) --- PASS: TestACL_HTTP/Token/Self (0.00s) @@ -47625,159 +42585,427 @@ --- PASS: TestACL_HTTP/Token/Update (0.00s) --- PASS: TestACL_HTTP/Token/CRUD_Missing_Token_Accessor_ID (0.00s) --- PASS: TestACL_HTTP/Token/Update_Accessor_Mismatch (0.00s) - --- PASS: TestACL_HTTP/Token/Delete (0.01s) + --- PASS: TestACL_HTTP/Token/Delete (0.00s) --- PASS: TestACL_HTTP/Token/List (0.00s) --- PASS: TestACL_HTTP/Token/List_by_Policy (0.00s) --- PASS: TestACL_HTTP/Token/Create_with_Accessor (0.00s) --- PASS: TestACL_HTTP/Token/Create_with_Secret (0.00s) - --- PASS: TestACL_HTTP/Token/Create_with_Accessor_and_Secret (0.02s) + --- PASS: TestACL_HTTP/Token/Create_with_Accessor_and_Secret (0.00s) --- PASS: TestACL_HTTP/Token/Create_with_Accessor_Dup (0.00s) - --- PASS: TestACL_HTTP/Token/Create_with_Secret_as_Accessor_Dup (0.01s) + --- PASS: TestACL_HTTP/Token/Create_with_Secret_as_Accessor_Dup (0.00s) --- PASS: TestACL_HTTP/Token/Create_with_Secret_Dup (0.00s) --- PASS: TestACL_HTTP/Token/Create_with_Accessor_as_Secret_Dup (0.00s) --- PASS: TestACL_HTTP/Token/Create_with_Reserved_Accessor (0.00s) --- PASS: TestACL_HTTP/Token/Create_with_Reserved_Secret (0.00s) -=== CONT TestParseWait_InvalidIndex ---- PASS: TestParseWait_InvalidIndex (0.00s) -=== CONT TestParseWait_InvalidTime ---- PASS: TestParseWait_InvalidTime (0.00s) -=== CONT TestAgent_InmemKeyrings -=== RUN TestAgent_InmemKeyrings/no_keys -2022-09-15T21:43:08.277Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:08.277Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:08.409Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:08.409Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:08.420 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0086c87a-50b1-0410-ab32-a493911d4f07 Address:127.0.0.1:13789}]" -43:08.421 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0086c87a-50b1-0410-ab32-a493911d4f07.dc1 127.0.0.1 -43:08.454 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13789 [Follower]" leader= -43:08.456 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0086c87a-50b1-0410-ab32-a493911d4f07 127.0.0.1 -2022-09-15T21:43:08.456Z [INFO] agent.router: Initializing LAN area manager -43:08.456 [INFO] TestAgent.server: Adding LAN server: server="Node-0086c87a-50b1-0410-ab32-a493911d4f07 (Addr: tcp/127.0.0.1:13789) (DC: dc1)" -43:08.456 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0086c87a-50b1-0410-ab32-a493911d4f07.dc1 area=wan -43:08.457 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13777 network=udp -43:08.458 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13777 network=tcp -43:08.459 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13785 network=tcp -43:08.459 [INFO] TestAgent: started state syncer -43:08.459 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13790 network=tcp -43:08.498 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:08.498 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13789 [Candidate]" term=2 -43:08.498 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:08.498 [DEBUG] TestAgent.server.raft: vote granted: from=0086c87a-50b1-0410-ab32-a493911d4f07 term=2 tally=1 -43:08.498 [INFO] TestAgent.server.raft: election won: tally=1 -43:08.498 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13789 [Leader]" -43:08.498 [INFO] TestAgent.server: cluster leadership acquired -43:08.499 [INFO] TestAgent.server: initializing acls -43:08.499 [INFO] TestAgent.server: Created ACL 'global-management' policy -43:08.499 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -43:08.500 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -43:08.500 [INFO] TestAgent.server: Created ACL anonymous token from configuration -43:08.500 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -43:08.500 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -43:08.501 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-0086c87a-50b1-0410-ab32-a493911d4f07 -43:08.501 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-0086c87a-50b1-0410-ab32-a493911d4f07.dc1 -43:08.502 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-0086c87a-50b1-0410-ab32-a493911d4f07.dc1 area=wan -43:08.502 [INFO] TestAgent.server: New leader elected: payload=Node-0086c87a-50b1-0410-ab32-a493911d4f07 -43:08.502 [INFO] TestAgent.server: Updating LAN server: server="Node-0086c87a-50b1-0410-ab32-a493911d4f07 (Addr: tcp/127.0.0.1:13789) (DC: dc1)" -43:08.502 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:08.505 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:08.505 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:08.505 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:08.505 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:08.505 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:08.505 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0086c87a-50b1-0410-ab32-a493911d4f07 -43:08.505 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0086c87a-50b1-0410-ab32-a493911d4f07 -43:08.506 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0086c87a-50b1-0410-ab32-a493911d4f07 -43:08.506 [INFO] TestAgent.server: federation state anti-entropy synced -43:08.679 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:08.679 [INFO] TestAgent: Synced node info -43:08.679 [DEBUG] TestAgent: Node info in sync -43:08.841 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-0086c87a-50b1-0410-ab32-a493911d4f07 -43:08.841 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-0086c87a-50b1-0410-ab32-a493911d4f07 -43:08.853 [INFO] TestAgent: Requesting shutdown -43:08.853 [INFO] TestAgent.server: shutting down server -43:08.853 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:08.853 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:08.853 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:08.853 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -43:08.853 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -43:08.853 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:08.853 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:08.853 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:08.853 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:08.854 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:08.854 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:08.854 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:08.854 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -43:08.854 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -43:08.854 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:08.858Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:08.858Z [INFO] agent.router.manager: shutting down -43:08.858 [INFO] TestAgent: consul server down -43:08.858 [INFO] TestAgent: shutdown complete -43:08.858 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13777 network=tcp -43:08.858 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13777 network=udp -43:08.858 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13785 network=tcp -43:09.370 [INFO] TestAgent: Waiting for endpoints to shut down -43:09.370 [INFO] TestAgent: Endpoints down ---- PASS: TestACL_Legacy_List (1.12s) -=== CONT TestKVSEndpoint_Recurse -2022-09-15T21:43:08.798Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:08.842Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:08.845 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6d1af033-7902-c1ac-8cd9-75ca3b3e83f6 Address:127.0.0.1:13747}]" -43:08.846 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13747 [Follower]" leader= -43:08.846 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6d1af033-7902-c1ac-8cd9-75ca3b3e83f6.dc1 127.0.0.1 -43:08.847 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6d1af033-7902-c1ac-8cd9-75ca3b3e83f6 127.0.0.1 -2022-09-15T21:43:08.847Z [INFO] agent.router: Initializing LAN area manager -43:08.848 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6d1af033-7902-c1ac-8cd9-75ca3b3e83f6.dc1 area=wan -43:08.848 [INFO] TestAgent.server: Adding LAN server: server="Node-6d1af033-7902-c1ac-8cd9-75ca3b3e83f6 (Addr: tcp/127.0.0.1:13747) (DC: dc1)" -43:08.852 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13826 network=udp -43:08.852 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13826 network=tcp -43:08.853 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13834 network=tcp -43:08.853 [INFO] TestAgent: started state syncer -43:08.853 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13748 network=tcp -43:08.890 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:08.890 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13747 [Candidate]" term=2 -43:08.890 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:08.890 [DEBUG] TestAgent.server.raft: vote granted: from=6d1af033-7902-c1ac-8cd9-75ca3b3e83f6 term=2 tally=1 -43:08.890 [INFO] TestAgent.server.raft: election won: tally=1 -43:08.890 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13747 [Leader]" -43:08.890 [INFO] TestAgent.server: cluster leadership acquired -43:08.891 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13747 -43:08.892 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:08.894 [INFO] TestAgent.server: New leader elected: payload=Node-6d1af033-7902-c1ac-8cd9-75ca3b3e83f6 -43:08.895 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:08.895 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:08.895 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:08.895 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:08.895 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:08.895 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6d1af033-7902-c1ac-8cd9-75ca3b3e83f6 -43:08.895 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6d1af033-7902-c1ac-8cd9-75ca3b3e83f6 -43:08.896 [INFO] TestAgent.server: federation state anti-entropy synced -43:08.932 [INFO] TestAgent: Requesting shutdown -43:08.932 [INFO] TestAgent.server: shutting down server -43:08.932 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:08.932 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:08.932 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:08.932 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:08.932 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:08.933 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:08.933 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:08.933 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:08.933 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:08.933 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:08.933 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:08.933 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:08.933Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:08.933Z [INFO] agent.router.manager: shutting down -43:08.933 [INFO] TestAgent: consul server down -43:08.933 [INFO] TestAgent: shutdown complete -43:08.933 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13826 network=tcp -43:08.933 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13826 network=udp -43:08.933 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13834 network=tcp -43:09.434 [INFO] TestAgent: Waiting for endpoints to shut down -43:09.434 [INFO] TestAgent: Endpoints down -=== CONT TestKVSEndpoint_PUT_GET_DELETE ---- PASS: TestACL_Disabled_Response (0.70s) +=== CONT TestUIResponseHeaders +=== RUN TestHTTP_wrap_obfuscateLog//some/url?token=secret1&token=secret2 +=== RUN TestHTTP_wrap_obfuscateLog//v1/acl/clone/secret1 +=== RUN TestHTTP_wrap_obfuscateLog//v1/acl/clone/secret1?token=secret2 +=== RUN TestHTTP_wrap_obfuscateLog//v1/acl/destroy/secret1 +=== RUN TestHTTP_wrap_obfuscateLog//v1/acl/destroy/secret1?token=secret2 +=== RUN TestHTTP_wrap_obfuscateLog//v1/acl/info/secret1 +=== RUN TestHTTP_wrap_obfuscateLog//v1/acl/info/secret1?token=secret2 +[WARN] freeport: 10 out of 46 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:35.635Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:35.655Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:35.658 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e5390df7-d05c-1eb4-2b6e-6cca53105055 Address:127.0.0.1:24420}]" +34:35.658 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e5390df7-d05c-1eb4-2b6e-6cca53105055.dc2 127.0.0.1 +34:35.659 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24420 [Follower]" leader= +34:35.660 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e5390df7-d05c-1eb4-2b6e-6cca53105055 127.0.0.1 +2021-08-13T15:34:35.660Z [INFO] agent.router: Initializing LAN area manager +34:35.660 [INFO] TestAgent.server: Adding LAN server: server="Node-e5390df7-d05c-1eb4-2b6e-6cca53105055 (Addr: tcp/127.0.0.1:24420) (DC: dc2)" +34:35.660 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e5390df7-d05c-1eb4-2b6e-6cca53105055.dc2 area=wan +34:35.660 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24464 network=udp +34:35.660 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24464 network=tcp +34:35.661 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24451 network=tcp +34:35.661 [INFO] TestAgent: started state syncer +34:35.661 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24428 network=tcp +34:35.711 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs, servers in acl datacenter are not yet upgraded: ACLDatacenter=dc1 mode=1 found=false +34:35.727 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:35.727 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24420 [Candidate]" term=2 +34:35.727 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:35.727 [DEBUG] TestAgent.server.raft: vote granted: from=e5390df7-d05c-1eb4-2b6e-6cca53105055 term=2 tally=1 +34:35.727 [INFO] TestAgent.server.raft: election won: tally=1 +34:35.727 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24420 [Leader]" +34:35.727 [INFO] TestAgent.server: cluster leadership acquired +34:35.728 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs, servers in acl datacenter are not yet upgraded: ACLDatacenter=dc1 mode=1 found=false +34:35.728 [INFO] TestAgent.leader: started routine: routine="legacy ACL replication" +34:35.728 [INFO] TestAgent.server: started legacy ACL replication +34:35.728 [WARN] TestAgent.server.connect: primary datacenter is configured but unreachable - deferring initialization of the secondary datacenter CA +34:35.728 [INFO] TestAgent.leader: started routine: routine="config entry replication" +34:35.728 [INFO] TestAgent.leader: started routine: routine="federation state replication" +34:35.728 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:35.728 [INFO] TestAgent.leader: started routine: routine="secondary CA roots watch" +34:35.728 [INFO] TestAgent.leader: started routine: routine="intention replication" +34:35.728 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:35.728 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:35.729 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e5390df7-d05c-1eb4-2b6e-6cca53105055 +34:35.729 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e5390df7-d05c-1eb4-2b6e-6cca53105055 +34:35.730 [INFO] TestAgent.server: New leader elected: payload=Node-e5390df7-d05c-1eb4-2b6e-6cca53105055 +34:35.730 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ACL.List +34:35.730 [WARN] TestAgent.server.replication.acl.legacy: Legacy ACL replication error (will retry if still leader): error="failed to retrieve remote ACLs: No path to datacenter" +34:35.730 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ACL.List +34:35.730 [WARN] TestAgent.server.replication.acl.legacy: Legacy ACL replication error (will retry if still leader): error="failed to retrieve remote ACLs: No path to datacenter" +34:35.730 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ACL.List +34:35.730 [WARN] TestAgent.server.replication.acl.legacy: Legacy ACL replication error (will retry if still leader): error="failed to retrieve remote ACLs: No path to datacenter" +34:35.730 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ACL.List +34:35.730 [WARN] TestAgent.server.replication.acl.legacy: Legacy ACL replication error (will retry if still leader): error="failed to retrieve remote ACLs: No path to datacenter" +34:35.730 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ACL.List +34:35.730 [WARN] TestAgent.server.replication.acl.legacy: Legacy ACL replication error (will retry if still leader): error="failed to retrieve remote ACLs: No path to datacenter" +34:35.730 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ConfigEntry.ListAll +34:35.730 [DEBUG] TestAgent.server: federation states are not enabled in the primary dc +34:35.730 [DEBUG] TestAgent.server: federation states are not enabled in the primary dc +34:35.730 [DEBUG] TestAgent.server: federation states are not enabled in the primary dc +34:35.730 [DEBUG] TestAgent.server: federation states are not enabled in the primary dc +34:35.730 [DEBUG] TestAgent.server: federation states are not enabled in the primary dc +34:35.730 [DEBUG] TestAgent.server: federation states are not enabled in the primary dc +34:35.730 [DEBUG] TestAgent.server.connect: starting Connect CA root replication from primary datacenter: primary=dc1 +34:35.730 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ConnectCA.Roots +34:35.730 [ERROR] TestAgent.server.connect: CA root replication failed, will retry: routine="secondary CA roots watch" error="Error retrieving the primary datacenter's roots: No path to datacenter" +34:35.730 [DEBUG] TestAgent.server.connect: starting Connect intention replication from primary datacenter: primary=dc1 +34:35.730 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=Intention.List +34:35.730 [ERROR] TestAgent.server.connect: error replicating intentions: routine="intention replication" error="No path to datacenter" +34:35.811 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs, servers in acl datacenter are not yet upgraded: ACLDatacenter=dc1 mode=1 found=false +34:35.939 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ACL.GetPolicy +34:35.939 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth +34:35.939 [WARN] TestAgent: Node info update blocked by ACLs: node=e5390df7-d05c-1eb4-2b6e-6cca53105055 accessorID=legacy-token +34:35.940 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ACL.GetPolicy +34:35.940 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-e5390df7-d05c-1eb4-2b6e-6cca53105055 +34:35.940 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:24433] +34:35.940 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:24433 +34:35.942 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ebab9b96-d633-a64e-239e-e860eb6145c5.dc1 127.0.0.1 +34:35.942 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 +34:35.942 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-e5390df7-d05c-1eb4-2b6e-6cca53105055 +34:35.942 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ebab9b96-d633-a64e-239e-e860eb6145c5.dc1 area=wan +34:35.968 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-e5390df7-d05c-1eb4-2b6e-6cca53105055 +34:35.994 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-e5390df7-d05c-1eb4-2b6e-6cca53105055 +34:36.011 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL replication" +34:36.011 [INFO] TestAgent.leader: started routine: routine="ACL policy replication" +34:36.011 [INFO] TestAgent.leader: started routine: routine="ACL role replication" +34:36.011 [INFO] TestAgent.leader: started routine: routine="ACL token replication" +34:36.011 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:36.011 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode +34:36.012 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-e5390df7-d05c-1eb4-2b6e-6cca53105055 +34:36.012 [INFO] TestAgent.server.replication.acl.role: started ACL Role replication +34:36.012 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-e5390df7-d05c-1eb4-2b6e-6cca53105055.dc2 +34:36.012 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-e5390df7-d05c-1eb4-2b6e-6cca53105055.dc2 area=wan +34:36.012 [INFO] TestAgent.server.replication.acl.token: started ACL Token replication +34:36.012 [INFO] TestAgent.server: Updating LAN server: server="Node-e5390df7-d05c-1eb4-2b6e-6cca53105055 (Addr: tcp/127.0.0.1:24420) (DC: dc2)" +34:36.012 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL replication" +34:36.012 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e5390df7-d05c-1eb4-2b6e-6cca53105055 +34:36.012 [INFO] TestAgent.server.replication.acl.policy: started ACL Policy replication +34:36.014 [DEBUG] TestAgent.server.replication.acl.token: finished fetching acls: amount=3 +34:36.014 [DEBUG] TestAgent.server.replication.acl.token: acl replication: local=0 remote=3 +34:36.014 [DEBUG] TestAgent.server.replication.acl.token: acl replication: deletions=0 updates=3 +34:36.015 [DEBUG] TestAgent.server.replication.acl.policy: finished fetching acls: amount=2 +34:36.015 [DEBUG] TestAgent.server.replication.acl.policy: acl replication: local=0 remote=2 +34:36.015 [DEBUG] TestAgent.server.replication.acl.policy: acl replication: deletions=0 updates=2 +34:36.015 [DEBUG] TestAgent.server.replication.acl.role: finished fetching acls: amount=0 +34:36.015 [DEBUG] TestAgent.server.replication.acl.role: acl replication: local=0 remote=0 +34:36.015 [DEBUG] TestAgent.server.replication.acl.role: acl replication: deletions=0 updates=0 +34:36.015 [DEBUG] TestAgent.server.replication.acl.role: ACL replication completed through remote index: index=1 +34:36.016 [DEBUG] TestAgent.server.replication.acl.token: acl replication - downloaded updates: amount=3 +34:36.016 [DEBUG] TestAgent.server.replication.acl.token: acl replication - performing updates +34:36.016 [DEBUG] TestAgent.server.replication.acl.policy: acl replication - downloaded updates: amount=2 +34:36.016 [DEBUG] TestAgent.server.replication.acl.policy: acl replication - performing updates +34:36.017 [DEBUG] TestAgent.server.replication.acl.token: acl replication - upserted batch: number_upserted=3 batch_size=442 +34:36.017 [DEBUG] TestAgent.server.replication.acl.token: acl replication - finished updates +34:36.017 [DEBUG] TestAgent.server.replication.acl.token: ACL replication completed through remote index: index=16 +34:36.017 [DEBUG] TestAgent.server.replication.acl.policy: acl replication - upserted batch: number_upserted=2 batch_size=675 +34:36.017 [DEBUG] TestAgent.server.replication.acl.policy: acl replication - finished updates +34:36.017 [DEBUG] TestAgent.server.replication.acl.policy: ACL replication completed through remote index: index=15 +34:36.033 [INFO] TestAgent: Requesting shutdown +34:36.033 [INFO] TestAgent.server: shutting down server +34:36.033 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:36.033 [DEBUG] TestAgent.leader: stopping routine: routine="intention replication" +34:36.033 [DEBUG] TestAgent.leader: stopping routine: routine="ACL policy replication" +34:36.033 [DEBUG] TestAgent.leader: stopping routine: routine="config entry replication" +34:36.033 [DEBUG] TestAgent.leader: stopping routine: routine="federation state replication" +34:36.033 [DEBUG] TestAgent.leader: stopping routine: routine="ACL role replication" +34:36.033 [DEBUG] TestAgent.leader: stopping routine: routine="ACL token replication" +34:36.033 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:36.033 [DEBUG] TestAgent.leader: stopping routine: routine="secondary CA roots watch" +34:36.033 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:36.033 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:36.033 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:36.033 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:36.033 [DEBUG] TestAgent.leader: stopped routine: routine="intention replication" +34:36.033 [INFO] TestAgent.server.replication.config_entry: stopped replication +34:36.033 [DEBUG] TestAgent.leader: stopped routine: routine="config entry replication" +34:36.033 [INFO] TestAgent.replication.federation_state: stopped replication +34:36.033 [DEBUG] TestAgent.leader: stopped routine: routine="federation state replication" +34:36.033 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:36.033 [DEBUG] TestAgent.leader: stopped routine: routine="secondary CA roots watch" +34:36.033 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:36.033 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:36.033 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:36.033Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:36.033Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:36.033Z [INFO] agent.router.manager: shutting down +34:36.034 [INFO] TestAgent: consul server down +34:36.034 [INFO] TestAgent: shutdown complete +34:36.034 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24464 network=tcp +34:36.034 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24464 network=udp +34:36.034 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24451 network=tcp +34:36.034 [ERROR] TestAgent.server.rpc: RPC failed to server in DC: server=127.0.0.1:24434 datacenter=dc1 method=ACL.RoleList error="rpc error making call: EOF" +34:36.034 [WARN] TestAgent.server.replication.acl.role: ACL replication error (will retry if still leader): error="failed to retrieve remote ACL roles: rpc error making call: EOF" +34:36.034 [DEBUG] TestAgent.leader: stopped routine: routine="ACL role replication" +34:36.034 [ERROR] TestAgent.server.rpc: RPC failed to server in DC: server=127.0.0.1:24434 datacenter=dc1 method=ACL.PolicyList error="rpc error making call: EOF" +34:36.034 [WARN] TestAgent.server.replication.acl.policy: ACL replication error (will retry if still leader): error="failed to retrieve remote ACL policies: rpc error making call: EOF" +34:36.034 [DEBUG] TestAgent.leader: stopped routine: routine="ACL policy replication" +34:36.034 [ERROR] TestAgent.server.rpc: RPC failed to server in DC: server=127.0.0.1:24434 datacenter=dc1 method=ACL.TokenList error="rpc error making call: EOF" +34:36.034 [WARN] TestAgent.server.replication.acl.token: ACL replication error (will retry if still leader): error="failed to retrieve remote ACL tokens: rpc error making call: EOF" +34:36.034 [DEBUG] TestAgent.leader: stopped routine: routine="ACL token replication" +34:36.534 [INFO] TestAgent: Waiting for endpoints to shut down +34:36.534 [INFO] TestAgent: Endpoints down +2021-08-13T15:34:35.360Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:35.378Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:35.389 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ebab9b96-d633-a64e-239e-e860eb6145c5 Address:127.0.0.1:24434}]" +34:35.390 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ebab9b96-d633-a64e-239e-e860eb6145c5.dc1 127.0.0.1 +34:35.391 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24434 [Follower]" leader= +34:35.392 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ebab9b96-d633-a64e-239e-e860eb6145c5 127.0.0.1 +2021-08-13T15:34:35.392Z [INFO] agent.router: Initializing LAN area manager +34:35.392 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24422 network=udp +34:35.392 [INFO] TestAgent.server: Adding LAN server: server="Node-ebab9b96-d633-a64e-239e-e860eb6145c5 (Addr: tcp/127.0.0.1:24434) (DC: dc1)" +34:35.393 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ebab9b96-d633-a64e-239e-e860eb6145c5.dc1 area=wan +34:35.393 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24422 network=tcp +34:35.394 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24402 network=tcp +34:35.394 [INFO] TestAgent: started state syncer +34:35.394 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24435 network=tcp +34:35.443 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:35.454 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:35.454 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24434 [Candidate]" term=2 +34:35.455 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:35.455 [DEBUG] TestAgent.server.raft: vote granted: from=ebab9b96-d633-a64e-239e-e860eb6145c5 term=2 tally=1 +34:35.455 [INFO] TestAgent.server.raft: election won: tally=1 +34:35.455 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24434 [Leader]" +34:35.455 [INFO] TestAgent.server: cluster leadership acquired +34:35.455 [INFO] TestAgent.server: New leader elected: payload=Node-ebab9b96-d633-a64e-239e-e860eb6145c5 +34:35.455 [INFO] TestAgent.server: initializing acls +34:35.456 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:35.456 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:35.457 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:35.457 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:35.457 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:35.457 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-ebab9b96-d633-a64e-239e-e860eb6145c5 +34:35.457 [INFO] TestAgent.server: Updating LAN server: server="Node-ebab9b96-d633-a64e-239e-e860eb6145c5 (Addr: tcp/127.0.0.1:24434) (DC: dc1)" +34:35.457 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-ebab9b96-d633-a64e-239e-e860eb6145c5.dc1 +34:35.458 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-ebab9b96-d633-a64e-239e-e860eb6145c5.dc1 area=wan +34:35.458 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:35.461 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:35.461 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:35.461 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:35.461 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:35.461 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:35.461 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ebab9b96-d633-a64e-239e-e860eb6145c5 +34:35.461 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ebab9b96-d633-a64e-239e-e860eb6145c5 +34:35.462 [INFO] TestAgent.server: federation state anti-entropy synced +34:35.462 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ebab9b96-d633-a64e-239e-e860eb6145c5 +34:35.616 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-ebab9b96-d633-a64e-239e-e860eb6145c5 +34:35.653 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:35.654 [INFO] TestAgent: Synced node info +34:35.654 [DEBUG] TestAgent: Node info in sync +34:35.775 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:35.775 [DEBUG] TestAgent: Node info in sync +34:35.941 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:46584 +34:35.941 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e5390df7-d05c-1eb4-2b6e-6cca53105055.dc2 127.0.0.1 +34:35.941 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e5390df7-d05c-1eb4-2b6e-6cca53105055.dc2 area=wan +34:36.534 [INFO] TestAgent: Requesting shutdown +34:36.534 [INFO] TestAgent.server: shutting down server +34:36.534 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:36.534 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:36.534 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:36.534 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:36.534 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:36.534 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:36.534 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:36.534 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:36.534 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:36.534 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:36.534 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:36.534 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:36.534 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:36.534 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:36.534 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:36.535Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:36.535Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:36.535Z [INFO] agent.router.manager: shutting down +34:36.535 [INFO] TestAgent: consul server down +34:36.535 [INFO] TestAgent: shutdown complete +34:36.535 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24422 network=tcp +34:36.535 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24422 network=udp +34:36.535 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24402 network=tcp +34:37.035 [INFO] TestAgent: Waiting for endpoints to shut down +34:37.035 [INFO] TestAgent: Endpoints down +--- PASS: TestACL_Authorize (1.69s) + --- PASS: TestACL_Authorize/master-token (0.00s) + --- PASS: TestACL_Authorize/master-token/dc1 (0.00s) + --- PASS: TestACL_Authorize/master-token/dc2 (0.00s) + --- PASS: TestACL_Authorize/custom-token (0.00s) + --- PASS: TestACL_Authorize/custom-token/dc1 (0.00s) + --- PASS: TestACL_Authorize/custom-token/dc2 (0.00s) + --- PASS: TestACL_Authorize/too-many-requests (0.00s) + --- PASS: TestACL_Authorize/decode-failure (0.00s) + --- PASS: TestACL_Authorize/acl-not-found (0.00s) + --- PASS: TestACL_Authorize/local-token-in-secondary-dc (0.01s) + --- PASS: TestACL_Authorize/local-token-wrong-dc (0.00s) +=== CONT TestACLReplicationStatus +=== RUN TestACL_Disabled_Response/ACLBootstrap +=== RUN TestACL_Disabled_Response/ACLReplicationStatus +=== RUN TestACL_Disabled_Response/AgentToken +=== RUN TestACL_Disabled_Response/ACLRulesTranslate +=== RUN TestACL_Disabled_Response/ACLRulesTranslateLegacyToken +=== RUN TestACL_Disabled_Response/ACLPolicyList +=== RUN TestACL_Disabled_Response/ACLPolicyCRUD +=== RUN TestACL_Disabled_Response/ACLPolicyCreate +=== RUN TestACL_Disabled_Response/ACLTokenList +=== RUN TestACL_Disabled_Response/ACLTokenCreate +=== RUN TestACL_Disabled_Response/ACLTokenSelf +=== RUN TestACL_Disabled_Response/ACLTokenCRUD +=== RUN TestACL_Disabled_Response/ACLRoleList +=== RUN TestACL_Disabled_Response/ACLRoleCreate +=== RUN TestACL_Disabled_Response/ACLRoleCRUD +=== RUN TestACL_Disabled_Response/ACLBindingRuleList +=== RUN TestACL_Disabled_Response/ACLBindingRuleCreate +=== RUN TestACL_Disabled_Response/ACLBindingRuleCRUD +=== RUN TestACL_Disabled_Response/ACLAuthMethodList +=== RUN TestACL_Disabled_Response/ACLAuthMethodCreate +=== RUN TestACL_Disabled_Response/ACLAuthMethodCRUD +=== RUN TestACL_Disabled_Response/ACLLogin +=== RUN TestACL_Disabled_Response/ACLLogout +=== RUN TestACL_Disabled_Response/ACLAuthorize +2021-08-13T15:34:35.223Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:35.240Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:35.243 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b88c7718-0380-9028-baeb-1ccb1491fc00 Address:127.0.0.1:24413}]" +34:35.243 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24413 [Follower]" leader= +34:35.244 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b88c7718-0380-9028-baeb-1ccb1491fc00.dc1 127.0.0.1 +34:35.245 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b88c7718-0380-9028-baeb-1ccb1491fc00 127.0.0.1 +2021-08-13T15:34:35.245Z [INFO] agent.router: Initializing LAN area manager +34:35.245 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b88c7718-0380-9028-baeb-1ccb1491fc00.dc1 area=wan +34:35.245 [INFO] TestAgent.server: Adding LAN server: server="Node-b88c7718-0380-9028-baeb-1ccb1491fc00 (Addr: tcp/127.0.0.1:24413) (DC: dc1)" +34:35.245 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24408 network=udp +34:35.245 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24408 network=tcp +34:35.246 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24423 network=tcp +34:35.246 [INFO] TestAgent: started state syncer +34:35.246 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24421 network=tcp +34:35.311 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:35.311 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24413 [Candidate]" term=2 +34:35.311 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:35.311 [DEBUG] TestAgent.server.raft: vote granted: from=b88c7718-0380-9028-baeb-1ccb1491fc00 term=2 tally=1 +34:35.311 [INFO] TestAgent.server.raft: election won: tally=1 +34:35.312 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24413 [Leader]" +34:35.312 [INFO] TestAgent.server: cluster leadership acquired +34:35.312 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24413 +34:35.316 [INFO] TestAgent.server: New leader elected: payload=Node-b88c7718-0380-9028-baeb-1ccb1491fc00 +34:35.323 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:35.339 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:35.339 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:35.339 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:35.339 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:35.339 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:35.339 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b88c7718-0380-9028-baeb-1ccb1491fc00 +34:35.339 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b88c7718-0380-9028-baeb-1ccb1491fc00 +34:35.340 [INFO] TestAgent.server: federation state anti-entropy synced +34:35.403 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:35.404 [INFO] TestAgent: Synced node info +34:35.748 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:35.748 [DEBUG] TestAgent: Node info in sync +34:35.748 [DEBUG] TestAgent: Node info in sync +34:36.680 [INFO] TestAgent: Requesting shutdown +34:36.681 [INFO] TestAgent.server: shutting down server +34:36.681 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:36.681 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:36.681 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:36.681 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:36.681 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:36.681 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:36.681 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:36.681 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:36.681 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:36.681 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:36.681 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:36.681Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:36.681Z [INFO] agent.router.manager: shutting down +34:36.681 [INFO] TestAgent: consul server down +34:36.681 [INFO] TestAgent: shutdown complete +34:36.681 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24408 network=tcp +34:36.681 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24408 network=udp +34:36.681 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24423 network=tcp +34:37.182 [INFO] TestAgent: Waiting for endpoints to shut down +34:37.182 [INFO] TestAgent: Endpoints down +--- PASS: TestPProfHandlers_EnableDebug (1.98s) +=== CONT TestHTTPAPIResponseHeaders +=== RUN TestACL_Bootstrap/bootstrap +=== RUN TestACL_Bootstrap/not_again +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +--- PASS: TestHTTP_wrap_obfuscateLog (0.70s) + --- PASS: TestHTTP_wrap_obfuscateLog//some/url?token=secret1&token=secret2 (0.00s) + --- PASS: TestHTTP_wrap_obfuscateLog//v1/acl/clone/secret1 (0.00s) + --- PASS: TestHTTP_wrap_obfuscateLog//v1/acl/clone/secret1?token=secret2 (0.00s) + --- PASS: TestHTTP_wrap_obfuscateLog//v1/acl/destroy/secret1 (0.00s) + --- PASS: TestHTTP_wrap_obfuscateLog//v1/acl/destroy/secret1?token=secret2 (0.00s) + --- PASS: TestHTTP_wrap_obfuscateLog//v1/acl/info/secret1 (0.00s) + --- PASS: TestHTTP_wrap_obfuscateLog//v1/acl/info/secret1?token=secret2 (0.00s) +=== CONT TestACL_Legacy_List +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:36.878Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:36.906Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:36.909 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1e974df2-b171-7809-1074-3f64f681f002 Address:127.0.0.1:24529}]" +34:36.909 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24529 [Follower]" leader= +34:36.910 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1e974df2-b171-7809-1074-3f64f681f002.dc1 127.0.0.1 +34:36.910 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1e974df2-b171-7809-1074-3f64f681f002 127.0.0.1 +2021-08-13T15:34:36.910Z [INFO] agent.router: Initializing LAN area manager +34:36.911 [INFO] TestAgent.server: Adding LAN server: server="Node-1e974df2-b171-7809-1074-3f64f681f002 (Addr: tcp/127.0.0.1:24529) (DC: dc1)" +34:36.911 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1e974df2-b171-7809-1074-3f64f681f002.dc1 area=wan +34:36.911 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24503 network=udp +34:36.911 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24503 network=tcp +34:36.912 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24525 network=tcp +34:36.912 [INFO] TestAgent: started state syncer +34:36.912 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24530 network=tcp +34:36.971 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:36.971 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24529 [Candidate]" term=2 +34:36.972 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:36.972 [DEBUG] TestAgent.server.raft: vote granted: from=1e974df2-b171-7809-1074-3f64f681f002 term=2 tally=1 +34:36.972 [INFO] TestAgent.server.raft: election won: tally=1 +34:36.972 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24529 [Leader]" +34:36.972 [INFO] TestAgent.server: cluster leadership acquired +34:36.972 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24529 +34:36.973 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:36.975 [INFO] TestAgent.server: New leader elected: payload=Node-1e974df2-b171-7809-1074-3f64f681f002 +34:36.976 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:36.976 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:36.976 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:36.976 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:36.976 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:36.976 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1e974df2-b171-7809-1074-3f64f681f002 +34:36.976 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1e974df2-b171-7809-1074-3f64f681f002 +34:36.977 [INFO] TestAgent.server: federation state anti-entropy synced +34:37.015 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:37.016 [INFO] TestAgent: Synced node info +34:37.016 [DEBUG] TestAgent: Node info in sync +34:37.146 [INFO] TestAgent: Requesting shutdown +34:37.146 [INFO] TestAgent.server: shutting down server +34:37.146 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:37.146 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:37.146 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:37.146 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:37.146 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:37.146 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:37.146 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:37.146 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:37.146 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:37.146 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:37.146 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:37.146Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:37.146Z [INFO] agent.router.manager: shutting down +34:37.146 [INFO] TestAgent: consul server down +34:37.146 [INFO] TestAgent: shutdown complete +34:37.147 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24503 network=tcp +34:37.147 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24503 network=udp +34:37.147 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24525 network=tcp +34:37.647 [INFO] TestAgent: Waiting for endpoints to shut down +34:37.647 [INFO] TestAgent: Endpoints down +--- PASS: TestACL_Disabled_Response (0.83s) --- PASS: TestACL_Disabled_Response/ACLBootstrap (0.00s) --- PASS: TestACL_Disabled_Response/ACLReplicationStatus (0.00s) --- PASS: TestACL_Disabled_Response/AgentToken (0.00s) @@ -47802,8836 +43030,13585 @@ --- PASS: TestACL_Disabled_Response/ACLLogin (0.00s) --- PASS: TestACL_Disabled_Response/ACLLogout (0.00s) --- PASS: TestACL_Disabled_Response/ACLAuthorize (0.00s) -[WARN] freeport: 4 out of 25 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestPProfHandlers_ACLs/case_0_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:200,_token:"master",_endpoint:"/debug/pprof/heap",_nilResponse:false}) -2022-09-15T21:43:08.745Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:08.745Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:08.821Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:08.821Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:08.832 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:425b03d1-1a7a-b734-8640-946885a1da20 Address:127.0.0.1:13824}]" -43:08.833 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-425b03d1-1a7a-b734-8640-946885a1da20.dc1 127.0.0.1 -43:08.834 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13824 [Follower]" leader= -43:08.839 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-425b03d1-1a7a-b734-8640-946885a1da20 127.0.0.1 -2022-09-15T21:43:08.839Z [INFO] agent.router: Initializing LAN area manager -43:08.839 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13812 network=udp -43:08.839 [INFO] TestAgent.server: Adding LAN server: server="Node-425b03d1-1a7a-b734-8640-946885a1da20 (Addr: tcp/127.0.0.1:13824) (DC: dc1)" -43:08.839 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-425b03d1-1a7a-b734-8640-946885a1da20.dc1 area=wan -43:08.840 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13812 network=tcp -43:08.841 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13813 network=tcp -43:08.841 [INFO] TestAgent: started state syncer -43:08.841 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13825 network=tcp -43:08.875 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:08.875 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13824 [Candidate]" term=2 -43:08.875 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:08.875 [DEBUG] TestAgent.server.raft: vote granted: from=425b03d1-1a7a-b734-8640-946885a1da20 term=2 tally=1 -43:08.875 [INFO] TestAgent.server.raft: election won: tally=1 -43:08.875 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13824 [Leader]" -43:08.875 [INFO] TestAgent.server: cluster leadership acquired -43:08.876 [INFO] TestAgent.server: initializing acls -43:08.876 [INFO] TestAgent.server: Created ACL 'global-management' policy -43:08.877 [INFO] TestAgent.server: Created ACL anonymous token from configuration -43:08.877 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -43:08.877 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -43:08.877 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-425b03d1-1a7a-b734-8640-946885a1da20 -43:08.877 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-425b03d1-1a7a-b734-8640-946885a1da20.dc1 -43:08.878 [INFO] TestAgent.server: New leader elected: payload=Node-425b03d1-1a7a-b734-8640-946885a1da20 -43:08.878 [INFO] TestAgent.server: Updating LAN server: server="Node-425b03d1-1a7a-b734-8640-946885a1da20 (Addr: tcp/127.0.0.1:13824) (DC: dc1)" -43:08.878 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-425b03d1-1a7a-b734-8640-946885a1da20.dc1 area=wan -43:08.879 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:08.881 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:08.881 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:08.881 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:08.881 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:08.881 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:08.881 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-425b03d1-1a7a-b734-8640-946885a1da20 -43:08.881 [INFO] TestAgent.server: member joined, marking health alive: member=Node-425b03d1-1a7a-b734-8640-946885a1da20 -43:08.894 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="ACL not found" -43:08.897 [INFO] TestAgent.server: federation state anti-entropy synced -43:08.898 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-425b03d1-1a7a-b734-8640-946885a1da20 -43:09.178 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-425b03d1-1a7a-b734-8640-946885a1da20 -43:09.178 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-425b03d1-1a7a-b734-8640-946885a1da20 -43:09.178 [WARN] TestAgent.server.acl: failed to remove bootstrap file: error="remove /tmp/consul-test/TestACL_Bootstrap-TestAgent-agent269989625/acl-bootstrap-reset: no such file or directory" -43:09.179 [INFO] TestAgent.server.acl: ACL bootstrap completed -43:09.179 [INFO] TestAgent: Requesting shutdown -43:09.180 [INFO] TestAgent.server: shutting down server -43:09.180 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:09.180 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:09.180 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:09.180 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:09.180 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -43:09.180 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -43:09.180 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:09.180 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -43:09.180 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:09.180 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:09.180 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:09.180 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:09.180 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:09.180 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -43:09.180 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:09.180Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:09.180Z [INFO] agent.router.manager: shutting down -43:09.180 [INFO] TestAgent: consul server down -43:09.180 [INFO] TestAgent: shutdown complete -43:09.180 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13812 network=tcp -43:09.180 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13812 network=udp -43:09.181 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13813 network=tcp -43:09.682 [INFO] TestAgent: Waiting for endpoints to shut down -43:09.682 [INFO] TestAgent: Endpoints down ---- PASS: TestACL_Bootstrap (1.01s) - --- PASS: TestACL_Bootstrap/bootstrap (0.00s) - --- PASS: TestACL_Bootstrap/not_again (0.00s) -=== CONT TestAgentKeyring_ACL -=== RUN TestPProfHandlers_ACLs/case_1_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:403,_token:"agent",_endpoint:"/debug/pprof/heap",_nilResponse:true}) -=== RUN TestPProfHandlers_ACLs/case_2_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:403,_token:"agent",_endpoint:"/debug/pprof/",_nilResponse:true}) -=== RUN TestPProfHandlers_ACLs/case_3_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:403,_token:"",_endpoint:"/debug/pprof/",_nilResponse:true}) -=== RUN TestPProfHandlers_ACLs/case_4_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:200,_token:"master",_endpoint:"/debug/pprof/heap",_nilResponse:false}) -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestPProfHandlers_ACLs/case_5_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:403,_token:"towel",_endpoint:"/debug/pprof/heap",_nilResponse:true}) -2022-09-15T21:43:09.174Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:09.174Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:09.216Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:09.216Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:09.228 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e4fb1f46-ff83-d59a-2a57-268824af035f Address:127.0.0.1:13803}]" -43:09.229 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e4fb1f46-ff83-d59a-2a57-268824af035f.dc1 127.0.0.1 -43:09.229 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13803 [Follower]" leader= -43:09.236 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e4fb1f46-ff83-d59a-2a57-268824af035f 127.0.0.1 -2022-09-15T21:43:09.237Z [INFO] agent.router: Initializing LAN area manager -43:09.237 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13749 network=udp -43:09.237 [INFO] TestAgent.server: Adding LAN server: server="Node-e4fb1f46-ff83-d59a-2a57-268824af035f (Addr: tcp/127.0.0.1:13803) (DC: dc1)" -43:09.237 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e4fb1f46-ff83-d59a-2a57-268824af035f.dc1 area=wan -43:09.238 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13749 network=tcp -43:09.238 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13764 network=tcp -43:09.238 [INFO] TestAgent: started state syncer -43:09.239 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13804 network=tcp -43:09.290 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -43:09.290 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:09.290 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13803 [Candidate]" term=2 -43:09.290 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:09.290 [DEBUG] TestAgent.server.raft: vote granted: from=e4fb1f46-ff83-d59a-2a57-268824af035f term=2 tally=1 -43:09.290 [INFO] TestAgent.server.raft: election won: tally=1 -43:09.290 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13803 [Leader]" -43:09.290 [INFO] TestAgent.server: cluster leadership acquired -43:09.291 [INFO] TestAgent.server: initializing acls -43:09.291 [INFO] TestAgent.server: Created ACL 'global-management' policy -43:09.291 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -43:09.292 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -43:09.293 [INFO] TestAgent.server: Created ACL anonymous token from configuration -43:09.293 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -43:09.293 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -43:09.293 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-e4fb1f46-ff83-d59a-2a57-268824af035f -43:09.293 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-e4fb1f46-ff83-d59a-2a57-268824af035f.dc1 -43:09.294 [INFO] TestAgent.server: New leader elected: payload=Node-e4fb1f46-ff83-d59a-2a57-268824af035f -43:09.294 [INFO] TestAgent.server: Updating LAN server: server="Node-e4fb1f46-ff83-d59a-2a57-268824af035f (Addr: tcp/127.0.0.1:13803) (DC: dc1)" -43:09.294 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-e4fb1f46-ff83-d59a-2a57-268824af035f.dc1 area=wan -43:09.294 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:09.297 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:09.297 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:09.297 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:09.297 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:09.297 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:09.297 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e4fb1f46-ff83-d59a-2a57-268824af035f -43:09.297 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e4fb1f46-ff83-d59a-2a57-268824af035f -43:09.298 [INFO] TestAgent.server: federation state anti-entropy synced -43:09.307 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e4fb1f46-ff83-d59a-2a57-268824af035f -43:09.320 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:09.321 [INFO] TestAgent: Synced node info -43:09.321 [DEBUG] TestAgent: Node info in sync -43:09.370 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-e4fb1f46-ff83-d59a-2a57-268824af035f -43:09.371 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-e4fb1f46-ff83-d59a-2a57-268824af035f -43:09.371 [INFO] TestAgent: Requesting shutdown -43:09.371 [INFO] TestAgent.server: shutting down server -43:09.371 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -43:09.371 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:09.371 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:09.371 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:09.371 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:09.371 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -43:09.371 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:09.372 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:09.372 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:09.372 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -43:09.372 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:09.372 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:09.372 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:09.372 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -43:09.372 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:09.373Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:09.373Z [INFO] agent.router.manager: shutting down -43:09.373 [INFO] TestAgent: consul server down -43:09.373 [INFO] TestAgent: shutdown complete -43:09.373 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13749 network=tcp -43:09.373 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13749 network=udp -43:09.373 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13764 network=tcp -43:09.874 [INFO] TestAgent: Waiting for endpoints to shut down -43:09.874 [INFO] TestAgent: Endpoints down ---- PASS: TestACLReplicationStatus (0.75s) -=== CONT TestAgent_InitKeyring ---- PASS: TestAgent_InitKeyring (0.00s) -=== CONT TestACL_Legacy_Destroy -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:09.454Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:09.503Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:09.510 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:46c98c98-ca73-0b3b-c7a2-bb3e3b8a0124 Address:127.0.0.1:13838}]" -43:09.512 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-46c98c98-ca73-0b3b-c7a2-bb3e3b8a0124.dc1 127.0.0.1 -43:09.512 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-46c98c98-ca73-0b3b-c7a2-bb3e3b8a0124 127.0.0.1 -2022-09-15T21:43:09.512Z [INFO] agent.router: Initializing LAN area manager -43:09.513 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13838 [Follower]" leader= -43:09.513 [INFO] TestAgent.server: Adding LAN server: server="Node-46c98c98-ca73-0b3b-c7a2-bb3e3b8a0124 (Addr: tcp/127.0.0.1:13838) (DC: dc1)" -43:09.513 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-46c98c98-ca73-0b3b-c7a2-bb3e3b8a0124.dc1 area=wan -43:09.514 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13833 network=tcp -43:09.514 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13833 network=udp -43:09.515 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13848 network=tcp -43:09.515 [INFO] TestAgent: started state syncer -43:09.515 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13839 network=tcp -43:09.550 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:09.550 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13838 [Candidate]" term=2 -43:09.550 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:09.550 [DEBUG] TestAgent.server.raft: vote granted: from=46c98c98-ca73-0b3b-c7a2-bb3e3b8a0124 term=2 tally=1 -43:09.550 [INFO] TestAgent.server.raft: election won: tally=1 -43:09.550 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13838 [Leader]" -43:09.550 [INFO] TestAgent.server: cluster leadership acquired -43:09.550 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13838 -43:09.552 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:09.554 [INFO] TestAgent.server: New leader elected: payload=Node-46c98c98-ca73-0b3b-c7a2-bb3e3b8a0124 -43:09.555 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:09.555 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:09.555 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:09.555 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:09.555 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:09.555 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-46c98c98-ca73-0b3b-c7a2-bb3e3b8a0124 -43:09.555 [INFO] TestAgent.server: member joined, marking health alive: member=Node-46c98c98-ca73-0b3b-c7a2-bb3e3b8a0124 -43:09.556 [INFO] TestAgent.server: federation state anti-entropy synced -43:09.596 [INFO] TestAgent: Requesting shutdown -43:09.596 [INFO] TestAgent.server: shutting down server -43:09.596 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:09.596 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:09.596 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:09.596 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:09.596 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:09.597 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:09.597 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:09.597 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:09.597 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:09.597 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:09.597 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:09.597 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:09.597Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:09.597Z [INFO] agent.router.manager: shutting down -43:09.597 [INFO] TestAgent: consul server down -43:09.597 [INFO] TestAgent: shutdown complete -43:09.597 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13833 network=tcp -43:09.597 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13833 network=udp -43:09.597 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13848 network=tcp -43:10.098 [INFO] TestAgent: Waiting for endpoints to shut down -43:10.098 [INFO] TestAgent: Endpoints down ---- PASS: TestKVSEndpoint_Recurse (0.73s) +=== CONT TestHTTPAPI_TranslateAddrHeader +2021-08-13T15:34:36.975Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:36.993Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:36.996 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c378806d-fa31-5736-fa49-8e88e826acfa Address:127.0.0.1:24515}]" +34:36.996 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24515 [Follower]" leader= +34:36.997 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c378806d-fa31-5736-fa49-8e88e826acfa.dc1 127.0.0.1 +34:36.997 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c378806d-fa31-5736-fa49-8e88e826acfa 127.0.0.1 +2021-08-13T15:34:36.998Z [INFO] agent.router: Initializing LAN area manager +34:36.998 [INFO] TestAgent.server: Adding LAN server: server="Node-c378806d-fa31-5736-fa49-8e88e826acfa (Addr: tcp/127.0.0.1:24515) (DC: dc1)" +34:36.998 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c378806d-fa31-5736-fa49-8e88e826acfa.dc1 area=wan +34:36.998 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24531 network=udp +34:36.998 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24531 network=tcp +34:36.999 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24504 network=tcp +34:36.999 [INFO] TestAgent: started state syncer +34:36.999 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24516 network=tcp +34:37.063 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:37.063 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24515 [Candidate]" term=2 +34:37.064 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:37.064 [DEBUG] TestAgent.server.raft: vote granted: from=c378806d-fa31-5736-fa49-8e88e826acfa term=2 tally=1 +34:37.064 [INFO] TestAgent.server.raft: election won: tally=1 +34:37.064 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24515 [Leader]" +34:37.064 [INFO] TestAgent.server: cluster leadership acquired +34:37.064 [INFO] TestAgent.server: New leader elected: payload=Node-c378806d-fa31-5736-fa49-8e88e826acfa +34:37.064 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24515 +34:37.065 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:37.068 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:37.068 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:37.068 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:37.068 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:37.068 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:37.068 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c378806d-fa31-5736-fa49-8e88e826acfa +34:37.068 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c378806d-fa31-5736-fa49-8e88e826acfa +34:37.069 [INFO] TestAgent.server: federation state anti-entropy synced +34:37.213 [DEBUG] TestAgent.http: Request finished: method=GET url=/ui from= latency=2.92µs +34:37.213 [INFO] TestAgent: Requesting shutdown +34:37.213 [INFO] TestAgent.server: shutting down server +34:37.213 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:37.213 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:37.213 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:37.213 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:37.213 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:37.213 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:37.213 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:37.213 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:37.213 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:37.213 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:37.213 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:37.214 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:37.214Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:37.214Z [INFO] agent.router.manager: shutting down +34:37.214 [INFO] TestAgent: consul server down +34:37.214 [INFO] TestAgent: shutdown complete +34:37.214 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24531 network=tcp +34:37.214 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24531 network=udp +34:37.214 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24504 network=tcp +34:37.714 [INFO] TestAgent: Waiting for endpoints to shut down +34:37.714 [INFO] TestAgent: Endpoints down +--- PASS: TestUIResponseHeaders (0.76s) === CONT TestACL_Legacy_Get === RUN TestACL_Legacy_Get/wrong_id -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:09.202Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:09.202Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:09.241Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:09.241Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:09.244 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:520f7f24-5c6f-bb24-68b4-b793f19d4e72 Address:127.0.0.1:13817}]" -43:09.245 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-520f7f24-5c6f-bb24-68b4-b793f19d4e72.dc1 127.0.0.1 -43:09.245 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-520f7f24-5c6f-bb24-68b4-b793f19d4e72 127.0.0.1 -43:09.246 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13817 [Follower]" leader= -2022-09-15T21:43:09.246Z [INFO] agent.router: Initializing LAN area manager -43:09.247 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13805 network=udp -43:09.247 [INFO] TestAgent.server: Adding LAN server: server="Node-520f7f24-5c6f-bb24-68b4-b793f19d4e72 (Addr: tcp/127.0.0.1:13817) (DC: dc1)" -43:09.247 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-520f7f24-5c6f-bb24-68b4-b793f19d4e72.dc1 area=wan -43:09.247 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13805 network=tcp -43:09.248 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13806 network=tcp -43:09.248 [INFO] TestAgent: started state syncer -43:09.248 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13818 network=tcp -43:09.298 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:09.298 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13817 [Candidate]" term=2 -43:09.298 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:09.298 [DEBUG] TestAgent.server.raft: vote granted: from=520f7f24-5c6f-bb24-68b4-b793f19d4e72 term=2 tally=1 -43:09.298 [INFO] TestAgent.server.raft: election won: tally=1 -43:09.298 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13817 [Leader]" -43:09.298 [INFO] TestAgent.server: cluster leadership acquired -43:09.298 [INFO] TestAgent.server: New leader elected: payload=Node-520f7f24-5c6f-bb24-68b4-b793f19d4e72 -43:09.298 [INFO] TestAgent.server: initializing acls -43:09.300 [INFO] TestAgent.server: Created ACL 'global-management' policy -43:09.300 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -43:09.301 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -43:09.301 [INFO] TestAgent.server: Created ACL anonymous token from configuration -43:09.301 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -43:09.301 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -43:09.298 [INFO] TestAgent.server: initializing acls -43:09.302 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -43:09.302 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-520f7f24-5c6f-bb24-68b4-b793f19d4e72 -43:09.302 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode -43:09.302 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-520f7f24-5c6f-bb24-68b4-b793f19d4e72.dc1 -43:09.302 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-520f7f24-5c6f-bb24-68b4-b793f19d4e72 -43:09.302 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-520f7f24-5c6f-bb24-68b4-b793f19d4e72.dc1 -43:09.302 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-520f7f24-5c6f-bb24-68b4-b793f19d4e72.dc1 area=wan -43:09.302 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-520f7f24-5c6f-bb24-68b4-b793f19d4e72.dc1 area=wan -43:09.302 [INFO] TestAgent.server: Updating LAN server: server="Node-520f7f24-5c6f-bb24-68b4-b793f19d4e72 (Addr: tcp/127.0.0.1:13817) (DC: dc1)" -43:09.302 [INFO] TestAgent.server: Updating LAN server: server="Node-520f7f24-5c6f-bb24-68b4-b793f19d4e72 (Addr: tcp/127.0.0.1:13817) (DC: dc1)" -43:09.303 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:09.305 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:09.305 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:09.305 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:09.305 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:09.305 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:09.305 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-520f7f24-5c6f-bb24-68b4-b793f19d4e72 -43:09.306 [INFO] TestAgent.server: member joined, marking health alive: member=Node-520f7f24-5c6f-bb24-68b4-b793f19d4e72 -43:09.306 [INFO] TestAgent.server: federation state anti-entropy synced -43:09.319 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-520f7f24-5c6f-bb24-68b4-b793f19d4e72 -43:09.319 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-520f7f24-5c6f-bb24-68b4-b793f19d4e72 -43:09.470 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="ACL not found" -43:09.622 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="ACL not found" -43:09.642 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-520f7f24-5c6f-bb24-68b4-b793f19d4e72 -43:09.642 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-520f7f24-5c6f-bb24-68b4-b793f19d4e72 -43:09.781 [INFO] TestAgent: Requesting shutdown -43:09.781 [INFO] TestAgent.server: shutting down server -43:09.781 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:09.781 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -43:09.781 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -43:09.781 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:09.781 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:09.781 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:09.781 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:09.781 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:09.781 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:09.781 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:09.781 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -43:09.781 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -43:09.781 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:09.781 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:09.781 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:09.782Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:09.782Z [INFO] agent.router.manager: shutting down -43:09.782 [INFO] TestAgent: consul server down -43:09.782 [INFO] TestAgent: shutdown complete -43:09.782 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13805 network=tcp -43:09.782 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13805 network=udp -43:09.782 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13806 network=tcp -43:10.294 [INFO] TestAgent: Waiting for endpoints to shut down -43:10.294 [INFO] TestAgent: Endpoints down ---- PASS: TestPProfHandlers_ACLs (1.14s) - --- PASS: TestPProfHandlers_ACLs/case_0_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:200,_token:"master",_endpoint:"/debug/pprof/heap",_nilResponse:false}) (0.08s) - --- PASS: TestPProfHandlers_ACLs/case_1_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:403,_token:"agent",_endpoint:"/debug/pprof/heap",_nilResponse:true}) (0.00s) - --- PASS: TestPProfHandlers_ACLs/case_2_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:403,_token:"agent",_endpoint:"/debug/pprof/",_nilResponse:true}) (0.00s) - --- PASS: TestPProfHandlers_ACLs/case_3_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:403,_token:"",_endpoint:"/debug/pprof/",_nilResponse:true}) (0.00s) - --- PASS: TestPProfHandlers_ACLs/case_4_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:200,_token:"master",_endpoint:"/debug/pprof/heap",_nilResponse:false}) (0.05s) - --- PASS: TestPProfHandlers_ACLs/case_5_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:403,_token:"towel",_endpoint:"/debug/pprof/heap",_nilResponse:true}) (0.00s) +2021-08-13T15:34:36.800Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:36.800Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:36.894Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:36.894Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:36.897 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:78f03789-8b53-5c4f-abee-a751a17e1f33 Address:127.0.0.1:24501}]" +34:36.897 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24501 [Follower]" leader= +34:36.898 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-78f03789-8b53-5c4f-abee-a751a17e1f33.dc1 127.0.0.1 +34:36.899 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-78f03789-8b53-5c4f-abee-a751a17e1f33 127.0.0.1 +2021-08-13T15:34:36.899Z [INFO] agent.router: Initializing LAN area manager +34:36.899 [INFO] TestAgent.server: Adding LAN server: server="Node-78f03789-8b53-5c4f-abee-a751a17e1f33 (Addr: tcp/127.0.0.1:24501) (DC: dc1)" +34:36.899 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-78f03789-8b53-5c4f-abee-a751a17e1f33.dc1 area=wan +34:36.903 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24510 network=tcp +34:36.903 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24510 network=udp +34:36.904 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24511 network=tcp +34:36.904 [INFO] TestAgent: started state syncer +34:36.904 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24502 network=tcp +34:36.949 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:36.955 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:36.955 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24501 [Candidate]" term=2 +34:36.955 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:36.955 [DEBUG] TestAgent.server.raft: vote granted: from=78f03789-8b53-5c4f-abee-a751a17e1f33 term=2 tally=1 +34:36.955 [INFO] TestAgent.server.raft: election won: tally=1 +34:36.955 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24501 [Leader]" +34:36.955 [INFO] TestAgent.server: cluster leadership acquired +34:36.955 [INFO] TestAgent.server: New leader elected: payload=Node-78f03789-8b53-5c4f-abee-a751a17e1f33 +34:36.955 [INFO] TestAgent.server: initializing acls +34:36.956 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:36.956 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:36.956 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:36.956 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:36.957 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-78f03789-8b53-5c4f-abee-a751a17e1f33 +34:36.957 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-78f03789-8b53-5c4f-abee-a751a17e1f33.dc1 +34:36.957 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-78f03789-8b53-5c4f-abee-a751a17e1f33.dc1 area=wan +34:36.957 [INFO] TestAgent.server: Updating LAN server: server="Node-78f03789-8b53-5c4f-abee-a751a17e1f33 (Addr: tcp/127.0.0.1:24501) (DC: dc1)" +34:36.958 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:36.960 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:36.960 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:36.960 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:36.960 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:36.960 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:36.960 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-78f03789-8b53-5c4f-abee-a751a17e1f33 +34:36.960 [INFO] TestAgent.server: member joined, marking health alive: member=Node-78f03789-8b53-5c4f-abee-a751a17e1f33 +34:36.961 [INFO] TestAgent.server: federation state anti-entropy synced +34:36.961 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-78f03789-8b53-5c4f-abee-a751a17e1f33 +34:37.100 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="ACL not found" +34:37.236 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-78f03789-8b53-5c4f-abee-a751a17e1f33 +34:37.236 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-78f03789-8b53-5c4f-abee-a751a17e1f33 +34:37.237 [WARN] TestAgent.server.acl: failed to remove bootstrap file: error="remove /tmp/consul-test/TestACL_Bootstrap-TestAgent-agent924842393/acl-bootstrap-reset: no such file or directory" +34:37.237 [INFO] TestAgent.server.acl: ACL bootstrap completed +34:37.238 [INFO] TestAgent: Requesting shutdown +34:37.238 [INFO] TestAgent.server: shutting down server +34:37.238 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:37.238 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:37.238 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:37.238 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:37.238 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:37.238 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:37.238 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:37.238 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:37.238 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:37.238 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:37.238 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:37.238 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:37.238 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:37.238 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:37.238 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:37.238Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:37.238Z [INFO] agent.router.manager: shutting down +34:37.238 [INFO] TestAgent: consul server down +34:37.238 [INFO] TestAgent: shutdown complete +34:37.238 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24510 network=tcp +34:37.238 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24510 network=udp +34:37.239 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24511 network=tcp +34:37.739 [INFO] TestAgent: Waiting for endpoints to shut down +34:37.739 [INFO] TestAgent: Endpoints down +--- PASS: TestACL_Bootstrap (0.96s) + --- PASS: TestACL_Bootstrap/bootstrap (0.00s) + --- PASS: TestACL_Bootstrap/not_again (0.00s) +=== CONT TestHTTPAPI_BlockEndpoints +2021-08-13T15:34:37.054Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:37.054Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:37.072Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:37.072Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:37.075 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b6f86881-96d5-ef56-ba88-654146c1c64e Address:127.0.0.1:24550}]" +34:37.075 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24550 [Follower]" leader= +34:37.076 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b6f86881-96d5-ef56-ba88-654146c1c64e.dc1 127.0.0.1 +34:37.076 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b6f86881-96d5-ef56-ba88-654146c1c64e 127.0.0.1 +2021-08-13T15:34:37.077Z [INFO] agent.router: Initializing LAN area manager +34:37.077 [INFO] TestAgent.server: Adding LAN server: server="Node-b6f86881-96d5-ef56-ba88-654146c1c64e (Addr: tcp/127.0.0.1:24550) (DC: dc1)" +34:37.077 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b6f86881-96d5-ef56-ba88-654146c1c64e.dc1 area=wan +34:37.077 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24524 network=udp +34:37.077 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24524 network=tcp +34:37.078 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24546 network=tcp +34:37.078 [INFO] TestAgent: started state syncer +34:37.078 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24551 network=tcp +34:37.125 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:37.125 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24550 [Candidate]" term=2 +34:37.125 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:37.125 [DEBUG] TestAgent.server.raft: vote granted: from=b6f86881-96d5-ef56-ba88-654146c1c64e term=2 tally=1 +34:37.125 [INFO] TestAgent.server.raft: election won: tally=1 +34:37.125 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24550 [Leader]" +34:37.125 [INFO] TestAgent.server: cluster leadership acquired +34:37.126 [INFO] TestAgent.server: New leader elected: payload=Node-b6f86881-96d5-ef56-ba88-654146c1c64e +34:37.126 [INFO] TestAgent.server: initializing acls +34:37.126 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:37.126 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:37.127 [INFO] TestAgent.server: initializing acls +34:37.127 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:37.127 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:37.128 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:37.128 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:37.128 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:37.128 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-b6f86881-96d5-ef56-ba88-654146c1c64e +34:37.128 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-b6f86881-96d5-ef56-ba88-654146c1c64e.dc1 +34:37.129 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:37.129 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode +34:37.129 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-b6f86881-96d5-ef56-ba88-654146c1c64e +34:37.129 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-b6f86881-96d5-ef56-ba88-654146c1c64e.dc1 +34:37.129 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-b6f86881-96d5-ef56-ba88-654146c1c64e.dc1 area=wan +34:37.129 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-b6f86881-96d5-ef56-ba88-654146c1c64e.dc1 area=wan +34:37.129 [INFO] TestAgent.server: Updating LAN server: server="Node-b6f86881-96d5-ef56-ba88-654146c1c64e (Addr: tcp/127.0.0.1:24550) (DC: dc1)" +34:37.129 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:37.129 [INFO] TestAgent.server: Updating LAN server: server="Node-b6f86881-96d5-ef56-ba88-654146c1c64e (Addr: tcp/127.0.0.1:24550) (DC: dc1)" +34:37.132 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:37.132 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:37.132 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:37.132 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:37.132 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:37.132 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b6f86881-96d5-ef56-ba88-654146c1c64e +34:37.132 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b6f86881-96d5-ef56-ba88-654146c1c64e +34:37.133 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b6f86881-96d5-ef56-ba88-654146c1c64e +34:37.133 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b6f86881-96d5-ef56-ba88-654146c1c64e +34:37.133 [INFO] TestAgent.server: federation state anti-entropy synced +34:37.244 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-b6f86881-96d5-ef56-ba88-654146c1c64e +34:37.244 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-b6f86881-96d5-ef56-ba88-654146c1c64e +34:37.244 [INFO] TestAgent: Requesting shutdown +34:37.244 [INFO] TestAgent.server: shutting down server +34:37.244 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:37.244 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:37.244 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:37.244 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:37.244 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:37.244 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:37.245 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:37.245 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:37.245 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:37.245 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:37.245 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:37.245 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:37.245 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:37.245 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:37.245 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:37.245 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:37.245Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:37.245Z [INFO] agent.router.manager: shutting down +34:37.245 [INFO] TestAgent: consul server down +34:37.245 [INFO] TestAgent: shutdown complete +34:37.245 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24524 network=tcp +34:37.245 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24524 network=udp +34:37.245 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24546 network=tcp +34:37.747 [INFO] TestAgent: Waiting for endpoints to shut down +34:37.747 [INFO] TestAgent: Endpoints down +--- PASS: TestACLReplicationStatus (0.71s) === CONT TestACL_Legacy_Clone -2022-09-15T21:43:09.326Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:09.375Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:09.388 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:118c1f03-1525-fe9d-0705-b44c732d783f Address:127.0.0.1:13831}]" -43:09.389 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-118c1f03-1525-fe9d-0705-b44c732d783f.dc1 127.0.0.1 -43:09.394 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13831 [Follower]" leader= -43:09.399 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-118c1f03-1525-fe9d-0705-b44c732d783f 127.0.0.1 -2022-09-15T21:43:09.399Z [INFO] agent.router: Initializing LAN area manager -43:09.400 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13819 network=udp -43:09.400 [INFO] TestAgent.server: Adding LAN server: server="Node-118c1f03-1525-fe9d-0705-b44c732d783f (Addr: tcp/127.0.0.1:13831) (DC: dc1)" -43:09.400 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-118c1f03-1525-fe9d-0705-b44c732d783f.dc1 area=wan -43:09.400 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13819 network=tcp -43:09.406 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13820 network=tcp -43:09.406 [INFO] TestAgent: started state syncer -43:09.407 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13832 network=tcp -43:09.442 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:09.442 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13831 [Candidate]" term=2 -43:09.442 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:09.442 [DEBUG] TestAgent.server.raft: vote granted: from=118c1f03-1525-fe9d-0705-b44c732d783f term=2 tally=1 -43:09.442 [INFO] TestAgent.server.raft: election won: tally=1 -43:09.442 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13831 [Leader]" -43:09.442 [INFO] TestAgent.server: cluster leadership acquired -43:09.443 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13831 -43:09.444 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:09.446 [INFO] TestAgent.server: New leader elected: payload=Node-118c1f03-1525-fe9d-0705-b44c732d783f -43:09.461 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:09.461 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:09.461 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:09.461 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:09.461 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:09.462 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-118c1f03-1525-fe9d-0705-b44c732d783f -43:09.462 [INFO] TestAgent.server: member joined, marking health alive: member=Node-118c1f03-1525-fe9d-0705-b44c732d783f -43:09.463 [INFO] TestAgent.server: federation state anti-entropy synced -43:09.477 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:09.482 [INFO] TestAgent: Synced node info -43:09.829 [INFO] TestAgent: Requesting shutdown -43:09.830 [INFO] TestAgent.server: shutting down server -43:09.830 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:09.830 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:09.830 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:09.830 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:09.830 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:09.830 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:09.830 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:09.830 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:09.830 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:09.830 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:09.830 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:09.830Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:09.830Z [INFO] agent.router.manager: shutting down -43:09.830 [INFO] TestAgent: consul server down -43:09.830 [INFO] TestAgent: shutdown complete -43:09.830 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13819 network=tcp -43:09.830 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13819 network=udp -43:09.831 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13820 network=tcp -43:10.331 [INFO] TestAgent: Waiting for endpoints to shut down -43:10.331 [INFO] TestAgent: Endpoints down -=== RUN TestAgent_InmemKeyrings/server_with_keys -2022-09-15T21:43:09.483Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:09.516Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:09.519 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ee161966-1c55-d9c2-fcd8-51e870f619ef Address:127.0.0.1:13852}]" -43:09.520 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ee161966-1c55-d9c2-fcd8-51e870f619ef.dc1 127.0.0.1 -43:09.521 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ee161966-1c55-d9c2-fcd8-51e870f619ef 127.0.0.1 -2022-09-15T21:43:09.521Z [INFO] agent.router: Initializing LAN area manager -43:09.521 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13840 network=udp -43:09.521 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13852 [Follower]" leader= -43:09.522 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ee161966-1c55-d9c2-fcd8-51e870f619ef.dc1 area=wan -43:09.522 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13840 network=tcp -43:09.523 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13827 network=tcp -43:09.523 [INFO] TestAgent: started state syncer -43:09.523 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13853 network=tcp -43:09.523 [INFO] TestAgent.server: Adding LAN server: server="Node-ee161966-1c55-d9c2-fcd8-51e870f619ef (Addr: tcp/127.0.0.1:13852) (DC: dc1)" -43:09.562 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:09.562 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13852 [Candidate]" term=2 -43:09.562 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:09.562 [DEBUG] TestAgent.server.raft: vote granted: from=ee161966-1c55-d9c2-fcd8-51e870f619ef term=2 tally=1 -43:09.562 [INFO] TestAgent.server.raft: election won: tally=1 -43:09.562 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13852 [Leader]" -43:09.562 [INFO] TestAgent.server: cluster leadership acquired -43:09.562 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13852 -43:09.564 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:09.566 [INFO] TestAgent.server: New leader elected: payload=Node-ee161966-1c55-d9c2-fcd8-51e870f619ef -43:09.566 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:09.566 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:09.566 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:09.566 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:09.566 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:09.566 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ee161966-1c55-d9c2-fcd8-51e870f619ef -43:09.566 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ee161966-1c55-d9c2-fcd8-51e870f619ef -43:09.568 [INFO] TestAgent.server: federation state anti-entropy synced -43:09.810 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:09.810 [INFO] TestAgent: Synced node info -43:09.810 [DEBUG] TestAgent: Node info in sync -43:09.881 [INFO] TestAgent: Requesting shutdown -43:09.881 [INFO] TestAgent.server: shutting down server -43:09.881 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:09.881 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:09.881 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:09.881 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:09.881 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:09.881 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:09.881 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:09.881 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:09.881 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:09.881 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:09.881 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:09.882Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:09.882Z [INFO] agent.router.manager: shutting down -43:09.882 [INFO] TestAgent: consul server down -43:09.882 [INFO] TestAgent: shutdown complete -43:09.882 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13840 network=tcp -43:09.882 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13840 network=udp -43:09.882 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13827 network=tcp -43:10.386 [INFO] TestAgent: Waiting for endpoints to shut down -43:10.386 [INFO] TestAgent: Endpoints down ---- PASS: TestKVSEndpoint_PUT_GET_DELETE (0.95s) -=== CONT TestPreparedQuery_List -=== RUN TestPreparedQuery_List/#00 -2022-09-15T21:43:09.721Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:09.721Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:09.757Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:09.757Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:09.764 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9b836052-0a74-3462-933a-3580b68cdb6d Address:127.0.0.1:13845}]" -43:09.764 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13845 [Follower]" leader= -43:09.771 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9b836052-0a74-3462-933a-3580b68cdb6d.dc1 127.0.0.1 -43:09.772 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9b836052-0a74-3462-933a-3580b68cdb6d 127.0.0.1 -2022-09-15T21:43:09.772Z [INFO] agent.router: Initializing LAN area manager -43:09.772 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13854 network=udp -43:09.773 [INFO] TestAgent.server: Adding LAN server: server="Node-9b836052-0a74-3462-933a-3580b68cdb6d (Addr: tcp/127.0.0.1:13845) (DC: dc1)" -43:09.773 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9b836052-0a74-3462-933a-3580b68cdb6d.dc1 area=wan -43:09.773 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13854 network=tcp -43:09.775 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13841 network=tcp -43:09.775 [INFO] TestAgent: started state syncer -43:09.775 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13846 network=tcp -43:09.816 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:09.816 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13845 [Candidate]" term=2 -43:09.816 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:09.816 [DEBUG] TestAgent.server.raft: vote granted: from=9b836052-0a74-3462-933a-3580b68cdb6d term=2 tally=1 -43:09.816 [INFO] TestAgent.server.raft: election won: tally=1 -43:09.816 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13845 [Leader]" -43:09.817 [INFO] TestAgent.server: cluster leadership acquired -43:09.817 [INFO] TestAgent.server: initializing acls -43:09.817 [INFO] TestAgent.server: Created ACL 'global-management' policy -43:09.817 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -43:09.818 [INFO] TestAgent.server: New leader elected: payload=Node-9b836052-0a74-3462-933a-3580b68cdb6d -43:09.818 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -43:09.819 [INFO] TestAgent.server: Created ACL anonymous token from configuration -43:09.819 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -43:09.819 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -43:09.819 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-9b836052-0a74-3462-933a-3580b68cdb6d -43:09.819 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-9b836052-0a74-3462-933a-3580b68cdb6d.dc1 -43:09.820 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:09.822 [INFO] TestAgent.server: Updating LAN server: server="Node-9b836052-0a74-3462-933a-3580b68cdb6d (Addr: tcp/127.0.0.1:13845) (DC: dc1)" -43:09.822 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-9b836052-0a74-3462-933a-3580b68cdb6d.dc1 area=wan -43:09.827 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:09.827 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:09.827 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:09.827 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:09.827 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:09.827 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9b836052-0a74-3462-933a-3580b68cdb6d -43:09.827 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9b836052-0a74-3462-933a-3580b68cdb6d -43:09.828 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9b836052-0a74-3462-933a-3580b68cdb6d -43:09.829 [INFO] TestAgent.server: federation state anti-entropy synced -43:09.839 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:09.840 [INFO] TestAgent: Synced node info -43:09.840 [DEBUG] TestAgent: Node info in sync -43:09.926 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-9b836052-0a74-3462-933a-3580b68cdb6d -43:09.926 [INFO] TestAgent.server.serf.wan: serf: Received list-keys query -43:09.930 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-9b836052-0a74-3462-933a-3580b68cdb6d.dc1 -43:09.930 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query -43:09.931 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-9b836052-0a74-3462-933a-3580b68cdb6d -43:09.931 [INFO] TestAgent.server.serf.wan: serf: Received install-key query -43:09.931 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-9b836052-0a74-3462-933a-3580b68cdb6d.dc1 -43:09.932 [INFO] TestAgent.server.serf.lan: serf: Received install-key query -43:09.932 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-9b836052-0a74-3462-933a-3580b68cdb6d -43:09.932 [INFO] TestAgent.server.serf.wan: serf: Received use-key query -43:09.933 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-9b836052-0a74-3462-933a-3580b68cdb6d.dc1 -43:09.933 [INFO] TestAgent.server.serf.lan: serf: Received use-key query -43:09.933 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-9b836052-0a74-3462-933a-3580b68cdb6d -43:09.933 [INFO] TestAgent.server.serf.wan: serf: Received remove-key query -43:09.934 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-9b836052-0a74-3462-933a-3580b68cdb6d.dc1 -43:09.934 [INFO] TestAgent.server.serf.lan: serf: Received remove-key query -43:09.934 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-9b836052-0a74-3462-933a-3580b68cdb6d -43:09.934 [INFO] TestAgent: Requesting shutdown -43:09.935 [INFO] TestAgent.server: shutting down server -43:09.935 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:09.935 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:09.935 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:09.935 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:09.935 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -43:09.935 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -43:09.935 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:09.935 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:09.935 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:09.935 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:09.935 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:09.935 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:09.935 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -43:09.935 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -43:09.935 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:09.935Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:09.935Z [INFO] agent.router.manager: shutting down -43:09.935 [INFO] TestAgent: consul server down -43:09.935 [INFO] TestAgent: shutdown complete -43:09.935 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13854 network=tcp -43:09.935 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13854 network=udp -43:09.936 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13841 network=tcp -43:10.436 [INFO] TestAgent: Waiting for endpoints to shut down -43:10.436 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentKeyring_ACL (0.75s) -=== CONT TestPreparedQuery_Delete -[WARN] freeport: 4 out of 32 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:09.896Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:09.896Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:09.914Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:09.914Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:09.917 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7974f5d8-0d20-3435-dab1-95f0bcf46de4 Address:127.0.0.1:13859}]" -43:09.917 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13859 [Follower]" leader= -43:09.919 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7974f5d8-0d20-3435-dab1-95f0bcf46de4.dc1 127.0.0.1 -43:09.919 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7974f5d8-0d20-3435-dab1-95f0bcf46de4 127.0.0.1 -2022-09-15T21:43:09.919Z [INFO] agent.router: Initializing LAN area manager -43:09.920 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13847 network=udp -43:09.920 [INFO] TestAgent.server: Adding LAN server: server="Node-7974f5d8-0d20-3435-dab1-95f0bcf46de4 (Addr: tcp/127.0.0.1:13859) (DC: dc1)" -43:09.920 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7974f5d8-0d20-3435-dab1-95f0bcf46de4.dc1 area=wan -43:09.922 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13847 network=tcp -43:09.923 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13862 network=tcp -43:09.923 [INFO] TestAgent: started state syncer -43:09.923 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13860 network=tcp -43:09.966 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:09.966 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13859 [Candidate]" term=2 -43:09.966 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:09.966 [DEBUG] TestAgent.server.raft: vote granted: from=7974f5d8-0d20-3435-dab1-95f0bcf46de4 term=2 tally=1 -43:09.966 [INFO] TestAgent.server.raft: election won: tally=1 -43:09.966 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13859 [Leader]" -43:09.966 [INFO] TestAgent.server: cluster leadership acquired -43:09.967 [INFO] TestAgent.server: initializing acls -43:09.967 [INFO] TestAgent.server: Created ACL 'global-management' policy -43:09.967 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -43:09.968 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -43:09.968 [INFO] TestAgent.server: Created ACL anonymous token from configuration -43:09.968 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -43:09.968 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -43:09.969 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-7974f5d8-0d20-3435-dab1-95f0bcf46de4 -43:09.969 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-7974f5d8-0d20-3435-dab1-95f0bcf46de4.dc1 -43:09.970 [INFO] TestAgent.server: New leader elected: payload=Node-7974f5d8-0d20-3435-dab1-95f0bcf46de4 -43:09.970 [INFO] TestAgent.server: Updating LAN server: server="Node-7974f5d8-0d20-3435-dab1-95f0bcf46de4 (Addr: tcp/127.0.0.1:13859) (DC: dc1)" -43:09.970 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-7974f5d8-0d20-3435-dab1-95f0bcf46de4.dc1 area=wan -43:09.971 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:09.973 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:09.973 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:09.973 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:09.973 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:09.973 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:09.973 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7974f5d8-0d20-3435-dab1-95f0bcf46de4 -43:09.973 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7974f5d8-0d20-3435-dab1-95f0bcf46de4 -43:09.975 [INFO] TestAgent.server: federation state anti-entropy synced -43:09.975 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7974f5d8-0d20-3435-dab1-95f0bcf46de4 -43:09.999 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:09.999 [INFO] TestAgent: Synced node info -43:10.106 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-7974f5d8-0d20-3435-dab1-95f0bcf46de4 -43:10.107 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-7974f5d8-0d20-3435-dab1-95f0bcf46de4 -43:10.108 [INFO] TestAgent: Requesting shutdown -43:10.108 [INFO] TestAgent.server: shutting down server -43:10.108 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -43:10.108 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -43:10.108 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:10.108 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:10.108 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:10.108 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:10.108 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:10.109 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -43:10.109 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:10.109 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:10.109 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -43:10.109 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:10.109 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:10.109 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:10.109 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:10.109Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:10.109Z [INFO] agent.router.manager: shutting down -43:10.109 [INFO] TestAgent: consul server down -43:10.109 [INFO] TestAgent: shutdown complete -43:10.109 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13847 network=tcp -43:10.109 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13847 network=udp -43:10.109 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13862 network=tcp -43:10.610 [INFO] TestAgent: Waiting for endpoints to shut down -43:10.610 [INFO] TestAgent: Endpoints down ---- PASS: TestACL_Legacy_Destroy (0.74s) -=== CONT TestPreparedQuery_Update -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:10.158Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:10.158Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:10.198Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:10.198Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:10.217 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ef6e3d3c-a5d7-fd22-bdf7-d444ff9b3eff Address:127.0.0.1:13677}]" -43:10.218 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13677 [Follower]" leader= -43:10.235 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ef6e3d3c-a5d7-fd22-bdf7-d444ff9b3eff.dc1 127.0.0.1 -43:10.236 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ef6e3d3c-a5d7-fd22-bdf7-d444ff9b3eff 127.0.0.1 -2022-09-15T21:43:10.236Z [INFO] agent.router: Initializing LAN area manager -43:10.237 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13861 network=udp -43:10.237 [INFO] TestAgent.server: Adding LAN server: server="Node-ef6e3d3c-a5d7-fd22-bdf7-d444ff9b3eff (Addr: tcp/127.0.0.1:13677) (DC: dc1)" -43:10.237 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ef6e3d3c-a5d7-fd22-bdf7-d444ff9b3eff.dc1 area=wan -43:10.237 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13861 network=tcp -43:10.251 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13878 network=tcp -43:10.252 [INFO] TestAgent: started state syncer -43:10.252 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13678 network=tcp -43:10.294 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:10.294 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13677 [Candidate]" term=2 -43:10.294 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:10.294 [DEBUG] TestAgent.server.raft: vote granted: from=ef6e3d3c-a5d7-fd22-bdf7-d444ff9b3eff term=2 tally=1 -43:10.294 [INFO] TestAgent.server.raft: election won: tally=1 -43:10.294 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13677 [Leader]" -43:10.294 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -43:10.294 [INFO] TestAgent.server: cluster leadership acquired -43:10.295 [INFO] TestAgent.server: initializing acls -43:10.295 [INFO] TestAgent.server: Created ACL 'global-management' policy -43:10.295 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -43:10.296 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -43:10.297 [INFO] TestAgent.server: Created ACL anonymous token from configuration -43:10.297 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -43:10.297 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -43:10.298 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-ef6e3d3c-a5d7-fd22-bdf7-d444ff9b3eff -43:10.298 [INFO] TestAgent.server: New leader elected: payload=Node-ef6e3d3c-a5d7-fd22-bdf7-d444ff9b3eff -43:10.298 [INFO] TestAgent.server: Updating LAN server: server="Node-ef6e3d3c-a5d7-fd22-bdf7-d444ff9b3eff (Addr: tcp/127.0.0.1:13677) (DC: dc1)" -43:10.306 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-ef6e3d3c-a5d7-fd22-bdf7-d444ff9b3eff.dc1 -43:10.308 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:10.317 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:10.317 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:10.317 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:10.317 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:10.317 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:10.317 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ef6e3d3c-a5d7-fd22-bdf7-d444ff9b3eff -43:10.317 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ef6e3d3c-a5d7-fd22-bdf7-d444ff9b3eff -43:10.318 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ef6e3d3c-a5d7-fd22-bdf7-d444ff9b3eff -43:10.318 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-ef6e3d3c-a5d7-fd22-bdf7-d444ff9b3eff.dc1 area=wan -43:10.318 [INFO] TestAgent.server: federation state anti-entropy synced -43:10.406 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-ef6e3d3c-a5d7-fd22-bdf7-d444ff9b3eff -43:10.406 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-ef6e3d3c-a5d7-fd22-bdf7-d444ff9b3eff -43:10.406 [INFO] TestAgent: Requesting shutdown -43:10.406 [INFO] TestAgent.server: shutting down server -43:10.406 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -43:10.406 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:10.406 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:10.406 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:10.406 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:10.406 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -43:10.406 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:10.407 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:10.407 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:10.407 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:10.407 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -43:10.407 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:10.407 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:10.407 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:10.407 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -43:10.407 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:10.407Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:10.407Z [INFO] agent.router.manager: shutting down -43:10.407 [INFO] TestAgent: consul server down -43:10.407 [INFO] TestAgent: shutdown complete -43:10.407 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13861 network=tcp -43:10.407 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13861 network=udp -43:10.407 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13878 network=tcp -43:10.907 [INFO] TestAgent: Waiting for endpoints to shut down -43:10.907 [INFO] TestAgent: Endpoints down -=== RUN TestACL_Legacy_Get/right_id -2022-09-15T21:43:10.341Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:10.341Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:10.378Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:10.378Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:10.381 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2e98c512-a5af-0952-9129-aa24bbc323c4 Address:127.0.0.1:13870}]" -43:10.381 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13870 [Follower]" leader= -43:10.391 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2e98c512-a5af-0952-9129-aa24bbc323c4.dc1 127.0.0.1 -43:10.391 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2e98c512-a5af-0952-9129-aa24bbc323c4 127.0.0.1 -2022-09-15T21:43:10.391Z [INFO] agent.router: Initializing LAN area manager -43:10.392 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13679 network=udp -43:10.392 [INFO] TestAgent.server: Adding LAN server: server="Node-2e98c512-a5af-0952-9129-aa24bbc323c4 (Addr: tcp/127.0.0.1:13870) (DC: dc1)" -43:10.392 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2e98c512-a5af-0952-9129-aa24bbc323c4.dc1 area=wan -43:10.393 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13679 network=tcp -43:10.393 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13680 network=tcp -43:10.394 [INFO] TestAgent: started state syncer -43:10.394 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13871 network=tcp -43:10.430 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:10.430 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13870 [Candidate]" term=2 -43:10.430 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:10.430 [DEBUG] TestAgent.server.raft: vote granted: from=2e98c512-a5af-0952-9129-aa24bbc323c4 term=2 tally=1 -43:10.430 [INFO] TestAgent.server.raft: election won: tally=1 -43:10.430 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13870 [Leader]" -43:10.430 [INFO] TestAgent.server: cluster leadership acquired -43:10.431 [INFO] TestAgent.server: initializing acls -43:10.431 [INFO] TestAgent.server: Created ACL 'global-management' policy -43:10.431 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -43:10.432 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -43:10.432 [INFO] TestAgent.server: Created ACL anonymous token from configuration -43:10.432 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -43:10.433 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -43:10.433 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-2e98c512-a5af-0952-9129-aa24bbc323c4 -43:10.433 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-2e98c512-a5af-0952-9129-aa24bbc323c4.dc1 -43:10.434 [INFO] TestAgent.server: New leader elected: payload=Node-2e98c512-a5af-0952-9129-aa24bbc323c4 -43:10.434 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-2e98c512-a5af-0952-9129-aa24bbc323c4.dc1 area=wan -43:10.434 [INFO] TestAgent.server: Updating LAN server: server="Node-2e98c512-a5af-0952-9129-aa24bbc323c4 (Addr: tcp/127.0.0.1:13870) (DC: dc1)" -43:10.451 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:10.463 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:10.463 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:10.463 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:10.463 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:10.463 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:10.463 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2e98c512-a5af-0952-9129-aa24bbc323c4 -43:10.463 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2e98c512-a5af-0952-9129-aa24bbc323c4 -43:10.464 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2e98c512-a5af-0952-9129-aa24bbc323c4 -43:10.465 [INFO] TestAgent.server: federation state anti-entropy synced -43:10.486 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-2e98c512-a5af-0952-9129-aa24bbc323c4 -43:10.489 [INFO] TestAgent: Requesting shutdown -43:10.489 [INFO] TestAgent.server: shutting down server -43:10.489 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -43:10.489 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -43:10.489 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:10.489 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:10.489 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:10.489 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:10.489 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:10.490 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:10.490 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:10.490 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -43:10.490 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:10.490 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -43:10.490 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:10.490 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:10.490 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:10.494 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:10.494Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:10.494Z [INFO] agent.router.manager: shutting down -43:10.494 [INFO] TestAgent: consul server down -43:10.494 [INFO] TestAgent: shutdown complete -43:10.494 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13679 network=tcp -43:10.494 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13679 network=udp -43:10.494 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13680 network=tcp -43:10.998 [INFO] TestAgent: Waiting for endpoints to shut down -43:10.998 [INFO] TestAgent: Endpoints down ---- PASS: TestACL_Legacy_Clone (0.70s) -=== CONT TestPreparedQuery_Get -=== RUN TestPreparedQuery_Get/#00 -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:42:59.349Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:59.350Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:59.367Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:42:59.367Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:59.370 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:db90ee44-32a3-fb6d-0544-4a984fcaa514 Address:127.0.0.1:13372}]" -42:59.370 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13372 [Follower]" leader= -42:59.371 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-db90ee44-32a3-fb6d-0544-4a984fcaa514.dc1 127.0.0.1 -42:59.372 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-db90ee44-32a3-fb6d-0544-4a984fcaa514 127.0.0.1 -2022-09-15T21:42:59.372Z [INFO] agent.router: Initializing LAN area manager -42:59.373 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13353 network=udp -42:59.373 [INFO] TestAgent.server: Adding LAN server: server="Node-db90ee44-32a3-fb6d-0544-4a984fcaa514 (Addr: tcp/127.0.0.1:13372) (DC: dc1)" -42:59.373 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-db90ee44-32a3-fb6d-0544-4a984fcaa514.dc1 area=wan -42:59.373 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13353 network=tcp -42:59.375 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13354 network=tcp -42:59.375 [INFO] TestAgent: started state syncer -42:59.375 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13373 network=tcp -42:59.422 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:59.422 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13372 [Candidate]" term=2 -42:59.423 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:59.423 [DEBUG] TestAgent.server.raft: vote granted: from=db90ee44-32a3-fb6d-0544-4a984fcaa514 term=2 tally=1 -42:59.423 [INFO] TestAgent.server.raft: election won: tally=1 -42:59.423 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13372 [Leader]" -42:59.423 [INFO] TestAgent.server: cluster leadership acquired -42:59.423 [INFO] TestAgent.server: initializing acls -42:59.424 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:59.424 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:59.424 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:59.425 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:59.425 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:59.425 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:59.425 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-db90ee44-32a3-fb6d-0544-4a984fcaa514 -42:59.425 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-db90ee44-32a3-fb6d-0544-4a984fcaa514.dc1 -42:59.426 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:59.427 [INFO] TestAgent.server: New leader elected: payload=Node-db90ee44-32a3-fb6d-0544-4a984fcaa514 -42:59.428 [INFO] TestAgent.server: Updating LAN server: server="Node-db90ee44-32a3-fb6d-0544-4a984fcaa514 (Addr: tcp/127.0.0.1:13372) (DC: dc1)" -42:59.428 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-db90ee44-32a3-fb6d-0544-4a984fcaa514.dc1 area=wan -42:59.430 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:59.430 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:59.430 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:59.430 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:59.430 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:59.430 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-db90ee44-32a3-fb6d-0544-4a984fcaa514 -42:59.430 [INFO] TestAgent.server: member joined, marking health alive: member=Node-db90ee44-32a3-fb6d-0544-4a984fcaa514 -42:59.431 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-db90ee44-32a3-fb6d-0544-4a984fcaa514 -42:59.431 [INFO] TestAgent.server: federation state anti-entropy synced -42:59.649 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-db90ee44-32a3-fb6d-0544-4a984fcaa514 -42:59.649 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-db90ee44-32a3-fb6d-0544-4a984fcaa514 -42:59.652 [INFO] TestAgent.server: server starting leave -42:59.652 [INFO] TestAgent.server.serf.wan: serf: EventMemberLeave: Node-db90ee44-32a3-fb6d-0544-4a984fcaa514.dc1 127.0.0.1 -42:59.652 [INFO] TestAgent.server: Handled event for server in area: event=member-leave server=Node-db90ee44-32a3-fb6d-0544-4a984fcaa514.dc1 area=wan -2022-09-15T21:42:59.652Z [INFO] agent.router.manager: shutting down -42:59.798 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:59.803 [INFO] TestAgent: Synced node info -43:02.061 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:02.061 [DEBUG] TestAgent: Node info in sync -43:02.061 [DEBUG] TestAgent: Node info in sync -43:02.652 [INFO] TestAgent.server.serf.lan: serf: EventMemberLeave: Node-db90ee44-32a3-fb6d-0544-4a984fcaa514 127.0.0.1 -43:02.652 [INFO] TestAgent.server: Removing LAN server: server="Node-db90ee44-32a3-fb6d-0544-4a984fcaa514 (Addr: tcp/127.0.0.1:13372) (DC: dc1)" -43:02.652 [WARN] TestAgent.server: deregistering self should be done by follower: name=Node-db90ee44-32a3-fb6d-0544-4a984fcaa514 -43:03.429 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" -43:05.430 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" -43:05.652 [INFO] TestAgent.server: Waiting to drain RPC traffic: drain_time=5s -43:07.430 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" -43:09.428 [ERROR] TestAgent.server.autopilot: Error promoting servers: error="error getting server raft protocol versions: No servers found" -43:09.428 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" -43:10.653 [INFO] TestAgent: Requesting shutdown -43:10.653 [INFO] TestAgent.server: shutting down server -43:10.653 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:10.653 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:10.653 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -43:10.653 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -43:10.653 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:10.653 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:10.653 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:10.653 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:10.653 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:10.653 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:10.653 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -43:10.653 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -43:10.653 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -2022-09-15T21:43:10.654Z [INFO] agent.router.manager: shutting down -43:10.654 [INFO] TestAgent: consul server down -43:10.654 [INFO] TestAgent: shutdown complete -43:10.654 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13353 network=tcp -43:10.654 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13353 network=udp -43:10.654 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13354 network=tcp -43:11.156 [INFO] TestAgent: Waiting for endpoints to shut down -43:11.156 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Leave_ACLDeny (11.83s) - --- PASS: TestAgent_Leave_ACLDeny/no_token (0.00s) - --- PASS: TestAgent_Leave_ACLDeny/read-only_token (0.00s) - --- PASS: TestAgent_Leave_ACLDeny/agent_master_token (11.00s) -=== CONT TestPreparedQuery_Explain -=== RUN TestPreparedQuery_Explain/#00 -2022-09-15T21:43:10.632Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:10.658Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:10.661 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2ac9b212-0258-e6fc-cdb1-5f351156ce22 Address:127.0.0.1:13926}]" -43:10.662 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13926 [Follower]" leader= -43:10.663 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2ac9b212-0258-e6fc-cdb1-5f351156ce22.dc1 127.0.0.1 -43:10.664 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2ac9b212-0258-e6fc-cdb1-5f351156ce22 127.0.0.1 -2022-09-15T21:43:10.665Z [INFO] agent.router: Initializing LAN area manager -43:10.665 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13907 network=udp -43:10.665 [INFO] TestAgent.server: Adding LAN server: server="Node-2ac9b212-0258-e6fc-cdb1-5f351156ce22 (Addr: tcp/127.0.0.1:13926) (DC: dc1)" -43:10.665 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2ac9b212-0258-e6fc-cdb1-5f351156ce22.dc1 area=wan -43:10.666 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13907 network=tcp -43:10.667 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13908 network=tcp -43:10.667 [INFO] TestAgent: started state syncer -43:10.667 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13927 network=tcp -43:10.722 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:10.722 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13926 [Candidate]" term=2 -43:10.722 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:10.722 [DEBUG] TestAgent.server.raft: vote granted: from=2ac9b212-0258-e6fc-cdb1-5f351156ce22 term=2 tally=1 -43:10.722 [INFO] TestAgent.server.raft: election won: tally=1 -43:10.722 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13926 [Leader]" -43:10.722 [INFO] TestAgent.server: cluster leadership acquired -43:10.723 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13926 -43:10.724 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:10.724 [INFO] TestAgent.server: New leader elected: payload=Node-2ac9b212-0258-e6fc-cdb1-5f351156ce22 -43:10.727 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:10.727 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:10.727 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:10.727 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:10.727 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:10.727 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2ac9b212-0258-e6fc-cdb1-5f351156ce22 -43:10.727 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2ac9b212-0258-e6fc-cdb1-5f351156ce22 -43:10.728 [INFO] TestAgent.server: federation state anti-entropy synced -43:10.741 [WARN] TestAgent.server: endpoint injected; this should only be used for testing -43:10.742 [INFO] TestAgent: Requesting shutdown -43:10.742 [INFO] TestAgent.server: shutting down server -43:10.742 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:10.742 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:10.742 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:10.742 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:10.742 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:10.742 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:10.742 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:10.742 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:10.742 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:10.742 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:10.742 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:10.742 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:10.743Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:10.743Z [INFO] agent.router.manager: shutting down -43:10.743 [INFO] TestAgent: consul server down -43:10.743 [INFO] TestAgent: shutdown complete -43:10.743 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13907 network=tcp -43:10.743 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13907 network=udp -43:10.743 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13908 network=tcp -43:11.245 [INFO] TestAgent: Waiting for endpoints to shut down -43:11.245 [INFO] TestAgent: Endpoints down ---- PASS: TestPreparedQuery_Update (0.63s) -=== CONT TestPreparedQuery_ExecuteCached -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:10.362Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:10.399Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:10.414 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:34adc3bc-82f3-0ef3-9f76-31c9ef65179d Address:127.0.0.1:13898}]" -43:10.415 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13898 [Follower]" leader= -43:10.423 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-34adc3bc-82f3-0ef3-9f76-31c9ef65179d.dc1 127.0.0.1 -43:10.424 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-34adc3bc-82f3-0ef3-9f76-31c9ef65179d 127.0.0.1 -2022-09-15T21:43:10.424Z [INFO] agent.router: Initializing LAN area manager -43:10.425 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13872 network=udp -43:10.425 [INFO] TestAgent.server: Adding LAN server: server="Node-34adc3bc-82f3-0ef3-9f76-31c9ef65179d (Addr: tcp/127.0.0.1:13898) (DC: dc1)" -43:10.425 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-34adc3bc-82f3-0ef3-9f76-31c9ef65179d.dc1 area=wan -43:10.425 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13872 network=tcp -43:10.436 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13873 network=tcp -43:10.436 [INFO] TestAgent: started state syncer -43:10.436 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13899 network=tcp -43:10.473 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:10.473 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13898 [Candidate]" term=2 -43:10.473 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:10.473 [DEBUG] TestAgent.server.raft: vote granted: from=34adc3bc-82f3-0ef3-9f76-31c9ef65179d term=2 tally=1 -43:10.473 [INFO] TestAgent.server.raft: election won: tally=1 -43:10.473 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13898 [Leader]" -43:10.473 [INFO] TestAgent.server: cluster leadership acquired -43:10.474 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13898 -43:10.475 [INFO] TestAgent.server: New leader elected: payload=Node-34adc3bc-82f3-0ef3-9f76-31c9ef65179d -43:10.475 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:10.482 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:10.482 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:10.482 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:10.482 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:10.482 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:10.482 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-34adc3bc-82f3-0ef3-9f76-31c9ef65179d -43:10.482 [INFO] TestAgent.server: member joined, marking health alive: member=Node-34adc3bc-82f3-0ef3-9f76-31c9ef65179d -43:10.483 [INFO] TestAgent.server: federation state anti-entropy synced -43:10.766 [INFO] TestAgent: Requesting shutdown -43:10.766 [INFO] TestAgent.server: shutting down server -43:10.766 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:10.766 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:10.766 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:10.766 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:10.766 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:10.769 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:10.769 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:10.769 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:10.769 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:10.769 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:10.769 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:10.774 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:10.775Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:10.775Z [INFO] agent.router.manager: shutting down -43:10.775 [INFO] TestAgent: consul server down -43:10.775 [INFO] TestAgent: shutdown complete -43:10.775 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13872 network=tcp -43:10.775 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13872 network=udp -43:10.775 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13873 network=tcp -43:11.278 [INFO] TestAgent: Waiting for endpoints to shut down -43:11.278 [INFO] TestAgent: Endpoints down -=== RUN TestAgent_InmemKeyrings/client_with_keys -2022-09-15T21:43:10.426Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:10.466Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:10.469 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9083c6bd-1aa1-6634-b3e5-6958e7ad668f Address:127.0.0.1:13891}]" -43:10.470 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13891 [Follower]" leader= -43:10.470 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9083c6bd-1aa1-6634-b3e5-6958e7ad668f.dc1 127.0.0.1 -43:10.471 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9083c6bd-1aa1-6634-b3e5-6958e7ad668f 127.0.0.1 -2022-09-15T21:43:10.471Z [INFO] agent.router: Initializing LAN area manager -43:10.471 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13900 network=udp -43:10.471 [INFO] TestAgent.server: Adding LAN server: server="Node-9083c6bd-1aa1-6634-b3e5-6958e7ad668f (Addr: tcp/127.0.0.1:13891) (DC: dc1)" -43:10.471 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9083c6bd-1aa1-6634-b3e5-6958e7ad668f.dc1 area=wan -43:10.472 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13900 network=tcp -43:10.472 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13901 network=tcp -43:10.473 [INFO] TestAgent: started state syncer -43:10.473 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13892 network=tcp -43:10.523 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:10.523 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13891 [Candidate]" term=2 -43:10.523 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:10.523 [DEBUG] TestAgent.server.raft: vote granted: from=9083c6bd-1aa1-6634-b3e5-6958e7ad668f term=2 tally=1 -43:10.523 [INFO] TestAgent.server.raft: election won: tally=1 -43:10.523 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13891 [Leader]" -43:10.523 [INFO] TestAgent.server: cluster leadership acquired -43:10.524 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13891 -43:10.525 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:10.526 [INFO] TestAgent.server: New leader elected: payload=Node-9083c6bd-1aa1-6634-b3e5-6958e7ad668f -43:10.528 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:10.528 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:10.528 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:10.528 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:10.528 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:10.528 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9083c6bd-1aa1-6634-b3e5-6958e7ad668f -43:10.528 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9083c6bd-1aa1-6634-b3e5-6958e7ad668f -43:10.532 [INFO] TestAgent.server: federation state anti-entropy synced -43:10.546 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:10.547 [INFO] TestAgent: Synced node info -43:10.547 [DEBUG] TestAgent: Node info in sync -43:10.809 [WARN] TestAgent.server: endpoint injected; this should only be used for testing -43:10.809 [INFO] TestAgent: Requesting shutdown -43:10.809 [INFO] TestAgent.server: shutting down server -43:10.809 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:10.809 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:10.809 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:10.809 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:10.809 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:10.809 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:10.809 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:10.809 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:10.809 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:10.809 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:10.810 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:10.810Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:10.810Z [INFO] agent.router.manager: shutting down -43:10.810 [INFO] TestAgent: consul server down -43:10.810 [INFO] TestAgent: shutdown complete -43:10.810 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13900 network=tcp -43:10.810 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13900 network=udp -43:10.810 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13901 network=tcp -43:11.314 [INFO] TestAgent: Waiting for endpoints to shut down -43:11.314 [INFO] TestAgent: Endpoints down -=== RUN TestPreparedQuery_List/#01 -2022-09-15T21:43:10.483Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:10.510Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:10.513 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8ce5e1ac-a222-e8fa-42e1-eccf19568bea Address:127.0.0.1:13905}]" -43:10.514 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13905 [Follower]" leader= -43:10.515 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8ce5e1ac-a222-e8fa-42e1-eccf19568bea.dc1 127.0.0.1 -43:10.516 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8ce5e1ac-a222-e8fa-42e1-eccf19568bea 127.0.0.1 -2022-09-15T21:43:10.516Z [INFO] agent.router: Initializing LAN area manager -43:10.517 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13893 network=udp -43:10.517 [INFO] TestAgent.server: Adding LAN server: server="Node-8ce5e1ac-a222-e8fa-42e1-eccf19568bea (Addr: tcp/127.0.0.1:13905) (DC: dc1)" -43:10.517 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8ce5e1ac-a222-e8fa-42e1-eccf19568bea.dc1 area=wan -43:10.518 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13893 network=tcp -43:10.519 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13894 network=tcp -43:10.519 [INFO] TestAgent: started state syncer -43:10.519 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13906 network=tcp -43:10.586 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:10.586 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13905 [Candidate]" term=2 -43:10.586 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:10.586 [DEBUG] TestAgent.server.raft: vote granted: from=8ce5e1ac-a222-e8fa-42e1-eccf19568bea term=2 tally=1 -43:10.586 [INFO] TestAgent.server.raft: election won: tally=1 -43:10.586 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13905 [Leader]" -43:10.586 [INFO] TestAgent.server: cluster leadership acquired -43:10.586 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13905 -43:10.588 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:10.590 [INFO] TestAgent.server: New leader elected: payload=Node-8ce5e1ac-a222-e8fa-42e1-eccf19568bea -43:10.591 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:10.591 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:10.591 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:10.591 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:10.591 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:10.591 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8ce5e1ac-a222-e8fa-42e1-eccf19568bea -43:10.591 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8ce5e1ac-a222-e8fa-42e1-eccf19568bea -43:10.593 [INFO] TestAgent.server: federation state anti-entropy synced -43:10.830 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:10.831 [INFO] TestAgent: Synced node info -43:10.838 [WARN] TestAgent.server: endpoint injected; this should only be used for testing -43:10.838 [INFO] TestAgent: Requesting shutdown -43:10.838 [INFO] TestAgent.server: shutting down server -43:10.838 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:10.838 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:10.838 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:10.838 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:10.838 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:10.838 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:10.838 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:10.839 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:10.839 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:10.839 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:10.839 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:10.839Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:10.839Z [INFO] agent.router.manager: shutting down -43:10.839 [INFO] TestAgent: consul server down -43:10.839 [INFO] TestAgent: shutdown complete -43:10.839 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13893 network=tcp -43:10.839 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13893 network=udp -43:10.839 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13894 network=tcp -43:11.342 [INFO] TestAgent: Waiting for endpoints to shut down -43:11.342 [INFO] TestAgent: Endpoints down ---- PASS: TestPreparedQuery_Delete (0.91s) -=== CONT TestPreparedQuery_Execute -=== RUN TestPreparedQuery_Execute/#00 -[WARN] freeport: 4 out of 25 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:10.947Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:10.947Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:10.974Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:10.974Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:10.977 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:60ec25d3-fd78-195e-f546-e252a920e2ce Address:127.0.0.1:13912}]" -43:10.978 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13912 [Follower]" leader= -43:10.981 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-60ec25d3-fd78-195e-f546-e252a920e2ce.dc1 127.0.0.1 -43:10.987 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-60ec25d3-fd78-195e-f546-e252a920e2ce 127.0.0.1 -2022-09-15T21:43:10.987Z [INFO] agent.router: Initializing LAN area manager -43:10.987 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13928 network=udp -43:10.987 [INFO] TestAgent.server: Adding LAN server: server="Node-60ec25d3-fd78-195e-f546-e252a920e2ce (Addr: tcp/127.0.0.1:13912) (DC: dc1)" -43:10.987 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-60ec25d3-fd78-195e-f546-e252a920e2ce.dc1 area=wan -43:10.988 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13928 network=tcp -43:10.989 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13929 network=tcp -43:10.989 [INFO] TestAgent: started state syncer -43:10.989 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13913 network=tcp -43:11.016 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:11.016 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13912 [Candidate]" term=2 -43:11.017 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:11.017 [DEBUG] TestAgent.server.raft: vote granted: from=60ec25d3-fd78-195e-f546-e252a920e2ce term=2 tally=1 -43:11.017 [INFO] TestAgent.server.raft: election won: tally=1 -43:11.017 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13912 [Leader]" -43:11.017 [INFO] TestAgent.server: cluster leadership acquired -43:11.017 [INFO] TestAgent.server: initializing acls -43:11.018 [INFO] TestAgent.server: Created ACL 'global-management' policy -43:11.018 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -43:11.018 [INFO] TestAgent.server: New leader elected: payload=Node-60ec25d3-fd78-195e-f546-e252a920e2ce -43:11.018 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -43:11.019 [INFO] TestAgent.server: Created ACL anonymous token from configuration -43:11.019 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -43:11.019 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -43:11.019 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-60ec25d3-fd78-195e-f546-e252a920e2ce -43:11.019 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-60ec25d3-fd78-195e-f546-e252a920e2ce.dc1 -43:11.021 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:11.022 [INFO] TestAgent.server: Updating LAN server: server="Node-60ec25d3-fd78-195e-f546-e252a920e2ce (Addr: tcp/127.0.0.1:13912) (DC: dc1)" -43:11.022 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-60ec25d3-fd78-195e-f546-e252a920e2ce.dc1 area=wan -43:11.052 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:11.052 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:11.052 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:11.052 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:11.052 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:11.052 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-60ec25d3-fd78-195e-f546-e252a920e2ce -43:11.052 [INFO] TestAgent.server: member joined, marking health alive: member=Node-60ec25d3-fd78-195e-f546-e252a920e2ce -43:11.058 [INFO] TestAgent.server: federation state anti-entropy synced -43:11.061 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-60ec25d3-fd78-195e-f546-e252a920e2ce -43:11.123 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-60ec25d3-fd78-195e-f546-e252a920e2ce -43:11.124 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-60ec25d3-fd78-195e-f546-e252a920e2ce -43:11.125 [INFO] TestAgent: Requesting shutdown -43:11.126 [INFO] TestAgent.server: shutting down server -43:11.126 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:11.126 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:11.126 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:11.126 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -43:11.126 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -43:11.126 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:11.126 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:11.126 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -43:11.126 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:11.126 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:11.127 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:11.127 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:11.127 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:11.127 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:11.127 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -43:11.127 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:11.127Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:11.127Z [INFO] agent.router.manager: shutting down -43:11.127 [INFO] TestAgent: consul server down -43:11.127 [INFO] TestAgent: shutdown complete -43:11.127 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13928 network=tcp -43:11.127 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13928 network=udp -43:11.127 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13929 network=tcp -43:11.628 [INFO] TestAgent: Waiting for endpoints to shut down -43:11.628 [INFO] TestAgent: Endpoints down ---- PASS: TestACL_Legacy_Get (1.53s) - --- PASS: TestACL_Legacy_Get/wrong_id (0.81s) - --- PASS: TestACL_Legacy_Get/right_id (0.72s) -=== CONT TestOperator_AutopilotCASConfiguration -2022-09-15T21:43:11.047Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:11.132Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:11.139 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:99699ebe-3cff-66ca-a16c-bfb7bced723c Address:127.0.0.1:13554}]" -43:11.140 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-99699ebe-3cff-66ca-a16c-bfb7bced723c.dc1 127.0.0.1 -43:11.141 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-99699ebe-3cff-66ca-a16c-bfb7bced723c 127.0.0.1 -2022-09-15T21:43:11.141Z [INFO] agent.router: Initializing LAN area manager -43:11.141 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13914 network=udp -43:11.141 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13554 [Follower]" leader= -43:11.142 [INFO] TestAgent.server: Adding LAN server: server="Node-99699ebe-3cff-66ca-a16c-bfb7bced723c (Addr: tcp/127.0.0.1:13554) (DC: dc1)" -43:11.142 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-99699ebe-3cff-66ca-a16c-bfb7bced723c.dc1 area=wan -43:11.142 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13914 network=tcp -43:11.143 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13915 network=tcp -43:11.143 [INFO] TestAgent: started state syncer -43:11.143 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13449 network=tcp -43:11.177 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:11.177 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13554 [Candidate]" term=2 -43:11.178 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:11.178 [DEBUG] TestAgent.server.raft: vote granted: from=99699ebe-3cff-66ca-a16c-bfb7bced723c term=2 tally=1 -43:11.178 [INFO] TestAgent.server.raft: election won: tally=1 -43:11.178 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13554 [Leader]" -43:11.178 [INFO] TestAgent.server: cluster leadership acquired -43:11.178 [INFO] TestAgent.server: New leader elected: payload=Node-99699ebe-3cff-66ca-a16c-bfb7bced723c -43:11.179 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13554 -43:11.180 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:11.183 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:11.183 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:11.183 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:11.183 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:11.183 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:11.183 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-99699ebe-3cff-66ca-a16c-bfb7bced723c -43:11.183 [INFO] TestAgent.server: member joined, marking health alive: member=Node-99699ebe-3cff-66ca-a16c-bfb7bced723c -43:11.184 [INFO] TestAgent.server: federation state anti-entropy synced -43:11.185 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:11.185 [INFO] TestAgent: Synced node info -43:11.264 [WARN] TestAgent.server: endpoint injected; this should only be used for testing -43:11.265 [INFO] TestAgent: Requesting shutdown -43:11.265 [INFO] TestAgent.server: shutting down server -43:11.265 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:11.265 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:11.265 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:11.265 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:11.265 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:11.265 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:11.265 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:11.265 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:11.265 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:11.265 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:11.265 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:11.265Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:11.265Z [INFO] agent.router.manager: shutting down -43:11.266 [INFO] TestAgent: consul server down -43:11.266 [INFO] TestAgent: shutdown complete -43:11.266 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13914 network=tcp -43:11.266 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13914 network=udp -43:11.266 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13915 network=tcp -43:11.766 [INFO] TestAgent: Waiting for endpoints to shut down -43:11.767 [INFO] TestAgent: Endpoints down -=== RUN TestPreparedQuery_Get/#01 -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:11.196Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:11.223Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:11.225 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8088602d-094a-640e-e6af-9992b487d4b3 Address:127.0.0.1:13625}]" -43:11.226 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13625 [Follower]" leader= -43:11.227 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8088602d-094a-640e-e6af-9992b487d4b3.dc1 127.0.0.1 -43:11.228 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8088602d-094a-640e-e6af-9992b487d4b3 127.0.0.1 -2022-09-15T21:43:11.228Z [INFO] agent.router: Initializing LAN area manager -43:11.228 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13620 network=udp -43:11.229 [INFO] TestAgent.server: Adding LAN server: server="Node-8088602d-094a-640e-e6af-9992b487d4b3 (Addr: tcp/127.0.0.1:13625) (DC: dc1)" -43:11.229 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8088602d-094a-640e-e6af-9992b487d4b3.dc1 area=wan -43:11.229 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13620 network=tcp -43:11.230 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13621 network=tcp -43:11.230 [INFO] TestAgent: started state syncer -43:11.231 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13626 network=tcp -43:11.271 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:11.271 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13625 [Candidate]" term=2 -43:11.272 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:11.272 [DEBUG] TestAgent.server.raft: vote granted: from=8088602d-094a-640e-e6af-9992b487d4b3 term=2 tally=1 -43:11.272 [INFO] TestAgent.server.raft: election won: tally=1 -43:11.272 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13625 [Leader]" -43:11.272 [INFO] TestAgent.server: cluster leadership acquired -43:11.272 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13625 -43:11.273 [INFO] TestAgent.server: New leader elected: payload=Node-8088602d-094a-640e-e6af-9992b487d4b3 -43:11.275 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:11.277 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:11.277 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:11.277 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:11.277 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:11.277 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:11.277 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8088602d-094a-640e-e6af-9992b487d4b3 -43:11.277 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8088602d-094a-640e-e6af-9992b487d4b3 -43:11.278 [INFO] TestAgent.server: federation state anti-entropy synced -43:11.370 [WARN] TestAgent.server: endpoint injected; this should only be used for testing -43:11.370 [INFO] TestAgent: Requesting shutdown -43:11.370 [INFO] TestAgent.server: shutting down server -43:11.370 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:11.370 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:11.370 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:11.370 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:11.370 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:11.370 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:11.370 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:11.370 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:11.370 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:11.371 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:11.371 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:11.371 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:11.371Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:11.371Z [INFO] agent.router.manager: shutting down -43:11.371 [INFO] TestAgent: consul server down -43:11.371 [INFO] TestAgent: shutdown complete -43:11.371 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13620 network=tcp -43:11.371 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13620 network=udp -43:11.371 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13621 network=tcp -43:11.871 [INFO] TestAgent: Waiting for endpoints to shut down -43:11.871 [INFO] TestAgent: Endpoints down -=== RUN TestPreparedQuery_Explain/#01 -43:11.392 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-7f98e10c-f197-cf58-1095-3db0c948c402 127.0.0.1 -2022-09-15T21:43:11.392Z [INFO] agent.router: Initializing LAN area manager -43:11.392 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13555 network=udp -43:11.393 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13555 network=tcp -43:11.398 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13557 network=tcp -43:11.398 [INFO] TestAgent: started state syncer -43:11.399 [INFO] TestAgent: Requesting shutdown -43:11.399 [INFO] TestAgent.client: shutting down client -43:11.399 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave -43:11.399 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13452 network=tcp -43:11.399 [ERROR] TestAgent: gRPC server failed: error="grpc: the server has been stopped" -2022-09-15T21:43:11.399Z [WARN] agent.router.manager: No servers available -43:11.399 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" -43:11.399 [INFO] TestAgent: consul client down -43:11.399 [INFO] TestAgent: shutdown complete -43:11.399 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13555 network=tcp -43:11.399 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13555 network=udp -43:11.399 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13557 network=tcp -43:11.900 [INFO] TestAgent: Waiting for endpoints to shut down -43:11.900 [INFO] TestAgent: Endpoints down -=== RUN TestAgent_InmemKeyrings/ignore_files -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:11.263Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:11.327Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:11.338 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8f791882-034c-6137-8fb8-c457597e55a5 Address:127.0.0.1:13597}]" -43:11.339 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8f791882-034c-6137-8fb8-c457597e55a5.dc1 127.0.0.1 -43:11.340 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8f791882-034c-6137-8fb8-c457597e55a5 127.0.0.1 -2022-09-15T21:43:11.340Z [INFO] agent.router: Initializing LAN area manager -43:11.341 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13592 network=udp -43:11.341 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13597 [Follower]" leader= -43:11.341 [INFO] TestAgent.server: Adding LAN server: server="Node-8f791882-034c-6137-8fb8-c457597e55a5 (Addr: tcp/127.0.0.1:13597) (DC: dc1)" -43:11.341 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8f791882-034c-6137-8fb8-c457597e55a5.dc1 area=wan -43:11.341 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13592 network=tcp -43:11.359 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13593 network=tcp -43:11.359 [INFO] TestAgent: started state syncer -43:11.359 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13598 network=tcp -43:11.413 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:11.413 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13597 [Candidate]" term=2 -43:11.413 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:11.413 [DEBUG] TestAgent.server.raft: vote granted: from=8f791882-034c-6137-8fb8-c457597e55a5 term=2 tally=1 -43:11.413 [INFO] TestAgent.server.raft: election won: tally=1 -43:11.413 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13597 [Leader]" -43:11.413 [INFO] TestAgent.server: cluster leadership acquired -43:11.414 [INFO] TestAgent.server: New leader elected: payload=Node-8f791882-034c-6137-8fb8-c457597e55a5 -43:11.414 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13597 -43:11.415 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:11.417 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:11.417 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:11.417 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:11.417 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:11.417 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:11.417 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8f791882-034c-6137-8fb8-c457597e55a5 -43:11.418 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8f791882-034c-6137-8fb8-c457597e55a5 -43:11.418 [INFO] TestAgent.server: federation state anti-entropy synced -43:11.521 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:11.522 [INFO] TestAgent: Synced node info -43:11.533 [WARN] TestAgent.server: endpoint injected; this should only be used for testing -43:11.534 [INFO] TestAgent: Requesting shutdown -43:11.534 [INFO] TestAgent.server: shutting down server -43:11.534 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:11.534 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:11.534 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:11.534 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:11.534 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:11.534 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:11.534 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:11.534 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:11.534 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:11.534 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:11.534 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:11.534Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:11.534Z [INFO] agent.router.manager: shutting down -43:11.535 [INFO] TestAgent: consul server down -43:11.535 [INFO] TestAgent: shutdown complete -43:11.535 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13592 network=tcp -43:11.535 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13592 network=udp -43:11.535 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13593 network=tcp -43:12.036 [INFO] TestAgent: Waiting for endpoints to shut down -43:12.036 [INFO] TestAgent: Endpoints down ---- PASS: TestPreparedQuery_ExecuteCached (0.79s) -=== CONT TestPreparedQuery_Create -2022-09-15T21:43:11.377Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:11.407Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:11.410 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6408c31e-d1e3-78a3-73ea-23d7abb904fe Address:127.0.0.1:13937}]" -43:11.411 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6408c31e-d1e3-78a3-73ea-23d7abb904fe.dc1 127.0.0.1 -43:11.412 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6408c31e-d1e3-78a3-73ea-23d7abb904fe 127.0.0.1 -2022-09-15T21:43:11.412Z [INFO] agent.router: Initializing LAN area manager -43:11.412 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13937 [Follower]" leader= -43:11.413 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6408c31e-d1e3-78a3-73ea-23d7abb904fe.dc1 area=wan -43:11.413 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13932 network=tcp -43:11.413 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13932 network=udp -43:11.413 [INFO] TestAgent.server: Adding LAN server: server="Node-6408c31e-d1e3-78a3-73ea-23d7abb904fe (Addr: tcp/127.0.0.1:13937) (DC: dc1)" -43:11.414 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13933 network=tcp -43:11.414 [INFO] TestAgent: started state syncer -43:11.414 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13938 network=tcp -43:11.481 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:11.481 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13937 [Candidate]" term=2 -43:11.481 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:11.481 [DEBUG] TestAgent.server.raft: vote granted: from=6408c31e-d1e3-78a3-73ea-23d7abb904fe term=2 tally=1 -43:11.481 [INFO] TestAgent.server.raft: election won: tally=1 -43:11.481 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13937 [Leader]" -43:11.482 [INFO] TestAgent.server: cluster leadership acquired -43:11.482 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13937 -43:11.483 [INFO] TestAgent.server: New leader elected: payload=Node-6408c31e-d1e3-78a3-73ea-23d7abb904fe -43:11.483 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:11.486 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:11.486 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:11.486 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:11.486 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:11.486 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:11.487 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6408c31e-d1e3-78a3-73ea-23d7abb904fe -43:11.487 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6408c31e-d1e3-78a3-73ea-23d7abb904fe -43:11.488 [INFO] TestAgent.server: federation state anti-entropy synced -43:11.550 [WARN] TestAgent.server: endpoint injected; this should only be used for testing -43:11.550 [INFO] TestAgent: Requesting shutdown -43:11.550 [INFO] TestAgent.server: shutting down server -43:11.550 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:11.550 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:11.550 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:11.550 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:11.550 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:11.550 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:11.550 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:11.551 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:11.551 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:11.551 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:11.551 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:11.551 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:11.551Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:11.551Z [INFO] agent.router.manager: shutting down -43:11.551 [INFO] TestAgent: consul server down -43:11.551 [INFO] TestAgent: shutdown complete -43:11.551 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13932 network=tcp -43:11.551 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13932 network=udp -43:11.551 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13933 network=tcp -43:12.052 [INFO] TestAgent: Waiting for endpoints to shut down -43:12.052 [INFO] TestAgent: Endpoints down -=== RUN TestPreparedQuery_Execute/#01 -2022-09-15T21:43:11.372Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:11.393Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:11.400 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ef39f8ba-7d4c-2162-bb35-a88f56e12b33 Address:127.0.0.1:13930}]" -43:11.401 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ef39f8ba-7d4c-2162-bb35-a88f56e12b33.dc1 127.0.0.1 -43:11.402 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13930 [Follower]" leader= -43:11.419 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ef39f8ba-7d4c-2162-bb35-a88f56e12b33 127.0.0.1 -2022-09-15T21:43:11.419Z [INFO] agent.router: Initializing LAN area manager -43:11.419 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13918 network=udp -43:11.419 [INFO] TestAgent.server: Adding LAN server: server="Node-ef39f8ba-7d4c-2162-bb35-a88f56e12b33 (Addr: tcp/127.0.0.1:13930) (DC: dc1)" -43:11.419 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ef39f8ba-7d4c-2162-bb35-a88f56e12b33.dc1 area=wan -43:11.420 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13918 network=tcp -43:11.420 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13919 network=tcp -43:11.421 [INFO] TestAgent: started state syncer -43:11.421 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13931 network=tcp -43:11.470 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:11.470 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13930 [Candidate]" term=2 -43:11.470 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:11.470 [DEBUG] TestAgent.server.raft: vote granted: from=ef39f8ba-7d4c-2162-bb35-a88f56e12b33 term=2 tally=1 -43:11.470 [INFO] TestAgent.server.raft: election won: tally=1 -43:11.470 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13930 [Leader]" -43:11.471 [INFO] TestAgent.server: cluster leadership acquired -43:11.471 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13930 -43:11.472 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:11.473 [INFO] TestAgent.server: New leader elected: payload=Node-ef39f8ba-7d4c-2162-bb35-a88f56e12b33 -43:11.476 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:11.476 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:11.476 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:11.476 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:11.476 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:11.476 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ef39f8ba-7d4c-2162-bb35-a88f56e12b33 -43:11.476 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ef39f8ba-7d4c-2162-bb35-a88f56e12b33 -43:11.477 [INFO] TestAgent.server: federation state anti-entropy synced -43:11.718 [WARN] TestAgent.server: endpoint injected; this should only be used for testing -43:11.719 [INFO] TestAgent: Requesting shutdown -43:11.719 [INFO] TestAgent.server: shutting down server -43:11.719 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:11.719 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:11.719 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:11.719 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:11.719 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:11.719 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:11.719 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:11.719 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:11.719 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:11.719 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:11.719 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:11.719 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:11.720Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:11.720Z [INFO] agent.router.manager: shutting down -43:11.720 [INFO] TestAgent: consul server down -43:11.720 [INFO] TestAgent: shutdown complete -43:11.720 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13918 network=tcp -43:11.720 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13918 network=udp -43:11.720 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13919 network=tcp -43:12.220 [INFO] TestAgent: Waiting for endpoints to shut down -43:12.220 [INFO] TestAgent: Endpoints down ---- PASS: TestPreparedQuery_List (1.83s) - --- PASS: TestPreparedQuery_List/#00 (0.93s) - --- PASS: TestPreparedQuery_List/#01 (0.91s) -=== CONT TestOperator_ServerHealth_Unhealthy -[WARN] freeport: 4 out of 25 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:11.661Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:11.680Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:11.687 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:432a1918-ffa6-1538-735f-40f7834458ac Address:127.0.0.1:13637}]" -43:11.688 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-432a1918-ffa6-1538-735f-40f7834458ac.dc1 127.0.0.1 -43:11.689 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-432a1918-ffa6-1538-735f-40f7834458ac 127.0.0.1 -2022-09-15T21:43:11.689Z [INFO] agent.router: Initializing LAN area manager -43:11.690 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13637 [Follower]" leader= -43:11.691 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13632 network=udp -43:11.691 [INFO] TestAgent.server: Adding LAN server: server="Node-432a1918-ffa6-1538-735f-40f7834458ac (Addr: tcp/127.0.0.1:13637) (DC: dc1)" -43:11.691 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-432a1918-ffa6-1538-735f-40f7834458ac.dc1 area=wan -43:11.691 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13632 network=tcp -43:11.692 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13633 network=tcp -43:11.692 [INFO] TestAgent: started state syncer -43:11.692 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13666 network=tcp -43:11.757 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:11.757 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13637 [Candidate]" term=2 -43:11.757 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:11.757 [DEBUG] TestAgent.server.raft: vote granted: from=432a1918-ffa6-1538-735f-40f7834458ac term=2 tally=1 -43:11.757 [INFO] TestAgent.server.raft: election won: tally=1 -43:11.757 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13637 [Leader]" -43:11.757 [INFO] TestAgent.server: cluster leadership acquired -43:11.758 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13637 -43:11.759 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:11.758 [INFO] TestAgent.server: New leader elected: payload=Node-432a1918-ffa6-1538-735f-40f7834458ac -43:11.761 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:11.761 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:11.761 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:11.761 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:11.761 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:11.761 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-432a1918-ffa6-1538-735f-40f7834458ac -43:11.761 [INFO] TestAgent.server: member joined, marking health alive: member=Node-432a1918-ffa6-1538-735f-40f7834458ac -43:11.762 [INFO] TestAgent.server: federation state anti-entropy synced -43:11.774 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:11.775 [INFO] TestAgent: Synced node info -43:11.775 [DEBUG] TestAgent: Node info in sync -43:11.859 [INFO] TestAgent: Requesting shutdown -43:11.859 [INFO] TestAgent.server: shutting down server -43:11.859 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:11.859 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:11.859 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:11.859 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:11.859 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:11.860 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:11.860 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:11.860 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:11.860 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:11.860 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:11.860 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:11.860Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:11.860Z [INFO] agent.router.manager: shutting down -43:11.860 [INFO] TestAgent: consul server down -43:11.860 [INFO] TestAgent: shutdown complete -43:11.860 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13632 network=tcp -43:11.860 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13632 network=udp -43:11.860 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13633 network=tcp -43:12.361 [INFO] TestAgent: Waiting for endpoints to shut down -43:12.361 [INFO] TestAgent: Endpoints down ---- PASS: TestOperator_AutopilotCASConfiguration (0.73s) -=== CONT TestOperator_ServerHealth -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:11.889Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:11.908Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:11.910 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:73105bb1-62e9-6742-6dc9-82903d54159f Address:127.0.0.1:13958}]" -43:11.912 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-73105bb1-62e9-6742-6dc9-82903d54159f.dc1 127.0.0.1 -43:11.912 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-73105bb1-62e9-6742-6dc9-82903d54159f 127.0.0.1 -2022-09-15T21:43:11.912Z [INFO] agent.router: Initializing LAN area manager -43:11.913 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13958 [Follower]" leader= -43:11.913 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13939 network=udp -43:11.913 [INFO] TestAgent.server: Adding LAN server: server="Node-73105bb1-62e9-6742-6dc9-82903d54159f (Addr: tcp/127.0.0.1:13958) (DC: dc1)" -43:11.913 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-73105bb1-62e9-6742-6dc9-82903d54159f.dc1 area=wan -43:11.913 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13939 network=tcp -43:11.914 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13940 network=tcp -43:11.914 [INFO] TestAgent: started state syncer -43:11.914 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13959 network=tcp -43:11.951 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:11.951 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13958 [Candidate]" term=2 -43:11.951 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:11.951 [DEBUG] TestAgent.server.raft: vote granted: from=73105bb1-62e9-6742-6dc9-82903d54159f term=2 tally=1 -43:11.951 [INFO] TestAgent.server.raft: election won: tally=1 -43:11.951 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13958 [Leader]" -43:11.952 [INFO] TestAgent.server: cluster leadership acquired -43:11.952 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13958 -43:11.952 [INFO] TestAgent.server: New leader elected: payload=Node-73105bb1-62e9-6742-6dc9-82903d54159f -43:11.953 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:11.956 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:11.956 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:11.956 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:11.956 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:11.956 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:11.956 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-73105bb1-62e9-6742-6dc9-82903d54159f -43:11.956 [INFO] TestAgent.server: member joined, marking health alive: member=Node-73105bb1-62e9-6742-6dc9-82903d54159f -43:11.957 [INFO] TestAgent.server: federation state anti-entropy synced -43:12.035 [INFO] TestAgent: Requesting shutdown -43:12.035 [INFO] TestAgent.server: shutting down server -43:12.035 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:12.035 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:12.035 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:12.035 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:12.035 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:12.035 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:12.035 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:12.035 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:12.035 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:12.035 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:12.036 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:12.036 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:12.036Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:12.036Z [INFO] agent.router.manager: shutting down -43:12.036 [INFO] TestAgent: consul server down -43:12.036 [INFO] TestAgent: shutdown complete -43:12.036 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13939 network=tcp -43:12.036 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13939 network=udp -43:12.036 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13940 network=tcp -43:12.536 [INFO] TestAgent: Waiting for endpoints to shut down -43:12.536 [INFO] TestAgent: Endpoints down -=== RUN TestPreparedQuery_Explain/#02 -2022-09-15T21:43:11.833Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:11.868Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:11.878 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7d07a9f7-53e5-2345-1589-1f47056b73db Address:127.0.0.1:13972}]" -43:11.879 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7d07a9f7-53e5-2345-1589-1f47056b73db.dc1 127.0.0.1 -43:11.880 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7d07a9f7-53e5-2345-1589-1f47056b73db 127.0.0.1 -2022-09-15T21:43:11.880Z [INFO] agent.router: Initializing LAN area manager -43:11.880 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13967 network=udp -43:11.880 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13972 [Follower]" leader= -43:11.881 [INFO] TestAgent.server: Adding LAN server: server="Node-7d07a9f7-53e5-2345-1589-1f47056b73db (Addr: tcp/127.0.0.1:13972) (DC: dc1)" -43:11.881 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7d07a9f7-53e5-2345-1589-1f47056b73db.dc1 area=wan -43:11.881 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13967 network=tcp -43:11.882 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13968 network=tcp -43:11.882 [INFO] TestAgent: started state syncer -43:11.883 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13973 network=tcp -43:11.928 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:11.928 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13972 [Candidate]" term=2 -43:11.928 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:11.928 [DEBUG] TestAgent.server.raft: vote granted: from=7d07a9f7-53e5-2345-1589-1f47056b73db term=2 tally=1 -43:11.928 [INFO] TestAgent.server.raft: election won: tally=1 -43:11.928 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13972 [Leader]" -43:11.928 [INFO] TestAgent.server: cluster leadership acquired -43:11.929 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13972 -43:11.930 [INFO] TestAgent.server: New leader elected: payload=Node-7d07a9f7-53e5-2345-1589-1f47056b73db -43:11.932 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:11.935 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:11.935 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:11.935 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:11.935 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:11.935 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:11.935 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7d07a9f7-53e5-2345-1589-1f47056b73db -43:11.935 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7d07a9f7-53e5-2345-1589-1f47056b73db -43:11.936 [INFO] TestAgent.server: federation state anti-entropy synced -43:12.106 [INFO] TestAgent: Requesting shutdown -43:12.106 [INFO] TestAgent.server: shutting down server -43:12.106 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:12.106 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:12.106 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:12.106 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:12.106 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:12.106 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:12.106 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:12.106 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:12.106 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:12.106 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:12.106 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:12.106 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:12.107Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:12.107Z [INFO] agent.router.manager: shutting down -43:12.107 [INFO] TestAgent: consul server down -43:12.107 [INFO] TestAgent: shutdown complete -43:12.107 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13967 network=tcp -43:12.107 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13967 network=udp -43:12.107 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13968 network=tcp -43:12.615 [INFO] TestAgent: Waiting for endpoints to shut down -43:12.615 [INFO] TestAgent: Endpoints down ---- PASS: TestPreparedQuery_Get (1.62s) - --- PASS: TestPreparedQuery_Get/#00 (0.77s) - --- PASS: TestPreparedQuery_Get/#01 (0.85s) -=== CONT TestOperator_AutopilotGetConfiguration -2022-09-15T21:43:11.919Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:11.937Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:11.940 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4eacaf9c-57bd-fbdf-3867-3252fd3d0222 Address:127.0.0.1:13944}]" -43:11.941 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4eacaf9c-57bd-fbdf-3867-3252fd3d0222.dc1 127.0.0.1 -43:11.941 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13944 [Follower]" leader= -43:11.942 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4eacaf9c-57bd-fbdf-3867-3252fd3d0222 127.0.0.1 -2022-09-15T21:43:11.942Z [INFO] agent.router: Initializing LAN area manager -43:11.943 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13960 network=udp -43:11.943 [INFO] TestAgent.server: Adding LAN server: server="Node-4eacaf9c-57bd-fbdf-3867-3252fd3d0222 (Addr: tcp/127.0.0.1:13944) (DC: dc1)" -43:11.943 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4eacaf9c-57bd-fbdf-3867-3252fd3d0222.dc1 area=wan -43:11.943 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13960 network=tcp -43:11.944 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13961 network=tcp -43:11.944 [INFO] TestAgent: started state syncer -43:11.944 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13945 network=tcp -43:11.984 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:11.985 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13944 [Candidate]" term=2 -43:11.985 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:11.985 [DEBUG] TestAgent.server.raft: vote granted: from=4eacaf9c-57bd-fbdf-3867-3252fd3d0222 term=2 tally=1 -43:11.985 [INFO] TestAgent.server.raft: election won: tally=1 -43:11.985 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13944 [Leader]" -43:11.985 [INFO] TestAgent.server: cluster leadership acquired -43:11.985 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13944 -43:11.987 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:11.989 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:11.989 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:11.989 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:11.989 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:11.990 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:11.990 [INFO] TestAgent.server: New leader elected: payload=Node-4eacaf9c-57bd-fbdf-3867-3252fd3d0222 -43:11.990 [INFO] TestAgent.server: federation state anti-entropy synced -43:11.991 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4eacaf9c-57bd-fbdf-3867-3252fd3d0222 -43:11.991 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4eacaf9c-57bd-fbdf-3867-3252fd3d0222 -43:12.168 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:12.168 [INFO] TestAgent: Synced node info -43:12.260 [INFO] TestAgent: Requesting shutdown -43:12.260 [INFO] TestAgent.server: shutting down server -43:12.260 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:12.260 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:12.260 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:12.260 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:12.260 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:12.260 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:12.260 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:12.260 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:12.260 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:12.260 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:12.260 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:12.261Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:12.261Z [INFO] agent.router.manager: shutting down -43:12.261 [INFO] TestAgent: consul server down -43:12.261 [INFO] TestAgent: shutdown complete -43:12.261 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13960 network=tcp -43:12.261 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13960 network=udp -43:12.261 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13961 network=tcp -43:12.761 [INFO] TestAgent: Waiting for endpoints to shut down -43:12.762 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_InmemKeyrings (3.46s) - --- PASS: TestAgent_InmemKeyrings/no_keys (1.03s) - --- PASS: TestAgent_InmemKeyrings/server_with_keys (0.95s) - --- PASS: TestAgent_InmemKeyrings/client_with_keys (0.62s) - --- PASS: TestAgent_InmemKeyrings/ignore_files (0.86s) -=== CONT TestOperator_AutopilotSetConfiguration -[WARN] freeport: 4 out of 25 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:12.070Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:12.089Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:12.092 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:768f8a8e-be82-189f-0c15-5d2c5fafffe7 Address:127.0.0.1:13979}]" -43:12.093 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13979 [Follower]" leader= -43:12.093 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-768f8a8e-be82-189f-0c15-5d2c5fafffe7.dc1 127.0.0.1 -43:12.095 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-768f8a8e-be82-189f-0c15-5d2c5fafffe7 127.0.0.1 -2022-09-15T21:43:12.095Z [INFO] agent.router: Initializing LAN area manager -43:12.095 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-768f8a8e-be82-189f-0c15-5d2c5fafffe7.dc1 area=wan -43:12.095 [INFO] TestAgent.server: Adding LAN server: server="Node-768f8a8e-be82-189f-0c15-5d2c5fafffe7 (Addr: tcp/127.0.0.1:13979) (DC: dc1)" -43:12.095 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13974 network=udp -43:12.096 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13974 network=tcp -43:12.097 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13975 network=tcp -43:12.097 [INFO] TestAgent: started state syncer -43:12.097 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13980 network=tcp -43:12.156 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:12.156 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13979 [Candidate]" term=2 -43:12.157 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:12.157 [DEBUG] TestAgent.server.raft: vote granted: from=768f8a8e-be82-189f-0c15-5d2c5fafffe7 term=2 tally=1 -43:12.157 [INFO] TestAgent.server.raft: election won: tally=1 -43:12.157 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13979 [Leader]" -43:12.157 [INFO] TestAgent.server: cluster leadership acquired -43:12.157 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13979 -43:12.158 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:12.160 [INFO] TestAgent.server: New leader elected: payload=Node-768f8a8e-be82-189f-0c15-5d2c5fafffe7 -43:12.161 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:12.161 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:12.161 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:12.161 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:12.161 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:12.161 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-768f8a8e-be82-189f-0c15-5d2c5fafffe7 -43:12.161 [INFO] TestAgent.server: member joined, marking health alive: member=Node-768f8a8e-be82-189f-0c15-5d2c5fafffe7 -43:12.163 [INFO] TestAgent.server: federation state anti-entropy synced -43:12.199 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:12.200 [INFO] TestAgent: Synced node info -43:12.200 [DEBUG] TestAgent: Node info in sync -43:12.378 [WARN] TestAgent.server: endpoint injected; this should only be used for testing -43:12.378 [INFO] TestAgent: Requesting shutdown -43:12.378 [INFO] TestAgent.server: shutting down server -43:12.378 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:12.378 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:12.378 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:12.378 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:12.378 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:12.378 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:12.378 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:12.378 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:12.379 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:12.379 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:12.379 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:12.379Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:12.379Z [INFO] agent.router.manager: shutting down -43:12.379 [INFO] TestAgent: consul server down -43:12.379 [INFO] TestAgent: shutdown complete -43:12.379 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13974 network=tcp -43:12.379 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13974 network=udp -43:12.379 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13975 network=tcp -43:12.882 [INFO] TestAgent: Waiting for endpoints to shut down -43:12.882 [INFO] TestAgent: Endpoints down -=== RUN TestPreparedQuery_Execute/#02 -2022-09-15T21:43:12.054Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:12.070Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:12.073 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9e1fa00c-cdf6-777e-1796-cfc464cf9c0f Address:127.0.0.1:13965}]" -43:12.074 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13965 [Follower]" leader= -43:12.074 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9e1fa00c-cdf6-777e-1796-cfc464cf9c0f.dc1 127.0.0.1 -43:12.075 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9e1fa00c-cdf6-777e-1796-cfc464cf9c0f 127.0.0.1 -2022-09-15T21:43:12.075Z [INFO] agent.router: Initializing LAN area manager -43:12.075 [INFO] TestAgent.server: Adding LAN server: server="Node-9e1fa00c-cdf6-777e-1796-cfc464cf9c0f (Addr: tcp/127.0.0.1:13965) (DC: dc1)" -43:12.075 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9e1fa00c-cdf6-777e-1796-cfc464cf9c0f.dc1 area=wan -43:12.076 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13946 network=udp -43:12.076 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13946 network=tcp -43:12.076 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13947 network=tcp -43:12.077 [INFO] TestAgent: started state syncer -43:12.077 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13966 network=tcp -43:12.132 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:12.132 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13965 [Candidate]" term=2 -43:12.133 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:12.133 [DEBUG] TestAgent.server.raft: vote granted: from=9e1fa00c-cdf6-777e-1796-cfc464cf9c0f term=2 tally=1 -43:12.133 [INFO] TestAgent.server.raft: election won: tally=1 -43:12.133 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13965 [Leader]" -43:12.133 [INFO] TestAgent.server: cluster leadership acquired -43:12.133 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13965 -43:12.134 [INFO] TestAgent.server: New leader elected: payload=Node-9e1fa00c-cdf6-777e-1796-cfc464cf9c0f -43:12.134 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:12.137 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:12.137 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:12.137 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:12.137 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:12.137 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:12.137 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9e1fa00c-cdf6-777e-1796-cfc464cf9c0f -43:12.137 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9e1fa00c-cdf6-777e-1796-cfc464cf9c0f -43:12.138 [INFO] TestAgent.server: federation state anti-entropy synced -43:12.379 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:12.380 [INFO] TestAgent: Synced node info -43:12.398 [WARN] TestAgent.server: endpoint injected; this should only be used for testing -43:12.398 [INFO] TestAgent: Requesting shutdown -43:12.398 [INFO] TestAgent.server: shutting down server -43:12.398 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:12.398 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:12.398 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:12.398 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:12.398 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:12.398 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:12.398 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:12.399 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:12.399 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:12.399 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:12.399 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:12.399Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:12.399Z [INFO] agent.router.manager: shutting down -43:12.399 [INFO] TestAgent: consul server down -43:12.399 [INFO] TestAgent: shutdown complete -43:12.399 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13946 network=tcp -43:12.399 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13946 network=udp -43:12.399 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13947 network=tcp -43:12.901 [INFO] TestAgent: Waiting for endpoints to shut down -43:12.901 [INFO] TestAgent: Endpoints down ---- PASS: TestPreparedQuery_Create (0.86s) -=== CONT TestTxnEndpoint_Bad_Size_Net -=== RUN TestTxnEndpoint_Bad_Size_Net/exceeds_default_limits -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 4 out of 4 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:12.557Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:12.578Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:12.581 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:17102122-704d-202c-e946-0bb00f57b42c Address:127.0.0.1:13993}]" -43:12.582 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13993 [Follower]" leader= -43:12.586 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-17102122-704d-202c-e946-0bb00f57b42c.dc1 127.0.0.1 -43:12.587 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-17102122-704d-202c-e946-0bb00f57b42c 127.0.0.1 -2022-09-15T21:43:12.587Z [INFO] agent.router: Initializing LAN area manager -43:12.588 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13988 network=udp -43:12.588 [INFO] TestAgent.server: Adding LAN server: server="Node-17102122-704d-202c-e946-0bb00f57b42c (Addr: tcp/127.0.0.1:13993) (DC: dc1)" -43:12.588 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-17102122-704d-202c-e946-0bb00f57b42c.dc1 area=wan -43:12.588 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13988 network=tcp -43:12.589 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13989 network=tcp -43:12.589 [INFO] TestAgent: started state syncer -43:12.589 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13994 network=tcp -43:12.620 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:12.620 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13993 [Candidate]" term=2 -43:12.620 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:12.620 [DEBUG] TestAgent.server.raft: vote granted: from=17102122-704d-202c-e946-0bb00f57b42c term=2 tally=1 -43:12.620 [INFO] TestAgent.server.raft: election won: tally=1 -43:12.620 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13993 [Leader]" -43:12.620 [INFO] TestAgent.server: cluster leadership acquired -43:12.620 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13993 -43:12.622 [INFO] TestAgent.server: New leader elected: payload=Node-17102122-704d-202c-e946-0bb00f57b42c -43:12.622 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:12.624 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:12.624 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:12.624 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:12.624 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:12.624 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:12.624 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-17102122-704d-202c-e946-0bb00f57b42c -43:12.624 [INFO] TestAgent.server: member joined, marking health alive: member=Node-17102122-704d-202c-e946-0bb00f57b42c -43:12.627 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:12.643 [INFO] TestAgent.server: federation state anti-entropy synced -43:12.643 [INFO] TestAgent: Synced node info -43:12.643 [DEBUG] TestAgent: Node info in sync -43:12.887 [WARN] TestAgent.server: endpoint injected; this should only be used for testing -43:12.887 [INFO] TestAgent: Requesting shutdown -43:12.887 [INFO] TestAgent.server: shutting down server -43:12.887 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:12.887 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:12.887 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:12.887 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:12.887 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:12.888 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:12.888 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:12.888 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:12.888 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:12.888 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:12.888 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:12.888Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:12.888Z [INFO] agent.router.manager: shutting down -43:12.888 [INFO] TestAgent: consul server down -43:12.888 [INFO] TestAgent: shutdown complete -43:12.888 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13988 network=tcp -43:12.888 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13988 network=udp -43:12.888 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13989 network=tcp -43:13.391 [INFO] TestAgent: Waiting for endpoints to shut down -43:13.391 [INFO] TestAgent: Endpoints down ---- PASS: TestPreparedQuery_Explain (2.23s) - --- PASS: TestPreparedQuery_Explain/#00 (0.72s) - --- PASS: TestPreparedQuery_Explain/#01 (0.66s) - --- PASS: TestPreparedQuery_Explain/#02 (0.85s) -=== CONT TestFireReceiveEvent -2022-09-15T21:43:12.806Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:12.823Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:12.826 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5cfaec97-5010-e240-21da-23b2c43c3a29 Address:127.0.0.1:14028}]" -43:12.827 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5cfaec97-5010-e240-21da-23b2c43c3a29.dc1 127.0.0.1 -43:12.828 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5cfaec97-5010-e240-21da-23b2c43c3a29 127.0.0.1 -2022-09-15T21:43:12.828Z [INFO] agent.router: Initializing LAN area manager -43:12.828 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14023 network=udp -43:12.828 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14028 [Follower]" leader= -43:12.829 [INFO] TestAgent.server: Adding LAN server: server="Node-5cfaec97-5010-e240-21da-23b2c43c3a29 (Addr: tcp/127.0.0.1:14028) (DC: dc1)" -43:12.829 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5cfaec97-5010-e240-21da-23b2c43c3a29.dc1 area=wan -43:12.829 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14023 network=tcp -43:12.830 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14024 network=tcp -43:12.830 [INFO] TestAgent: started state syncer -43:12.830 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14029 network=tcp -43:12.886 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:12.886 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14028 [Candidate]" term=2 -43:12.887 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:12.887 [DEBUG] TestAgent.server.raft: vote granted: from=5cfaec97-5010-e240-21da-23b2c43c3a29 term=2 tally=1 -43:12.887 [INFO] TestAgent.server.raft: election won: tally=1 -43:12.887 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14028 [Leader]" -43:12.887 [INFO] TestAgent.server: cluster leadership acquired -43:12.887 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14028 -43:12.888 [INFO] TestAgent.server: New leader elected: payload=Node-5cfaec97-5010-e240-21da-23b2c43c3a29 -43:12.889 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:12.891 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:12.891 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:12.891 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:12.891 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:12.891 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:12.891 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5cfaec97-5010-e240-21da-23b2c43c3a29 -43:12.891 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5cfaec97-5010-e240-21da-23b2c43c3a29 -43:12.893 [INFO] TestAgent.server: federation state anti-entropy synced -43:12.906 [INFO] TestAgent: Requesting shutdown -43:12.906 [INFO] TestAgent.server: shutting down server -43:12.906 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:12.906 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:12.906 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:12.906 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:12.906 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:12.907 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:12.907 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:12.907 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:12.907 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:12.907 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:12.907 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:12.907 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:12.907Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:12.907Z [INFO] agent.router.manager: shutting down -43:12.907 [INFO] TestAgent: consul server down -43:12.907 [INFO] TestAgent: shutdown complete -43:12.907 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14023 network=tcp -43:12.907 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14023 network=udp -43:12.907 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14024 network=tcp -43:13.409 [INFO] TestAgent: Waiting for endpoints to shut down -43:13.409 [INFO] TestAgent: Endpoints down ---- PASS: TestOperator_AutopilotSetConfiguration (0.65s) -=== CONT TestIngestUserEvent -2022-09-15T21:43:12.655Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:12.673Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:12.676 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:853ddab2-5c4d-e205-ceae-ecac26a4095c Address:127.0.0.1:14014}]" -43:12.676 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14014 [Follower]" leader= -43:12.677 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-853ddab2-5c4d-e205-ceae-ecac26a4095c.dc1 127.0.0.1 -43:12.677 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-853ddab2-5c4d-e205-ceae-ecac26a4095c 127.0.0.1 -2022-09-15T21:43:12.677Z [INFO] agent.router: Initializing LAN area manager -43:12.678 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13995 network=udp -43:12.678 [INFO] TestAgent.server: Adding LAN server: server="Node-853ddab2-5c4d-e205-ceae-ecac26a4095c (Addr: tcp/127.0.0.1:14014) (DC: dc1)" -43:12.678 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-853ddab2-5c4d-e205-ceae-ecac26a4095c.dc1 area=wan -43:12.678 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13995 network=tcp -43:12.679 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13996 network=tcp -43:12.679 [INFO] TestAgent: started state syncer -43:12.679 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14015 network=tcp -43:12.724 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:12.724 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14014 [Candidate]" term=2 -43:12.724 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:12.724 [DEBUG] TestAgent.server.raft: vote granted: from=853ddab2-5c4d-e205-ceae-ecac26a4095c term=2 tally=1 -43:12.724 [INFO] TestAgent.server.raft: election won: tally=1 -43:12.724 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14014 [Leader]" -43:12.724 [INFO] TestAgent.server: cluster leadership acquired -43:12.725 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14014 -43:12.725 [INFO] TestAgent.server: New leader elected: payload=Node-853ddab2-5c4d-e205-ceae-ecac26a4095c -43:12.726 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:12.728 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:12.728 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:12.728 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:12.728 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:12.728 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:12.729 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-853ddab2-5c4d-e205-ceae-ecac26a4095c -43:12.729 [INFO] TestAgent.server: member joined, marking health alive: member=Node-853ddab2-5c4d-e205-ceae-ecac26a4095c -43:12.735 [INFO] TestAgent.server: federation state anti-entropy synced -43:12.922 [INFO] TestAgent: Requesting shutdown -43:12.922 [INFO] TestAgent.server: shutting down server -43:12.922 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:12.922 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:12.922 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:12.922 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:12.922 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:12.922 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:12.922 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:12.923 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:12.923 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:12.923 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:12.923 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:12.923 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:12.923Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:12.923Z [INFO] agent.router.manager: shutting down -43:12.923 [INFO] TestAgent: consul server down -43:12.923 [INFO] TestAgent: shutdown complete -43:12.923 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13995 network=tcp -43:12.923 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13995 network=udp -43:12.923 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13996 network=tcp -43:13.426 [INFO] TestAgent: Waiting for endpoints to shut down -43:13.426 [INFO] TestAgent: Endpoints down ---- PASS: TestOperator_AutopilotGetConfiguration (0.81s) -=== CONT TestShouldProcessUserEvent -[WARN] freeport: 4 out of 25 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:12.901Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:12.944Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:12.955 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:dcf17898-77f1-f3b1-797f-bc0fe97a4027 Address:127.0.0.1:14035}]" -43:12.956 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-dcf17898-77f1-f3b1-797f-bc0fe97a4027.dc1 127.0.0.1 -43:12.957 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-dcf17898-77f1-f3b1-797f-bc0fe97a4027 127.0.0.1 -2022-09-15T21:43:12.957Z [INFO] agent.router: Initializing LAN area manager -43:12.957 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14030 network=udp -43:12.957 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14035 [Follower]" leader= -43:12.958 [INFO] TestAgent.server: Adding LAN server: server="Node-dcf17898-77f1-f3b1-797f-bc0fe97a4027 (Addr: tcp/127.0.0.1:14035) (DC: dc1)" -43:12.958 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-dcf17898-77f1-f3b1-797f-bc0fe97a4027.dc1 area=wan -43:12.958 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14030 network=tcp -43:12.963 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14031 network=tcp -43:12.964 [INFO] TestAgent: started state syncer -43:12.964 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14036 network=tcp -43:13.010 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:13.010 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14035 [Candidate]" term=2 -43:13.010 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:13.010 [DEBUG] TestAgent.server.raft: vote granted: from=dcf17898-77f1-f3b1-797f-bc0fe97a4027 term=2 tally=1 -43:13.010 [INFO] TestAgent.server.raft: election won: tally=1 -43:13.010 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14035 [Leader]" -43:13.010 [INFO] TestAgent.server: cluster leadership acquired -43:13.010 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14035 -43:13.012 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:13.014 [INFO] TestAgent.server: New leader elected: payload=Node-dcf17898-77f1-f3b1-797f-bc0fe97a4027 -43:13.018 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:13.019 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:13.019 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:13.019 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:13.019 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:13.019 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-dcf17898-77f1-f3b1-797f-bc0fe97a4027 -43:13.019 [INFO] TestAgent.server: member joined, marking health alive: member=Node-dcf17898-77f1-f3b1-797f-bc0fe97a4027 -43:13.020 [INFO] TestAgent.server: federation state anti-entropy synced -43:13.049 [WARN] TestAgent.server: endpoint injected; this should only be used for testing -43:13.049 [INFO] TestAgent: Requesting shutdown -43:13.050 [INFO] TestAgent.server: shutting down server -43:13.050 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:13.050 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:13.050 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:13.050 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:13.050 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:13.050 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:13.050 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:13.050 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:13.050 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:13.050 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:13.050 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:13.050 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:13.050Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:13.050Z [INFO] agent.router.manager: shutting down -43:13.050 [INFO] TestAgent: consul server down -43:13.050 [INFO] TestAgent: shutdown complete -43:13.050 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14030 network=tcp -43:13.051 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14030 network=udp -43:13.051 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14031 network=tcp -43:13.551 [INFO] TestAgent: Waiting for endpoints to shut down -43:13.551 [INFO] TestAgent: Endpoints down -=== RUN TestPreparedQuery_Execute/#03 -2022-09-15T21:42:59.334Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:59.356Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:59.363 [INFO] Agent2.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ac85ac6f-09fd-7224-6a2f-a3539b2a90ce Address:127.0.0.1:13351}]" -42:59.363 [INFO] Agent2.server.raft: entering follower state: follower="Node at 127.0.0.1:13351 [Follower]" leader= -42:59.364 [INFO] Agent2.server.serf.wan: serf: EventMemberJoin: Node-ac85ac6f-09fd-7224-6a2f-a3539b2a90ce.dc1 127.0.0.1 -42:59.365 [INFO] Agent2.server.serf.lan: serf: EventMemberJoin: Node-ac85ac6f-09fd-7224-6a2f-a3539b2a90ce 127.0.0.1 -2022-09-15T21:42:59.365Z [INFO] agent.router: Initializing LAN area manager -42:59.365 [INFO] Agent2: Started DNS server: address=127.0.0.1:13360 network=udp -42:59.365 [INFO] Agent2.server: Adding LAN server: server="Node-ac85ac6f-09fd-7224-6a2f-a3539b2a90ce (Addr: tcp/127.0.0.1:13351) (DC: dc1)" -42:59.365 [INFO] Agent2.server: Handled event for server in area: event=member-join server=Node-ac85ac6f-09fd-7224-6a2f-a3539b2a90ce.dc1 area=wan -42:59.368 [INFO] Agent2: Started DNS server: address=127.0.0.1:13360 network=tcp -42:59.369 [INFO] Agent2: Started HTTP server: address=127.0.0.1:13361 network=tcp -42:59.370 [INFO] Agent2: started state syncer -42:59.370 [INFO] Agent2: Started gRPC server: address=127.0.0.1:13352 network=tcp -42:59.402 [WARN] Agent2.server.raft: heartbeat timeout reached, starting election: last-leader= -42:59.402 [INFO] Agent2.server.raft: entering candidate state: node="Node at 127.0.0.1:13351 [Candidate]" term=2 -42:59.402 [DEBUG] Agent2.server.raft: votes: needed=1 -42:59.402 [DEBUG] Agent2.server.raft: vote granted: from=ac85ac6f-09fd-7224-6a2f-a3539b2a90ce term=2 tally=1 -42:59.402 [INFO] Agent2.server.raft: election won: tally=1 -42:59.402 [INFO] Agent2.server.raft: entering leader state: leader="Node at 127.0.0.1:13351 [Leader]" -42:59.402 [INFO] Agent2.server: cluster leadership acquired -42:59.403 [DEBUG] Agent2.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13351 -42:59.404 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:59.405 [INFO] Agent2.server: New leader elected: payload=Node-ac85ac6f-09fd-7224-6a2f-a3539b2a90ce -42:59.406 [INFO] Agent2.server.connect: initialized primary datacenter CA with provider: provider=consul -42:59.407 [INFO] Agent2.leader: started routine: routine="federation state anti-entropy" -42:59.407 [INFO] Agent2.leader: started routine: routine="federation state pruning" -42:59.407 [INFO] Agent2.leader: started routine: routine="intermediate cert renew watch" -42:59.407 [INFO] Agent2.leader: started routine: routine="CA root pruning" -42:59.407 [DEBUG] Agent2.server: Skipping self join check for node since the cluster is too small: node=Node-ac85ac6f-09fd-7224-6a2f-a3539b2a90ce -42:59.407 [INFO] Agent2.server: member joined, marking health alive: member=Node-ac85ac6f-09fd-7224-6a2f-a3539b2a90ce -42:59.408 [INFO] Agent2.server: federation state anti-entropy synced -42:59.449 [DEBUG] Agent2: Skipping remote check since it is managed automatically: check=serfHealth -42:59.449 [INFO] Agent2: Synced node info -42:59.449 [DEBUG] Agent2: Node info in sync -42:59.734 [DEBUG] Agent2.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:52990 -42:59.735 [INFO] Agent2.server.serf.lan: serf: EventMemberJoin: Node-7b911bd4-5115-4094-d5fc-12be00f2e856 127.0.0.1 -42:59.735 [INFO] Agent2.server: Adding LAN server: server="Node-7b911bd4-5115-4094-d5fc-12be00f2e856 (Addr: tcp/127.0.0.1:13365) (DC: dc1)" -42:59.736 [INFO] Agent2.server: New leader elected: payload=Node-7b911bd4-5115-4094-d5fc-12be00f2e856 -42:59.736 [ERROR] Agent2.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-7b911bd4-5115-4094-d5fc-12be00f2e856 other=Node-ac85ac6f-09fd-7224-6a2f-a3539b2a90ce -42:59.736 [INFO] Agent2.server: member joined, marking health alive: member=Node-7b911bd4-5115-4094-d5fc-12be00f2e856 -42:59.737 [DEBUG] Agent2.server.memberlist.wan: memberlist: Initiating push/pull sync with: Node-7b911bd4-5115-4094-d5fc-12be00f2e856.dc1 127.0.0.1:13343 -42:59.738 [INFO] Agent2: Requesting shutdown -42:59.739 [INFO] Agent2.server: shutting down server -42:59.739 [DEBUG] Agent2.leader: stopping routine: routine="intermediate cert renew watch" -42:59.739 [DEBUG] Agent2.leader: stopping routine: routine="CA root pruning" -42:59.739 [DEBUG] Agent2.leader: stopping routine: routine="federation state anti-entropy" -42:59.739 [DEBUG] Agent2.leader: stopping routine: routine="federation state pruning" -42:59.739 [WARN] Agent2.server.serf.lan: serf: Shutdown without a Leave -42:59.740 [ERROR] Agent2.server: error performing anti-entropy sync of federation state: error="context canceled" -42:59.740 [DEBUG] Agent2.leader: stopped routine: routine="federation state anti-entropy" -42:59.740 [DEBUG] Agent2.leader: stopped routine: routine="intermediate cert renew watch" -42:59.740 [DEBUG] Agent2.leader: stopped routine: routine="CA root pruning" -42:59.740 [DEBUG] Agent2.leader: stopped routine: routine="federation state pruning" -42:59.740 [WARN] Agent2.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:59.741Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:59.741Z [INFO] agent.router.manager: shutting down -42:59.741 [INFO] Agent2: consul server down -42:59.741 [INFO] Agent2: shutdown complete -42:59.741 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:13360 network=tcp -42:59.741 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:13360 network=udp -42:59.741 [INFO] Agent2: Stopping server: protocol=HTTP address=127.0.0.1:13361 network=tcp -42:59.758 [INFO] Agent2.server.serf.wan: serf: EventMemberJoin: Node-7b911bd4-5115-4094-d5fc-12be00f2e856.dc1 127.0.0.1 -42:59.758 [DEBUG] Agent2.server: Successfully performed flood-join for server at address: server=Node-7b911bd4-5115-4094-d5fc-12be00f2e856.dc1 address=127.0.0.1:13343 -43:00.242 [INFO] Agent2: Waiting for endpoints to shut down -43:00.242 [INFO] Agent2: Endpoints down -2022-09-15T21:42:59.009Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:59.036Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:59.054 [INFO] Agent1.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7b911bd4-5115-4094-d5fc-12be00f2e856 Address:127.0.0.1:13365}]" -42:59.056 [INFO] Agent1.server.serf.wan: serf: EventMemberJoin: Node-7b911bd4-5115-4094-d5fc-12be00f2e856.dc1 127.0.0.1 -42:59.057 [INFO] Agent1.server.serf.lan: serf: EventMemberJoin: Node-7b911bd4-5115-4094-d5fc-12be00f2e856 127.0.0.1 -2022-09-15T21:42:59.057Z [INFO] agent.router: Initializing LAN area manager -42:59.057 [INFO] Agent1: Started DNS server: address=127.0.0.1:13339 network=udp -42:59.057 [INFO] Agent1.server.raft: entering follower state: follower="Node at 127.0.0.1:13365 [Follower]" leader= -42:59.062 [INFO] Agent1.server: Adding LAN server: server="Node-7b911bd4-5115-4094-d5fc-12be00f2e856 (Addr: tcp/127.0.0.1:13365) (DC: dc1)" -42:59.062 [INFO] Agent1.server: Handled event for server in area: event=member-join server=Node-7b911bd4-5115-4094-d5fc-12be00f2e856.dc1 area=wan -42:59.062 [INFO] Agent1: Started DNS server: address=127.0.0.1:13339 network=tcp -42:59.064 [INFO] Agent1: Started HTTP server: address=127.0.0.1:13340 network=tcp -42:59.064 [INFO] Agent1: started state syncer -42:59.064 [INFO] Agent1: Started gRPC server: address=127.0.0.1:13366 network=tcp -42:59.116 [WARN] Agent1.server.raft: heartbeat timeout reached, starting election: last-leader= -42:59.116 [INFO] Agent1.server.raft: entering candidate state: node="Node at 127.0.0.1:13365 [Candidate]" term=2 -42:59.116 [DEBUG] Agent1.server.raft: votes: needed=1 -42:59.116 [DEBUG] Agent1.server.raft: vote granted: from=7b911bd4-5115-4094-d5fc-12be00f2e856 term=2 tally=1 -42:59.116 [INFO] Agent1.server.raft: election won: tally=1 -42:59.116 [INFO] Agent1.server.raft: entering leader state: leader="Node at 127.0.0.1:13365 [Leader]" -42:59.117 [INFO] Agent1.server: cluster leadership acquired -42:59.117 [INFO] Agent1.server: New leader elected: payload=Node-7b911bd4-5115-4094-d5fc-12be00f2e856 -42:59.117 [DEBUG] Agent1.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13365 -42:59.118 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:59.121 [INFO] Agent1.server.connect: initialized primary datacenter CA with provider: provider=consul -42:59.121 [INFO] Agent1.leader: started routine: routine="federation state anti-entropy" -42:59.121 [INFO] Agent1.leader: started routine: routine="federation state pruning" -42:59.121 [INFO] Agent1.leader: started routine: routine="intermediate cert renew watch" -42:59.121 [INFO] Agent1.leader: started routine: routine="CA root pruning" -42:59.121 [DEBUG] Agent1.server: Skipping self join check for node since the cluster is too small: node=Node-7b911bd4-5115-4094-d5fc-12be00f2e856 -42:59.121 [INFO] Agent1.server: member joined, marking health alive: member=Node-7b911bd4-5115-4094-d5fc-12be00f2e856 -42:59.122 [INFO] Agent1.server: federation state anti-entropy synced -42:59.447 [DEBUG] Agent1: Skipping remote check since it is managed automatically: check=serfHealth -42:59.448 [INFO] Agent1: Synced node info -42:59.726 [INFO] Agent1: (LAN) joining: lan_addresses=[127.0.0.1:13363] -42:59.734 [DEBUG] Agent1.server.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:13363 -42:59.738 [DEBUG] Agent1.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:47552 -42:59.738 [INFO] Agent1.server.serf.lan: serf: EventMemberJoin: Node-ac85ac6f-09fd-7224-6a2f-a3539b2a90ce 127.0.0.1 -42:59.738 [INFO] Agent1: (LAN) joined: number_of_nodes=1 -42:59.738 [DEBUG] Agent1: systemd notify failed: error="No socket" -42:59.739 [INFO] Agent1.server: Adding LAN server: server="Node-ac85ac6f-09fd-7224-6a2f-a3539b2a90ce (Addr: tcp/127.0.0.1:13351) (DC: dc1)" -42:59.739 [ERROR] Agent1.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-ac85ac6f-09fd-7224-6a2f-a3539b2a90ce other=Node-7b911bd4-5115-4094-d5fc-12be00f2e856 -42:59.739 [INFO] Agent1.server: member joined, marking health alive: member=Node-ac85ac6f-09fd-7224-6a2f-a3539b2a90ce -42:59.741 [DEBUG] Agent1.server.memberlist.wan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:13364: connect: connection refused -42:59.741 [DEBUG] Agent1.server: Failed to flood-join server at address: server=Node-ac85ac6f-09fd-7224-6a2f-a3539b2a90ce.dc1 address=127.0.0.1:13364 error="1 error occurred: - * Failed to join 127.0.0.1: dial tcp 127.0.0.1:13364: connect: connection refused +[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:36.865Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:36.881Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:36.884 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c809de4f-797d-bae6-081f-fb824440f8ee Address:127.0.0.1:24522}]" +34:36.885 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24522 [Follower]" leader= +34:36.885 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c809de4f-797d-bae6-081f-fb824440f8ee.dc1 127.0.0.1 +34:36.886 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c809de4f-797d-bae6-081f-fb824440f8ee 127.0.0.1 +2021-08-13T15:34:36.886Z [INFO] agent.router: Initializing LAN area manager +34:36.887 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24517 network=udp +34:36.887 [INFO] TestAgent.server: Adding LAN server: server="Node-c809de4f-797d-bae6-081f-fb824440f8ee (Addr: tcp/127.0.0.1:24522) (DC: dc1)" +34:36.887 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c809de4f-797d-bae6-081f-fb824440f8ee.dc1 area=wan +34:36.887 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24517 network=tcp +34:36.888 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24518 network=tcp +34:36.888 [INFO] TestAgent: started state syncer +34:36.888 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24523 network=tcp +34:36.939 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:36.939 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24522 [Candidate]" term=2 +34:36.940 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:36.940 [DEBUG] TestAgent.server.raft: vote granted: from=c809de4f-797d-bae6-081f-fb824440f8ee term=2 tally=1 +34:36.940 [INFO] TestAgent.server.raft: election won: tally=1 +34:36.940 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24522 [Leader]" +34:36.940 [INFO] TestAgent.server: cluster leadership acquired +34:36.940 [INFO] TestAgent.server: New leader elected: payload=Node-c809de4f-797d-bae6-081f-fb824440f8ee +34:36.940 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24522 +34:36.942 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:36.944 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:36.944 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:36.944 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:36.944 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:36.944 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:36.944 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c809de4f-797d-bae6-081f-fb824440f8ee +34:36.944 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c809de4f-797d-bae6-081f-fb824440f8ee +34:36.945 [INFO] TestAgent.server: federation state anti-entropy synced +34:37.041 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:37.042 [INFO] TestAgent: Synced node info +34:37.283 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/key from= latency=55.001µs +34:37.283 [INFO] TestAgent: Requesting shutdown +34:37.283 [INFO] TestAgent.server: shutting down server +34:37.283 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:37.283 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:37.283 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:37.283 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:37.283 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:37.283 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:37.283 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:37.284 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:37.284 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:37.284 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:37.284 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:37.284Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:37.284Z [INFO] agent.router.manager: shutting down +34:37.284 [INFO] TestAgent: consul server down +34:37.284 [INFO] TestAgent: shutdown complete +34:37.284 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24517 network=tcp +34:37.284 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24517 network=udp +34:37.284 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24518 network=tcp +34:37.786 [INFO] TestAgent: Waiting for endpoints to shut down +34:37.786 [INFO] TestAgent: Endpoints down +--- PASS: TestContentTypeIsJSON (0.99s) +=== CONT TestSetMeta +--- PASS: TestSetMeta (0.00s) +=== CONT TestACL_Legacy_Destroy +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:37.201Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:37.219Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:37.222 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f7ac72a1-19ac-31a2-e469-d0e88446fa60 Address:127.0.0.1:24449}]" +34:37.222 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24449 [Follower]" leader= +34:37.223 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f7ac72a1-19ac-31a2-e469-d0e88446fa60.dc1 127.0.0.1 +34:37.224 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f7ac72a1-19ac-31a2-e469-d0e88446fa60 127.0.0.1 +2021-08-13T15:34:37.224Z [INFO] agent.router: Initializing LAN area manager +34:37.224 [INFO] TestAgent.server: Adding LAN server: server="Node-f7ac72a1-19ac-31a2-e469-d0e88446fa60 (Addr: tcp/127.0.0.1:24449) (DC: dc1)" +34:37.224 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f7ac72a1-19ac-31a2-e469-d0e88446fa60.dc1 area=wan +34:37.224 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24538 network=tcp +34:37.224 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24538 network=udp +34:37.225 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24445 network=tcp +34:37.225 [INFO] TestAgent: started state syncer +34:37.225 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24450 network=tcp +34:37.267 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:37.267 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24449 [Candidate]" term=2 +34:37.267 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:37.267 [DEBUG] TestAgent.server.raft: vote granted: from=f7ac72a1-19ac-31a2-e469-d0e88446fa60 term=2 tally=1 +34:37.267 [INFO] TestAgent.server.raft: election won: tally=1 +34:37.267 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24449 [Leader]" +34:37.268 [INFO] TestAgent.server: cluster leadership acquired +34:37.268 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24449 +34:37.269 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:37.271 [INFO] TestAgent.server: New leader elected: payload=Node-f7ac72a1-19ac-31a2-e469-d0e88446fa60 +34:37.272 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:37.272 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:37.272 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:37.272 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:37.272 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:37.272 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f7ac72a1-19ac-31a2-e469-d0e88446fa60 +34:37.272 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f7ac72a1-19ac-31a2-e469-d0e88446fa60 +34:37.273 [INFO] TestAgent.server: federation state anti-entropy synced +34:37.480 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:37.481 [INFO] TestAgent: Synced node info +34:37.575 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from= latency=2.84µs +34:37.575 [INFO] TestAgent: Requesting shutdown +34:37.575 [INFO] TestAgent.server: shutting down server +34:37.575 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:37.575 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:37.575 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:37.575 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:37.575 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:37.575 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:37.575 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:37.575 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:37.575 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:37.576 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:37.576 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:37.576Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:37.576Z [INFO] agent.router.manager: shutting down +34:37.576 [INFO] TestAgent: consul server down +34:37.576 [INFO] TestAgent: shutdown complete +34:37.576 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24538 network=tcp +34:37.576 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24538 network=udp +34:37.576 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24445 network=tcp +34:38.083 [INFO] TestAgent: Waiting for endpoints to shut down +34:38.083 [INFO] TestAgent: Endpoints down +--- PASS: TestHTTPAPIResponseHeaders (0.90s) +=== CONT TestSetLastContact +=== RUN TestSetLastContact/neg +=== RUN TestSetLastContact/zero +=== RUN TestSetLastContact/pos +=== RUN TestSetLastContact/pos_ms_only +--- PASS: TestSetLastContact (0.00s) + --- PASS: TestSetLastContact/neg (0.00s) + --- PASS: TestSetLastContact/zero (0.00s) + --- PASS: TestSetLastContact/pos (0.00s) + --- PASS: TestSetLastContact/pos_ms_only (0.00s) +=== CONT TestACL_Legacy_UpdateUpsert +2021-08-13T15:34:23.936Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:23.955Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:23.958 [INFO] Agent2.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7fc0e1f6-df7c-3a56-a157-e211c1f28f95 Address:127.0.0.1:24893}]" +34:23.958 [INFO] Agent2.server.raft: entering follower state: follower="Node at 127.0.0.1:24893 [Follower]" leader= +34:23.959 [INFO] Agent2.server.serf.wan: serf: EventMemberJoin: Node-7fc0e1f6-df7c-3a56-a157-e211c1f28f95.dc1 127.0.0.1 +34:23.960 [INFO] Agent2.server.serf.lan: serf: EventMemberJoin: Node-7fc0e1f6-df7c-3a56-a157-e211c1f28f95 127.0.0.1 +2021-08-13T15:34:23.960Z [INFO] agent.router: Initializing LAN area manager +34:23.960 [INFO] Agent2: Started DNS server: address=127.0.0.1:24874 network=udp +34:23.960 [INFO] Agent2.server: Adding LAN server: server="Node-7fc0e1f6-df7c-3a56-a157-e211c1f28f95 (Addr: tcp/127.0.0.1:24893) (DC: dc1)" +34:23.960 [INFO] Agent2.server: Handled event for server in area: event=member-join server=Node-7fc0e1f6-df7c-3a56-a157-e211c1f28f95.dc1 area=wan +34:23.961 [INFO] Agent2: Started DNS server: address=127.0.0.1:24874 network=tcp +34:23.961 [INFO] Agent2: Started HTTP server: address=127.0.0.1:24875 network=tcp +34:23.961 [INFO] Agent2: started state syncer +34:23.962 [INFO] Agent2: Started gRPC server: address=127.0.0.1:24915 network=tcp +34:24.027 [WARN] Agent2.server.raft: heartbeat timeout reached, starting election: last-leader= +34:24.027 [INFO] Agent2.server.raft: entering candidate state: node="Node at 127.0.0.1:24893 [Candidate]" term=2 +34:24.027 [DEBUG] Agent2.server.raft: votes: needed=1 +34:24.027 [DEBUG] Agent2.server.raft: vote granted: from=7fc0e1f6-df7c-3a56-a157-e211c1f28f95 term=2 tally=1 +34:24.027 [INFO] Agent2.server.raft: election won: tally=1 +34:24.027 [INFO] Agent2.server.raft: entering leader state: leader="Node at 127.0.0.1:24893 [Leader]" +34:24.027 [INFO] Agent2.server: cluster leadership acquired +34:24.028 [INFO] Agent2.server: New leader elected: payload=Node-7fc0e1f6-df7c-3a56-a157-e211c1f28f95 +34:24.028 [DEBUG] Agent2.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24893 +34:24.029 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:24.032 [INFO] Agent2.server.connect: initialized primary datacenter CA with provider: provider=consul +34:24.032 [INFO] Agent2.leader: started routine: routine="federation state anti-entropy" +34:24.032 [INFO] Agent2.leader: started routine: routine="federation state pruning" +34:24.032 [INFO] Agent2.leader: started routine: routine="intermediate cert renew watch" +34:24.032 [INFO] Agent2.leader: started routine: routine="CA root pruning" +34:24.032 [DEBUG] Agent2.server: Skipping self join check for node since the cluster is too small: node=Node-7fc0e1f6-df7c-3a56-a157-e211c1f28f95 +34:24.032 [INFO] Agent2.server: member joined, marking health alive: member=Node-7fc0e1f6-df7c-3a56-a157-e211c1f28f95 +34:24.033 [INFO] Agent2.server: federation state anti-entropy synced +34:24.254 [DEBUG] Agent2: Skipping remote check since it is managed automatically: check=serfHealth +34:24.255 [INFO] Agent2: Synced node info +34:24.354 [DEBUG] Agent2.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:34634 +34:24.355 [INFO] Agent2.server.serf.lan: serf: EventMemberJoin: Node-66e29c96-a5cb-5494-2960-033fd20c4e1f 127.0.0.1 +34:24.355 [INFO] Agent2.server: Adding LAN server: server="Node-66e29c96-a5cb-5494-2960-033fd20c4e1f (Addr: tcp/127.0.0.1:24907) (DC: dc1)" +34:24.355 [INFO] Agent2: Requesting shutdown +34:24.355 [INFO] Agent2.server: New leader elected: payload=Node-66e29c96-a5cb-5494-2960-033fd20c4e1f +34:24.355 [ERROR] Agent2.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-66e29c96-a5cb-5494-2960-033fd20c4e1f other=Node-7fc0e1f6-df7c-3a56-a157-e211c1f28f95 +34:24.355 [INFO] Agent2.server: shutting down server +34:24.355 [DEBUG] Agent2.leader: stopping routine: routine="CA root pruning" +34:24.355 [INFO] Agent2.server: member joined, marking health alive: member=Node-66e29c96-a5cb-5494-2960-033fd20c4e1f +34:24.355 [DEBUG] Agent2.leader: stopped routine: routine="CA root pruning" +34:24.356 [DEBUG] Agent2.server.memberlist.wan: memberlist: Initiating push/pull sync with: Node-66e29c96-a5cb-5494-2960-033fd20c4e1f.dc1 127.0.0.1:24906 +34:24.355 [DEBUG] Agent2.leader: stopping routine: routine="federation state anti-entropy" +34:24.356 [DEBUG] Agent2.leader: stopping routine: routine="federation state pruning" +34:24.356 [DEBUG] Agent2.leader: stopping routine: routine="intermediate cert renew watch" +34:24.356 [WARN] Agent2.server.serf.lan: serf: Shutdown without a Leave +34:24.357 [ERROR] Agent2.server: error performing anti-entropy sync of federation state: error="context canceled" +34:24.357 [DEBUG] Agent2.leader: stopped routine: routine="federation state anti-entropy" +34:24.357 [DEBUG] Agent2.leader: stopped routine: routine="federation state pruning" +34:24.357 [DEBUG] Agent2.leader: stopped routine: routine="intermediate cert renew watch" +34:24.357 [WARN] Agent2.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:24.357Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:24.357Z [INFO] agent.router.manager: shutting down +34:24.357 [INFO] Agent2: consul server down +34:24.357 [INFO] Agent2: shutdown complete +34:24.357 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:24874 network=tcp +34:24.357 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:24874 network=udp +34:24.357 [INFO] Agent2: Stopping server: protocol=HTTP address=127.0.0.1:24875 network=tcp +34:24.357 [INFO] Agent2.server.serf.wan: serf: EventMemberJoin: Node-66e29c96-a5cb-5494-2960-033fd20c4e1f.dc1 127.0.0.1 +34:24.357 [DEBUG] Agent2.server: Successfully performed flood-join for server at address: server=Node-66e29c96-a5cb-5494-2960-033fd20c4e1f.dc1 address=127.0.0.1:24906 +34:24.857 [INFO] Agent2: Waiting for endpoints to shut down +34:24.857 [INFO] Agent2: Endpoints down +2021-08-13T15:34:23.584Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:23.599Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:23.602 [INFO] Agent1.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:66e29c96-a5cb-5494-2960-033fd20c4e1f Address:127.0.0.1:24907}]" +34:23.602 [INFO] Agent1.server.raft: entering follower state: follower="Node at 127.0.0.1:24907 [Follower]" leader= +34:23.603 [INFO] Agent1.server.serf.wan: serf: EventMemberJoin: Node-66e29c96-a5cb-5494-2960-033fd20c4e1f.dc1 127.0.0.1 +34:23.604 [INFO] Agent1.server.serf.lan: serf: EventMemberJoin: Node-66e29c96-a5cb-5494-2960-033fd20c4e1f 127.0.0.1 +2021-08-13T15:34:23.604Z [INFO] agent.router: Initializing LAN area manager +34:23.604 [INFO] Agent1.server: Adding LAN server: server="Node-66e29c96-a5cb-5494-2960-033fd20c4e1f (Addr: tcp/127.0.0.1:24907) (DC: dc1)" +34:23.604 [INFO] Agent1.server: Handled event for server in area: event=member-join server=Node-66e29c96-a5cb-5494-2960-033fd20c4e1f.dc1 area=wan +34:23.604 [INFO] Agent1: Started DNS server: address=127.0.0.1:24881 network=tcp +34:23.604 [INFO] Agent1: Started DNS server: address=127.0.0.1:24881 network=udp +34:23.605 [INFO] Agent1: Started HTTP server: address=127.0.0.1:24882 network=tcp +34:23.605 [INFO] Agent1: started state syncer +34:23.605 [INFO] Agent1: Started gRPC server: address=127.0.0.1:24901 network=tcp +34:23.658 [WARN] Agent1.server.raft: heartbeat timeout reached, starting election: last-leader= +34:23.658 [INFO] Agent1.server.raft: entering candidate state: node="Node at 127.0.0.1:24907 [Candidate]" term=2 +34:23.659 [DEBUG] Agent1.server.raft: votes: needed=1 +34:23.659 [DEBUG] Agent1.server.raft: vote granted: from=66e29c96-a5cb-5494-2960-033fd20c4e1f term=2 tally=1 +34:23.659 [INFO] Agent1.server.raft: election won: tally=1 +34:23.659 [INFO] Agent1.server.raft: entering leader state: leader="Node at 127.0.0.1:24907 [Leader]" +34:23.659 [INFO] Agent1.server: cluster leadership acquired +34:23.659 [DEBUG] Agent1.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24907 +34:23.660 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:23.662 [INFO] Agent1.server: New leader elected: payload=Node-66e29c96-a5cb-5494-2960-033fd20c4e1f +34:23.662 [INFO] Agent1.server.connect: initialized primary datacenter CA with provider: provider=consul +34:23.663 [INFO] Agent1.leader: started routine: routine="federation state anti-entropy" +34:23.663 [INFO] Agent1.leader: started routine: routine="federation state pruning" +34:23.663 [INFO] Agent1.leader: started routine: routine="intermediate cert renew watch" +34:23.663 [INFO] Agent1.leader: started routine: routine="CA root pruning" +34:23.663 [DEBUG] Agent1.server: Skipping self join check for node since the cluster is too small: node=Node-66e29c96-a5cb-5494-2960-033fd20c4e1f +34:23.663 [INFO] Agent1.server: member joined, marking health alive: member=Node-66e29c96-a5cb-5494-2960-033fd20c4e1f +34:23.664 [INFO] Agent1.server: federation state anti-entropy synced +34:23.704 [DEBUG] Agent1: Skipping remote check since it is managed automatically: check=serfHealth +34:23.704 [INFO] Agent1: Synced node info +34:24.353 [INFO] Agent1: (LAN) joining: lan_addresses=[127.0.0.1:24891] +34:24.354 [DEBUG] Agent1.server.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:24891 +34:24.355 [INFO] Agent1.server.serf.lan: serf: EventMemberJoin: Node-7fc0e1f6-df7c-3a56-a157-e211c1f28f95 127.0.0.1 +34:24.355 [INFO] Agent1: (LAN) joined: number_of_nodes=1 +34:24.355 [DEBUG] Agent1: systemd notify failed: error="No socket" +34:24.355 [INFO] Agent1.server: Adding LAN server: server="Node-7fc0e1f6-df7c-3a56-a157-e211c1f28f95 (Addr: tcp/127.0.0.1:24893) (DC: dc1)" +34:24.355 [ERROR] Agent1.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-7fc0e1f6-df7c-3a56-a157-e211c1f28f95 other=Node-66e29c96-a5cb-5494-2960-033fd20c4e1f +34:24.356 [INFO] Agent1.server: member joined, marking health alive: member=Node-7fc0e1f6-df7c-3a56-a157-e211c1f28f95 +34:24.356 [DEBUG] Agent1.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:38620 +34:24.357 [INFO] Agent1.server.serf.wan: serf: EventMemberJoin: Node-7fc0e1f6-df7c-3a56-a157-e211c1f28f95.dc1 127.0.0.1 +34:24.357 [DEBUG] Agent1.server.memberlist.wan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:24892: connect: connection reset by peer +34:24.357 [DEBUG] Agent1.server: Failed to flood-join server at address: server=Node-7fc0e1f6-df7c-3a56-a157-e211c1f28f95.dc1 address=127.0.0.1:24892 error="1 error occurred: + * Failed to join 127.0.0.1: dial tcp 127.0.0.1:24892: connect: connection reset by peer " -42:59.742 [INFO] Agent1.server.serf.wan: serf: EventMemberJoin: Node-ac85ac6f-09fd-7224-6a2f-a3539b2a90ce.dc1 127.0.0.1 -42:59.743 [INFO] Agent1.server: Handled event for server in area: event=member-join server=Node-ac85ac6f-09fd-7224-6a2f-a3539b2a90ce.dc1 area=wan -43:00.558 [DEBUG] Agent1.server.memberlist.lan: memberlist: Failed ping: Node-ac85ac6f-09fd-7224-6a2f-a3539b2a90ce (timeout reached) -43:01.059 [INFO] Agent1.server.memberlist.lan: memberlist: Suspect Node-ac85ac6f-09fd-7224-6a2f-a3539b2a90ce has failed, no acks received -43:01.134 [WARN] Agent1: error getting server health from server: server=Node-ac85ac6f-09fd-7224-6a2f-a3539b2a90ce error="rpc error getting client: failed to get conn: dial tcp 127.0.0.1:0->127.0.0.1:13351: connect: connection refused" -43:01.277 [DEBUG] Agent1: Skipping remote check since it is managed automatically: check=serfHealth -43:01.277 [DEBUG] Agent1: Node info in sync -43:01.277 [DEBUG] Agent1: Node info in sync -43:02.134 [WARN] Agent1: error getting server health from server: server=Node-ac85ac6f-09fd-7224-6a2f-a3539b2a90ce error="context deadline exceeded" -43:02.558 [DEBUG] Agent1.server.memberlist.lan: memberlist: Failed ping: Node-ac85ac6f-09fd-7224-6a2f-a3539b2a90ce (timeout reached) -43:03.119 [WARN] Agent1: error getting server health from server: server=Node-ac85ac6f-09fd-7224-6a2f-a3539b2a90ce error="rpc error getting client: failed to get conn: dial tcp 127.0.0.1:0->127.0.0.1:13351: connect: connection refused" -43:04.057 [INFO] Agent1.server.memberlist.lan: memberlist: Suspect Node-ac85ac6f-09fd-7224-6a2f-a3539b2a90ce has failed, no acks received -43:04.118 [WARN] Agent1: error getting server health from server: server=Node-ac85ac6f-09fd-7224-6a2f-a3539b2a90ce error="context deadline exceeded" -43:05.062 [INFO] Agent1.server.memberlist.lan: memberlist: Marking Node-ac85ac6f-09fd-7224-6a2f-a3539b2a90ce as failed, suspect timeout reached (0 peer confirmations) -43:05.062 [INFO] Agent1.server.serf.lan: serf: EventMemberFailed: Node-ac85ac6f-09fd-7224-6a2f-a3539b2a90ce 127.0.0.1 -43:05.062 [INFO] Agent1.server: Removing LAN server: server="Node-ac85ac6f-09fd-7224-6a2f-a3539b2a90ce (Addr: tcp/127.0.0.1:13351) (DC: dc1)" -43:05.062 [INFO] Agent1.server: member failed, marking health critical: member=Node-ac85ac6f-09fd-7224-6a2f-a3539b2a90ce -43:05.075 [INFO] Agent1: Force leaving node: node=Node-ac85ac6f-09fd-7224-6a2f-a3539b2a90ce -43:05.075 [INFO] Agent1.server.serf.lan: serf: EventMemberLeave (forced): Node-ac85ac6f-09fd-7224-6a2f-a3539b2a90ce 127.0.0.1 -43:05.075 [INFO] Agent1.server.serf.lan: serf: EventMemberReap (forced): Node-ac85ac6f-09fd-7224-6a2f-a3539b2a90ce 127.0.0.1 -43:05.076 [INFO] Agent1.server: Removing LAN server: server="Node-ac85ac6f-09fd-7224-6a2f-a3539b2a90ce (Addr: tcp/127.0.0.1:13351) (DC: dc1)" -43:05.076 [INFO] Agent1.server: Removing LAN server: server="Node-ac85ac6f-09fd-7224-6a2f-a3539b2a90ce (Addr: tcp/127.0.0.1:13351) (DC: dc1)" -43:05.076 [INFO] Agent1.server: deregistering member: member=Node-ac85ac6f-09fd-7224-6a2f-a3539b2a90ce reason=left -43:07.058 [DEBUG] Agent1.server.memberlist.wan: memberlist: Failed ping: Node-ac85ac6f-09fd-7224-6a2f-a3539b2a90ce.dc1 (timeout reached) -43:09.058 [INFO] Agent1.server.memberlist.wan: memberlist: Suspect Node-ac85ac6f-09fd-7224-6a2f-a3539b2a90ce.dc1 has failed, no acks received -43:13.076 [INFO] Agent1.server.serf.wan: serf: EventMemberReap (forced): Node-ac85ac6f-09fd-7224-6a2f-a3539b2a90ce.dc1 127.0.0.1 -43:13.076 [INFO] Agent1: Requesting shutdown -43:13.077 [INFO] Agent1.server: shutting down server -43:13.077 [DEBUG] Agent1.leader: stopping routine: routine="intermediate cert renew watch" -43:13.077 [DEBUG] Agent1.leader: stopping routine: routine="CA root pruning" -43:13.077 [DEBUG] Agent1.leader: stopping routine: routine="federation state anti-entropy" -43:13.077 [DEBUG] Agent1.leader: stopping routine: routine="federation state pruning" -43:13.077 [WARN] Agent1.server.serf.lan: serf: Shutdown without a Leave -43:13.077 [INFO] Agent1.server: Handled event for server in area: event=member-reap server=Node-ac85ac6f-09fd-7224-6a2f-a3539b2a90ce.dc1 area=wan -43:13.077 [ERROR] Agent1.server: error performing anti-entropy sync of federation state: error="context canceled" -43:13.077 [DEBUG] Agent1.leader: stopped routine: routine="federation state anti-entropy" -43:13.077 [DEBUG] Agent1.leader: stopped routine: routine="intermediate cert renew watch" -43:13.077 [DEBUG] Agent1.leader: stopped routine: routine="CA root pruning" -43:13.077 [DEBUG] Agent1.leader: stopped routine: routine="federation state pruning" -43:13.077 [WARN] Agent1.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:13.077Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:13.077Z [INFO] agent.router.manager: shutting down -43:13.082 [INFO] Agent1: consul server down -43:13.082 [INFO] Agent1: shutdown complete -43:13.082 [INFO] Agent1: Stopping server: protocol=DNS address=127.0.0.1:13339 network=tcp -43:13.082 [INFO] Agent1: Stopping server: protocol=DNS address=127.0.0.1:13339 network=udp -43:13.082 [INFO] Agent1: Stopping server: protocol=HTTP address=127.0.0.1:13340 network=tcp -43:13.586 [INFO] Agent1: Waiting for endpoints to shut down -43:13.586 [INFO] Agent1: Endpoints down ---- PASS: TestAgent_ForceLeavePrune (14.60s) +34:24.357 [INFO] Agent1.server: Handled event for server in area: event=member-join server=Node-7fc0e1f6-df7c-3a56-a157-e211c1f28f95.dc1 area=wan +34:25.106 [DEBUG] Agent1.server.memberlist.lan: memberlist: Failed ping: Node-7fc0e1f6-df7c-3a56-a157-e211c1f28f95 (timeout reached) +34:25.605 [INFO] Agent1.server.memberlist.lan: memberlist: Suspect Node-7fc0e1f6-df7c-3a56-a157-e211c1f28f95 has failed, no acks received +34:25.663 [WARN] Agent1: error getting server health from server: server=Node-7fc0e1f6-df7c-3a56-a157-e211c1f28f95 error="rpc error getting client: failed to get conn: dial tcp 127.0.0.1:0->127.0.0.1:24893: connect: connection refused" +34:25.935 [DEBUG] Agent1: Skipping remote check since it is managed automatically: check=serfHealth +34:25.935 [DEBUG] Agent1: Node info in sync +34:25.935 [DEBUG] Agent1: Node info in sync +34:26.663 [WARN] Agent1: error getting server health from server: server=Node-7fc0e1f6-df7c-3a56-a157-e211c1f28f95 error="context deadline exceeded" +34:27.104 [DEBUG] Agent1.server.memberlist.lan: memberlist: Failed ping: Node-7fc0e1f6-df7c-3a56-a157-e211c1f28f95 (timeout reached) +34:27.663 [WARN] Agent1: error getting server health from server: server=Node-7fc0e1f6-df7c-3a56-a157-e211c1f28f95 error="rpc error getting client: failed to get conn: dial tcp 127.0.0.1:0->127.0.0.1:24893: connect: connection refused" +34:28.607 [INFO] Agent1.server.memberlist.lan: memberlist: Suspect Node-7fc0e1f6-df7c-3a56-a157-e211c1f28f95 has failed, no acks received +34:28.662 [WARN] Agent1: error getting server health from server: server=Node-7fc0e1f6-df7c-3a56-a157-e211c1f28f95 error="context deadline exceeded" +34:29.607 [INFO] Agent1.server.memberlist.lan: memberlist: Marking Node-7fc0e1f6-df7c-3a56-a157-e211c1f28f95 as failed, suspect timeout reached (0 peer confirmations) +34:29.608 [INFO] Agent1.server.serf.lan: serf: EventMemberFailed: Node-7fc0e1f6-df7c-3a56-a157-e211c1f28f95 127.0.0.1 +34:29.608 [INFO] Agent1.server: Removing LAN server: server="Node-7fc0e1f6-df7c-3a56-a157-e211c1f28f95 (Addr: tcp/127.0.0.1:24893) (DC: dc1)" +34:29.608 [INFO] Agent1.server: member failed, marking health critical: member=Node-7fc0e1f6-df7c-3a56-a157-e211c1f28f95 +34:29.609 [INFO] Agent1: Force leaving node: node=Node-7fc0e1f6-df7c-3a56-a157-e211c1f28f95 +34:29.609 [INFO] Agent1.server.serf.lan: serf: EventMemberLeave (forced): Node-7fc0e1f6-df7c-3a56-a157-e211c1f28f95 127.0.0.1 +34:29.609 [INFO] Agent1.server.serf.lan: serf: EventMemberReap (forced): Node-7fc0e1f6-df7c-3a56-a157-e211c1f28f95 127.0.0.1 +34:29.609 [INFO] Agent1.server: Removing LAN server: server="Node-7fc0e1f6-df7c-3a56-a157-e211c1f28f95 (Addr: tcp/127.0.0.1:24893) (DC: dc1)" +34:29.609 [INFO] Agent1.server: Removing LAN server: server="Node-7fc0e1f6-df7c-3a56-a157-e211c1f28f95 (Addr: tcp/127.0.0.1:24893) (DC: dc1)" +34:29.609 [INFO] Agent1.server: deregistering member: member=Node-7fc0e1f6-df7c-3a56-a157-e211c1f28f95 reason=left +34:30.107 [DEBUG] Agent1.server.memberlist.lan: memberlist: Failed ping: Node-7fc0e1f6-df7c-3a56-a157-e211c1f28f95 (timeout reached) +34:31.607 [DEBUG] Agent1.server.memberlist.wan: memberlist: Failed ping: Node-7fc0e1f6-df7c-3a56-a157-e211c1f28f95.dc1 (timeout reached) +34:32.607 [INFO] Agent1.server.memberlist.lan: memberlist: Suspect Node-7fc0e1f6-df7c-3a56-a157-e211c1f28f95 has failed, no acks received +34:33.607 [INFO] Agent1.server.memberlist.wan: memberlist: Suspect Node-7fc0e1f6-df7c-3a56-a157-e211c1f28f95.dc1 has failed, no acks received +34:37.609 [INFO] Agent1.server.serf.wan: serf: EventMemberReap (forced): Node-7fc0e1f6-df7c-3a56-a157-e211c1f28f95.dc1 127.0.0.1 +34:37.609 [INFO] Agent1: Requesting shutdown +34:37.609 [INFO] Agent1.server: shutting down server +34:37.609 [DEBUG] Agent1.leader: stopping routine: routine="federation state anti-entropy" +34:37.609 [DEBUG] Agent1.leader: stopping routine: routine="federation state pruning" +34:37.609 [DEBUG] Agent1.leader: stopping routine: routine="intermediate cert renew watch" +34:37.609 [DEBUG] Agent1.leader: stopping routine: routine="CA root pruning" +34:37.609 [WARN] Agent1.server.serf.lan: serf: Shutdown without a Leave +34:37.609 [INFO] Agent1.server: Handled event for server in area: event=member-reap server=Node-7fc0e1f6-df7c-3a56-a157-e211c1f28f95.dc1 area=wan +34:37.609 [DEBUG] Agent1.leader: stopped routine: routine="federation state pruning" +34:37.609 [DEBUG] Agent1.leader: stopped routine: routine="intermediate cert renew watch" +34:37.609 [ERROR] Agent1.server: error performing anti-entropy sync of federation state: error="context canceled" +34:37.609 [DEBUG] Agent1.leader: stopped routine: routine="CA root pruning" +34:37.609 [DEBUG] Agent1.leader: stopped routine: routine="federation state anti-entropy" +34:37.609 [WARN] Agent1.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:37.609Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:37.609Z [INFO] agent.router.manager: shutting down +34:37.610 [INFO] Agent1: consul server down +34:37.610 [INFO] Agent1: shutdown complete +34:37.610 [INFO] Agent1: Stopping server: protocol=DNS address=127.0.0.1:24881 network=tcp +34:37.610 [INFO] Agent1: Stopping server: protocol=DNS address=127.0.0.1:24881 network=udp +34:37.610 [INFO] Agent1: Stopping server: protocol=HTTP address=127.0.0.1:24882 network=tcp +34:38.110 [INFO] Agent1: Waiting for endpoints to shut down +34:38.110 [INFO] Agent1: Endpoints down +--- PASS: TestAgent_ForceLeavePrune (14.54s) +=== CONT TestSetKnownLeader +--- PASS: TestSetKnownLeader (0.00s) +=== CONT TestACL_Legacy_Update +2021-08-13T15:34:37.517Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:37.517Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:37.543Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:37.543Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:37.546 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bd12542b-54e2-0860-3327-880769c13ddb Address:127.0.0.1:24536}]" +34:37.546 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24536 [Follower]" leader= +34:37.547 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bd12542b-54e2-0860-3327-880769c13ddb.dc1 127.0.0.1 +34:37.548 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bd12542b-54e2-0860-3327-880769c13ddb 127.0.0.1 +2021-08-13T15:34:37.548Z [INFO] agent.router: Initializing LAN area manager +34:37.549 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24458 network=udp +34:37.549 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bd12542b-54e2-0860-3327-880769c13ddb.dc1 area=wan +34:37.549 [INFO] TestAgent.server: Adding LAN server: server="Node-bd12542b-54e2-0860-3327-880769c13ddb (Addr: tcp/127.0.0.1:24536) (DC: dc1)" +34:37.549 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24458 network=tcp +34:37.550 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24532 network=tcp +34:37.550 [INFO] TestAgent: started state syncer +34:37.550 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24537 network=tcp +34:37.582 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:37.582 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24536 [Candidate]" term=2 +34:37.582 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:37.582 [DEBUG] TestAgent.server.raft: vote granted: from=bd12542b-54e2-0860-3327-880769c13ddb term=2 tally=1 +34:37.582 [INFO] TestAgent.server.raft: election won: tally=1 +34:37.582 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24536 [Leader]" +34:37.583 [INFO] TestAgent.server: cluster leadership acquired +34:37.583 [INFO] TestAgent.server: New leader elected: payload=Node-bd12542b-54e2-0860-3327-880769c13ddb +34:37.583 [INFO] TestAgent.server: initializing acls +34:37.583 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:37.583 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:37.584 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:37.585 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:37.585 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:37.585 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:37.585 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-bd12542b-54e2-0860-3327-880769c13ddb +34:37.585 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-bd12542b-54e2-0860-3327-880769c13ddb.dc1 +34:37.585 [INFO] TestAgent.server: Updating LAN server: server="Node-bd12542b-54e2-0860-3327-880769c13ddb (Addr: tcp/127.0.0.1:24536) (DC: dc1)" +34:37.585 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-bd12542b-54e2-0860-3327-880769c13ddb.dc1 area=wan +34:37.586 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:37.588 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:37.588 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:37.588 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:37.588 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:37.588 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:37.589 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bd12542b-54e2-0860-3327-880769c13ddb +34:37.589 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bd12542b-54e2-0860-3327-880769c13ddb +34:37.590 [INFO] TestAgent.server: federation state anti-entropy synced +34:37.590 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bd12542b-54e2-0860-3327-880769c13ddb +34:37.697 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-bd12542b-54e2-0860-3327-880769c13ddb +34:37.697 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-bd12542b-54e2-0860-3327-880769c13ddb +34:37.704 [INFO] TestAgent: Requesting shutdown +34:37.704 [INFO] TestAgent.server: shutting down server +34:37.704 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:37.704 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:37.704 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:37.704 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:37.704 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:37.704 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:37.704 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:37.704 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:37.704 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:37.704 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:37.704 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:37.704 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:37.704 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:37.704 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:37.704 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:37.704 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:37.704Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:37.704Z [INFO] agent.router.manager: shutting down +34:37.704 [INFO] TestAgent: consul server down +34:37.704 [INFO] TestAgent: shutdown complete +34:37.704 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24458 network=tcp +34:37.704 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24458 network=udp +34:37.704 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24532 network=tcp +34:38.205 [INFO] TestAgent: Waiting for endpoints to shut down +34:38.205 [INFO] TestAgent: Endpoints down +--- PASS: TestACL_Legacy_List (0.73s) +=== CONT TestSetIndex +--- PASS: TestSetIndex (0.00s) +=== CONT TestHTTPServer_UnixSocket_FileExists +[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:37.764Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:37.764Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:37.779Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:37.779Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:37.782 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e8d17156-7372-7015-7ba1-f8d61df41cce Address:127.0.0.1:24578}]" +34:37.782 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24578 [Follower]" leader= +34:37.783 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e8d17156-7372-7015-7ba1-f8d61df41cce.dc1 127.0.0.1 +34:37.784 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e8d17156-7372-7015-7ba1-f8d61df41cce 127.0.0.1 +2021-08-13T15:34:37.784Z [INFO] agent.router: Initializing LAN area manager +34:37.784 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24566 network=udp +34:37.784 [INFO] TestAgent.server: Adding LAN server: server="Node-e8d17156-7372-7015-7ba1-f8d61df41cce (Addr: tcp/127.0.0.1:24578) (DC: dc1)" +34:37.784 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e8d17156-7372-7015-7ba1-f8d61df41cce.dc1 area=wan +34:37.784 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24566 network=tcp +34:37.785 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24567 network=tcp +34:37.785 [INFO] TestAgent: started state syncer +34:37.785 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24579 network=tcp +34:37.830 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:37.830 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24578 [Candidate]" term=2 +34:37.830 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:37.830 [DEBUG] TestAgent.server.raft: vote granted: from=e8d17156-7372-7015-7ba1-f8d61df41cce term=2 tally=1 +34:37.830 [INFO] TestAgent.server.raft: election won: tally=1 +34:37.830 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24578 [Leader]" +34:37.830 [INFO] TestAgent.server: cluster leadership acquired +34:37.831 [INFO] TestAgent.server: New leader elected: payload=Node-e8d17156-7372-7015-7ba1-f8d61df41cce +34:37.831 [INFO] TestAgent.server: initializing acls +34:37.831 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:37.831 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:37.832 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:37.832 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:37.832 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:37.832 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:37.833 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-e8d17156-7372-7015-7ba1-f8d61df41cce +34:37.833 [INFO] TestAgent.server: Updating LAN server: server="Node-e8d17156-7372-7015-7ba1-f8d61df41cce (Addr: tcp/127.0.0.1:24578) (DC: dc1)" +34:37.833 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-e8d17156-7372-7015-7ba1-f8d61df41cce.dc1 +34:37.833 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-e8d17156-7372-7015-7ba1-f8d61df41cce.dc1 area=wan +34:37.834 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:37.836 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:37.836 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:37.836 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:37.836 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:37.836 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:37.837 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e8d17156-7372-7015-7ba1-f8d61df41cce +34:37.837 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e8d17156-7372-7015-7ba1-f8d61df41cce +34:37.838 [INFO] TestAgent.server: federation state anti-entropy synced +34:37.838 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e8d17156-7372-7015-7ba1-f8d61df41cce +34:38.057 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-e8d17156-7372-7015-7ba1-f8d61df41cce +34:38.057 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-e8d17156-7372-7015-7ba1-f8d61df41cce +34:38.064 [INFO] TestAgent: Requesting shutdown +34:38.064 [INFO] TestAgent.server: shutting down server +34:38.064 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:38.064 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:38.064 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:38.064 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:38.064 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:38.064 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:38.064 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:38.064 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:38.064 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:38.064 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:38.064 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:38.064 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:38.064 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:38.064 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:38.065 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:38.065 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:38.065Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:38.065Z [INFO] agent.router.manager: shutting down +34:38.065 [INFO] TestAgent: consul server down +34:38.065 [INFO] TestAgent: shutdown complete +34:38.065 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24566 network=tcp +34:38.065 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24566 network=udp +34:38.065 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24567 network=tcp +34:38.565 [INFO] TestAgent: Waiting for endpoints to shut down +34:38.565 [INFO] TestAgent: Endpoints down +--- PASS: TestACL_Legacy_Clone (0.82s) +=== CONT TestHTTPServer_UnixSocket +2021-08-13T15:34:37.804Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:37.804Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:37.820Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:37.820Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:37.823 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0d234ed3-7b10-e5db-0cbd-e2a0bf17cd39 Address:127.0.0.1:24592}]" +34:37.823 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24592 [Follower]" leader= +34:37.824 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0d234ed3-7b10-e5db-0cbd-e2a0bf17cd39.dc1 127.0.0.1 +34:37.825 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0d234ed3-7b10-e5db-0cbd-e2a0bf17cd39 127.0.0.1 +2021-08-13T15:34:37.825Z [INFO] agent.router: Initializing LAN area manager +34:37.825 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0d234ed3-7b10-e5db-0cbd-e2a0bf17cd39.dc1 area=wan +34:37.825 [INFO] TestAgent.server: Adding LAN server: server="Node-0d234ed3-7b10-e5db-0cbd-e2a0bf17cd39 (Addr: tcp/127.0.0.1:24592) (DC: dc1)" +34:37.825 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24580 network=udp +34:37.825 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24580 network=tcp +34:37.826 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24588 network=tcp +34:37.826 [INFO] TestAgent: started state syncer +34:37.826 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24593 network=tcp +34:37.875 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:37.889 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:37.889 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24592 [Candidate]" term=2 +34:37.889 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:37.889 [DEBUG] TestAgent.server.raft: vote granted: from=0d234ed3-7b10-e5db-0cbd-e2a0bf17cd39 term=2 tally=1 +34:37.889 [INFO] TestAgent.server.raft: election won: tally=1 +34:37.889 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24592 [Leader]" +34:37.889 [INFO] TestAgent.server: cluster leadership acquired +34:37.890 [INFO] TestAgent.server: New leader elected: payload=Node-0d234ed3-7b10-e5db-0cbd-e2a0bf17cd39 +34:37.890 [INFO] TestAgent.server: initializing acls +34:37.890 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:37.890 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:37.891 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:37.891 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:37.891 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:37.891 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:37.892 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-0d234ed3-7b10-e5db-0cbd-e2a0bf17cd39 +34:37.892 [INFO] TestAgent.server: Updating LAN server: server="Node-0d234ed3-7b10-e5db-0cbd-e2a0bf17cd39 (Addr: tcp/127.0.0.1:24592) (DC: dc1)" +34:37.892 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-0d234ed3-7b10-e5db-0cbd-e2a0bf17cd39.dc1 +34:37.892 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-0d234ed3-7b10-e5db-0cbd-e2a0bf17cd39.dc1 area=wan +34:37.893 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:37.895 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:37.895 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:37.895 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:37.895 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:37.895 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:37.896 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0d234ed3-7b10-e5db-0cbd-e2a0bf17cd39 +34:37.896 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0d234ed3-7b10-e5db-0cbd-e2a0bf17cd39 +34:37.897 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0d234ed3-7b10-e5db-0cbd-e2a0bf17cd39 +34:37.897 [INFO] TestAgent.server: federation state anti-entropy synced +34:38.103 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-0d234ed3-7b10-e5db-0cbd-e2a0bf17cd39 +34:38.103 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-0d234ed3-7b10-e5db-0cbd-e2a0bf17cd39 +34:38.105 [INFO] TestAgent: Requesting shutdown +34:38.106 [INFO] TestAgent.server: shutting down server +34:38.106 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:38.106 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:38.106 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:38.106 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:38.106 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:38.106 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:38.106 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:38.106 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:38.106 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:38.106 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:38.106 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:38.106 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:38.106 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:38.106 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:38.106 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:38.106 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:38.106Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:38.106Z [INFO] agent.router.manager: shutting down +34:38.106 [INFO] TestAgent: consul server down +34:38.106 [INFO] TestAgent: shutdown complete +34:38.106 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24580 network=tcp +34:38.106 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24580 network=udp +34:38.106 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24588 network=tcp +34:38.607 [INFO] TestAgent: Waiting for endpoints to shut down +34:38.607 [INFO] TestAgent: Endpoints down +--- PASS: TestACL_Legacy_Destroy (0.82s) +=== CONT TestFilterNonPassing +--- PASS: TestFilterNonPassing (0.00s) +=== CONT TestHealthConnectServiceNodes_PassingFilter +2021-08-13T15:34:37.732Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:37.732Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:37.748Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:37.748Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:37.751 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0240ad09-160c-2958-a819-08b0afe64f74 Address:127.0.0.1:24564}]" +34:37.752 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0240ad09-160c-2958-a819-08b0afe64f74.dc1 127.0.0.1 +34:37.753 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0240ad09-160c-2958-a819-08b0afe64f74 127.0.0.1 +2021-08-13T15:34:37.753Z [INFO] agent.router: Initializing LAN area manager +34:37.753 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24559 network=udp +34:37.753 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24564 [Follower]" leader= +34:37.754 [INFO] TestAgent.server: Adding LAN server: server="Node-0240ad09-160c-2958-a819-08b0afe64f74 (Addr: tcp/127.0.0.1:24564) (DC: dc1)" +34:37.754 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0240ad09-160c-2958-a819-08b0afe64f74.dc1 area=wan +34:37.754 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24559 network=tcp +34:37.755 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24560 network=tcp +34:37.755 [INFO] TestAgent: started state syncer +34:37.755 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24565 network=tcp +34:37.804 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:37.805 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:37.805 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24564 [Candidate]" term=2 +34:37.806 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:37.806 [DEBUG] TestAgent.server.raft: vote granted: from=0240ad09-160c-2958-a819-08b0afe64f74 term=2 tally=1 +34:37.806 [INFO] TestAgent.server.raft: election won: tally=1 +34:37.806 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24564 [Leader]" +34:37.806 [INFO] TestAgent.server: cluster leadership acquired +34:37.806 [INFO] TestAgent.server: New leader elected: payload=Node-0240ad09-160c-2958-a819-08b0afe64f74 +34:37.806 [INFO] TestAgent.server: initializing acls +34:37.807 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:37.807 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:37.807 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:37.808 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:37.808 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:37.808 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:37.808 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-0240ad09-160c-2958-a819-08b0afe64f74 +34:37.808 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-0240ad09-160c-2958-a819-08b0afe64f74.dc1 +34:37.808 [INFO] TestAgent.server: Updating LAN server: server="Node-0240ad09-160c-2958-a819-08b0afe64f74 (Addr: tcp/127.0.0.1:24564) (DC: dc1)" +34:37.808 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-0240ad09-160c-2958-a819-08b0afe64f74.dc1 area=wan +34:37.809 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:37.811 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:37.811 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:37.811 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:37.811 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:37.811 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:37.811 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0240ad09-160c-2958-a819-08b0afe64f74 +34:37.812 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0240ad09-160c-2958-a819-08b0afe64f74 +34:37.813 [INFO] TestAgent.server: federation state anti-entropy synced +34:37.813 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0240ad09-160c-2958-a819-08b0afe64f74 +34:37.820 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:37.821 [INFO] TestAgent: Synced node info +34:37.821 [DEBUG] TestAgent: Node info in sync +34:38.119 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-0240ad09-160c-2958-a819-08b0afe64f74 +34:38.123 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-0240ad09-160c-2958-a819-08b0afe64f74 +34:38.123 [INFO] TestAgent: Requesting shutdown +34:38.123 [INFO] TestAgent.server: shutting down server +34:38.123 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:38.123 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:38.123 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:38.123 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:38.123 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:38.123 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:38.123 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:38.124 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:38.124 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:38.124 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:38.124 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:38.124 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:38.124 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:38.124 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:38.124 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:38.124Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:38.124Z [INFO] agent.router.manager: shutting down +34:38.124 [INFO] TestAgent: consul server down +34:38.124 [INFO] TestAgent: shutdown complete +34:38.124 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24559 network=tcp +34:38.124 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24559 network=udp +34:38.124 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24560 network=tcp +34:38.625 [INFO] TestAgent: Waiting for endpoints to shut down +34:38.625 [INFO] TestAgent: Endpoints down +=== RUN TestACL_Legacy_Get/right_id +2021-08-13T15:34:37.757Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:37.773Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:37.776 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a0cca61f-bf0c-c36e-118c-f861f3f962a6 Address:127.0.0.1:24557}]" +34:37.776 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24557 [Follower]" leader= +34:37.777 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a0cca61f-bf0c-c36e-118c-f861f3f962a6.dc1 127.0.0.1 +34:37.778 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a0cca61f-bf0c-c36e-118c-f861f3f962a6 127.0.0.1 +2021-08-13T15:34:37.778Z [INFO] agent.router: Initializing LAN area manager +34:37.778 [INFO] TestAgent.server: Adding LAN server: server="Node-a0cca61f-bf0c-c36e-118c-f861f3f962a6 (Addr: tcp/127.0.0.1:24557) (DC: dc1)" +34:37.778 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a0cca61f-bf0c-c36e-118c-f861f3f962a6.dc1 area=wan +34:37.778 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24552 network=udp +34:37.778 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24552 network=tcp +34:37.779 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24553 network=tcp +34:37.779 [INFO] TestAgent: started state syncer +34:37.779 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24558 network=tcp +34:37.820 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:37.820 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24557 [Candidate]" term=2 +34:37.820 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:37.820 [DEBUG] TestAgent.server.raft: vote granted: from=a0cca61f-bf0c-c36e-118c-f861f3f962a6 term=2 tally=1 +34:37.820 [INFO] TestAgent.server.raft: election won: tally=1 +34:37.820 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24557 [Leader]" +34:37.821 [INFO] TestAgent.server: cluster leadership acquired +34:37.821 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24557 +34:37.822 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:37.823 [INFO] TestAgent.server: New leader elected: payload=Node-a0cca61f-bf0c-c36e-118c-f861f3f962a6 +34:37.825 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:37.825 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:37.825 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:37.825 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:37.825 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:37.825 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a0cca61f-bf0c-c36e-118c-f861f3f962a6 +34:37.825 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a0cca61f-bf0c-c36e-118c-f861f3f962a6 +34:37.826 [INFO] TestAgent.server: federation state anti-entropy synced +34:37.889 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:37.890 [INFO] TestAgent: Synced node info +34:37.890 [DEBUG] TestAgent: Node info in sync +34:38.133 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/self from= error="Endpoint is blocked by agent configuration" +34:38.133 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/checks from= latency=2.321µs +34:38.133 [INFO] TestAgent: Requesting shutdown +34:38.133 [INFO] TestAgent.server: shutting down server +34:38.133 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:38.133 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:38.133 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:38.133 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:38.133 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:38.133 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:38.133 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:38.133 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:38.133 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:38.133 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:38.133 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:38.134Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:38.134Z [INFO] agent.router.manager: shutting down +34:38.134 [INFO] TestAgent: consul server down +34:38.134 [INFO] TestAgent: shutdown complete +34:38.134 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24552 network=tcp +34:38.134 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24552 network=udp +34:38.134 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24553 network=tcp +34:38.635 [INFO] TestAgent: Waiting for endpoints to shut down +34:38.635 [INFO] TestAgent: Endpoints down +--- PASS: TestHTTPAPI_BlockEndpoints (0.90s) +=== CONT TestHealthChecksInState_DistanceSort +[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:38.109Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:38.109Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:38.127Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:38.127Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:38.130 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7870b11c-d390-161a-2d57-deabdf0f9024 Address:127.0.0.1:24613}]" +34:38.131 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7870b11c-d390-161a-2d57-deabdf0f9024.dc1 127.0.0.1 +34:38.131 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24613 [Follower]" leader= +34:38.132 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7870b11c-d390-161a-2d57-deabdf0f9024 127.0.0.1 +2021-08-13T15:34:38.132Z [INFO] agent.router: Initializing LAN area manager +34:38.132 [INFO] TestAgent.server: Adding LAN server: server="Node-7870b11c-d390-161a-2d57-deabdf0f9024 (Addr: tcp/127.0.0.1:24613) (DC: dc1)" +34:38.132 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7870b11c-d390-161a-2d57-deabdf0f9024.dc1 area=wan +34:38.133 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24601 network=tcp +34:38.133 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24601 network=udp +34:38.134 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24609 network=tcp +34:38.134 [INFO] TestAgent: started state syncer +34:38.134 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24614 network=tcp +34:38.179 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:38.179 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24613 [Candidate]" term=2 +34:38.179 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:38.179 [DEBUG] TestAgent.server.raft: vote granted: from=7870b11c-d390-161a-2d57-deabdf0f9024 term=2 tally=1 +34:38.179 [INFO] TestAgent.server.raft: election won: tally=1 +34:38.179 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24613 [Leader]" +34:38.180 [INFO] TestAgent.server: cluster leadership acquired +34:38.180 [INFO] TestAgent.server: New leader elected: payload=Node-7870b11c-d390-161a-2d57-deabdf0f9024 +34:38.180 [INFO] TestAgent.server: initializing acls +34:38.180 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:38.180 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:38.181 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:38.181 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:38.181 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:38.181 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:38.182 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-7870b11c-d390-161a-2d57-deabdf0f9024 +34:38.182 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-7870b11c-d390-161a-2d57-deabdf0f9024.dc1 +34:38.182 [INFO] TestAgent.server: Updating LAN server: server="Node-7870b11c-d390-161a-2d57-deabdf0f9024 (Addr: tcp/127.0.0.1:24613) (DC: dc1)" +34:38.182 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-7870b11c-d390-161a-2d57-deabdf0f9024.dc1 area=wan +34:38.183 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:38.185 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:38.185 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:38.185 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:38.185 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:38.185 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:38.185 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7870b11c-d390-161a-2d57-deabdf0f9024 +34:38.185 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7870b11c-d390-161a-2d57-deabdf0f9024 +34:38.186 [INFO] TestAgent.server: federation state anti-entropy synced +34:38.186 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7870b11c-d390-161a-2d57-deabdf0f9024 +34:38.221 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:38.222 [INFO] TestAgent: Synced node info +34:38.282 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-7870b11c-d390-161a-2d57-deabdf0f9024 +34:38.282 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-7870b11c-d390-161a-2d57-deabdf0f9024 +34:38.283 [INFO] TestAgent: Requesting shutdown +34:38.283 [INFO] TestAgent.server: shutting down server +34:38.283 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:38.283 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:38.283 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:38.283 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:38.283 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:38.283 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:38.283 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:38.283 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:38.283 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:38.283 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:38.283 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:38.283 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:38.283 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:38.283 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:38.283 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:38.283Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:38.283Z [INFO] agent.router.manager: shutting down +34:38.283 [INFO] TestAgent: consul server down +34:38.283 [INFO] TestAgent: shutdown complete +34:38.283 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24601 network=tcp +34:38.283 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24601 network=udp +34:38.284 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24609 network=tcp +34:38.784 [INFO] TestAgent: Waiting for endpoints to shut down +34:38.784 [INFO] TestAgent: Endpoints down +--- PASS: TestACL_Legacy_UpdateUpsert (0.70s) +=== CONT TestHealthConnectServiceNodes_Filter +2021-08-13T15:34:38.223Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:38.240Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:38.243 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9da19c20-c647-e40e-e023-5a891cb9e5c3 Address:127.0.0.1:24571}]" +34:38.243 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24571 [Follower]" leader= +34:38.244 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9da19c20-c647-e40e-e023-5a891cb9e5c3.dc1 127.0.0.1 +34:38.245 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9da19c20-c647-e40e-e023-5a891cb9e5c3 127.0.0.1 +2021-08-13T15:34:38.245Z [INFO] agent.router: Initializing LAN area manager +34:38.245 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24615 network=udp +34:38.245 [INFO] TestAgent.server: Adding LAN server: server="Node-9da19c20-c647-e40e-e023-5a891cb9e5c3 (Addr: tcp/127.0.0.1:24571) (DC: dc1)" +34:38.245 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9da19c20-c647-e40e-e023-5a891cb9e5c3.dc1 area=wan +34:38.246 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24615 network=tcp +34:38.246 [WARN] TestAgent: Replacing socket: path=/tmp/consul-test/TestHTTPServer_UnixSocket_FileExists-consul979834738/test.sock +34:38.247 [INFO] TestAgent: Started HTTP server: address=/tmp/consul-test/TestHTTPServer_UnixSocket_FileExists-consul979834738/test.sock network=unix +34:38.247 [INFO] TestAgent: started state syncer +34:38.247 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24572 network=tcp +34:38.299 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:38.299 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24571 [Candidate]" term=2 +34:38.299 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:38.299 [DEBUG] TestAgent.server.raft: vote granted: from=9da19c20-c647-e40e-e023-5a891cb9e5c3 term=2 tally=1 +34:38.299 [INFO] TestAgent.server.raft: election won: tally=1 +34:38.299 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24571 [Leader]" +34:38.299 [INFO] TestAgent.server: cluster leadership acquired +34:38.299 [INFO] TestAgent.server: New leader elected: payload=Node-9da19c20-c647-e40e-e023-5a891cb9e5c3 +34:38.300 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24571 +34:38.301 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:38.302 [INFO] TestAgent: Requesting shutdown +34:38.302 [INFO] TestAgent.server: shutting down server +34:38.302 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:38.302 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:38.302 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:38.302Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:38.302Z [INFO] agent.router.manager: shutting down +34:38.303 [INFO] TestAgent: consul server down +34:38.303 [INFO] TestAgent: shutdown complete +34:38.303 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24615 network=tcp +34:38.303 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24615 network=udp +34:38.303 [INFO] TestAgent: Stopping server: protocol=HTTP address=/tmp/consul-test/TestHTTPServer_UnixSocket_FileExists-consul979834738/test.sock network=unix +34:38.303 [ERROR] TestAgent.server.connect: Raft apply failed: error="raft is already shutdown" +34:38.303 [ERROR] TestAgent.server: failed to establish leadership: error="raft is already shutdown" +34:38.804 [INFO] TestAgent: Waiting for endpoints to shut down +34:38.804 [INFO] TestAgent: Endpoints down +--- PASS: TestHTTPServer_UnixSocket_FileExists (0.60s) +=== CONT TestHealthChecksInState_Filter +2021-08-13T15:34:38.128Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:38.128Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:38.144Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:38.144Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:38.147 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:934bafde-14e8-bf49-b8ef-dcaaeca70ede Address:127.0.0.1:24648}]" +34:38.147 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24648 [Follower]" leader= +34:38.148 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-934bafde-14e8-bf49-b8ef-dcaaeca70ede.dc1 127.0.0.1 +34:38.148 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-934bafde-14e8-bf49-b8ef-dcaaeca70ede 127.0.0.1 +2021-08-13T15:34:38.148Z [INFO] agent.router: Initializing LAN area manager +34:38.149 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24622 network=udp +34:38.149 [INFO] TestAgent.server: Adding LAN server: server="Node-934bafde-14e8-bf49-b8ef-dcaaeca70ede (Addr: tcp/127.0.0.1:24648) (DC: dc1)" +34:38.149 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-934bafde-14e8-bf49-b8ef-dcaaeca70ede.dc1 area=wan +34:38.149 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24622 network=tcp +34:38.150 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24637 network=tcp +34:38.150 [INFO] TestAgent: started state syncer +34:38.150 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24649 network=tcp +34:38.199 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:38.203 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:38.203 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24648 [Candidate]" term=2 +34:38.203 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:38.203 [DEBUG] TestAgent.server.raft: vote granted: from=934bafde-14e8-bf49-b8ef-dcaaeca70ede term=2 tally=1 +34:38.203 [INFO] TestAgent.server.raft: election won: tally=1 +34:38.203 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24648 [Leader]" +34:38.203 [INFO] TestAgent.server: cluster leadership acquired +34:38.203 [INFO] TestAgent.server: New leader elected: payload=Node-934bafde-14e8-bf49-b8ef-dcaaeca70ede +34:38.204 [INFO] TestAgent.server: initializing acls +34:38.204 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:38.204 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:38.205 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:38.205 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:38.205 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:38.205 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:38.205 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-934bafde-14e8-bf49-b8ef-dcaaeca70ede +34:38.206 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-934bafde-14e8-bf49-b8ef-dcaaeca70ede.dc1 +34:38.207 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:38.207 [INFO] TestAgent.server: Updating LAN server: server="Node-934bafde-14e8-bf49-b8ef-dcaaeca70ede (Addr: tcp/127.0.0.1:24648) (DC: dc1)" +34:38.207 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-934bafde-14e8-bf49-b8ef-dcaaeca70ede.dc1 area=wan +34:38.209 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:38.209 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:38.209 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:38.209 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:38.209 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:38.209 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-934bafde-14e8-bf49-b8ef-dcaaeca70ede +34:38.209 [INFO] TestAgent.server: member joined, marking health alive: member=Node-934bafde-14e8-bf49-b8ef-dcaaeca70ede +34:38.210 [INFO] TestAgent.server: federation state anti-entropy synced +34:38.211 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-934bafde-14e8-bf49-b8ef-dcaaeca70ede +34:38.379 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-934bafde-14e8-bf49-b8ef-dcaaeca70ede +34:38.380 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-934bafde-14e8-bf49-b8ef-dcaaeca70ede +34:38.382 [INFO] TestAgent: Requesting shutdown +34:38.382 [INFO] TestAgent.server: shutting down server +34:38.382 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:38.382 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:38.382 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:38.382 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:38.382 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:38.382 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:38.382 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:38.382 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:38.382 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:38.382 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:38.382 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:38.382 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:38.382 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:38.382 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:38.382 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:38.382 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:38.382Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:38.382Z [INFO] agent.router.manager: shutting down +34:38.383 [INFO] TestAgent: consul server down +34:38.383 [INFO] TestAgent: shutdown complete +34:38.383 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24622 network=tcp +34:38.383 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24622 network=udp +34:38.383 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24637 network=tcp +34:38.883 [INFO] TestAgent: Waiting for endpoints to shut down +34:38.883 [INFO] TestAgent: Endpoints down +--- PASS: TestACL_Legacy_Update (0.77s) +=== CONT TestHealthIngressServiceNodes +=== RUN TestHealthConnectServiceNodes_PassingFilter/bc_no_query_value +=== RUN TestHealthConnectServiceNodes_PassingFilter/passing_true +=== RUN TestHealthConnectServiceNodes_PassingFilter/passing_false +=== RUN TestHealthConnectServiceNodes_PassingFilter/passing_bad +=== RUN TestHealthIngressServiceNodes/associated_service +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestHealthIngressServiceNodes/non-associated_service +2021-08-13T15:34:38.585Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:38.602Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:38.605 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:89ee36f4-91ea-0eca-1f2c-d6fdb4c07eb8 Address:127.0.0.1:24627}]" +34:38.605 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24627 [Follower]" leader= +34:38.606 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-89ee36f4-91ea-0eca-1f2c-d6fdb4c07eb8.dc1 127.0.0.1 +34:38.606 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-89ee36f4-91ea-0eca-1f2c-d6fdb4c07eb8 127.0.0.1 +2021-08-13T15:34:38.606Z [INFO] agent.router: Initializing LAN area manager +34:38.607 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24594 network=udp +34:38.607 [INFO] TestAgent.server: Adding LAN server: server="Node-89ee36f4-91ea-0eca-1f2c-d6fdb4c07eb8 (Addr: tcp/127.0.0.1:24627) (DC: dc1)" +34:38.607 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-89ee36f4-91ea-0eca-1f2c-d6fdb4c07eb8.dc1 area=wan +34:38.607 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24594 network=tcp +34:38.608 [INFO] TestAgent: Started HTTP server: address=/tmp/consul-test/TestHTTPServer_UnixSocket-consul188828532/test.sock network=unix +34:38.609 [INFO] TestAgent: started state syncer +34:38.609 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24628 network=tcp +34:38.667 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:38.667 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24627 [Candidate]" term=2 +34:38.667 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:38.667 [DEBUG] TestAgent.server.raft: vote granted: from=89ee36f4-91ea-0eca-1f2c-d6fdb4c07eb8 term=2 tally=1 +34:38.667 [INFO] TestAgent.server.raft: election won: tally=1 +34:38.667 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24627 [Leader]" +34:38.668 [INFO] TestAgent.server: cluster leadership acquired +34:38.669 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24627 +34:38.671 [INFO] TestAgent.server: New leader elected: payload=Node-89ee36f4-91ea-0eca-1f2c-d6fdb4c07eb8 +34:38.672 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:38.680 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:38.680 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:38.680 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:38.680 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:38.680 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:38.680 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-89ee36f4-91ea-0eca-1f2c-d6fdb4c07eb8 +34:38.680 [INFO] TestAgent.server: member joined, marking health alive: member=Node-89ee36f4-91ea-0eca-1f2c-d6fdb4c07eb8 +34:38.691 [INFO] TestAgent.server: federation state anti-entropy synced +34:38.739 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=@ latency=1.889705ms +34:38.740 [INFO] TestAgent: Requesting shutdown +34:38.740 [INFO] TestAgent.server: shutting down server +34:38.740 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:38.740 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:38.740 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:38.740 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:38.740 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:38.740 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:38.741 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:38.741 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:38.741 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:38.741 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:38.741 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:38.741 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:38.741Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:38.741Z [INFO] agent.router.manager: shutting down +34:38.741 [INFO] TestAgent: consul server down +34:38.741 [INFO] TestAgent: shutdown complete +34:38.741 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24594 network=tcp +34:38.741 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24594 network=udp +34:38.741 [INFO] TestAgent: Stopping server: protocol=HTTP address=/tmp/consul-test/TestHTTPServer_UnixSocket-consul188828532/test.sock network=unix +34:39.241 [INFO] TestAgent: Waiting for endpoints to shut down +34:39.241 [INFO] TestAgent: Endpoints down +--- PASS: TestHTTPServer_UnixSocket (0.68s) +=== CONT TestHealthChecksInState_NodeMetaFilter +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:38.691Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:38.691Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:38.711Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:38.711Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:38.713 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f78c29f1-e5b1-945d-9004-c4375444e466 Address:127.0.0.1:24655}]" +34:38.714 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f78c29f1-e5b1-945d-9004-c4375444e466.dc1 127.0.0.1 +34:38.715 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f78c29f1-e5b1-945d-9004-c4375444e466 127.0.0.1 +2021-08-13T15:34:38.715Z [INFO] agent.router: Initializing LAN area manager +34:38.715 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24655 [Follower]" leader= +34:38.715 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24643 network=udp +34:38.715 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f78c29f1-e5b1-945d-9004-c4375444e466.dc1 area=wan +34:38.715 [INFO] TestAgent.server: Adding LAN server: server="Node-f78c29f1-e5b1-945d-9004-c4375444e466 (Addr: tcp/127.0.0.1:24655) (DC: dc1)" +34:38.715 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24643 network=tcp +34:38.716 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24644 network=tcp +34:38.716 [INFO] TestAgent: started state syncer +34:38.716 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24656 network=tcp +34:38.765 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:38.774 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:38.774 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24655 [Candidate]" term=2 +34:38.774 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:38.774 [DEBUG] TestAgent.server.raft: vote granted: from=f78c29f1-e5b1-945d-9004-c4375444e466 term=2 tally=1 +34:38.774 [INFO] TestAgent.server.raft: election won: tally=1 +34:38.774 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24655 [Leader]" +34:38.774 [INFO] TestAgent.server: cluster leadership acquired +34:38.774 [INFO] TestAgent.server: New leader elected: payload=Node-f78c29f1-e5b1-945d-9004-c4375444e466 +34:38.774 [INFO] TestAgent.server: initializing acls +34:38.775 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:38.775 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:38.776 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:38.776 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:38.776 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:38.776 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:38.776 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-f78c29f1-e5b1-945d-9004-c4375444e466 +34:38.776 [INFO] TestAgent.server: Updating LAN server: server="Node-f78c29f1-e5b1-945d-9004-c4375444e466 (Addr: tcp/127.0.0.1:24655) (DC: dc1)" +34:38.777 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-f78c29f1-e5b1-945d-9004-c4375444e466.dc1 +34:38.777 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-f78c29f1-e5b1-945d-9004-c4375444e466.dc1 area=wan +34:38.778 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:38.780 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:38.780 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:38.780 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:38.780 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:38.780 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:38.780 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f78c29f1-e5b1-945d-9004-c4375444e466 +34:38.780 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f78c29f1-e5b1-945d-9004-c4375444e466 +34:38.781 [INFO] TestAgent.server: federation state anti-entropy synced +34:38.781 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f78c29f1-e5b1-945d-9004-c4375444e466 +34:38.815 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-f78c29f1-e5b1-945d-9004-c4375444e466 +34:38.815 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-f78c29f1-e5b1-945d-9004-c4375444e466 +34:38.816 [INFO] TestAgent: Requesting shutdown +34:38.817 [INFO] TestAgent.server: shutting down server +34:38.817 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:38.817 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:38.817 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:38.817 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:38.817 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:38.817 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:38.817 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:38.817 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:38.817 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:38.817 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:38.817 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:38.817 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:38.817 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:38.817 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:38.817 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:38.817 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:38.817Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:38.817Z [INFO] agent.router.manager: shutting down +34:38.817 [INFO] TestAgent: consul server down +34:38.817 [INFO] TestAgent: shutdown complete +34:38.817 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24643 network=tcp +34:38.817 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24643 network=udp +34:38.817 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24644 network=tcp +34:39.318 [INFO] TestAgent: Waiting for endpoints to shut down +34:39.318 [INFO] TestAgent: Endpoints down +--- PASS: TestACL_Legacy_Get (1.60s) + --- PASS: TestACL_Legacy_Get/wrong_id (0.91s) + --- PASS: TestACL_Legacy_Get/right_id (0.69s) +=== CONT TestHealthConnectServiceNodes +2021-08-13T15:34:38.692Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:38.708Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:38.711 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:72c3f1c5-1aac-fce3-cf56-85b5f240624a Address:127.0.0.1:24599}]" +34:38.711 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24599 [Follower]" leader= +34:38.712 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-72c3f1c5-1aac-fce3-cf56-85b5f240624a.dc1 127.0.0.1 +34:38.713 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-72c3f1c5-1aac-fce3-cf56-85b5f240624a 127.0.0.1 +2021-08-13T15:34:38.713Z [INFO] agent.router: Initializing LAN area manager +34:38.713 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24664 network=udp +34:38.714 [INFO] TestAgent.server: Adding LAN server: server="Node-72c3f1c5-1aac-fce3-cf56-85b5f240624a (Addr: tcp/127.0.0.1:24599) (DC: dc1)" +34:38.714 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-72c3f1c5-1aac-fce3-cf56-85b5f240624a.dc1 area=wan +34:38.714 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24664 network=tcp +34:38.715 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24595 network=tcp +34:38.715 [INFO] TestAgent: started state syncer +34:38.715 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24600 network=tcp +34:38.778 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:38.778 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24599 [Candidate]" term=2 +34:38.778 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:38.778 [DEBUG] TestAgent.server.raft: vote granted: from=72c3f1c5-1aac-fce3-cf56-85b5f240624a term=2 tally=1 +34:38.778 [INFO] TestAgent.server.raft: election won: tally=1 +34:38.778 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24599 [Leader]" +34:38.778 [INFO] TestAgent.server: cluster leadership acquired +34:38.778 [INFO] TestAgent.server: New leader elected: payload=Node-72c3f1c5-1aac-fce3-cf56-85b5f240624a +34:38.779 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24599 +34:38.780 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:38.782 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:38.782 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:38.782 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:38.782 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:38.782 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:38.782 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-72c3f1c5-1aac-fce3-cf56-85b5f240624a +34:38.782 [INFO] TestAgent.server: member joined, marking health alive: member=Node-72c3f1c5-1aac-fce3-cf56-85b5f240624a +34:38.783 [INFO] TestAgent.server: federation state anti-entropy synced +34:38.929 [INFO] TestAgent: Requesting shutdown +34:38.929 [INFO] TestAgent.server: shutting down server +34:38.929 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:38.929 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:38.929 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:38.929 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:38.929 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:38.929 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:38.929 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:38.929 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:38.929 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:38.929 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:38.929 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:38.929 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:38.929Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:38.929Z [INFO] agent.router.manager: shutting down +34:38.930 [INFO] TestAgent: consul server down +34:38.930 [INFO] TestAgent: shutdown complete +34:38.930 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24664 network=tcp +34:38.930 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24664 network=udp +34:38.930 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24595 network=tcp +34:39.430 [INFO] TestAgent: Waiting for endpoints to shut down +34:39.430 [INFO] TestAgent: Endpoints down +--- PASS: TestHealthChecksInState_DistanceSort (0.79s) +=== CONT TestUUIDToUint64 +--- PASS: TestUUIDToUint64 (0.00s) +=== CONT TestHealthServiceNodes_WanTranslation +2021-08-13T15:34:38.624Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:38.678Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:38.695 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cede716e-ca5c-2237-ff3b-1a818eb4c604 Address:127.0.0.1:24585}]" +34:38.696 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-cede716e-ca5c-2237-ff3b-1a818eb4c604.dc1 127.0.0.1 +34:38.697 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-cede716e-ca5c-2237-ff3b-1a818eb4c604 127.0.0.1 +2021-08-13T15:34:38.697Z [INFO] agent.router: Initializing LAN area manager +34:38.698 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24573 network=udp +34:38.698 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24585 [Follower]" leader= +34:38.698 [INFO] TestAgent.server: Adding LAN server: server="Node-cede716e-ca5c-2237-ff3b-1a818eb4c604 (Addr: tcp/127.0.0.1:24585) (DC: dc1)" +34:38.698 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-cede716e-ca5c-2237-ff3b-1a818eb4c604.dc1 area=wan +34:38.699 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24573 network=tcp +34:38.700 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24574 network=tcp +34:38.700 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24586 network=tcp +34:38.700 [INFO] TestAgent: started state syncer +34:38.763 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:38.763 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24585 [Candidate]" term=2 +34:38.763 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:38.763 [DEBUG] TestAgent.server.raft: vote granted: from=cede716e-ca5c-2237-ff3b-1a818eb4c604 term=2 tally=1 +34:38.763 [INFO] TestAgent.server.raft: election won: tally=1 +34:38.763 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24585 [Leader]" +34:38.763 [INFO] TestAgent.server: cluster leadership acquired +34:38.763 [INFO] TestAgent.server: New leader elected: payload=Node-cede716e-ca5c-2237-ff3b-1a818eb4c604 +34:38.763 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24585 +34:38.765 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:38.767 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:38.767 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:38.767 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:38.767 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:38.767 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:38.767 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cede716e-ca5c-2237-ff3b-1a818eb4c604 +34:38.767 [INFO] TestAgent.server: member joined, marking health alive: member=Node-cede716e-ca5c-2237-ff3b-1a818eb4c604 +34:38.768 [INFO] TestAgent.server: federation state anti-entropy synced +34:39.002 [INFO] TestAgent: Requesting shutdown +34:39.002 [INFO] TestAgent.server: shutting down server +34:39.002 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:39.002 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:39.002 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:39.002 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:39.002 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:39.002 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:39.002 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:39.002 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:39.002 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:39.002 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:39.002 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:39.003 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:39.003Z [INFO] agent.router.manager: shutting down +34:39.003 [INFO] TestAgent: consul server down +34:39.003 [INFO] TestAgent: shutdown complete +34:39.003 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24573 network=tcp +34:39.003 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24573 network=udp +34:39.003 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24574 network=tcp +2021-08-13T15:34:39.003Z [INFO] agent.router.manager: shutting down +34:39.503 [INFO] TestAgent: Waiting for endpoints to shut down +34:39.503 [INFO] TestAgent: Endpoints down +--- PASS: TestHealthConnectServiceNodes_PassingFilter (0.90s) + --- PASS: TestHealthConnectServiceNodes_PassingFilter/bc_no_query_value (0.00s) + --- PASS: TestHealthConnectServiceNodes_PassingFilter/passing_true (0.00s) + --- PASS: TestHealthConnectServiceNodes_PassingFilter/passing_false (0.00s) + --- PASS: TestHealthConnectServiceNodes_PassingFilter/passing_bad (0.00s) +=== CONT TestEventList_EventBufOrder +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:38.916Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:38.934Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:38.937 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:45040844-d0e3-ece5-0096-d652e5ceba28 Address:127.0.0.1:24295}]" +34:38.938 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-45040844-d0e3-ece5-0096-d652e5ceba28.dc1 127.0.0.1 +34:38.939 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-45040844-d0e3-ece5-0096-d652e5ceba28 127.0.0.1 +2021-08-13T15:34:38.939Z [INFO] agent.router: Initializing LAN area manager +34:38.939 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24295 [Follower]" leader= +34:38.939 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24332 network=udp +34:38.939 [INFO] TestAgent.server: Adding LAN server: server="Node-45040844-d0e3-ece5-0096-d652e5ceba28 (Addr: tcp/127.0.0.1:24295) (DC: dc1)" +34:38.939 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-45040844-d0e3-ece5-0096-d652e5ceba28.dc1 area=wan +34:38.940 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24332 network=tcp +34:38.940 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25311 network=tcp +34:38.941 [INFO] TestAgent: started state syncer +34:38.941 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24296 network=tcp +34:39.004 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:39.004 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24295 [Candidate]" term=2 +34:39.004 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:39.004 [DEBUG] TestAgent.server.raft: vote granted: from=45040844-d0e3-ece5-0096-d652e5ceba28 term=2 tally=1 +34:39.004 [INFO] TestAgent.server.raft: election won: tally=1 +34:39.004 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24295 [Leader]" +34:39.004 [INFO] TestAgent.server: cluster leadership acquired +34:39.004 [INFO] TestAgent.server: New leader elected: payload=Node-45040844-d0e3-ece5-0096-d652e5ceba28 +34:39.004 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24295 +34:39.006 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:39.008 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:39.008 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:39.008 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:39.009 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:39.009 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:39.009 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-45040844-d0e3-ece5-0096-d652e5ceba28 +34:39.009 [INFO] TestAgent.server: member joined, marking health alive: member=Node-45040844-d0e3-ece5-0096-d652e5ceba28 +34:39.010 [INFO] TestAgent.server: federation state anti-entropy synced +34:39.023 [INFO] TestAgent: Requesting shutdown +34:39.023 [INFO] TestAgent.server: shutting down server +34:39.023 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:39.023 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:39.023 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:39.023 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:39.023 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:39.023 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:39.023 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:39.023 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:39.023 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:39.023 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:39.023 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:39.024 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:39.024Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:39.024Z [INFO] agent.router.manager: shutting down +34:39.024 [INFO] TestAgent: consul server down +34:39.024 [INFO] TestAgent: shutdown complete +34:39.024 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24332 network=tcp +34:39.024 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24332 network=udp +34:39.024 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25311 network=tcp +34:39.525 [INFO] TestAgent: Waiting for endpoints to shut down +34:39.525 [INFO] TestAgent: Endpoints down +--- PASS: TestHealthIngressServiceNodes (0.64s) + --- PASS: TestHealthIngressServiceNodes/associated_service (0.00s) + --- PASS: TestHealthIngressServiceNodes/non-associated_service (0.00s) +=== CONT TestHealthServiceNodes_CheckType +2021-08-13T15:34:38.821Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:38.837Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:38.840 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7161c2ac-5d49-a44d-e739-6aff3f5023fa Address:127.0.0.1:24351}]" +34:38.840 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24351 [Follower]" leader= +34:38.841 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7161c2ac-5d49-a44d-e739-6aff3f5023fa.dc1 127.0.0.1 +34:38.841 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7161c2ac-5d49-a44d-e739-6aff3f5023fa 127.0.0.1 +2021-08-13T15:34:38.841Z [INFO] agent.router: Initializing LAN area manager +34:38.842 [INFO] TestAgent.server: Adding LAN server: server="Node-7161c2ac-5d49-a44d-e739-6aff3f5023fa (Addr: tcp/127.0.0.1:24351) (DC: dc1)" +34:38.842 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7161c2ac-5d49-a44d-e739-6aff3f5023fa.dc1 area=wan +34:38.842 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25075 network=udp +34:38.842 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25075 network=tcp +34:38.843 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24340 network=tcp +34:38.843 [INFO] TestAgent: started state syncer +34:38.843 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24352 network=tcp +34:38.893 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:38.893 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24351 [Candidate]" term=2 +34:38.894 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:38.894 [DEBUG] TestAgent.server.raft: vote granted: from=7161c2ac-5d49-a44d-e739-6aff3f5023fa term=2 tally=1 +34:38.894 [INFO] TestAgent.server.raft: election won: tally=1 +34:38.894 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24351 [Leader]" +34:38.895 [INFO] TestAgent.server: cluster leadership acquired +34:38.895 [INFO] TestAgent.server: New leader elected: payload=Node-7161c2ac-5d49-a44d-e739-6aff3f5023fa +34:38.899 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24351 +34:38.900 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:38.907 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:38.907 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:38.907 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:38.907 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:38.907 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:38.908 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7161c2ac-5d49-a44d-e739-6aff3f5023fa +34:38.908 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7161c2ac-5d49-a44d-e739-6aff3f5023fa +34:38.909 [INFO] TestAgent.server: federation state anti-entropy synced +34:39.030 [INFO] TestAgent: Requesting shutdown +34:39.030 [INFO] TestAgent.server: shutting down server +34:39.030 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:39.030 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:39.030 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:39.030 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:39.030 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:39.030 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:39.030 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:39.030 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:39.030 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:39.030 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +34:39.030 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:39.030 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +2021-08-13T15:34:39.030Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:39.030Z [INFO] agent.router.manager: shutting down +34:39.030 [INFO] TestAgent: consul server down +34:39.030 [INFO] TestAgent: shutdown complete +34:39.030 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25075 network=tcp +34:39.031 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25075 network=udp +34:39.031 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24340 network=tcp +34:39.531 [INFO] TestAgent: Waiting for endpoints to shut down +34:39.531 [INFO] TestAgent: Endpoints down +--- PASS: TestHealthChecksInState_Filter (0.73s) +=== CONT TestEventList_Blocking +2021-08-13T15:34:38.126Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:38.144Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:38.147 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b9098c88-0f36-4c6b-18f5-07df5b8410ac Address:127.0.0.1:24620}]" +34:38.147 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24620 [Follower]" leader= +34:38.148 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b9098c88-0f36-4c6b-18f5-07df5b8410ac.dc1 127.0.0.1 +34:38.148 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b9098c88-0f36-4c6b-18f5-07df5b8410ac 127.0.0.1 +2021-08-13T15:34:38.148Z [INFO] agent.router: Initializing LAN area manager +34:38.149 [INFO] TestAgent.server: Adding LAN server: server="Node-b9098c88-0f36-4c6b-18f5-07df5b8410ac (Addr: tcp/127.0.0.1:24620) (DC: dc1)" +34:38.149 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b9098c88-0f36-4c6b-18f5-07df5b8410ac.dc1 area=wan +34:38.149 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24636 network=udp +34:38.149 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24636 network=tcp +34:38.150 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24616 network=tcp +34:38.150 [INFO] TestAgent: started state syncer +34:38.150 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24621 network=tcp +34:38.189 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:38.189 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24620 [Candidate]" term=2 +34:38.189 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:38.189 [DEBUG] TestAgent.server.raft: vote granted: from=b9098c88-0f36-4c6b-18f5-07df5b8410ac term=2 tally=1 +34:38.189 [INFO] TestAgent.server.raft: election won: tally=1 +34:38.189 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24620 [Leader]" +34:38.189 [INFO] TestAgent.server: cluster leadership acquired +34:38.189 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24620 +34:38.190 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:38.191 [INFO] TestAgent.server: New leader elected: payload=Node-b9098c88-0f36-4c6b-18f5-07df5b8410ac +34:38.193 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:38.193 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:38.193 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:38.193 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:38.193 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:38.193 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b9098c88-0f36-4c6b-18f5-07df5b8410ac +34:38.193 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b9098c88-0f36-4c6b-18f5-07df5b8410ac +34:38.194 [INFO] TestAgent.server: federation state anti-entropy synced +34:38.333 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:38.334 [INFO] TestAgent: Synced node info +34:38.552 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from= latency=3.14µs +34:38.552 [INFO] TestAgent: Requesting shutdown +34:38.552 [INFO] TestAgent.server: shutting down server +34:38.552 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:38.552 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:38.552 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:38.552 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:38.552 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:38.553 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:38.553 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:38.553 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:38.553 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:38.553 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:38.553 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:38.553Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:38.553Z [INFO] agent.router.manager: shutting down +34:38.553 [INFO] TestAgent: consul server down +34:38.553 [INFO] TestAgent: shutdown complete +34:38.553 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24636 network=tcp +34:38.553 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24636 network=udp +34:38.553 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24616 network=tcp +34:39.055 [INFO] TestAgent: Waiting for endpoints to shut down +34:39.055 [INFO] TestAgent: Endpoints down +2021-08-13T15:34:37.664Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:37.681Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:37.683 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0ce117e3-7851-14e0-6146-3720f12e86c2 Address:127.0.0.1:24543}]" +34:37.684 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24543 [Follower]" leader= +34:37.684 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0ce117e3-7851-14e0-6146-3720f12e86c2.dc1 127.0.0.1 +34:37.685 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0ce117e3-7851-14e0-6146-3720f12e86c2 127.0.0.1 +2021-08-13T15:34:37.685Z [INFO] agent.router: Initializing LAN area manager +34:37.685 [INFO] TestAgent.server: Adding LAN server: server="Node-0ce117e3-7851-14e0-6146-3720f12e86c2 (Addr: tcp/127.0.0.1:24543) (DC: dc1)" +34:37.686 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0ce117e3-7851-14e0-6146-3720f12e86c2.dc1 area=wan +34:37.686 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24545 network=udp +34:37.686 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24545 network=tcp +34:37.687 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24539 network=tcp +34:37.687 [INFO] TestAgent: started state syncer +34:37.687 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24544 network=tcp +34:37.741 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:37.741 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24543 [Candidate]" term=2 +34:37.741 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:37.741 [DEBUG] TestAgent.server.raft: vote granted: from=0ce117e3-7851-14e0-6146-3720f12e86c2 term=2 tally=1 +34:37.741 [INFO] TestAgent.server.raft: election won: tally=1 +34:37.741 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24543 [Leader]" +34:37.741 [INFO] TestAgent.server: cluster leadership acquired +34:37.741 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24543 +34:37.743 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:37.743 [INFO] TestAgent.server: New leader elected: payload=Node-0ce117e3-7851-14e0-6146-3720f12e86c2 +34:37.745 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:37.745 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:37.745 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:37.745 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:37.745 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:37.745 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0ce117e3-7851-14e0-6146-3720f12e86c2 +34:37.745 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0ce117e3-7851-14e0-6146-3720f12e86c2 +34:37.747 [INFO] TestAgent.server: federation state anti-entropy synced +34:38.027 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:38.027 [INFO] TestAgent: Synced node info +34:38.109 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from= latency=4.24µs +34:38.127 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:38.127 [DEBUG] TestAgent: Node info in sync +34:38.127 [DEBUG] TestAgent: Node info in sync +34:39.055 [INFO] TestAgent: Requesting shutdown +34:39.055 [INFO] TestAgent.server: shutting down server +34:39.055 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:39.055 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:39.055 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:39.055 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:39.055 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:39.055 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:39.055 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:39.055 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:39.055 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:39.055 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:39.055 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:39.056Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:39.056Z [INFO] agent.router.manager: shutting down +34:39.056 [INFO] TestAgent: consul server down +34:39.056 [INFO] TestAgent: shutdown complete +34:39.056 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24545 network=tcp +34:39.056 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24545 network=udp +34:39.056 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24539 network=tcp +34:39.556 [INFO] TestAgent: Waiting for endpoints to shut down +34:39.556 [INFO] TestAgent: Endpoints down +--- PASS: TestHTTPAPI_TranslateAddrHeader (1.91s) +=== CONT TestHealthServiceNodes_PassingFilter +2021-08-13T15:34:38.800Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:38.816Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:38.819 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c0974c08-5306-1803-9d9e-20773b3aa73d Address:127.0.0.1:25073}]" +34:38.819 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25073 [Follower]" leader= +34:38.820 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c0974c08-5306-1803-9d9e-20773b3aa73d.dc1 127.0.0.1 +34:38.821 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c0974c08-5306-1803-9d9e-20773b3aa73d 127.0.0.1 +2021-08-13T15:34:38.821Z [INFO] agent.router: Initializing LAN area manager +34:38.821 [INFO] TestAgent.server: Adding LAN server: server="Node-c0974c08-5306-1803-9d9e-20773b3aa73d (Addr: tcp/127.0.0.1:25073) (DC: dc1)" +34:38.821 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c0974c08-5306-1803-9d9e-20773b3aa73d.dc1 area=wan +34:38.821 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24587 network=udp +34:38.822 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24587 network=tcp +34:38.822 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25083 network=tcp +34:38.823 [INFO] TestAgent: started state syncer +34:38.823 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25074 network=tcp +34:38.887 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:38.887 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25073 [Candidate]" term=2 +34:38.887 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:38.887 [DEBUG] TestAgent.server.raft: vote granted: from=c0974c08-5306-1803-9d9e-20773b3aa73d term=2 tally=1 +34:38.887 [INFO] TestAgent.server.raft: election won: tally=1 +34:38.887 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25073 [Leader]" +34:38.888 [INFO] TestAgent.server: cluster leadership acquired +34:38.888 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25073 +34:38.889 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:38.891 [INFO] TestAgent.server: New leader elected: payload=Node-c0974c08-5306-1803-9d9e-20773b3aa73d +34:38.892 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:38.892 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:38.892 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:38.892 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:38.892 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:38.892 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c0974c08-5306-1803-9d9e-20773b3aa73d +34:38.893 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c0974c08-5306-1803-9d9e-20773b3aa73d +34:38.894 [INFO] TestAgent.server: federation state anti-entropy synced +34:39.039 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:39.040 [INFO] TestAgent: Synced node info +34:39.040 [DEBUG] TestAgent: Node info in sync +34:39.161 [INFO] TestAgent: Requesting shutdown +34:39.161 [INFO] TestAgent.server: shutting down server +34:39.162 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:39.162 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:39.162 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:39.162 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:39.162 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:39.162 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:39.162 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:39.162 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:39.162 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:39.162 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:39.162 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:39.162Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:39.162Z [INFO] agent.router.manager: shutting down +34:39.162 [INFO] TestAgent: consul server down +34:39.162 [INFO] TestAgent: shutdown complete +34:39.162 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24587 network=tcp +34:39.162 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24587 network=udp +34:39.162 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25083 network=tcp +34:39.662 [INFO] TestAgent: Waiting for endpoints to shut down +34:39.662 [INFO] TestAgent: Endpoints down +--- PASS: TestHealthConnectServiceNodes_Filter (0.88s) +=== CONT TestEventList_ACLFilter +[WARN] freeport: 10 out of 45 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:39.258Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:39.273Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:39.276 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e80a0405-d5ab-6a08-364d-b231155f851b Address:127.0.0.1:24662}]" +34:39.276 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24662 [Follower]" leader= +34:39.277 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e80a0405-d5ab-6a08-364d-b231155f851b.dc1 127.0.0.1 +34:39.277 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e80a0405-d5ab-6a08-364d-b231155f851b 127.0.0.1 +2021-08-13T15:34:39.277Z [INFO] agent.router: Initializing LAN area manager +34:39.278 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24297 network=udp +34:39.278 [INFO] TestAgent.server: Adding LAN server: server="Node-e80a0405-d5ab-6a08-364d-b231155f851b (Addr: tcp/127.0.0.1:24662) (DC: dc1)" +34:39.278 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e80a0405-d5ab-6a08-364d-b231155f851b.dc1 area=wan +34:39.278 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24297 network=tcp +34:39.279 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24658 network=tcp +34:39.279 [INFO] TestAgent: started state syncer +34:39.279 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24663 network=tcp +34:39.314 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:39.314 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24662 [Candidate]" term=2 +34:39.314 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:39.314 [DEBUG] TestAgent.server.raft: vote granted: from=e80a0405-d5ab-6a08-364d-b231155f851b term=2 tally=1 +34:39.314 [INFO] TestAgent.server.raft: election won: tally=1 +34:39.314 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24662 [Leader]" +34:39.314 [INFO] TestAgent.server: cluster leadership acquired +34:39.314 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24662 +34:39.315 [INFO] TestAgent.server: New leader elected: payload=Node-e80a0405-d5ab-6a08-364d-b231155f851b +34:39.316 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:39.318 [INFO] TestAgent: Requesting shutdown +34:39.318 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:39.318 [INFO] TestAgent.server: shutting down server +34:39.318 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:39.318 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:39.318 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:39.318 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:39.318 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:39.318 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:39.318 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:39.318 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:39.318 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:39.318 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:39.318 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e80a0405-d5ab-6a08-364d-b231155f851b +34:39.318 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e80a0405-d5ab-6a08-364d-b231155f851b +34:39.318 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:39.318 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:39.318Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:39.318Z [INFO] agent.router.manager: shutting down +34:39.318 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:39.318 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:39.318 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:39.319 [INFO] TestAgent: consul server down +34:39.319 [INFO] TestAgent: shutdown complete +34:39.319 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24297 network=tcp +34:39.319 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24297 network=udp +34:39.319 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24658 network=tcp +34:39.819 [INFO] TestAgent: Waiting for endpoints to shut down +34:39.819 [INFO] TestAgent: Endpoints down +--- PASS: TestHealthChecksInState_NodeMetaFilter (0.58s) +=== CONT TestHealthServiceNodes_DistanceSort +=== RUN TestHealthServiceNodes_PassingFilter/bc_no_query_value +=== RUN TestHealthServiceNodes_PassingFilter/passing_true +=== RUN TestHealthServiceNodes_PassingFilter/passing_false +=== RUN TestHealthServiceNodes_PassingFilter/passing_bad +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestEventList_ACLFilter/no_token +=== RUN TestEventList_ACLFilter/root_token +2021-08-13T15:34:39.350Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:39.404Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:39.407 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:82e168e7-ba16-d85d-977f-96d43203ad3f Address:127.0.0.1:24606}]" +34:39.407 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24606 [Follower]" leader= +34:39.408 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-82e168e7-ba16-d85d-977f-96d43203ad3f.dc1 127.0.0.1 +34:39.408 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-82e168e7-ba16-d85d-977f-96d43203ad3f 127.0.0.1 +2021-08-13T15:34:39.408Z [INFO] agent.router: Initializing LAN area manager +34:39.409 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-82e168e7-ba16-d85d-977f-96d43203ad3f.dc1 area=wan +34:39.409 [INFO] TestAgent.server: Adding LAN server: server="Node-82e168e7-ba16-d85d-977f-96d43203ad3f (Addr: tcp/127.0.0.1:24606) (DC: dc1)" +34:39.409 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24692 network=udp +34:39.409 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24692 network=tcp +34:39.410 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24602 network=tcp +34:39.410 [INFO] TestAgent: started state syncer +34:39.410 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24607 network=tcp +34:39.457 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:39.457 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24606 [Candidate]" term=2 +34:39.458 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:39.458 [DEBUG] TestAgent.server.raft: vote granted: from=82e168e7-ba16-d85d-977f-96d43203ad3f term=2 tally=1 +34:39.458 [INFO] TestAgent.server.raft: election won: tally=1 +34:39.458 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24606 [Leader]" +34:39.458 [INFO] TestAgent.server: cluster leadership acquired +34:39.458 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24606 +34:39.459 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:39.462 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:39.462 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:39.462 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:39.462 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:39.462 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:39.462 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-82e168e7-ba16-d85d-977f-96d43203ad3f +34:39.462 [INFO] TestAgent.server: member joined, marking health alive: member=Node-82e168e7-ba16-d85d-977f-96d43203ad3f +34:39.463 [INFO] TestAgent.server: New leader elected: payload=Node-82e168e7-ba16-d85d-977f-96d43203ad3f +34:39.464 [INFO] TestAgent.server: federation state anti-entropy synced +34:39.586 [INFO] TestAgent: Requesting shutdown +34:39.586 [INFO] TestAgent.server: shutting down server +34:39.586 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:39.586 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:39.586 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:39.586 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:39.587 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:39.587 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:39.587 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:39.587 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:39.587 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:39.587 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:39.587 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:39.587 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:39.587Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:39.587Z [INFO] agent.router.manager: shutting down +34:39.587 [INFO] TestAgent: consul server down +34:39.587 [INFO] TestAgent: shutdown complete +34:39.587 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24692 network=tcp +34:39.587 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24692 network=udp +34:39.587 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24602 network=tcp +34:40.088 [INFO] TestAgent: Waiting for endpoints to shut down +34:40.088 [INFO] TestAgent: Endpoints down +--- PASS: TestHealthConnectServiceNodes (0.77s) +=== CONT TestEventList_Filter +2021-08-13T15:34:39.520Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:39.536Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:39.538 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:70991225-5b3c-9903-146a-97adb75f892a Address:127.0.0.1:24683}]" +34:39.539 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24683 [Follower]" leader= +34:39.540 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-70991225-5b3c-9903-146a-97adb75f892a.dc1 127.0.0.1 +34:39.540 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-70991225-5b3c-9903-146a-97adb75f892a 127.0.0.1 +2021-08-13T15:34:39.540Z [INFO] agent.router: Initializing LAN area manager +34:39.541 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24699 network=udp +34:39.541 [INFO] TestAgent.server: Adding LAN server: server="Node-70991225-5b3c-9903-146a-97adb75f892a (Addr: tcp/127.0.0.1:24683) (DC: dc1)" +34:39.541 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-70991225-5b3c-9903-146a-97adb75f892a.dc1 area=wan +34:39.541 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24699 network=tcp +34:39.542 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24700 network=tcp +34:39.542 [INFO] TestAgent: started state syncer +34:39.542 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24684 network=tcp +34:39.600 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:39.600 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24683 [Candidate]" term=2 +34:39.601 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:39.601 [DEBUG] TestAgent.server.raft: vote granted: from=70991225-5b3c-9903-146a-97adb75f892a term=2 tally=1 +34:39.601 [INFO] TestAgent.server.raft: election won: tally=1 +34:39.601 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24683 [Leader]" +34:39.601 [INFO] TestAgent.server: cluster leadership acquired +34:39.601 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24683 +34:39.602 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:39.603 [INFO] TestAgent.server: New leader elected: payload=Node-70991225-5b3c-9903-146a-97adb75f892a +34:39.605 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:39.605 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:39.605 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:39.605 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:39.605 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:39.605 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-70991225-5b3c-9903-146a-97adb75f892a +34:39.605 [INFO] TestAgent.server: member joined, marking health alive: member=Node-70991225-5b3c-9903-146a-97adb75f892a +34:39.606 [INFO] TestAgent.server: federation state anti-entropy synced +34:39.716 [DEBUG] TestAgent.server: User event: event=foo +34:39.716 [DEBUG] TestAgent.server: User event: event=bar +34:39.716 [DEBUG] TestAgent.server: User event: event=foo +34:39.716 [DEBUG] TestAgent.server: User event: event=foo +34:39.716 [DEBUG] TestAgent.server: User event: event=bar +34:39.716 [DEBUG] TestAgent: new event: event_name=foo event_id=1136e596-a5cd-c4e8-e376-d67f1ebe0894 +34:39.716 [DEBUG] TestAgent: new event: event_name=bar event_id=3d7d8852-f37f-e2c1-cb1e-020e64885c5b +34:39.716 [DEBUG] TestAgent: new event: event_name=foo event_id=a6e6d48e-1778-5b0f-49dc-f6c2a31dd354 +34:39.716 [DEBUG] TestAgent: new event: event_name=foo event_id=f801f6e3-eb3a-16c2-16ac-042e1dd95254 +34:39.716 [DEBUG] TestAgent: new event: event_name=bar event_id=864796d8-6290-f432-65e1-0497c7f05cc2 +34:39.742 [INFO] TestAgent: Requesting shutdown +34:39.742 [INFO] TestAgent.server: shutting down server +34:39.742 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:39.742 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:39.742 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:39.743 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:39.743 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:39.743 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:39.743 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:39.743 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:39.743 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:39.743 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:39.743 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:39.748 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:39.748Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:39.748Z [INFO] agent.router.manager: shutting down +34:39.749 [INFO] TestAgent: consul server down +34:39.749 [INFO] TestAgent: shutdown complete +34:39.749 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24699 network=tcp +34:39.749 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24699 network=udp +34:39.749 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24700 network=tcp +34:40.249 [INFO] TestAgent: Waiting for endpoints to shut down +34:40.249 [INFO] TestAgent: Endpoints down +--- PASS: TestEventList_EventBufOrder (0.75s) +=== CONT TestHealthServiceNodes_Filter +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:39.557Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:39.576Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:39.579 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9ae4ef76-555e-f937-b013-fb07c008d80d Address:127.0.0.1:24690}]" +34:39.579 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24690 [Follower]" leader= +34:39.579 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9ae4ef76-555e-f937-b013-fb07c008d80d.dc1 127.0.0.1 +34:39.580 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9ae4ef76-555e-f937-b013-fb07c008d80d 127.0.0.1 +2021-08-13T15:34:39.580Z [INFO] agent.router: Initializing LAN area manager +34:39.581 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24678 network=udp +34:39.581 [INFO] TestAgent.server: Adding LAN server: server="Node-9ae4ef76-555e-f937-b013-fb07c008d80d (Addr: tcp/127.0.0.1:24690) (DC: dc1)" +34:39.581 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9ae4ef76-555e-f937-b013-fb07c008d80d.dc1 area=wan +34:39.581 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24678 network=tcp +34:39.582 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24665 network=tcp +34:39.582 [INFO] TestAgent: started state syncer +34:39.582 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24691 network=tcp +34:39.616 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:39.616 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24690 [Candidate]" term=2 +34:39.616 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:39.616 [DEBUG] TestAgent.server.raft: vote granted: from=9ae4ef76-555e-f937-b013-fb07c008d80d term=2 tally=1 +34:39.616 [INFO] TestAgent.server.raft: election won: tally=1 +34:39.616 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24690 [Leader]" +34:39.616 [INFO] TestAgent.server: cluster leadership acquired +34:39.617 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24690 +34:39.617 [INFO] TestAgent.server: New leader elected: payload=Node-9ae4ef76-555e-f937-b013-fb07c008d80d +34:39.618 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:39.620 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:39.620 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:39.620 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:39.620 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:39.620 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:39.620 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9ae4ef76-555e-f937-b013-fb07c008d80d +34:39.620 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9ae4ef76-555e-f937-b013-fb07c008d80d +34:39.621 [INFO] TestAgent: Synced node info +34:39.621 [INFO] TestAgent.server: federation state anti-entropy synced +34:39.780 [INFO] TestAgent: Requesting shutdown +34:39.780 [INFO] TestAgent.server: shutting down server +34:39.780 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:39.780 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:39.780 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:39.780 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:39.780 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:39.780 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:39.780 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:39.780 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:39.780 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:39.780 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:39.780 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:39.780Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:39.780Z [INFO] agent.router.manager: shutting down +34:39.780 [INFO] TestAgent: consul server down +34:39.780 [INFO] TestAgent: shutdown complete +34:39.780 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24678 network=tcp +34:39.780 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24678 network=udp +34:39.780 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24665 network=tcp +34:40.280 [INFO] TestAgent: Waiting for endpoints to shut down +34:40.280 [INFO] TestAgent: Endpoints down +--- PASS: TestHealthServiceNodes_CheckType (0.76s) +=== CONT TestHealthServiceNodes_NodeMetaFilter +2021-08-13T15:34:39.589Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:39.605Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:39.607 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3b733849-6bd6-37ad-c58a-c11aa06845ca Address:127.0.0.1:24732}]" +34:39.607 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24732 [Follower]" leader= +34:39.608 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3b733849-6bd6-37ad-c58a-c11aa06845ca.dc1 127.0.0.1 +34:39.609 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3b733849-6bd6-37ad-c58a-c11aa06845ca 127.0.0.1 +2021-08-13T15:34:39.609Z [INFO] agent.router: Initializing LAN area manager +34:39.609 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24727 network=udp +34:39.610 [INFO] TestAgent.server: Adding LAN server: server="Node-3b733849-6bd6-37ad-c58a-c11aa06845ca (Addr: tcp/127.0.0.1:24732) (DC: dc1)" +34:39.610 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3b733849-6bd6-37ad-c58a-c11aa06845ca.dc1 area=wan +34:39.610 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24727 network=tcp +34:39.611 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24728 network=tcp +34:39.611 [INFO] TestAgent: started state syncer +34:39.611 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24733 network=tcp +34:39.652 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:39.652 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24732 [Candidate]" term=2 +34:39.653 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:39.653 [DEBUG] TestAgent.server.raft: vote granted: from=3b733849-6bd6-37ad-c58a-c11aa06845ca term=2 tally=1 +34:39.653 [INFO] TestAgent.server.raft: election won: tally=1 +34:39.653 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24732 [Leader]" +34:39.653 [INFO] TestAgent.server: cluster leadership acquired +34:39.653 [INFO] TestAgent.server: New leader elected: payload=Node-3b733849-6bd6-37ad-c58a-c11aa06845ca +34:39.653 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24732 +34:39.654 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:39.656 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:39.656 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:39.656 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:39.656 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:39.656 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:39.656 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3b733849-6bd6-37ad-c58a-c11aa06845ca +34:39.656 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3b733849-6bd6-37ad-c58a-c11aa06845ca +34:39.657 [INFO] TestAgent.server: federation state anti-entropy synced +34:39.748 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:39.749 [INFO] TestAgent: Synced node info +34:39.877 [INFO] TestAgent: Requesting shutdown +34:39.877 [INFO] TestAgent.server: shutting down server +34:39.877 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:39.877 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:39.877 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:39.877 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:39.877 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:39.879 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:39.879 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:39.879 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:39.879 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +34:39.879 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:39.879 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +2021-08-13T15:34:39.879Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:39.879Z [INFO] agent.router.manager: shutting down +34:39.879 [INFO] TestAgent: consul server down +34:39.879 [INFO] TestAgent: shutdown complete +34:39.879 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24727 network=tcp +34:39.879 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24727 network=udp +34:39.879 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24728 network=tcp +34:40.380 [INFO] TestAgent: Waiting for endpoints to shut down +34:40.380 [INFO] TestAgent: Endpoints down +--- PASS: TestHealthServiceNodes_PassingFilter (0.82s) + --- PASS: TestHealthServiceNodes_PassingFilter/bc_no_query_value (0.00s) + --- PASS: TestHealthServiceNodes_PassingFilter/passing_true (0.00s) + --- PASS: TestHealthServiceNodes_PassingFilter/passing_false (0.00s) + --- PASS: TestHealthServiceNodes_PassingFilter/passing_bad (0.00s) +=== CONT TestEventList +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:39.559Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:39.578Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:39.584 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:dba681c0-e6c2-cfcc-de77-ae0e122ca867 Address:127.0.0.1:24697}]" +34:39.585 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24697 [Follower]" leader= +34:39.585 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-dba681c0-e6c2-cfcc-de77-ae0e122ca867.dc1 127.0.0.1 +34:39.586 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-dba681c0-e6c2-cfcc-de77-ae0e122ca867 127.0.0.1 +2021-08-13T15:34:39.586Z [INFO] agent.router: Initializing LAN area manager +34:39.586 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24657 network=udp +34:39.587 [INFO] TestAgent.server: Adding LAN server: server="Node-dba681c0-e6c2-cfcc-de77-ae0e122ca867 (Addr: tcp/127.0.0.1:24697) (DC: dc1)" +34:39.588 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24657 network=tcp +34:39.588 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24679 network=tcp +34:39.589 [INFO] TestAgent: started state syncer +34:39.589 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24698 network=tcp +34:39.595 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-dba681c0-e6c2-cfcc-de77-ae0e122ca867.dc1 area=wan +34:39.646 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:39.647 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24697 [Candidate]" term=2 +34:39.647 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:39.647 [DEBUG] TestAgent.server.raft: vote granted: from=dba681c0-e6c2-cfcc-de77-ae0e122ca867 term=2 tally=1 +34:39.647 [INFO] TestAgent.server.raft: election won: tally=1 +34:39.647 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24697 [Leader]" +34:39.647 [INFO] TestAgent.server: cluster leadership acquired +34:39.647 [INFO] TestAgent.server: New leader elected: payload=Node-dba681c0-e6c2-cfcc-de77-ae0e122ca867 +34:39.647 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24697 +34:39.648 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:39.650 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:39.651 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:39.651 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:39.651 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:39.651 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:39.651 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-dba681c0-e6c2-cfcc-de77-ae0e122ca867 +34:39.651 [INFO] TestAgent.server: member joined, marking health alive: member=Node-dba681c0-e6c2-cfcc-de77-ae0e122ca867 +34:39.652 [INFO] TestAgent.server: federation state anti-entropy synced +34:39.975 [DEBUG] TestAgent.server: User event: event=test +34:39.975 [DEBUG] TestAgent: new event: event_name=test event_id=7e83dc60-0577-dff9-1f70-1b6d892fb981 +34:40.002 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:40.002 [INFO] TestAgent: Synced node info +34:40.026 [DEBUG] TestAgent.server: User event: event=second +34:40.026 [DEBUG] TestAgent: new event: event_name=second event_id=839c673e-75d5-f16e-9c7c-d0af262b5d93 +34:40.026 [INFO] TestAgent: Requesting shutdown +34:40.027 [INFO] TestAgent.server: shutting down server +34:40.027 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:40.027 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:40.027 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:40.027 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:40.027 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:40.027 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:40.027 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:40.027 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:40.027 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:40.027 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:40.027 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:40.027Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:40.027Z [INFO] agent.router.manager: shutting down +34:40.028 [INFO] TestAgent: consul server down +34:40.028 [INFO] TestAgent: shutdown complete +34:40.028 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24657 network=tcp +34:40.028 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24657 network=udp +34:40.028 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24679 network=tcp +34:40.528 [INFO] TestAgent: Waiting for endpoints to shut down +34:40.528 [INFO] TestAgent: Endpoints down +--- PASS: TestEventList_Blocking (1.00s) +=== CONT TestHealthServiceNodes +2021-08-13T15:34:39.679Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:39.679Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:39.695Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:39.695Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:39.698 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e797029e-f75a-99ac-2a72-c7675270d327 Address:127.0.0.1:24711}]" +34:39.698 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24711 [Follower]" leader= +34:39.699 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e797029e-f75a-99ac-2a72-c7675270d327.dc1 127.0.0.1 +34:39.699 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e797029e-f75a-99ac-2a72-c7675270d327 127.0.0.1 +2021-08-13T15:34:39.699Z [INFO] agent.router: Initializing LAN area manager +34:39.699 [INFO] TestAgent.server: Adding LAN server: server="Node-e797029e-f75a-99ac-2a72-c7675270d327 (Addr: tcp/127.0.0.1:24711) (DC: dc1)" +34:39.700 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e797029e-f75a-99ac-2a72-c7675270d327.dc1 area=wan +34:39.700 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24671 network=udp +34:39.700 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24671 network=tcp +34:39.703 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24672 network=tcp +34:39.704 [INFO] TestAgent: started state syncer +34:39.705 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24712 network=tcp +34:39.750 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:39.767 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:39.767 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24711 [Candidate]" term=2 +34:39.768 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:39.768 [DEBUG] TestAgent.server.raft: vote granted: from=e797029e-f75a-99ac-2a72-c7675270d327 term=2 tally=1 +34:39.768 [INFO] TestAgent.server.raft: election won: tally=1 +34:39.768 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24711 [Leader]" +34:39.768 [INFO] TestAgent.server: cluster leadership acquired +34:39.768 [INFO] TestAgent.server: initializing acls +34:39.768 [INFO] TestAgent.server: New leader elected: payload=Node-e797029e-f75a-99ac-2a72-c7675270d327 +34:39.768 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:39.768 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:39.769 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:39.769 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:39.769 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:39.769 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:39.770 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-e797029e-f75a-99ac-2a72-c7675270d327 +34:39.770 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-e797029e-f75a-99ac-2a72-c7675270d327.dc1 +34:39.771 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:39.771 [INFO] TestAgent.server: Updating LAN server: server="Node-e797029e-f75a-99ac-2a72-c7675270d327 (Addr: tcp/127.0.0.1:24711) (DC: dc1)" +34:39.771 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-e797029e-f75a-99ac-2a72-c7675270d327.dc1 area=wan +34:39.773 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:39.773 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:39.773 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:39.773 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:39.773 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:39.773 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e797029e-f75a-99ac-2a72-c7675270d327 +34:39.773 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e797029e-f75a-99ac-2a72-c7675270d327 +34:39.774 [INFO] TestAgent.server: federation state anti-entropy synced +34:39.774 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e797029e-f75a-99ac-2a72-c7675270d327 +34:40.027 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:40.028 [INFO] TestAgent: Synced node info +34:40.070 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-e797029e-f75a-99ac-2a72-c7675270d327 +34:40.070 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-e797029e-f75a-99ac-2a72-c7675270d327 +34:40.071 [DEBUG] TestAgent.server: User event: event=foo +34:40.071 [DEBUG] TestAgent: new event: event_name=foo event_id=db7b38c1-1915-d80b-0c1b-d28e151a27ed +34:40.071 [DEBUG] TestAgent: dropping event from result due to ACLs: event=foo +34:40.071 [INFO] TestAgent: Requesting shutdown +34:40.071 [INFO] TestAgent.server: shutting down server +34:40.071 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:40.071 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:40.071 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:40.071 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:40.071 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:40.071 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:40.071 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:40.071 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:40.071 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:40.071 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:40.071 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:40.071 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:40.071 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:40.071 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:40.072 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:40.072Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:40.072Z [INFO] agent.router.manager: shutting down +34:40.072 [INFO] TestAgent: consul server down +34:40.072 [INFO] TestAgent: shutdown complete +34:40.072 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24671 network=tcp +34:40.072 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24671 network=udp +34:40.072 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24672 network=tcp +34:40.572 [INFO] TestAgent: Waiting for endpoints to shut down +34:40.572 [INFO] TestAgent: Endpoints down +--- PASS: TestEventList_ACLFilter (0.91s) + --- PASS: TestEventList_ACLFilter/no_token (0.00s) + --- PASS: TestEventList_ACLFilter/root_token (0.00s) +=== CONT TestEventFire_token +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:39.836Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:39.851Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:39.853 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ffe0cc2a-7cf3-64a3-6d79-de46f8b07d31 Address:127.0.0.1:24739}]" +34:39.853 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24739 [Follower]" leader= +34:39.854 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ffe0cc2a-7cf3-64a3-6d79-de46f8b07d31.dc1 127.0.0.1 +34:39.855 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ffe0cc2a-7cf3-64a3-6d79-de46f8b07d31 127.0.0.1 +2021-08-13T15:34:39.855Z [INFO] agent.router: Initializing LAN area manager +34:39.855 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24734 network=udp +34:39.856 [INFO] TestAgent.server: Adding LAN server: server="Node-ffe0cc2a-7cf3-64a3-6d79-de46f8b07d31 (Addr: tcp/127.0.0.1:24739) (DC: dc1)" +34:39.856 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ffe0cc2a-7cf3-64a3-6d79-de46f8b07d31.dc1 area=wan +34:39.856 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24734 network=tcp +34:39.857 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24735 network=tcp +34:39.857 [INFO] TestAgent: started state syncer +34:39.857 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24740 network=tcp +34:39.897 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:39.897 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24739 [Candidate]" term=2 +34:39.897 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:39.897 [DEBUG] TestAgent.server.raft: vote granted: from=ffe0cc2a-7cf3-64a3-6d79-de46f8b07d31 term=2 tally=1 +34:39.897 [INFO] TestAgent.server.raft: election won: tally=1 +34:39.897 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24739 [Leader]" +34:39.897 [INFO] TestAgent.server: cluster leadership acquired +34:39.897 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24739 +34:39.898 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:39.901 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:39.901 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:39.901 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:39.901 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:39.901 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:39.901 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ffe0cc2a-7cf3-64a3-6d79-de46f8b07d31 +34:39.901 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ffe0cc2a-7cf3-64a3-6d79-de46f8b07d31 +34:39.902 [INFO] TestAgent.server: New leader elected: payload=Node-ffe0cc2a-7cf3-64a3-6d79-de46f8b07d31 +34:39.902 [INFO] TestAgent.server: federation state anti-entropy synced +34:39.966 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:39.966 [INFO] TestAgent: Synced node info +34:39.966 [DEBUG] TestAgent: Node info in sync +34:40.275 [INFO] TestAgent: Requesting shutdown +34:40.275 [INFO] TestAgent.server: shutting down server +34:40.275 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:40.275 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:40.275 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:40.275 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:40.275 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:40.276 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:40.276 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:40.276 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:40.276 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:40.276 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:40.276 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:40.276Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:40.276Z [INFO] agent.router.manager: shutting down +34:40.276 [INFO] TestAgent: consul server down +34:40.276 [INFO] TestAgent: shutdown complete +34:40.276 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24734 network=tcp +34:40.276 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24734 network=udp +34:40.276 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24735 network=tcp +34:40.776 [INFO] TestAgent: Waiting for endpoints to shut down +34:40.776 [INFO] TestAgent: Endpoints down +--- PASS: TestHealthServiceNodes_DistanceSort (0.96s) +=== CONT TestHealthServiceChecks_DistanceSort +2021-08-13T15:34:40.104Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:40.119Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:40.121 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d6ff0e01-1255-6fa4-f84a-207dbe765c52 Address:127.0.0.1:24718}]" +34:40.122 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24718 [Follower]" leader= +34:40.122 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d6ff0e01-1255-6fa4-f84a-207dbe765c52.dc1 127.0.0.1 +34:40.123 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d6ff0e01-1255-6fa4-f84a-207dbe765c52 127.0.0.1 +2021-08-13T15:34:40.123Z [INFO] agent.router: Initializing LAN area manager +34:40.123 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24706 network=udp +34:40.124 [INFO] TestAgent.server: Adding LAN server: server="Node-d6ff0e01-1255-6fa4-f84a-207dbe765c52 (Addr: tcp/127.0.0.1:24718) (DC: dc1)" +34:40.124 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d6ff0e01-1255-6fa4-f84a-207dbe765c52.dc1 area=wan +34:40.124 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24706 network=tcp +34:40.125 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24707 network=tcp +34:40.125 [INFO] TestAgent: started state syncer +34:40.125 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24719 network=tcp +34:40.177 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:40.177 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24718 [Candidate]" term=2 +34:40.177 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:40.177 [DEBUG] TestAgent.server.raft: vote granted: from=d6ff0e01-1255-6fa4-f84a-207dbe765c52 term=2 tally=1 +34:40.177 [INFO] TestAgent.server.raft: election won: tally=1 +34:40.177 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24718 [Leader]" +34:40.177 [INFO] TestAgent.server: cluster leadership acquired +34:40.177 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24718 +34:40.178 [INFO] TestAgent.server: New leader elected: payload=Node-d6ff0e01-1255-6fa4-f84a-207dbe765c52 +34:40.179 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:40.181 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:40.181 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:40.181 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:40.181 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:40.181 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:40.181 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d6ff0e01-1255-6fa4-f84a-207dbe765c52 +34:40.181 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d6ff0e01-1255-6fa4-f84a-207dbe765c52 +34:40.182 [INFO] TestAgent.server: federation state anti-entropy synced +34:40.249 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:40.250 [INFO] TestAgent: Synced node info +34:40.268 [DEBUG] TestAgent.server: User event: event=test +34:40.268 [DEBUG] TestAgent.server: User event: event=foo +34:40.268 [DEBUG] TestAgent: new event: event_name=test event_id=1be01ac6-cb7d-3bf0-cc8e-12681013792d +34:40.268 [DEBUG] TestAgent: new event: event_name=foo event_id=4b902495-ffdf-16bf-6eb0-b731919bc2fc +34:40.294 [INFO] TestAgent: Requesting shutdown +34:40.294 [INFO] TestAgent.server: shutting down server +34:40.294 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:40.294 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:40.294 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:40.294 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:40.294 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:40.294 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:40.294 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:40.294 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:40.295 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:40.295 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:40.295 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:40.295Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:40.295Z [INFO] agent.router.manager: shutting down +34:40.295 [INFO] TestAgent: consul server down +34:40.295 [INFO] TestAgent: shutdown complete +34:40.295 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24706 network=tcp +34:40.295 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24706 network=udp +34:40.295 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24707 network=tcp +34:40.796 [INFO] TestAgent: Waiting for endpoints to shut down +34:40.796 [INFO] TestAgent: Endpoints down +--- PASS: TestEventList_Filter (0.71s) +=== CONT TestDNS_Compression_trimUDPResponse +--- PASS: TestDNS_Compression_trimUDPResponse (0.01s) +=== CONT TestHealthServiceChecks_Filtering +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:40.296Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:40.311Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:40.314 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:956d7505-9f4f-fa92-1a40-19ecbe2993a0 Address:127.0.0.1:24746}]" +34:40.315 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-956d7505-9f4f-fa92-1a40-19ecbe2993a0.dc1 127.0.0.1 +34:40.315 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24746 [Follower]" leader= +34:40.318 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-956d7505-9f4f-fa92-1a40-19ecbe2993a0 127.0.0.1 +2021-08-13T15:34:40.318Z [INFO] agent.router: Initializing LAN area manager +34:40.319 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24741 network=udp +34:40.319 [INFO] TestAgent.server: Adding LAN server: server="Node-956d7505-9f4f-fa92-1a40-19ecbe2993a0 (Addr: tcp/127.0.0.1:24746) (DC: dc1)" +34:40.319 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-956d7505-9f4f-fa92-1a40-19ecbe2993a0.dc1 area=wan +34:40.319 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24741 network=tcp +34:40.320 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24742 network=tcp +34:40.320 [INFO] TestAgent: started state syncer +34:40.320 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24747 network=tcp +34:40.379 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:40.379 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24746 [Candidate]" term=2 +34:40.380 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:40.380 [DEBUG] TestAgent.server.raft: vote granted: from=956d7505-9f4f-fa92-1a40-19ecbe2993a0 term=2 tally=1 +34:40.380 [INFO] TestAgent.server.raft: election won: tally=1 +34:40.380 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24746 [Leader]" +34:40.380 [INFO] TestAgent.server: cluster leadership acquired +34:40.380 [INFO] TestAgent.server: New leader elected: payload=Node-956d7505-9f4f-fa92-1a40-19ecbe2993a0 +34:40.380 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24746 +34:40.381 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:40.384 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:40.384 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:40.384 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:40.384 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:40.384 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:40.384 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-956d7505-9f4f-fa92-1a40-19ecbe2993a0 +34:40.384 [INFO] TestAgent.server: member joined, marking health alive: member=Node-956d7505-9f4f-fa92-1a40-19ecbe2993a0 +34:40.385 [INFO] TestAgent.server: federation state anti-entropy synced +34:40.562 [INFO] TestAgent: Requesting shutdown +34:40.562 [INFO] TestAgent.server: shutting down server +34:40.562 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:40.562 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:40.562 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:40.562 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:40.562 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:40.562 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:40.563 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:40.563 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:40.563 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +34:40.563 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:40.563 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:40.563 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2021-08-13T15:34:40.563Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:40.563Z [INFO] agent.router.manager: shutting down +34:40.563 [INFO] TestAgent: consul server down +34:40.563 [INFO] TestAgent: shutdown complete +34:40.563 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24741 network=tcp +34:40.563 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24741 network=udp +34:40.563 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24742 network=tcp +34:41.063 [INFO] TestAgent: Waiting for endpoints to shut down +34:41.063 [INFO] TestAgent: Endpoints down +--- PASS: TestHealthServiceNodes_NodeMetaFilter (0.78s) +=== CONT TestEventFire +2021-08-13T15:34:40.396Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:40.412Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:40.425 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ede49df3-79d0-c71b-98f4-705aa5c54443 Address:127.0.0.1:24704}]" +34:40.427 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ede49df3-79d0-c71b-98f4-705aa5c54443.dc1 127.0.0.1 +34:40.427 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24704 [Follower]" leader= +34:40.429 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ede49df3-79d0-c71b-98f4-705aa5c54443 127.0.0.1 +2021-08-13T15:34:40.429Z [INFO] agent.router: Initializing LAN area manager +34:40.431 [INFO] TestAgent.server: Adding LAN server: server="Node-ede49df3-79d0-c71b-98f4-705aa5c54443 (Addr: tcp/127.0.0.1:24704) (DC: dc1)" +34:40.432 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ede49df3-79d0-c71b-98f4-705aa5c54443.dc1 area=wan +34:40.439 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24685 network=udp +34:40.439 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24685 network=tcp +34:40.441 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24686 network=tcp +34:40.441 [INFO] TestAgent: started state syncer +34:40.441 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24705 network=tcp +34:40.484 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:40.484 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24704 [Candidate]" term=2 +34:40.484 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:40.484 [DEBUG] TestAgent.server.raft: vote granted: from=ede49df3-79d0-c71b-98f4-705aa5c54443 term=2 tally=1 +34:40.484 [INFO] TestAgent.server.raft: election won: tally=1 +34:40.484 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24704 [Leader]" +34:40.484 [INFO] TestAgent.server: cluster leadership acquired +34:40.485 [INFO] TestAgent.server: New leader elected: payload=Node-ede49df3-79d0-c71b-98f4-705aa5c54443 +34:40.485 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24704 +34:40.486 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:40.489 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:40.489 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:40.489 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:40.489 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:40.489 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:40.489 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ede49df3-79d0-c71b-98f4-705aa5c54443 +34:40.489 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ede49df3-79d0-c71b-98f4-705aa5c54443 +34:40.490 [INFO] TestAgent.server: federation state anti-entropy synced +34:40.571 [DEBUG] TestAgent.server: User event: event=test +34:40.571 [DEBUG] TestAgent: new event: event_name=test event_id=88f7a71f-fe1f-511e-200b-85d35b6ecb3d +34:40.597 [INFO] TestAgent: Requesting shutdown +34:40.597 [INFO] TestAgent.server: shutting down server +34:40.597 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:40.597 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:40.597 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:40.597 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:40.597 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:40.597 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:40.597 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:40.597 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:40.597 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:40.597 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:40.597 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:40.597 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:40.598Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:40.598Z [INFO] agent.router.manager: shutting down +34:40.598 [INFO] TestAgent: consul server down +34:40.598 [INFO] TestAgent: shutdown complete +34:40.598 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24685 network=tcp +34:40.598 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24685 network=udp +34:40.598 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24686 network=tcp +34:41.098 [INFO] TestAgent: Waiting for endpoints to shut down +34:41.098 [INFO] TestAgent: Endpoints down +--- PASS: TestEventList (0.72s) +=== CONT TestHealthServiceChecks_NodeMetaFilter +2021-08-13T15:34:40.265Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:40.281Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:40.284 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f26aa774-b762-1e74-6550-9be24978b035 Address:127.0.0.1:24725}]" +34:40.285 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f26aa774-b762-1e74-6550-9be24978b035.dc1 127.0.0.1 +34:40.286 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f26aa774-b762-1e74-6550-9be24978b035 127.0.0.1 +2021-08-13T15:34:40.286Z [INFO] agent.router: Initializing LAN area manager +34:40.286 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24720 network=udp +34:40.286 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24725 [Follower]" leader= +34:40.287 [INFO] TestAgent.server: Adding LAN server: server="Node-f26aa774-b762-1e74-6550-9be24978b035 (Addr: tcp/127.0.0.1:24725) (DC: dc1)" +34:40.287 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f26aa774-b762-1e74-6550-9be24978b035.dc1 area=wan +34:40.287 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24720 network=tcp +34:40.288 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24721 network=tcp +34:40.288 [INFO] TestAgent: started state syncer +34:40.288 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24726 network=tcp +34:40.326 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:40.326 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24725 [Candidate]" term=2 +34:40.326 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:40.326 [DEBUG] TestAgent.server.raft: vote granted: from=f26aa774-b762-1e74-6550-9be24978b035 term=2 tally=1 +34:40.326 [INFO] TestAgent.server.raft: election won: tally=1 +34:40.326 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24725 [Leader]" +34:40.326 [INFO] TestAgent.server: cluster leadership acquired +34:40.327 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24725 +34:40.328 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:40.330 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:40.330 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:40.330 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:40.330 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:40.330 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:40.330 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f26aa774-b762-1e74-6550-9be24978b035 +34:40.330 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f26aa774-b762-1e74-6550-9be24978b035 +34:40.331 [INFO] TestAgent.server: New leader elected: payload=Node-f26aa774-b762-1e74-6550-9be24978b035 +34:40.332 [INFO] TestAgent.server: federation state anti-entropy synced +34:40.683 [INFO] TestAgent: Requesting shutdown +34:40.683 [INFO] TestAgent.server: shutting down server +34:40.683 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:40.683 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:40.683 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:40.683 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:40.683 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:40.683 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:40.683 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:40.683 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:40.683 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:40.683 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:40.683 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:40.683 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:40.683Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:40.683Z [INFO] agent.router.manager: shutting down +34:40.684 [INFO] TestAgent: consul server down +34:40.684 [INFO] TestAgent: shutdown complete +34:40.684 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24720 network=tcp +34:40.684 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24720 network=udp +34:40.684 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24721 network=tcp +34:41.187 [INFO] TestAgent: Waiting for endpoints to shut down +34:41.187 [INFO] TestAgent: Endpoints down +--- PASS: TestHealthServiceNodes_Filter (0.94s) +=== CONT TestDNS_ReloadConfig_DuringQuery +2021-08-13T15:34:40.590Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:40.590Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:40.606Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:40.606Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:40.609 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4655f1af-c39d-ae09-0a0c-696b2dd36c1e Address:127.0.0.1:24753}]" +34:40.609 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24753 [Follower]" leader= +34:40.610 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4655f1af-c39d-ae09-0a0c-696b2dd36c1e.dc1 127.0.0.1 +34:40.610 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4655f1af-c39d-ae09-0a0c-696b2dd36c1e 127.0.0.1 +2021-08-13T15:34:40.611Z [INFO] agent.router: Initializing LAN area manager +34:40.611 [INFO] TestAgent.server: Adding LAN server: server="Node-4655f1af-c39d-ae09-0a0c-696b2dd36c1e (Addr: tcp/127.0.0.1:24753) (DC: dc1)" +34:40.611 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4655f1af-c39d-ae09-0a0c-696b2dd36c1e.dc1 area=wan +34:40.611 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24776 network=udp +34:40.611 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24776 network=tcp +34:40.612 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24749 network=tcp +34:40.612 [INFO] TestAgent: started state syncer +34:40.612 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24754 network=tcp +34:40.660 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:40.660 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24753 [Candidate]" term=2 +34:40.660 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:40.660 [DEBUG] TestAgent.server.raft: vote granted: from=4655f1af-c39d-ae09-0a0c-696b2dd36c1e term=2 tally=1 +34:40.660 [INFO] TestAgent.server.raft: election won: tally=1 +34:40.660 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24753 [Leader]" +34:40.660 [INFO] TestAgent.server: cluster leadership acquired +34:40.661 [INFO] TestAgent.server: New leader elected: payload=Node-4655f1af-c39d-ae09-0a0c-696b2dd36c1e +34:40.661 [INFO] TestAgent.server: initializing acls +34:40.661 [INFO] TestAgent.server: initializing acls +34:40.661 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:40.661 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:40.661 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:40.661 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:40.662 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:40.662 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:40.662 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:40.662 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:40.662 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:40.662 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode +34:40.662 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:40.663 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-4655f1af-c39d-ae09-0a0c-696b2dd36c1e +34:40.663 [INFO] TestAgent.server: Updating LAN server: server="Node-4655f1af-c39d-ae09-0a0c-696b2dd36c1e (Addr: tcp/127.0.0.1:24753) (DC: dc1)" +34:40.663 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-4655f1af-c39d-ae09-0a0c-696b2dd36c1e.dc1 +34:40.663 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-4655f1af-c39d-ae09-0a0c-696b2dd36c1e.dc1 area=wan +34:40.663 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-4655f1af-c39d-ae09-0a0c-696b2dd36c1e +34:40.663 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-4655f1af-c39d-ae09-0a0c-696b2dd36c1e.dc1 +34:40.663 [INFO] TestAgent.server: Updating LAN server: server="Node-4655f1af-c39d-ae09-0a0c-696b2dd36c1e (Addr: tcp/127.0.0.1:24753) (DC: dc1)" +34:40.663 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-4655f1af-c39d-ae09-0a0c-696b2dd36c1e.dc1 area=wan +34:40.664 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:40.666 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:40.666 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:40.666 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:40.666 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:40.666 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:40.666 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4655f1af-c39d-ae09-0a0c-696b2dd36c1e +34:40.666 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4655f1af-c39d-ae09-0a0c-696b2dd36c1e +34:40.668 [INFO] TestAgent.server: federation state anti-entropy synced +34:40.668 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4655f1af-c39d-ae09-0a0c-696b2dd36c1e +34:40.668 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4655f1af-c39d-ae09-0a0c-696b2dd36c1e +34:40.686 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-4655f1af-c39d-ae09-0a0c-696b2dd36c1e +34:40.686 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-4655f1af-c39d-ae09-0a0c-696b2dd36c1e +34:40.688 [WARN] TestAgent.server.internal: user event blocked by ACLs: event=foo accessorID= +34:40.688 [WARN] TestAgent.server.internal: user event blocked by ACLs: event=bar accessorID= +34:40.688 [INFO] TestAgent: Requesting shutdown +34:40.688 [DEBUG] TestAgent.server: User event: event=baz +34:40.688 [INFO] TestAgent.server: shutting down server +34:40.688 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:40.688 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:40.688 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:40.688 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:40.688 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:40.688 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:40.688 [DEBUG] TestAgent: new event: event_name=baz event_id=b0357241-5293-7f60-a27a-c35592e5d763 +34:40.688 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:40.688 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:40.688 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:40.689 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:40.688 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:40.689 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:40.688 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:40.689 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:40.689 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:40.689 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:40.689Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:40.689Z [INFO] agent.router.manager: shutting down +34:40.689 [INFO] TestAgent: consul server down +34:40.689 [INFO] TestAgent: shutdown complete +34:40.689 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24776 network=tcp +34:40.689 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24776 network=udp +34:40.689 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24749 network=tcp +34:41.191 [INFO] TestAgent: Waiting for endpoints to shut down +34:41.191 [INFO] TestAgent: Endpoints down +--- PASS: TestEventFire_token (0.62s) +=== CONT TestHealthServiceChecks +2021-08-13T15:34:39.878Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:39.878Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:39.895Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:39.895Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:39.898 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5a0d3675-37a1-1201-eedc-26c5eab9e4f6 Address:127.0.0.1:24676}]" +34:39.898 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5a0d3675-37a1-1201-eedc-26c5eab9e4f6.dc2 127.0.0.1 +34:39.899 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5a0d3675-37a1-1201-eedc-26c5eab9e4f6 127.0.0.1 +2021-08-13T15:34:39.899Z [INFO] agent.router: Initializing LAN area manager +34:39.900 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24676 [Follower]" leader= +34:39.900 [INFO] TestAgent.server: Adding LAN server: server="Node-5a0d3675-37a1-1201-eedc-26c5eab9e4f6 (Addr: tcp/127.0.0.1:24676) (DC: dc2)" +34:39.900 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5a0d3675-37a1-1201-eedc-26c5eab9e4f6.dc2 area=wan +34:39.900 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24713 network=tcp +34:39.900 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24713 network=udp +34:39.901 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24714 network=tcp +34:39.901 [INFO] TestAgent: started state syncer +34:39.902 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24677 network=tcp +34:39.950 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:39.956 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:39.956 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24676 [Candidate]" term=2 +34:39.956 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:39.956 [DEBUG] TestAgent.server.raft: vote granted: from=5a0d3675-37a1-1201-eedc-26c5eab9e4f6 term=2 tally=1 +34:39.956 [INFO] TestAgent.server.raft: election won: tally=1 +34:39.956 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24676 [Leader]" +34:39.956 [INFO] TestAgent.server: cluster leadership acquired +34:39.956 [INFO] TestAgent.server: initializing acls +34:39.957 [INFO] TestAgent.server: New leader elected: payload=Node-5a0d3675-37a1-1201-eedc-26c5eab9e4f6 +34:39.957 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:39.957 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:39.957 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:39.957 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:39.958 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-5a0d3675-37a1-1201-eedc-26c5eab9e4f6 +34:39.958 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-5a0d3675-37a1-1201-eedc-26c5eab9e4f6.dc2 +34:39.958 [INFO] TestAgent.server: Updating LAN server: server="Node-5a0d3675-37a1-1201-eedc-26c5eab9e4f6 (Addr: tcp/127.0.0.1:24676) (DC: dc2)" +34:39.958 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-5a0d3675-37a1-1201-eedc-26c5eab9e4f6.dc2 area=wan +34:39.959 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:39.961 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:39.961 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:39.961 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:39.961 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:39.961 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:39.961 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5a0d3675-37a1-1201-eedc-26c5eab9e4f6 +34:39.961 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5a0d3675-37a1-1201-eedc-26c5eab9e4f6 +34:39.962 [INFO] TestAgent.server: federation state anti-entropy synced +34:39.962 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5a0d3675-37a1-1201-eedc-26c5eab9e4f6 +34:40.023 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:40.024 [INFO] TestAgent: Synced node info +34:40.255 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:24668] +34:40.256 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:24668 +34:40.257 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b82722fc-9087-40d8-f631-fdd7ddd3f218.dc1 127.0.0.1 +34:40.257 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 +34:40.257 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b82722fc-9087-40d8-f631-fdd7ddd3f218.dc1 area=wan +34:40.259 [INFO] TestAgent: Requesting shutdown +34:40.259 [INFO] TestAgent.server: shutting down server +34:40.259 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:40.259 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:40.259 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:40.259 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:40.260 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:40.260 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:40.260 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:40.260 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:40.260 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:40.260 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:40.260 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:40.260 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:40.260 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:40.260 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:40.260 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:40.260Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:40.260Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:40.260Z [INFO] agent.router.manager: shutting down +34:40.260 [INFO] TestAgent: consul server down +34:40.260 [INFO] TestAgent: shutdown complete +34:40.260 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24713 network=tcp +34:40.260 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24713 network=udp +34:40.260 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24714 network=tcp +34:40.760 [INFO] TestAgent: Waiting for endpoints to shut down +34:40.760 [INFO] TestAgent: Endpoints down +2021-08-13T15:34:39.447Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:39.447Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:39.463Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:39.463Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:39.466 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b82722fc-9087-40d8-f631-fdd7ddd3f218 Address:127.0.0.1:24669}]" +34:39.467 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b82722fc-9087-40d8-f631-fdd7ddd3f218.dc1 127.0.0.1 +34:39.467 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24669 [Follower]" leader= +34:39.467 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b82722fc-9087-40d8-f631-fdd7ddd3f218 127.0.0.1 +2021-08-13T15:34:39.468Z [INFO] agent.router: Initializing LAN area manager +34:39.468 [INFO] TestAgent.server: Adding LAN server: server="Node-b82722fc-9087-40d8-f631-fdd7ddd3f218 (Addr: tcp/127.0.0.1:24669) (DC: dc1)" +34:39.468 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b82722fc-9087-40d8-f631-fdd7ddd3f218.dc1 area=wan +34:39.468 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24608 network=udp +34:39.468 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24608 network=tcp +34:39.469 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24693 network=tcp +34:39.469 [INFO] TestAgent: started state syncer +34:39.469 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24670 network=tcp +34:39.518 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:39.534 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:39.534 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24669 [Candidate]" term=2 +34:39.534 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:39.534 [DEBUG] TestAgent.server.raft: vote granted: from=b82722fc-9087-40d8-f631-fdd7ddd3f218 term=2 tally=1 +34:39.534 [INFO] TestAgent.server.raft: election won: tally=1 +34:39.534 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24669 [Leader]" +34:39.535 [INFO] TestAgent.server: cluster leadership acquired +34:39.535 [INFO] TestAgent.server: initializing acls +34:39.536 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:39.536 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:39.536 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:39.536 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:39.536 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-b82722fc-9087-40d8-f631-fdd7ddd3f218 +34:39.536 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-b82722fc-9087-40d8-f631-fdd7ddd3f218.dc1 +34:39.537 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:39.539 [INFO] TestAgent.server: New leader elected: payload=Node-b82722fc-9087-40d8-f631-fdd7ddd3f218 +34:39.539 [INFO] TestAgent.server: Updating LAN server: server="Node-b82722fc-9087-40d8-f631-fdd7ddd3f218 (Addr: tcp/127.0.0.1:24669) (DC: dc1)" +34:39.539 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-b82722fc-9087-40d8-f631-fdd7ddd3f218.dc1 area=wan +34:39.540 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:39.540 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:39.540 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:39.540 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:39.540 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:39.540 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b82722fc-9087-40d8-f631-fdd7ddd3f218 +34:39.540 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b82722fc-9087-40d8-f631-fdd7ddd3f218 +34:39.541 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b82722fc-9087-40d8-f631-fdd7ddd3f218 +34:39.542 [INFO] TestAgent.server: federation state anti-entropy synced +34:39.847 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:39.848 [INFO] TestAgent: Synced node info +34:40.256 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:44264 +34:40.257 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5a0d3675-37a1-1201-eedc-26c5eab9e4f6.dc2 127.0.0.1 +34:40.257 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5a0d3675-37a1-1201-eedc-26c5eab9e4f6.dc2 area=wan +34:40.760 [INFO] TestAgent: Requesting shutdown +34:40.761 [INFO] TestAgent.server: shutting down server +34:40.761 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:40.761 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:40.761 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:40.761 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:40.761 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:40.761 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:40.761 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:40.761 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:40.761 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:40.761 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:40.761 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:40.761 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:40.761 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:40.761 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:40.761 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:40.761Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:40.761Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:40.761Z [INFO] agent.router.manager: shutting down +34:40.761 [INFO] TestAgent: consul server down +34:40.761 [INFO] TestAgent: shutdown complete +34:40.761 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24608 network=tcp +34:40.761 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24608 network=udp +34:40.761 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24693 network=tcp +34:41.262 [INFO] TestAgent: Waiting for endpoints to shut down +34:41.262 [INFO] TestAgent: Endpoints down +--- PASS: TestHealthServiceNodes_WanTranslation (1.83s) +=== CONT TestDNS_ConfigReload +[WARN] freeport: 10 out of 52 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:40.545Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:40.562Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:40.564 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:215b6d01-6f9f-e364-3ebf-6d6041638392 Address:127.0.0.1:24760}]" +34:40.565 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24760 [Follower]" leader= +34:40.565 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-215b6d01-6f9f-e364-3ebf-6d6041638392.dc1 127.0.0.1 +34:40.566 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-215b6d01-6f9f-e364-3ebf-6d6041638392 127.0.0.1 +2021-08-13T15:34:40.566Z [INFO] agent.router: Initializing LAN area manager +34:40.566 [INFO] TestAgent.server: Adding LAN server: server="Node-215b6d01-6f9f-e364-3ebf-6d6041638392 (Addr: tcp/127.0.0.1:24760) (DC: dc1)" +34:40.566 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-215b6d01-6f9f-e364-3ebf-6d6041638392.dc1 area=wan +34:40.566 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24748 network=tcp +34:40.566 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24748 network=udp +34:40.567 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24770 network=tcp +34:40.567 [INFO] TestAgent: started state syncer +34:40.567 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24761 network=tcp +34:40.619 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:40.620 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24760 [Candidate]" term=2 +34:40.620 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:40.620 [DEBUG] TestAgent.server.raft: vote granted: from=215b6d01-6f9f-e364-3ebf-6d6041638392 term=2 tally=1 +34:40.620 [INFO] TestAgent.server.raft: election won: tally=1 +34:40.620 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24760 [Leader]" +34:40.620 [INFO] TestAgent.server: cluster leadership acquired +34:40.620 [INFO] TestAgent.server: New leader elected: payload=Node-215b6d01-6f9f-e364-3ebf-6d6041638392 +34:40.620 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24760 +34:40.621 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:40.624 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:40.624 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:40.624 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:40.624 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:40.624 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:40.624 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-215b6d01-6f9f-e364-3ebf-6d6041638392 +34:40.624 [INFO] TestAgent.server: member joined, marking health alive: member=Node-215b6d01-6f9f-e364-3ebf-6d6041638392 +34:40.625 [INFO] TestAgent.server: federation state anti-entropy synced +34:40.833 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:40.833 [INFO] TestAgent: Synced node info +34:40.887 [INFO] TestAgent: Requesting shutdown +34:40.887 [INFO] TestAgent.server: shutting down server +34:40.887 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:40.887 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:40.887 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:40.887 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:40.887 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:40.887 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:40.887 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:40.887 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:40.887 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:40.887 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:40.887 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:40.887Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:40.887Z [INFO] agent.router.manager: shutting down +34:40.888 [INFO] TestAgent: consul server down +34:40.888 [INFO] TestAgent: shutdown complete +34:40.888 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24748 network=tcp +34:40.888 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24748 network=udp +34:40.888 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24770 network=tcp +34:41.391 [INFO] TestAgent: Waiting for endpoints to shut down +34:41.391 [INFO] TestAgent: Endpoints down +--- PASS: TestHealthServiceNodes (0.86s) +=== CONT TestDNS_syncExtra +--- PASS: TestDNS_syncExtra (0.00s) +=== CONT TestDNS_Compression_Recurse +2021-08-13T15:34:40.823Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:40.838Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:40.841 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:29661b8a-83cc-5db9-3213-86124d32e1db Address:127.0.0.1:24815}]" +34:40.841 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24815 [Follower]" leader= +34:40.841 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-29661b8a-83cc-5db9-3213-86124d32e1db.dc1 127.0.0.1 +34:40.842 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-29661b8a-83cc-5db9-3213-86124d32e1db 127.0.0.1 +2021-08-13T15:34:40.842Z [INFO] agent.router: Initializing LAN area manager +34:40.842 [INFO] TestAgent.server: Adding LAN server: server="Node-29661b8a-83cc-5db9-3213-86124d32e1db (Addr: tcp/127.0.0.1:24815) (DC: dc1)" +34:40.843 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-29661b8a-83cc-5db9-3213-86124d32e1db.dc1 area=wan +34:40.843 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24783 network=udp +34:40.843 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24783 network=tcp +34:40.844 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24804 network=tcp +34:40.844 [INFO] TestAgent: started state syncer +34:40.844 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24816 network=tcp +34:40.887 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:40.887 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24815 [Candidate]" term=2 +34:40.887 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:40.887 [DEBUG] TestAgent.server.raft: vote granted: from=29661b8a-83cc-5db9-3213-86124d32e1db term=2 tally=1 +34:40.887 [INFO] TestAgent.server.raft: election won: tally=1 +34:40.887 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24815 [Leader]" +34:40.887 [INFO] TestAgent.server: cluster leadership acquired +34:40.887 [INFO] TestAgent.server: New leader elected: payload=Node-29661b8a-83cc-5db9-3213-86124d32e1db +34:40.887 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24815 +34:40.889 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:40.891 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:40.891 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:40.891 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:40.891 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:40.891 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:40.891 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-29661b8a-83cc-5db9-3213-86124d32e1db +34:40.891 [INFO] TestAgent.server: member joined, marking health alive: member=Node-29661b8a-83cc-5db9-3213-86124d32e1db +34:40.892 [INFO] TestAgent.server: federation state anti-entropy synced +34:40.944 [INFO] TestAgent: Requesting shutdown +34:40.944 [INFO] TestAgent.server: shutting down server +34:40.944 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:40.944 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:40.944 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:40.944 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:40.944 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:40.944 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:40.944 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:40.944 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:40.944 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:40.944 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:40.944 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:40.944 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:40.945Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:40.945Z [INFO] agent.router.manager: shutting down +34:40.945 [INFO] TestAgent: consul server down +34:40.945 [INFO] TestAgent: shutdown complete +34:40.945 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24783 network=tcp +34:40.945 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24783 network=udp +34:40.945 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24804 network=tcp +34:41.445 [INFO] TestAgent: Waiting for endpoints to shut down +34:41.445 [INFO] TestAgent: Endpoints down +--- PASS: TestHealthServiceChecks_Filtering (0.64s) +=== CONT TestCatalogServiceNodes_ConnectProxy +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:41.080Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:41.095Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:41.098 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:db668e69-0163-46ff-26f3-b4a30f834126 Address:127.0.0.1:24801}]" +34:41.098 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24801 [Follower]" leader= +34:41.099 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-db668e69-0163-46ff-26f3-b4a30f834126.dc1 127.0.0.1 +34:41.100 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-db668e69-0163-46ff-26f3-b4a30f834126 127.0.0.1 +2021-08-13T15:34:41.100Z [INFO] agent.router: Initializing LAN area manager +34:41.100 [INFO] TestAgent.server: Adding LAN server: server="Node-db668e69-0163-46ff-26f3-b4a30f834126 (Addr: tcp/127.0.0.1:24801) (DC: dc1)" +34:41.100 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-db668e69-0163-46ff-26f3-b4a30f834126.dc1 area=wan +34:41.100 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24769 network=udp +34:41.100 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24769 network=tcp +34:41.101 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24797 network=tcp +34:41.101 [INFO] TestAgent: started state syncer +34:41.101 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24802 network=tcp +34:41.165 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:41.165 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24801 [Candidate]" term=2 +34:41.165 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:41.165 [DEBUG] TestAgent.server.raft: vote granted: from=db668e69-0163-46ff-26f3-b4a30f834126 term=2 tally=1 +34:41.165 [INFO] TestAgent.server.raft: election won: tally=1 +34:41.165 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24801 [Leader]" +34:41.165 [INFO] TestAgent.server: cluster leadership acquired +34:41.166 [INFO] TestAgent.server: New leader elected: payload=Node-db668e69-0163-46ff-26f3-b4a30f834126 +34:41.166 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24801 +34:41.167 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:41.170 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:41.170 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:41.170 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:41.170 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:41.170 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:41.170 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-db668e69-0163-46ff-26f3-b4a30f834126 +34:41.170 [INFO] TestAgent.server: member joined, marking health alive: member=Node-db668e69-0163-46ff-26f3-b4a30f834126 +34:41.173 [INFO] TestAgent.server: federation state anti-entropy synced +34:41.215 [INFO] TestAgent: Requesting shutdown +34:41.216 [INFO] TestAgent.server: shutting down server +34:41.216 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:41.216 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:41.216 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:41.216 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:41.216 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:41.216 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:41.216 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:41.216 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:41.216 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:41.216 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +34:41.216 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:41.216 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2021-08-13T15:34:41.216Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:41.216Z [INFO] agent.router.manager: shutting down +34:41.216 [INFO] TestAgent: consul server down +34:41.216 [INFO] TestAgent: shutdown complete +34:41.216 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24769 network=tcp +34:41.216 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24769 network=udp +34:41.216 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24797 network=tcp +34:41.716 [INFO] TestAgent: Waiting for endpoints to shut down +34:41.716 [INFO] TestAgent: Endpoints down +--- PASS: TestEventFire (0.65s) +=== CONT TestDNS_Compression_ReverseLookup +2021-08-13T15:34:40.794Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:40.811Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:40.814 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fee85493-953c-7217-b8fc-7a134d6e405a Address:127.0.0.1:24794}]" +34:40.814 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24794 [Follower]" leader= +34:40.815 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fee85493-953c-7217-b8fc-7a134d6e405a.dc1 127.0.0.1 +34:40.816 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fee85493-953c-7217-b8fc-7a134d6e405a 127.0.0.1 +2021-08-13T15:34:40.816Z [INFO] agent.router: Initializing LAN area manager +34:40.816 [INFO] TestAgent.server: Adding LAN server: server="Node-fee85493-953c-7217-b8fc-7a134d6e405a (Addr: tcp/127.0.0.1:24794) (DC: dc1)" +34:40.816 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fee85493-953c-7217-b8fc-7a134d6e405a.dc1 area=wan +34:40.816 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24803 network=udp +34:40.816 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24803 network=tcp +34:40.817 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24777 network=tcp +34:40.817 [INFO] TestAgent: started state syncer +34:40.817 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24795 network=tcp +34:40.855 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:40.855 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24794 [Candidate]" term=2 +34:40.856 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:40.856 [DEBUG] TestAgent.server.raft: vote granted: from=fee85493-953c-7217-b8fc-7a134d6e405a term=2 tally=1 +34:40.856 [INFO] TestAgent.server.raft: election won: tally=1 +34:40.856 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24794 [Leader]" +34:40.856 [INFO] TestAgent.server: cluster leadership acquired +34:40.856 [INFO] TestAgent.server: New leader elected: payload=Node-fee85493-953c-7217-b8fc-7a134d6e405a +34:40.856 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24794 +34:40.857 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:40.860 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:40.860 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:40.860 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:40.860 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:40.860 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:40.860 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fee85493-953c-7217-b8fc-7a134d6e405a +34:40.860 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fee85493-953c-7217-b8fc-7a134d6e405a +34:40.860 [INFO] TestAgent.server: federation state anti-entropy synced +34:41.094 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:41.094 [INFO] TestAgent: Synced node info +34:41.228 [INFO] TestAgent: Requesting shutdown +34:41.229 [INFO] TestAgent.server: shutting down server +34:41.229 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:41.229 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:41.229 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:41.229 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:41.229 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:41.229 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:41.229 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:41.229 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:41.229 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:41.229 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:41.229 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:41.229Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:41.229Z [INFO] agent.router.manager: shutting down +34:41.229 [INFO] TestAgent: consul server down +34:41.229 [INFO] TestAgent: shutdown complete +34:41.229 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24803 network=tcp +34:41.229 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24803 network=udp +34:41.229 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24777 network=tcp +34:41.730 [INFO] TestAgent: Waiting for endpoints to shut down +34:41.730 [INFO] TestAgent: Endpoints down +--- PASS: TestHealthServiceChecks_DistanceSort (0.95s) +=== CONT TestHealthNodeChecks_Filtering +2021-08-13T15:34:41.115Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:41.131Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:41.134 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:745dd897-4ea3-4447-0b47-3e6758d69766 Address:127.0.0.1:24781}]" +34:41.134 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24781 [Follower]" leader= +34:41.135 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-745dd897-4ea3-4447-0b47-3e6758d69766.dc1 127.0.0.1 +34:41.135 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-745dd897-4ea3-4447-0b47-3e6758d69766 127.0.0.1 +2021-08-13T15:34:41.135Z [INFO] agent.router: Initializing LAN area manager +34:41.136 [INFO] TestAgent.server: Adding LAN server: server="Node-745dd897-4ea3-4447-0b47-3e6758d69766 (Addr: tcp/127.0.0.1:24781) (DC: dc1)" +34:41.136 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-745dd897-4ea3-4447-0b47-3e6758d69766.dc1 area=wan +34:41.136 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24762 network=udp +34:41.136 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24762 network=tcp +34:41.137 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24763 network=tcp +34:41.137 [INFO] TestAgent: started state syncer +34:41.137 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24782 network=tcp +34:41.193 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:41.193 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24781 [Candidate]" term=2 +34:41.193 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:41.193 [DEBUG] TestAgent.server.raft: vote granted: from=745dd897-4ea3-4447-0b47-3e6758d69766 term=2 tally=1 +34:41.193 [INFO] TestAgent.server.raft: election won: tally=1 +34:41.193 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24781 [Leader]" +34:41.193 [INFO] TestAgent.server: cluster leadership acquired +34:41.194 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24781 +34:41.194 [INFO] TestAgent.server: New leader elected: payload=Node-745dd897-4ea3-4447-0b47-3e6758d69766 +34:41.195 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:41.199 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:41.199 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:41.199 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:41.199 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:41.199 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:41.199 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-745dd897-4ea3-4447-0b47-3e6758d69766 +34:41.199 [INFO] TestAgent.server: member joined, marking health alive: member=Node-745dd897-4ea3-4447-0b47-3e6758d69766 +34:41.209 [INFO] TestAgent.server: federation state anti-entropy synced +34:41.251 [INFO] TestAgent: Requesting shutdown +34:41.251 [INFO] TestAgent.server: shutting down server +34:41.251 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:41.251 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:41.251 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:41.251 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:41.251 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:41.251 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:41.251 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:41.251 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:41.251 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:41.251 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +34:41.251 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:41.251 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2021-08-13T15:34:41.251Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:41.251Z [INFO] agent.router.manager: shutting down +34:41.252 [INFO] TestAgent: consul server down +34:41.252 [INFO] TestAgent: shutdown complete +34:41.252 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24762 network=tcp +34:41.252 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24762 network=udp +34:41.252 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24763 network=tcp +34:41.753 [INFO] TestAgent: Waiting for endpoints to shut down +34:41.753 [INFO] TestAgent: Endpoints down +--- PASS: TestHealthServiceChecks_NodeMetaFilter (0.66s) +=== CONT TestDNS_Compression_Query +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:41.221Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:41.238Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:41.241 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bf79c3eb-e91d-917c-9153-203ba1ee4455 Address:127.0.0.1:24767}]" +34:41.241 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24767 [Follower]" leader= +34:41.242 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bf79c3eb-e91d-917c-9153-203ba1ee4455.dc1 127.0.0.1 +34:41.243 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bf79c3eb-e91d-917c-9153-203ba1ee4455 127.0.0.1 +2021-08-13T15:34:41.243Z [INFO] agent.router: Initializing LAN area manager +34:41.243 [INFO] TestAgent.server: Adding LAN server: server="Node-bf79c3eb-e91d-917c-9153-203ba1ee4455 (Addr: tcp/127.0.0.1:24767) (DC: dc1)" +34:41.243 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bf79c3eb-e91d-917c-9153-203ba1ee4455.dc1 area=wan +34:41.243 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24796 network=udp +34:41.243 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24796 network=tcp +34:41.244 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24756 network=tcp +34:41.244 [INFO] TestAgent: started state syncer +34:41.244 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24768 network=tcp +34:41.309 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:41.309 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24767 [Candidate]" term=2 +34:41.309 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:41.309 [DEBUG] TestAgent.server.raft: vote granted: from=bf79c3eb-e91d-917c-9153-203ba1ee4455 term=2 tally=1 +34:41.309 [INFO] TestAgent.server.raft: election won: tally=1 +34:41.309 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24767 [Leader]" +34:41.309 [INFO] TestAgent.server: cluster leadership acquired +34:41.310 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24767 +34:41.310 [INFO] TestAgent.server: New leader elected: payload=Node-bf79c3eb-e91d-917c-9153-203ba1ee4455 +34:41.311 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:41.313 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:41.313 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:41.313 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:41.313 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:41.313 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:41.313 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bf79c3eb-e91d-917c-9153-203ba1ee4455 +34:41.313 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bf79c3eb-e91d-917c-9153-203ba1ee4455 +34:41.314 [INFO] TestAgent.server: federation state anti-entropy synced +34:41.419 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:41.420 [INFO] TestAgent: Synced node info +34:41.436 [INFO] TestAgent: Requesting shutdown +34:41.436 [INFO] TestAgent.server: shutting down server +34:41.436 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:41.436 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:41.436 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:41.436 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:41.436 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:41.436 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:41.436 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:41.436 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:41.436 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:41.436 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:41.436 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:41.436Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:41.436Z [INFO] agent.router.manager: shutting down +34:41.436 [INFO] TestAgent: consul server down +34:41.436 [INFO] TestAgent: shutdown complete +34:41.436 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24796 network=tcp +34:41.437 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24796 network=udp +34:41.437 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24756 network=tcp +34:41.937 [INFO] TestAgent: Waiting for endpoints to shut down +34:41.937 [INFO] TestAgent: Endpoints down +--- PASS: TestHealthServiceChecks (0.75s) +=== CONT TestHealthNodeChecks +2021-08-13T15:34:41.218Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:41.250Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:41.256 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:aecbf125-a533-e8fe-dfe9-4f1d41987bda Address:127.0.0.1:24774}]" +34:41.256 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24774 [Follower]" leader= +34:41.257 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-aecbf125-a533-e8fe-dfe9-4f1d41987bda.dc1 127.0.0.1 +34:41.257 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-aecbf125-a533-e8fe-dfe9-4f1d41987bda 127.0.0.1 +2021-08-13T15:34:41.257Z [INFO] agent.router: Initializing LAN area manager +34:41.258 [INFO] TestAgent.server: Adding LAN server: server="Node-aecbf125-a533-e8fe-dfe9-4f1d41987bda (Addr: tcp/127.0.0.1:24774) (DC: dc1)" +34:41.258 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-aecbf125-a533-e8fe-dfe9-4f1d41987bda.dc1 area=wan +34:41.258 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24755 network=udp +34:41.258 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24755 network=tcp +34:41.259 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24784 network=tcp +34:41.259 [INFO] TestAgent: started state syncer +34:41.259 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24775 network=tcp +34:41.305 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:41.305 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24774 [Candidate]" term=2 +34:41.305 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:41.305 [DEBUG] TestAgent.server.raft: vote granted: from=aecbf125-a533-e8fe-dfe9-4f1d41987bda term=2 tally=1 +34:41.305 [INFO] TestAgent.server.raft: election won: tally=1 +34:41.305 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24774 [Leader]" +34:41.306 [INFO] TestAgent.server: cluster leadership acquired +34:41.306 [INFO] TestAgent.server: New leader elected: payload=Node-aecbf125-a533-e8fe-dfe9-4f1d41987bda +34:41.306 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24774 +34:41.307 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:41.309 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:41.309 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:41.309 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:41.309 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:41.309 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:41.309 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-aecbf125-a533-e8fe-dfe9-4f1d41987bda +34:41.309 [INFO] TestAgent.server: member joined, marking health alive: member=Node-aecbf125-a533-e8fe-dfe9-4f1d41987bda +34:41.310 [INFO] TestAgent.server: federation state anti-entropy synced +34:41.343 [WARN] TestAgent.server: endpoint injected; this should only be used for testing +34:41.351 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:41.352 [INFO] TestAgent: Synced node info +34:41.352 [DEBUG] TestAgent: Node info in sync +34:41.444 [INFO] TestAgent: Requesting shutdown +34:41.444 [INFO] TestAgent.server: shutting down server +34:41.445 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:41.445 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:41.445 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:41.445Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:41.445Z [INFO] agent.router.manager: shutting down +34:41.445 [INFO] TestAgent: consul server down +34:41.445 [INFO] TestAgent: shutdown complete +34:41.445 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24755 network=tcp +34:41.445 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24755 network=udp +34:41.445 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24784 network=tcp +34:41.947 [INFO] TestAgent: Waiting for endpoints to shut down +34:41.947 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ReloadConfig_DuringQuery (0.76s) +=== CONT TestDNS_trimUDPResponse_TrimSizeEDNS +--- PASS: TestDNS_trimUDPResponse_TrimSizeEDNS (0.02s) +=== CONT TestDNS_trimUDPResponse_TrimSize +--- PASS: TestDNS_trimUDPResponse_TrimSize (0.03s) +=== CONT TestDNS_trimUDPResponse_TrimLimit +--- PASS: TestDNS_trimUDPResponse_TrimLimit (0.02s) +=== CONT TestDNS_AddressLookupIPV6 +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:41.462Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:41.486Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:41.497 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:de6bbad4-642c-c152-c0cd-bd79bf312b40 Address:127.0.0.1:24843}]" +34:41.497 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24843 [Follower]" leader= +34:41.498 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-de6bbad4-642c-c152-c0cd-bd79bf312b40.dc1 127.0.0.1 +34:41.499 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-de6bbad4-642c-c152-c0cd-bd79bf312b40 127.0.0.1 +2021-08-13T15:34:41.499Z [INFO] agent.router: Initializing LAN area manager +34:41.499 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-de6bbad4-642c-c152-c0cd-bd79bf312b40.dc1 area=wan +34:41.499 [INFO] TestAgent.server: Adding LAN server: server="Node-de6bbad4-642c-c152-c0cd-bd79bf312b40 (Addr: tcp/127.0.0.1:24843) (DC: dc1)" +34:41.499 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24817 network=udp +34:41.499 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24817 network=tcp +34:41.500 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24839 network=tcp +34:41.500 [INFO] TestAgent: started state syncer +34:41.500 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24844 network=tcp +34:41.543 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:41.543 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24843 [Candidate]" term=2 +34:41.543 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:41.543 [DEBUG] TestAgent.server.raft: vote granted: from=de6bbad4-642c-c152-c0cd-bd79bf312b40 term=2 tally=1 +34:41.543 [INFO] TestAgent.server.raft: election won: tally=1 +34:41.543 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24843 [Leader]" +34:41.544 [INFO] TestAgent.server: cluster leadership acquired +34:41.544 [INFO] TestAgent.server: New leader elected: payload=Node-de6bbad4-642c-c152-c0cd-bd79bf312b40 +34:41.544 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24843 +34:41.545 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:41.548 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:41.548 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:41.548 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:41.548 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:41.548 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:41.548 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-de6bbad4-642c-c152-c0cd-bd79bf312b40 +34:41.548 [INFO] TestAgent.server: member joined, marking health alive: member=Node-de6bbad4-642c-c152-c0cd-bd79bf312b40 +34:41.549 [INFO] TestAgent.server: federation state anti-entropy synced +34:41.655 [INFO] TestAgent: Requesting shutdown +34:41.656 [INFO] TestAgent.server: shutting down server +34:41.656 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:41.656 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:41.656 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:41.656 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:41.656 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:41.656 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:41.656 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:41.656 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +34:41.656 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:41.656 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:41.656 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:41.656 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +2021-08-13T15:34:41.656Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:41.656Z [INFO] agent.router.manager: shutting down +34:41.656 [INFO] TestAgent: consul server down +34:41.656 [INFO] TestAgent: shutdown complete +34:41.656 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24817 network=tcp +34:41.656 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24817 network=udp +34:41.656 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24839 network=tcp +34:42.156 [INFO] TestAgent: Waiting for endpoints to shut down +34:42.157 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogServiceNodes_ConnectProxy (0.71s) +=== CONT TestDNS_trimUDPResponse_NoTrim +2021-08-13T15:34:41.282Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:41.300Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:41.303 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1b9b0e77-b4c0-bc00-1c64-a2bcd1153180 Address:127.0.0.1:24808}]" +34:41.303 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24808 [Follower]" leader= +34:41.304 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1b9b0e77-b4c0-bc00-1c64-a2bcd1153180.dc1 127.0.0.1 +34:41.305 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1b9b0e77-b4c0-bc00-1c64-a2bcd1153180 127.0.0.1 +2021-08-13T15:34:41.305Z [INFO] agent.router: Initializing LAN area manager +34:41.305 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1b9b0e77-b4c0-bc00-1c64-a2bcd1153180.dc1 area=wan +34:41.305 [INFO] TestAgent.server: Adding LAN server: server="Node-1b9b0e77-b4c0-bc00-1c64-a2bcd1153180 (Addr: tcp/127.0.0.1:24808) (DC: dc1)" +34:41.305 [DEBUG] TestAgent.dns: recursor enabled +34:41.305 [DEBUG] TestAgent.dns: recursor enabled +34:41.305 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24810 network=udp +34:41.306 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24810 network=tcp +34:41.306 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24818 network=tcp +34:41.307 [INFO] TestAgent: started state syncer +34:41.307 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24809 network=tcp +34:41.368 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:41.368 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24808 [Candidate]" term=2 +34:41.369 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:41.369 [DEBUG] TestAgent.server.raft: vote granted: from=1b9b0e77-b4c0-bc00-1c64-a2bcd1153180 term=2 tally=1 +34:41.369 [INFO] TestAgent.server.raft: election won: tally=1 +34:41.369 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24808 [Leader]" +34:41.369 [INFO] TestAgent.server: cluster leadership acquired +34:41.369 [INFO] TestAgent.server: New leader elected: payload=Node-1b9b0e77-b4c0-bc00-1c64-a2bcd1153180 +34:41.369 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24808 +34:41.371 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:41.373 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:41.373 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:41.373 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:41.373 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:41.373 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:41.373 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1b9b0e77-b4c0-bc00-1c64-a2bcd1153180 +34:41.373 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1b9b0e77-b4c0-bc00-1c64-a2bcd1153180 +34:41.380 [INFO] TestAgent.server: federation state anti-entropy synced +34:41.502 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:41.502 [INFO] TestAgent: Synced node info +34:41.659 [INFO] TestAgent: Requesting shutdown +34:41.659 [INFO] TestAgent.server: shutting down server +34:41.659 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:41.659 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:41.659 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:41.659Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:41.659Z [INFO] agent.router.manager: shutting down +34:41.659 [INFO] TestAgent: consul server down +34:41.659 [INFO] TestAgent: shutdown complete +34:41.659 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24810 network=tcp +34:41.659 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24810 network=udp +34:41.659 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24818 network=tcp +34:42.160 [INFO] TestAgent: Waiting for endpoints to shut down +34:42.160 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ConfigReload (0.90s) +=== CONT TestDNS_AddressLookup +--- PASS: TestDNS_trimUDPResponse_NoTrim (0.01s) +=== CONT TestDNS_PreparedQuery_AgentSource +2021-08-13T15:34:41.410Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:41.427Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:41.430 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2fa8ef3f-e230-f3f3-91c6-015fb668fa98 Address:127.0.0.1:24822}]" +34:41.431 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24822 [Follower]" leader= +34:41.431 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2fa8ef3f-e230-f3f3-91c6-015fb668fa98.dc1 127.0.0.1 +34:41.432 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2fa8ef3f-e230-f3f3-91c6-015fb668fa98 127.0.0.1 +2021-08-13T15:34:41.432Z [INFO] agent.router: Initializing LAN area manager +34:41.432 [DEBUG] TestAgent.dns: recursor enabled +34:41.433 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24838 network=udp +34:41.433 [INFO] TestAgent.server: Adding LAN server: server="Node-2fa8ef3f-e230-f3f3-91c6-015fb668fa98 (Addr: tcp/127.0.0.1:24822) (DC: dc1)" +34:41.433 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2fa8ef3f-e230-f3f3-91c6-015fb668fa98.dc1 area=wan +34:41.433 [DEBUG] TestAgent.dns: recursor enabled +34:41.433 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24838 network=tcp +34:41.434 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24811 network=tcp +34:41.434 [INFO] TestAgent: started state syncer +34:41.434 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24823 network=tcp +34:41.487 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:41.487 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24822 [Candidate]" term=2 +34:41.487 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:41.487 [DEBUG] TestAgent.server.raft: vote granted: from=2fa8ef3f-e230-f3f3-91c6-015fb668fa98 term=2 tally=1 +34:41.487 [INFO] TestAgent.server.raft: election won: tally=1 +34:41.487 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24822 [Leader]" +34:41.488 [INFO] TestAgent.server: cluster leadership acquired +34:41.488 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24822 +34:41.489 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:41.491 [INFO] TestAgent.server: New leader elected: payload=Node-2fa8ef3f-e230-f3f3-91c6-015fb668fa98 +34:41.492 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:41.492 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:41.492 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:41.492 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:41.492 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:41.492 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2fa8ef3f-e230-f3f3-91c6-015fb668fa98 +34:41.492 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2fa8ef3f-e230-f3f3-91c6-015fb668fa98 +34:41.493 [INFO] TestAgent.server: federation state anti-entropy synced +34:41.624 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:41.624 [INFO] TestAgent: Synced node info +34:41.677 [DEBUG] TestAgent.dns: recurse succeeded for question: question="{apple.com. 255 1}" rtt=196.06µs recursor=127.0.0.1:55466 +34:41.677 [DEBUG] TestAgent.dns: request served from client: question="{apple.com. 255 1}" network=udp latency=425.181µs client=127.0.0.1:45337 client_network=udp +34:41.677 [DEBUG] TestAgent.dns: recurse succeeded for question: question="{apple.com. 255 1}" rtt=142.02µs recursor=127.0.0.1:55466 +34:41.677 [DEBUG] TestAgent.dns: request served from client: question="{apple.com. 255 1}" network=udp latency=290.28µs client=127.0.0.1:45337 client_network=udp +34:41.677 [INFO] TestAgent: Requesting shutdown +34:41.677 [INFO] TestAgent.server: shutting down server +34:41.677 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:41.677 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:41.677 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:41.677 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:41.677 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:41.677 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:41.677 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:41.677 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:41.677 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:41.677 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:41.678 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:41.678Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:41.678Z [INFO] agent.router.manager: shutting down +34:41.678 [INFO] TestAgent: consul server down +34:41.678 [INFO] TestAgent: shutdown complete +34:41.678 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24838 network=tcp +34:41.678 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24838 network=udp +34:41.678 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24811 network=tcp +34:42.179 [INFO] TestAgent: Waiting for endpoints to shut down +34:42.179 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_Compression_Recurse (0.79s) +=== CONT TestDNS_ServiceLookup_FilterACL +=== RUN TestDNS_ServiceLookup_FilterACL/ACLToken_==_root +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:41.733Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:41.748Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:41.751 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2a11921c-f8c5-1bdf-77f7-d0b622afe76f Address:127.0.0.1:24829}]" +34:41.751 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24829 [Follower]" leader= +34:41.752 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2a11921c-f8c5-1bdf-77f7-d0b622afe76f.dc1 127.0.0.1 +34:41.752 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2a11921c-f8c5-1bdf-77f7-d0b622afe76f 127.0.0.1 +2021-08-13T15:34:41.752Z [INFO] agent.router: Initializing LAN area manager +34:41.753 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24824 network=udp +34:41.753 [INFO] TestAgent.server: Adding LAN server: server="Node-2a11921c-f8c5-1bdf-77f7-d0b622afe76f (Addr: tcp/127.0.0.1:24829) (DC: dc1)" +34:41.753 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2a11921c-f8c5-1bdf-77f7-d0b622afe76f.dc1 area=wan +34:41.753 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24824 network=tcp +34:41.754 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24825 network=tcp +34:41.754 [INFO] TestAgent: started state syncer +34:41.754 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24830 network=tcp +34:41.787 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:41.787 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24829 [Candidate]" term=2 +34:41.788 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:41.788 [DEBUG] TestAgent.server.raft: vote granted: from=2a11921c-f8c5-1bdf-77f7-d0b622afe76f term=2 tally=1 +34:41.788 [INFO] TestAgent.server.raft: election won: tally=1 +34:41.788 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24829 [Leader]" +34:41.788 [INFO] TestAgent.server: cluster leadership acquired +34:41.788 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24829 +34:41.790 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:41.791 [INFO] TestAgent.server: New leader elected: payload=Node-2a11921c-f8c5-1bdf-77f7-d0b622afe76f +34:41.802 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:41.802 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:41.802 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:41.802 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:41.802 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:41.802 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2a11921c-f8c5-1bdf-77f7-d0b622afe76f +34:41.802 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2a11921c-f8c5-1bdf-77f7-d0b622afe76f +34:41.810 [INFO] TestAgent.server: federation state anti-entropy synced +34:41.826 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:41.826 [INFO] TestAgent: Synced node info +34:41.956 [DEBUG] TestAgent.dns: request served from client: question="{2.0.0.127.in-addr.arpa. 255 1}" latency=143.181µs client=127.0.0.1:53573 client_network=udp +34:41.957 [DEBUG] TestAgent.dns: request served from client: question="{2.0.0.127.in-addr.arpa. 255 1}" latency=90.78µs client=127.0.0.1:53573 client_network=udp +34:41.957 [INFO] TestAgent: Requesting shutdown +34:41.957 [INFO] TestAgent.server: shutting down server +34:41.957 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:41.957 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:41.957 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:41.957 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:41.957 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:41.957 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:41.957 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:41.957 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:41.957 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:41.957 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:41.957 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:41.957Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:41.957Z [INFO] agent.router.manager: shutting down +34:41.958 [INFO] TestAgent: consul server down +34:41.958 [INFO] TestAgent: shutdown complete +34:41.958 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24824 network=tcp +34:41.958 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24824 network=udp +34:41.958 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24825 network=tcp +34:42.458 [INFO] TestAgent: Waiting for endpoints to shut down +34:42.458 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_Compression_ReverseLookup (0.74s) +=== CONT TestDNS_InvalidQueries +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:41.746Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:41.816Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:41.819 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:202ea5b6-54cc-463c-d7e2-8a8f23e48160 Address:127.0.0.1:24864}]" +34:41.819 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24864 [Follower]" leader= +34:41.820 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-202ea5b6-54cc-463c-d7e2-8a8f23e48160.dc1 127.0.0.1 +34:41.820 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-202ea5b6-54cc-463c-d7e2-8a8f23e48160 127.0.0.1 +2021-08-13T15:34:41.820Z [INFO] agent.router: Initializing LAN area manager +34:41.821 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24859 network=udp +34:41.821 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-202ea5b6-54cc-463c-d7e2-8a8f23e48160.dc1 area=wan +34:41.821 [INFO] TestAgent.server: Adding LAN server: server="Node-202ea5b6-54cc-463c-d7e2-8a8f23e48160 (Addr: tcp/127.0.0.1:24864) (DC: dc1)" +34:41.821 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24859 network=tcp +34:41.822 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24860 network=tcp +34:41.822 [INFO] TestAgent: started state syncer +34:41.822 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24865 network=tcp +34:41.875 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:41.875 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24864 [Candidate]" term=2 +34:41.876 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:41.876 [DEBUG] TestAgent.server.raft: vote granted: from=202ea5b6-54cc-463c-d7e2-8a8f23e48160 term=2 tally=1 +34:41.876 [INFO] TestAgent.server.raft: election won: tally=1 +34:41.876 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24864 [Leader]" +34:41.876 [INFO] TestAgent.server: cluster leadership acquired +34:41.876 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24864 +34:41.877 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:41.879 [INFO] TestAgent.server: New leader elected: payload=Node-202ea5b6-54cc-463c-d7e2-8a8f23e48160 +34:41.880 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:41.880 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:41.880 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:41.880 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:41.880 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:41.880 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-202ea5b6-54cc-463c-d7e2-8a8f23e48160 +34:41.880 [INFO] TestAgent.server: member joined, marking health alive: member=Node-202ea5b6-54cc-463c-d7e2-8a8f23e48160 +34:41.881 [INFO] TestAgent.server: federation state anti-entropy synced +34:41.967 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:41.968 [INFO] TestAgent: Synced node info +34:41.968 [DEBUG] TestAgent: Node info in sync +34:42.045 [INFO] TestAgent: Requesting shutdown +34:42.045 [INFO] TestAgent.server: shutting down server +34:42.045 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:42.045 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:42.045 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:42.045 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:42.045 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:42.045 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:42.045 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:42.045 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:42.045 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:42.045 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:42.045 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:42.046Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:42.046Z [INFO] agent.router.manager: shutting down +34:42.046 [INFO] TestAgent: consul server down +34:42.046 [INFO] TestAgent: shutdown complete +34:42.046 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24859 network=tcp +34:42.046 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24859 network=udp +34:42.046 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24860 network=tcp +34:42.546 [INFO] TestAgent: Waiting for endpoints to shut down +34:42.546 [INFO] TestAgent: Endpoints down +--- PASS: TestHealthNodeChecks_Filtering (0.82s) +=== CONT TestDNS_ServiceLookup_SRV_RFC_TCP_Default +2021-08-13T15:34:41.815Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:41.832Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:41.835 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:11f067e7-e8cf-db4f-fe90-aca27b036308 Address:127.0.0.1:24871}]" +34:41.835 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24871 [Follower]" leader= +34:41.836 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-11f067e7-e8cf-db4f-fe90-aca27b036308.dc1 127.0.0.1 +34:41.837 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-11f067e7-e8cf-db4f-fe90-aca27b036308 127.0.0.1 +2021-08-13T15:34:41.837Z [INFO] agent.router: Initializing LAN area manager +34:41.837 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-11f067e7-e8cf-db4f-fe90-aca27b036308.dc1 area=wan +34:41.837 [INFO] TestAgent.server: Adding LAN server: server="Node-11f067e7-e8cf-db4f-fe90-aca27b036308 (Addr: tcp/127.0.0.1:24871) (DC: dc1)" +34:41.837 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24866 network=tcp +34:41.837 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24866 network=udp +34:41.838 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24867 network=tcp +34:41.838 [INFO] TestAgent: started state syncer +34:41.838 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24872 network=tcp +34:41.887 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:41.887 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24871 [Candidate]" term=2 +34:41.887 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:41.887 [DEBUG] TestAgent.server.raft: vote granted: from=11f067e7-e8cf-db4f-fe90-aca27b036308 term=2 tally=1 +34:41.887 [INFO] TestAgent.server.raft: election won: tally=1 +34:41.887 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24871 [Leader]" +34:41.887 [INFO] TestAgent.server: cluster leadership acquired +34:41.888 [INFO] TestAgent.server: New leader elected: payload=Node-11f067e7-e8cf-db4f-fe90-aca27b036308 +34:41.888 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24871 +34:41.889 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:41.891 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:41.891 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:41.891 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:41.891 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:41.891 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:41.891 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-11f067e7-e8cf-db4f-fe90-aca27b036308 +34:41.891 [INFO] TestAgent.server: member joined, marking health alive: member=Node-11f067e7-e8cf-db4f-fe90-aca27b036308 +34:41.892 [INFO] TestAgent.server: federation state anti-entropy synced +34:42.007 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:42.008 [INFO] TestAgent: Synced node info +34:42.086 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=283µs client=127.0.0.1:45111 client_network=udp +34:42.086 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=218.64µs client=127.0.0.1:45111 client_network=udp +34:42.087 [DEBUG] TestAgent.dns: request served from client: name=bbe79eb4-751c-879f-d3ca-5a6caa0c22a1.query.consul. type=SRV class=IN latency=187.32µs client=127.0.0.1:36940 client_network=udp +34:42.087 [DEBUG] TestAgent.dns: request served from client: name=bbe79eb4-751c-879f-d3ca-5a6caa0c22a1.query.consul. type=SRV class=IN latency=165.441µs client=127.0.0.1:36940 client_network=udp +34:42.087 [INFO] TestAgent: Requesting shutdown +34:42.087 [INFO] TestAgent.server: shutting down server +34:42.087 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:42.087 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:42.087 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:42.087 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:42.087 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:42.087 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:42.087 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:42.087 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:42.087 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:42.087 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:42.088 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:42.088Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:42.088Z [INFO] agent.router.manager: shutting down +34:42.088 [INFO] TestAgent: consul server down +34:42.088 [INFO] TestAgent: shutdown complete +34:42.088 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24866 network=tcp +34:42.088 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24866 network=udp +34:42.088 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24867 network=tcp +34:42.588 [INFO] TestAgent: Waiting for endpoints to shut down +34:42.588 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_Compression_Query (0.83s) +=== CONT TestDNS_PreparedQuery_AllowStale +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:42.195Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:42.195Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:42.236Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:42.236Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:42.253 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:60fde58b-3eb3-54c7-f444-88dabc5eee22 Address:127.0.0.1:24899}]" +34:42.254 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-60fde58b-3eb3-54c7-f444-88dabc5eee22.dc1 127.0.0.1 +34:42.255 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24899 [Follower]" leader= +34:42.255 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-60fde58b-3eb3-54c7-f444-88dabc5eee22 127.0.0.1 +2021-08-13T15:34:42.255Z [INFO] agent.router: Initializing LAN area manager +34:42.256 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24880 network=udp +34:42.256 [INFO] TestAgent.server: Adding LAN server: server="Node-60fde58b-3eb3-54c7-f444-88dabc5eee22 (Addr: tcp/127.0.0.1:24899) (DC: dc1)" +34:42.256 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-60fde58b-3eb3-54c7-f444-88dabc5eee22.dc1 area=wan +34:42.256 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24880 network=tcp +34:42.257 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24888 network=tcp +34:42.257 [INFO] TestAgent: started state syncer +34:42.257 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24900 network=tcp +34:42.306 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:42.306 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24899 [Candidate]" term=2 +34:42.306 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:42.307 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:42.307 [DEBUG] TestAgent.server.raft: vote granted: from=60fde58b-3eb3-54c7-f444-88dabc5eee22 term=2 tally=1 +34:42.307 [INFO] TestAgent.server.raft: election won: tally=1 +34:42.307 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24899 [Leader]" +34:42.307 [INFO] TestAgent.server: cluster leadership acquired +34:42.307 [INFO] TestAgent.server: New leader elected: payload=Node-60fde58b-3eb3-54c7-f444-88dabc5eee22 +34:42.307 [INFO] TestAgent.server: initializing acls +34:42.307 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:42.307 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:42.308 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:42.309 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:42.309 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:42.309 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:42.309 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-60fde58b-3eb3-54c7-f444-88dabc5eee22 +34:42.309 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-60fde58b-3eb3-54c7-f444-88dabc5eee22.dc1 +34:42.309 [INFO] TestAgent.server: Updating LAN server: server="Node-60fde58b-3eb3-54c7-f444-88dabc5eee22 (Addr: tcp/127.0.0.1:24899) (DC: dc1)" +34:42.309 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-60fde58b-3eb3-54c7-f444-88dabc5eee22.dc1 area=wan +34:42.310 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:42.312 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:42.312 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:42.312 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:42.312 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:42.312 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:42.313 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-60fde58b-3eb3-54c7-f444-88dabc5eee22 +34:42.313 [INFO] TestAgent.server: member joined, marking health alive: member=Node-60fde58b-3eb3-54c7-f444-88dabc5eee22 +34:42.313 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-60fde58b-3eb3-54c7-f444-88dabc5eee22 +34:42.314 [INFO] TestAgent.server: federation state anti-entropy synced +34:42.331 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:42.332 [INFO] TestAgent: Synced node info +34:42.332 [DEBUG] TestAgent: Node info in sync +34:42.336 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-60fde58b-3eb3-54c7-f444-88dabc5eee22 +34:42.336 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-60fde58b-3eb3-54c7-f444-88dabc5eee22 +34:42.338 [DEBUG] TestAgent.dns: request served from client: name=foo.service.consul. type=A class=IN latency=487.041µs client=127.0.0.1:50569 client_network=udp +34:42.338 [INFO] TestAgent: Requesting shutdown +34:42.338 [INFO] TestAgent.server: shutting down server +34:42.338 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:42.338 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:42.338 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:42.338 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:42.338 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:42.338 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:42.338 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:42.338 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:42.338 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:42.338 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:42.338 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:42.338 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:42.338 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:42.338 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:42.338 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:42.339Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:42.339Z [INFO] agent.router.manager: shutting down +34:42.339 [INFO] TestAgent: consul server down +34:42.339 [INFO] TestAgent: shutdown complete +34:42.339 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24880 network=tcp +34:42.339 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24880 network=udp +34:42.339 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24888 network=tcp +34:42.839 [INFO] TestAgent: Waiting for endpoints to shut down +34:42.839 [INFO] TestAgent: Endpoints down +=== RUN TestDNS_ServiceLookup_FilterACL/ACLToken_==_anonymous +2021-08-13T15:34:41.967Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:42.009Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:42.012 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9aaa151e-6cf2-90a0-c7f3-b2cc068c4073 Address:127.0.0.1:24850}]" +34:42.012 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24850 [Follower]" leader= +34:42.013 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9aaa151e-6cf2-90a0-c7f3-b2cc068c4073.dc1 127.0.0.1 +34:42.014 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9aaa151e-6cf2-90a0-c7f3-b2cc068c4073 127.0.0.1 +2021-08-13T15:34:42.014Z [INFO] agent.router: Initializing LAN area manager +34:42.014 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24845 network=udp +34:42.015 [INFO] TestAgent.server: Adding LAN server: server="Node-9aaa151e-6cf2-90a0-c7f3-b2cc068c4073 (Addr: tcp/127.0.0.1:24850) (DC: dc1)" +34:42.016 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24845 network=tcp +34:42.016 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24846 network=tcp +34:42.017 [INFO] TestAgent: started state syncer +34:42.017 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24851 network=tcp +34:42.023 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9aaa151e-6cf2-90a0-c7f3-b2cc068c4073.dc1 area=wan +34:42.059 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:42.059 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24850 [Candidate]" term=2 +34:42.059 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:42.059 [DEBUG] TestAgent.server.raft: vote granted: from=9aaa151e-6cf2-90a0-c7f3-b2cc068c4073 term=2 tally=1 +34:42.059 [INFO] TestAgent.server.raft: election won: tally=1 +34:42.059 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24850 [Leader]" +34:42.060 [INFO] TestAgent.server: cluster leadership acquired +34:42.060 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24850 +34:42.061 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:42.063 [INFO] TestAgent.server: New leader elected: payload=Node-9aaa151e-6cf2-90a0-c7f3-b2cc068c4073 +34:42.064 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:42.064 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:42.064 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:42.064 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:42.064 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:42.064 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9aaa151e-6cf2-90a0-c7f3-b2cc068c4073 +34:42.064 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9aaa151e-6cf2-90a0-c7f3-b2cc068c4073 +34:42.065 [INFO] TestAgent.server: federation state anti-entropy synced +34:42.359 [INFO] TestAgent: Requesting shutdown +34:42.359 [INFO] TestAgent.server: shutting down server +34:42.359 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:42.359 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:42.359 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:42.359 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:42.359 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:42.359 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:42.359 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:42.359 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:42.359 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:42.359 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:42.359 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:42.359 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:42.360Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:42.360Z [INFO] agent.router.manager: shutting down +34:42.360 [INFO] TestAgent: consul server down +34:42.360 [INFO] TestAgent: shutdown complete +34:42.360 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24845 network=tcp +34:42.360 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24845 network=udp +34:42.360 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24846 network=tcp +34:42.861 [INFO] TestAgent: Waiting for endpoints to shut down +34:42.861 [INFO] TestAgent: Endpoints down +--- PASS: TestHealthNodeChecks (0.92s) +=== CONT TestDNS_ServiceLookup_SRV_RFC +2021-08-13T15:34:42.192Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:42.263Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:42.266 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b1e19204-8523-86c3-e666-32afbbf341c0 Address:127.0.0.1:24885}]" +34:42.266 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24885 [Follower]" leader= +34:42.267 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b1e19204-8523-86c3-e666-32afbbf341c0.dc1 127.0.0.1 +34:42.268 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b1e19204-8523-86c3-e666-32afbbf341c0 127.0.0.1 +2021-08-13T15:34:42.268Z [INFO] agent.router: Initializing LAN area manager +34:42.268 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24894 network=udp +34:42.268 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b1e19204-8523-86c3-e666-32afbbf341c0.dc1 area=wan +34:42.268 [INFO] TestAgent.server: Adding LAN server: server="Node-b1e19204-8523-86c3-e666-32afbbf341c0 (Addr: tcp/127.0.0.1:24885) (DC: dc1)" +34:42.268 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24894 network=tcp +34:42.269 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24895 network=tcp +34:42.269 [INFO] TestAgent: started state syncer +34:42.269 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24886 network=tcp +34:42.332 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:42.332 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24885 [Candidate]" term=2 +34:42.332 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:42.332 [DEBUG] TestAgent.server.raft: vote granted: from=b1e19204-8523-86c3-e666-32afbbf341c0 term=2 tally=1 +34:42.332 [INFO] TestAgent.server.raft: election won: tally=1 +34:42.332 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24885 [Leader]" +34:42.332 [INFO] TestAgent.server: cluster leadership acquired +34:42.332 [INFO] TestAgent.server: New leader elected: payload=Node-b1e19204-8523-86c3-e666-32afbbf341c0 +34:42.332 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24885 +34:42.333 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:42.336 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:42.336 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:42.336 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:42.336 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:42.336 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:42.336 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b1e19204-8523-86c3-e666-32afbbf341c0 +34:42.336 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b1e19204-8523-86c3-e666-32afbbf341c0 +34:42.338 [INFO] TestAgent.server: federation state anti-entropy synced +34:42.410 [WARN] TestAgent.server: endpoint injected; this should only be used for testing +34:42.410 [DEBUG] TestAgent.dns: request served from client: name=foo.query.consul. type=SRV class=IN latency=110.68µs client=127.0.0.1:59815 client_network=udp +34:42.411 [INFO] TestAgent: Requesting shutdown +34:42.411 [INFO] TestAgent.server: shutting down server +34:42.411 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:42.411 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:42.411 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:42.411 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:42.411 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:42.411 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:42.411 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:42.411 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:42.411 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:42.411 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:42.411 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:42.411 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:42.411Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:42.411Z [INFO] agent.router.manager: shutting down +34:42.411 [INFO] TestAgent: consul server down +34:42.411 [INFO] TestAgent: shutdown complete +34:42.411 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24894 network=tcp +34:42.411 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24894 network=udp +34:42.412 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24895 network=tcp +34:42.915 [INFO] TestAgent: Waiting for endpoints to shut down +34:42.915 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_PreparedQuery_AgentSource (0.75s) +=== CONT TestDNS_AltDomains_Overlap +2021-08-13T15:34:42.041Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:42.057Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:42.060 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:078dc7a2-30c0-c786-f74f-042250b6d904 Address:127.0.0.1:24836}]" +34:42.061 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-078dc7a2-30c0-c786-f74f-042250b6d904.dc1 127.0.0.1 +34:42.062 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-078dc7a2-30c0-c786-f74f-042250b6d904 127.0.0.1 +2021-08-13T15:34:42.062Z [INFO] agent.router: Initializing LAN area manager +34:42.063 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24831 network=udp +34:42.063 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24836 [Follower]" leader= +34:42.063 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-078dc7a2-30c0-c786-f74f-042250b6d904.dc1 area=wan +34:42.063 [INFO] TestAgent.server: Adding LAN server: server="Node-078dc7a2-30c0-c786-f74f-042250b6d904 (Addr: tcp/127.0.0.1:24836) (DC: dc1)" +34:42.063 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24831 network=tcp +34:42.064 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24832 network=tcp +34:42.064 [INFO] TestAgent: started state syncer +34:42.064 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24837 network=tcp +34:42.127 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:42.127 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24836 [Candidate]" term=2 +34:42.127 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:42.127 [DEBUG] TestAgent.server.raft: vote granted: from=078dc7a2-30c0-c786-f74f-042250b6d904 term=2 tally=1 +34:42.127 [INFO] TestAgent.server.raft: election won: tally=1 +34:42.127 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24836 [Leader]" +34:42.127 [INFO] TestAgent.server: cluster leadership acquired +34:42.127 [INFO] TestAgent.server: New leader elected: payload=Node-078dc7a2-30c0-c786-f74f-042250b6d904 +34:42.127 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24836 +34:42.129 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:42.131 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:42.131 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:42.131 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:42.131 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:42.131 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:42.132 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-078dc7a2-30c0-c786-f74f-042250b6d904 +34:42.132 [INFO] TestAgent.server: member joined, marking health alive: member=Node-078dc7a2-30c0-c786-f74f-042250b6d904 +34:42.132 [INFO] TestAgent.server: federation state anti-entropy synced +34:42.447 [DEBUG] TestAgent.dns: request served from client: name=2607002040050808000000000000200e.addr.consul. type=SRV class=IN latency=88.521µs client=127.0.0.1:58955 client_network=udp +34:42.447 [DEBUG] TestAgent.dns: request served from client: name=2607112040051808ffffffffffff200e.addr.consul. type=SRV class=IN latency=78.3µs client=127.0.0.1:44115 client_network=udp +34:42.447 [INFO] TestAgent: Requesting shutdown +34:42.447 [INFO] TestAgent.server: shutting down server +34:42.447 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:42.447 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:42.447 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:42.447 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:42.447 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:42.447 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:42.447 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:42.447 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:42.447 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:42.447 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:42.447 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:42.447 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:42.448Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:42.448Z [INFO] agent.router.manager: shutting down +34:42.448 [INFO] TestAgent: consul server down +34:42.448 [INFO] TestAgent: shutdown complete +34:42.448 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24831 network=tcp +34:42.448 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24831 network=udp +34:42.448 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24832 network=tcp +34:42.951 [INFO] TestAgent: Waiting for endpoints to shut down +34:42.951 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_AddressLookupIPV6 (0.93s) +=== CONT TestDNS_PreparedQuery_Failover +[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:42.474Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:42.489Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:42.492 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:80efb610-4eac-edc1-a6c6-e64065ed9e41 Address:127.0.0.1:24913}]" +34:42.492 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24913 [Follower]" leader= +34:42.493 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-80efb610-4eac-edc1-a6c6-e64065ed9e41.dc1 127.0.0.1 +34:42.494 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-80efb610-4eac-edc1-a6c6-e64065ed9e41 127.0.0.1 +2021-08-13T15:34:42.494Z [INFO] agent.router: Initializing LAN area manager +34:42.494 [INFO] TestAgent.server: Adding LAN server: server="Node-80efb610-4eac-edc1-a6c6-e64065ed9e41 (Addr: tcp/127.0.0.1:24913) (DC: dc1)" +34:42.494 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-80efb610-4eac-edc1-a6c6-e64065ed9e41.dc1 area=wan +34:42.494 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24873 network=udp +34:42.494 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24873 network=tcp +34:42.495 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24909 network=tcp +34:42.495 [INFO] TestAgent: started state syncer +34:42.495 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24914 network=tcp +34:42.555 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:42.555 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24913 [Candidate]" term=2 +34:42.555 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:42.555 [DEBUG] TestAgent.server.raft: vote granted: from=80efb610-4eac-edc1-a6c6-e64065ed9e41 term=2 tally=1 +34:42.555 [INFO] TestAgent.server.raft: election won: tally=1 +34:42.555 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24913 [Leader]" +34:42.555 [INFO] TestAgent.server: cluster leadership acquired +34:42.555 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24913 +34:42.557 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:42.559 [INFO] TestAgent.server: New leader elected: payload=Node-80efb610-4eac-edc1-a6c6-e64065ed9e41 +34:42.567 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:42.567 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:42.567 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:42.567 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:42.567 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:42.567 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-80efb610-4eac-edc1-a6c6-e64065ed9e41 +34:42.567 [INFO] TestAgent.server: member joined, marking health alive: member=Node-80efb610-4eac-edc1-a6c6-e64065ed9e41 +34:42.568 [INFO] TestAgent.server: federation state anti-entropy synced +34:42.577 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:42.578 [INFO] TestAgent: Synced node info +34:42.578 [DEBUG] TestAgent: Node info in sync +34:42.587 [WARN] TestAgent.dns: QName invalid: qname= +34:42.587 [DEBUG] TestAgent.dns: request served from client: name=consul. type=SRV class=IN latency=90.98µs client=127.0.0.1:52965 client_network=udp +34:42.587 [WARN] TestAgent.dns: QName invalid: qname=node. +34:42.587 [DEBUG] TestAgent.dns: request served from client: name=node.consul. type=SRV class=IN latency=64.54µs client=127.0.0.1:45098 client_network=udp +34:42.587 [WARN] TestAgent.dns: QName invalid: qname=service. +34:42.587 [DEBUG] TestAgent.dns: request served from client: name=service.consul. type=SRV class=IN latency=66.4µs client=127.0.0.1:38999 client_network=udp +34:42.587 [WARN] TestAgent.dns: QName invalid: qname=query. +34:42.587 [DEBUG] TestAgent.dns: request served from client: name=query.consul. type=SRV class=IN latency=62.88µs client=127.0.0.1:39822 client_network=udp +34:42.588 [WARN] TestAgent.dns: QName invalid: qname=foo.node.dc1.extra.more. +34:42.588 [DEBUG] TestAgent.dns: request served from client: name=foo.node.dc1.extra.more.consul. type=SRV class=IN latency=63.98µs client=127.0.0.1:38164 client_network=udp +34:42.588 [WARN] TestAgent.dns: QName invalid: qname=foo.service.dc1.extra.more. +34:42.588 [DEBUG] TestAgent.dns: request served from client: name=foo.service.dc1.extra.more.consul. type=SRV class=IN latency=63.94µs client=127.0.0.1:36840 client_network=udp +34:42.588 [WARN] TestAgent.dns: QName invalid: qname=foo.query.dc1.extra.more. +34:42.588 [DEBUG] TestAgent.dns: request served from client: name=foo.query.dc1.extra.more.consul. type=SRV class=IN latency=67.92µs client=127.0.0.1:57104 client_network=udp +34:42.588 [INFO] TestAgent: Requesting shutdown +34:42.588 [INFO] TestAgent.server: shutting down server +34:42.588 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:42.588 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:42.588 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:42.588 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:42.588 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:42.588 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:42.588 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:42.588 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:42.588 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:42.589 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:42.589 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:42.589Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:42.589Z [INFO] agent.router.manager: shutting down +34:42.589 [INFO] TestAgent: consul server down +34:42.589 [INFO] TestAgent: shutdown complete +34:42.589 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24873 network=tcp +34:42.589 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24873 network=udp +34:42.589 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24909 network=tcp +34:43.089 [INFO] TestAgent: Waiting for endpoints to shut down +34:43.089 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_InvalidQueries (0.63s) +=== CONT TestDNS_AltDomains_SOA +2021-08-13T15:34:42.176Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:42.203Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:42.206 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1db42aff-cf22-dffb-3852-318cd9434d3b Address:127.0.0.1:24857}]" +34:42.206 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24857 [Follower]" leader= +34:42.213 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1db42aff-cf22-dffb-3852-318cd9434d3b.dc1 127.0.0.1 +34:42.232 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1db42aff-cf22-dffb-3852-318cd9434d3b 127.0.0.1 +2021-08-13T15:34:42.232Z [INFO] agent.router: Initializing LAN area manager +34:42.240 [INFO] TestAgent.server: Adding LAN server: server="Node-1db42aff-cf22-dffb-3852-318cd9434d3b (Addr: tcp/127.0.0.1:24857) (DC: dc1)" +34:42.240 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24852 network=udp +34:42.241 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24852 network=tcp +34:42.245 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24853 network=tcp +34:42.245 [INFO] TestAgent: started state syncer +34:42.245 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24858 network=tcp +34:42.247 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1db42aff-cf22-dffb-3852-318cd9434d3b.dc1 area=wan +34:42.255 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:42.255 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24857 [Candidate]" term=2 +34:42.255 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:42.255 [DEBUG] TestAgent.server.raft: vote granted: from=1db42aff-cf22-dffb-3852-318cd9434d3b term=2 tally=1 +34:42.255 [INFO] TestAgent.server.raft: election won: tally=1 +34:42.255 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24857 [Leader]" +34:42.255 [INFO] TestAgent.server: cluster leadership acquired +34:42.256 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24857 +34:42.257 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:42.258 [INFO] TestAgent.server: New leader elected: payload=Node-1db42aff-cf22-dffb-3852-318cd9434d3b +34:42.261 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:42.261 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:42.261 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:42.261 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:42.261 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:42.261 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1db42aff-cf22-dffb-3852-318cd9434d3b +34:42.261 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1db42aff-cf22-dffb-3852-318cd9434d3b +34:42.262 [INFO] TestAgent.server: federation state anti-entropy synced +34:42.300 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:42.300 [INFO] TestAgent: Synced node info +34:42.300 [DEBUG] TestAgent: Node info in sync +34:42.598 [DEBUG] TestAgent.dns: request served from client: name=7f000001.addr.dc1.consul. type=SRV class=IN latency=67.44µs client=127.0.0.1:55727 client_network=udp +34:42.598 [INFO] TestAgent: Requesting shutdown +34:42.598 [INFO] TestAgent.server: shutting down server +34:42.598 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:42.598 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:42.598 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:42.598 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:42.598 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:42.599 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:42.599 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:42.599 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:42.599 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:42.599 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:42.599 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:42.599Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:42.599Z [INFO] agent.router.manager: shutting down +34:42.599 [INFO] TestAgent: consul server down +34:42.599 [INFO] TestAgent: shutdown complete +34:42.599 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24852 network=tcp +34:42.599 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24852 network=udp +34:42.599 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24853 network=tcp +34:43.100 [INFO] TestAgent: Waiting for endpoints to shut down +34:43.100 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_AddressLookup (0.94s) +=== CONT TestDNS_PreparedQuery_TTL +2021-08-13T15:34:42.563Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:42.579Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:42.582 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:33d5fb8c-f1f5-c76f-9453-c78fd0d321da Address:127.0.0.1:24878}]" +34:42.582 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24878 [Follower]" leader= +34:42.583 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-33d5fb8c-f1f5-c76f-9453-c78fd0d321da.dc1 127.0.0.1 +34:42.583 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-33d5fb8c-f1f5-c76f-9453-c78fd0d321da 127.0.0.1 +2021-08-13T15:34:42.583Z [INFO] agent.router: Initializing LAN area manager +34:42.584 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24887 network=udp +34:42.584 [INFO] TestAgent.server: Adding LAN server: server="Node-33d5fb8c-f1f5-c76f-9453-c78fd0d321da (Addr: tcp/127.0.0.1:24878) (DC: dc1)" +34:42.584 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-33d5fb8c-f1f5-c76f-9453-c78fd0d321da.dc1 area=wan +34:42.584 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24887 network=tcp +34:42.585 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24902 network=tcp +34:42.585 [INFO] TestAgent: started state syncer +34:42.585 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24879 network=tcp +34:42.647 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:42.647 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24878 [Candidate]" term=2 +34:42.647 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:42.647 [DEBUG] TestAgent.server.raft: vote granted: from=33d5fb8c-f1f5-c76f-9453-c78fd0d321da term=2 tally=1 +34:42.647 [INFO] TestAgent.server.raft: election won: tally=1 +34:42.647 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24878 [Leader]" +34:42.647 [INFO] TestAgent.server: cluster leadership acquired +34:42.648 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24878 +34:42.649 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:42.651 [INFO] TestAgent.server: New leader elected: payload=Node-33d5fb8c-f1f5-c76f-9453-c78fd0d321da +34:42.651 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:42.651 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:42.651 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:42.651 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:42.651 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:42.651 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-33d5fb8c-f1f5-c76f-9453-c78fd0d321da +34:42.651 [INFO] TestAgent.server: member joined, marking health alive: member=Node-33d5fb8c-f1f5-c76f-9453-c78fd0d321da +34:42.652 [INFO] TestAgent.server: federation state anti-entropy synced +34:42.713 [DEBUG] TestAgent.dns: request served from client: name=_db._tcp.service.dc1.consul. type=SRV class=IN latency=274.92µs client=127.0.0.1:58762 client_network=udp +34:42.713 [DEBUG] TestAgent.dns: request served from client: name=_db._tcp.service.consul. type=SRV class=IN latency=188.84µs client=127.0.0.1:35464 client_network=udp +34:42.713 [DEBUG] TestAgent.dns: request served from client: name=_db._tcp.dc1.consul. type=SRV class=IN latency=182.98µs client=127.0.0.1:44072 client_network=udp +34:42.714 [DEBUG] TestAgent.dns: request served from client: name=_db._tcp.consul. type=SRV class=IN latency=173.76µs client=127.0.0.1:57342 client_network=udp +34:42.714 [INFO] TestAgent: Requesting shutdown +34:42.714 [INFO] TestAgent.server: shutting down server +34:42.714 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:42.714 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:42.714 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:42.714 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:42.714 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:42.714 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:42.714 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:42.714 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:42.714 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:42.714 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:42.714 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:42.714 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:42.715Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:42.715Z [INFO] agent.router.manager: shutting down +34:42.715 [INFO] TestAgent: consul server down +34:42.715 [INFO] TestAgent: shutdown complete +34:42.715 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24887 network=tcp +34:42.715 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24887 network=udp +34:42.715 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24902 network=tcp +34:43.215 [INFO] TestAgent: Waiting for endpoints to shut down +34:43.215 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup_SRV_RFC_TCP_Default (0.67s) +=== CONT TestDNS_AltDomains_Service +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:42.605Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:42.621Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:42.623 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d6677160-7644-76cb-788b-571cc1044caa Address:127.0.0.1:24920}]" +34:42.624 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24920 [Follower]" leader= +34:42.624 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d6677160-7644-76cb-788b-571cc1044caa.dc1 127.0.0.1 +34:42.625 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d6677160-7644-76cb-788b-571cc1044caa 127.0.0.1 +2021-08-13T15:34:42.625Z [INFO] agent.router: Initializing LAN area manager +34:42.625 [INFO] TestAgent.server: Adding LAN server: server="Node-d6677160-7644-76cb-788b-571cc1044caa (Addr: tcp/127.0.0.1:24920) (DC: dc1)" +34:42.625 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d6677160-7644-76cb-788b-571cc1044caa.dc1 area=wan +34:42.625 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24908 network=udp +34:42.626 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24908 network=tcp +34:42.626 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24916 network=tcp +34:42.627 [INFO] TestAgent: started state syncer +34:42.627 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24921 network=tcp +34:42.681 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:42.681 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24920 [Candidate]" term=2 +34:42.681 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:42.681 [DEBUG] TestAgent.server.raft: vote granted: from=d6677160-7644-76cb-788b-571cc1044caa term=2 tally=1 +34:42.681 [INFO] TestAgent.server.raft: election won: tally=1 +34:42.681 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24920 [Leader]" +34:42.681 [INFO] TestAgent.server: cluster leadership acquired +34:42.681 [INFO] TestAgent.server: New leader elected: payload=Node-d6677160-7644-76cb-788b-571cc1044caa +34:42.681 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24920 +34:42.683 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:42.685 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:42.685 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:42.685 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:42.685 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:42.685 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:42.685 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d6677160-7644-76cb-788b-571cc1044caa +34:42.685 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d6677160-7644-76cb-788b-571cc1044caa +34:42.686 [INFO] TestAgent.server: federation state anti-entropy synced +34:42.697 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:42.698 [INFO] TestAgent: Synced node info +34:42.942 [WARN] TestAgent.server: endpoint injected; this should only be used for testing +34:42.943 [WARN] TestAgent.dns: Query results too stale, re-requesting +34:42.943 [DEBUG] TestAgent.dns: request served from client: name=nope.query.consul. type=SRV class=IN latency=153.34µs client=127.0.0.1:40325 client_network=udp +34:42.943 [INFO] TestAgent: Requesting shutdown +34:42.943 [INFO] TestAgent.server: shutting down server +34:42.943 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:42.943 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:42.943 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:42.943 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:42.943 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:42.944 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:42.944 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:42.944 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:42.944 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:42.944 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:42.944 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:42.944Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:42.944Z [INFO] agent.router.manager: shutting down +34:42.944 [INFO] TestAgent: consul server down +34:42.944 [INFO] TestAgent: shutdown complete +34:42.944 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24908 network=tcp +34:42.944 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24908 network=udp +34:42.944 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24916 network=tcp +34:43.445 [INFO] TestAgent: Waiting for endpoints to shut down +34:43.445 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_PreparedQuery_AllowStale (0.86s) +=== CONT TestDNS_ServiceLookup_TTL +2021-08-13T15:34:42.878Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:42.894Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:42.911 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:dec9365c-c449-4ef9-f0be-11eb8e805e91 Address:127.0.0.1:24969}]" +34:42.913 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-dec9365c-c449-4ef9-f0be-11eb8e805e91.dc1 127.0.0.1 +34:42.914 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-dec9365c-c449-4ef9-f0be-11eb8e805e91 127.0.0.1 +2021-08-13T15:34:42.914Z [INFO] agent.router: Initializing LAN area manager +34:42.914 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24943 network=udp +34:42.914 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24969 [Follower]" leader= +34:42.915 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-dec9365c-c449-4ef9-f0be-11eb8e805e91.dc1 area=wan +34:42.915 [INFO] TestAgent.server: Adding LAN server: server="Node-dec9365c-c449-4ef9-f0be-11eb8e805e91 (Addr: tcp/127.0.0.1:24969) (DC: dc1)" +34:42.915 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24943 network=tcp +34:42.917 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24944 network=tcp +34:42.917 [INFO] TestAgent: started state syncer +34:42.917 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24970 network=tcp +34:42.953 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:42.953 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24969 [Candidate]" term=2 +34:42.954 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:42.954 [DEBUG] TestAgent.server.raft: vote granted: from=dec9365c-c449-4ef9-f0be-11eb8e805e91 term=2 tally=1 +34:42.954 [INFO] TestAgent.server.raft: election won: tally=1 +34:42.954 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24969 [Leader]" +34:42.954 [INFO] TestAgent.server: cluster leadership acquired +34:42.954 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24969 +34:42.955 [INFO] TestAgent.server: New leader elected: payload=Node-dec9365c-c449-4ef9-f0be-11eb8e805e91 +34:42.956 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:42.960 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:42.960 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:42.960 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:42.960 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:42.960 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:42.960 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-dec9365c-c449-4ef9-f0be-11eb8e805e91 +34:42.960 [INFO] TestAgent.server: member joined, marking health alive: member=Node-dec9365c-c449-4ef9-f0be-11eb8e805e91 +34:42.961 [INFO] TestAgent.server: federation state anti-entropy synced +34:42.991 [DEBUG] TestAgent.dns: request served from client: name=_db._primary.service.dc1.consul. type=SRV class=IN latency=258.14µs client=127.0.0.1:60558 client_network=udp +34:42.992 [DEBUG] TestAgent.dns: request served from client: name=_db._primary.service.consul. type=SRV class=IN latency=144.26µs client=127.0.0.1:33226 client_network=udp +34:42.992 [DEBUG] TestAgent.dns: request served from client: name=_db._primary.dc1.consul. type=SRV class=IN latency=139.26µs client=127.0.0.1:34481 client_network=udp +34:42.993 [DEBUG] TestAgent.dns: request served from client: name=_db._primary.consul. type=SRV class=IN latency=137.62µs client=127.0.0.1:46337 client_network=udp +34:42.993 [INFO] TestAgent: Requesting shutdown +34:42.993 [INFO] TestAgent.server: shutting down server +34:42.993 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:42.993 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:42.993 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:42.993 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:42.993 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:42.993 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:42.993 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:42.993 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:42.993 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:42.993 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:42.993 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:42.993 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:42.993Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:42.993Z [INFO] agent.router.manager: shutting down +34:42.993 [INFO] TestAgent: consul server down +34:42.993 [INFO] TestAgent: shutdown complete +34:42.994 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24943 network=tcp +34:42.994 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24943 network=udp +34:42.994 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24944 network=tcp +34:43.494 [INFO] TestAgent: Waiting for endpoints to shut down +34:43.494 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup_SRV_RFC (0.63s) +=== CONT TestDNS_NonExistingLookupEmptyAorAAAA +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestDNS_PreparedQuery_TTL/db.query.consul. +=== RUN TestDNS_PreparedQuery_TTL/db-ttl.query.consul. +=== RUN TestDNS_PreparedQuery_TTL/dblb.query.consul. +=== RUN TestDNS_PreparedQuery_TTL/dblb-ttl.query.consul. +=== RUN TestDNS_PreparedQuery_TTL/dk.query.consul. +=== RUN TestDNS_PreparedQuery_TTL/dk-ttl.query.consul. +=== RUN TestDNS_PreparedQuery_TTL/api.query.consul. +=== RUN TestDNS_PreparedQuery_TTL/api-ttl.query.consul. +=== RUN TestDNS_ServiceLookup_TTL/db.service.consul. +2021-08-13T15:34:42.856Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:42.856Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:42.872Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:42.872Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:42.875 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fc91ca12-bec5-e863-6603-2e85926c3788 Address:127.0.0.1:24927}]" +34:42.875 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24927 [Follower]" leader= +34:42.876 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fc91ca12-bec5-e863-6603-2e85926c3788.dc1 127.0.0.1 +34:42.877 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fc91ca12-bec5-e863-6603-2e85926c3788 127.0.0.1 +2021-08-13T15:34:42.877Z [INFO] agent.router: Initializing LAN area manager +34:42.877 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24929 network=udp +34:42.877 [INFO] TestAgent.server: Adding LAN server: server="Node-fc91ca12-bec5-e863-6603-2e85926c3788 (Addr: tcp/127.0.0.1:24927) (DC: dc1)" +34:42.877 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fc91ca12-bec5-e863-6603-2e85926c3788.dc1 area=wan +34:42.878 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24929 network=tcp +34:42.878 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24923 network=tcp +34:42.878 [INFO] TestAgent: started state syncer +34:42.879 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24928 network=tcp +34:42.931 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:42.931 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24927 [Candidate]" term=2 +34:42.931 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:42.931 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:42.931 [DEBUG] TestAgent.server.raft: vote granted: from=fc91ca12-bec5-e863-6603-2e85926c3788 term=2 tally=1 +34:42.931 [INFO] TestAgent.server.raft: election won: tally=1 +34:42.931 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24927 [Leader]" +34:42.932 [INFO] TestAgent.server: cluster leadership acquired +34:42.932 [INFO] TestAgent.server: initializing acls +34:42.932 [INFO] TestAgent.server: New leader elected: payload=Node-fc91ca12-bec5-e863-6603-2e85926c3788 +34:42.933 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:42.933 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:42.933 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:42.934 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:42.934 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:42.934 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:42.935 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-fc91ca12-bec5-e863-6603-2e85926c3788 +34:42.939 [INFO] TestAgent.server: Updating LAN server: server="Node-fc91ca12-bec5-e863-6603-2e85926c3788 (Addr: tcp/127.0.0.1:24927) (DC: dc1)" +34:42.939 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-fc91ca12-bec5-e863-6603-2e85926c3788.dc1 +34:42.941 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:42.943 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-fc91ca12-bec5-e863-6603-2e85926c3788.dc1 area=wan +34:42.944 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:42.944 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:42.944 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:42.944 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:42.944 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:42.944 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fc91ca12-bec5-e863-6603-2e85926c3788 +34:42.945 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fc91ca12-bec5-e863-6603-2e85926c3788 +34:42.946 [INFO] TestAgent.server: federation state anti-entropy synced +34:42.946 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fc91ca12-bec5-e863-6603-2e85926c3788 +34:43.113 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-fc91ca12-bec5-e863-6603-2e85926c3788 +34:43.114 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-fc91ca12-bec5-e863-6603-2e85926c3788 +34:43.116 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=foo +34:43.116 [DEBUG] TestAgent.dns: request served from client: name=foo.service.consul. type=A class=IN latency=305.78µs client=127.0.0.1:48198 client_network=udp +34:43.116 [INFO] TestAgent: Requesting shutdown +34:43.116 [INFO] TestAgent.server: shutting down server +34:43.116 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:43.116 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:43.116 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:43.116 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:43.116 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:43.116 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:43.116 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:43.116 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:43.116 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:43.116 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:43.116 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:43.116 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:43.116 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:43.116 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:43.116 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:43.116 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:43.116Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:43.116Z [INFO] agent.router.manager: shutting down +34:43.117 [INFO] TestAgent: consul server down +34:43.117 [INFO] TestAgent: shutdown complete +34:43.117 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24929 network=tcp +34:43.117 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24929 network=udp +34:43.117 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24923 network=tcp +34:43.617 [INFO] TestAgent: Waiting for endpoints to shut down +34:43.617 [INFO] TestAgent: Endpoints down +=== RUN TestDNS_ServiceLookup_TTL/dblb.service.consul. +--- PASS: TestDNS_ServiceLookup_FilterACL (1.44s) + --- PASS: TestDNS_ServiceLookup_FilterACL/ACLToken_==_root (0.66s) + --- PASS: TestDNS_ServiceLookup_FilterACL/ACLToken_==_anonymous (0.78s) +=== CONT TestDNS_NodeLookup_TTL +=== RUN TestDNS_ServiceLookup_TTL/dk.service.consul. +=== RUN TestDNS_ServiceLookup_TTL/api.service.consul. +2021-08-13T15:34:42.966Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:42.996Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:42.999 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:350252b0-4b81-a836-4a16-2025fb22349c Address:127.0.0.1:24955}]" +34:42.999 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24955 [Follower]" leader= +34:43.000 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +34:43.001 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2021-08-13T15:34:43.001Z [INFO] agent.router: Initializing LAN area manager +34:43.001 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24950 network=udp +34:43.001 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:24955) (DC: dc1)" +34:43.001 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +34:43.002 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24950 network=tcp +34:43.003 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24937 network=tcp +34:43.003 [INFO] TestAgent: started state syncer +34:43.003 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24956 network=tcp +34:43.047 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:43.047 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24955 [Candidate]" term=2 +34:43.047 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:43.047 [DEBUG] TestAgent.server.raft: vote granted: from=350252b0-4b81-a836-4a16-2025fb22349c term=2 tally=1 +34:43.047 [INFO] TestAgent.server.raft: election won: tally=1 +34:43.047 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24955 [Leader]" +34:43.047 [INFO] TestAgent.server: cluster leadership acquired +34:43.048 [INFO] TestAgent.server: New leader elected: payload=test-node +34:43.048 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24955 +34:43.049 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:43.051 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:43.051 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:43.051 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:43.051 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:43.051 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:43.051 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +34:43.051 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +34:43.052 [INFO] TestAgent.server: federation state anti-entropy synced +34:43.227 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:43.228 [INFO] TestAgent: Synced node info +34:43.246 [DEBUG] TestAgent.dns: request served from client: name=test-node.node.consul. type=A class=IN latency=195.101µs client=127.0.0.1:49845 client_network=udp +34:43.246 [DEBUG] TestAgent.dns: request served from client: name=test-node.node.test.consul. type=A class=IN latency=137.321µs client=127.0.0.1:55966 client_network=udp +34:43.246 [DEBUG] TestAgent.dns: request served from client: name=test-node.node.dc1.consul. type=A class=IN latency=165.02µs client=127.0.0.1:53627 client_network=udp +34:43.247 [DEBUG] TestAgent.dns: request served from client: name=test-node.node.dc1.test.consul. type=A class=IN latency=130.76µs client=127.0.0.1:45544 client_network=udp +34:43.247 [INFO] TestAgent: Requesting shutdown +34:43.247 [INFO] TestAgent.server: shutting down server +34:43.247 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:43.247 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:43.247 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:43.247 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:43.247 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:43.247 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:43.247 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:43.247 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:43.247 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:43.247 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:43.247 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:43.247Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:43.247Z [INFO] agent.router.manager: shutting down +34:43.248 [INFO] TestAgent: consul server down +34:43.248 [INFO] TestAgent: shutdown complete +34:43.248 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24950 network=tcp +34:43.248 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24950 network=udp +34:43.248 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24937 network=tcp +34:43.748 [INFO] TestAgent: Waiting for endpoints to shut down +34:43.748 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_AltDomains_Overlap (0.83s) +=== CONT TestDNS_NonExistingLookup +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:43.106Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:43.127Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:43.129 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8ebead8a-2b8c-9d30-d963-95196d6d7944 Address:127.0.0.1:25004}]" +34:43.130 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +34:43.131 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25004 [Follower]" leader= +34:43.131 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2021-08-13T15:34:43.131Z [INFO] agent.router: Initializing LAN area manager +34:43.132 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24992 network=udp +34:43.132 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:25004) (DC: dc1)" +34:43.132 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +34:43.132 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24992 network=tcp +34:43.133 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25014 network=tcp +34:43.133 [INFO] TestAgent: started state syncer +34:43.133 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25005 network=tcp +34:43.185 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:43.185 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25004 [Candidate]" term=2 +34:43.186 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:43.186 [DEBUG] TestAgent.server.raft: vote granted: from=8ebead8a-2b8c-9d30-d963-95196d6d7944 term=2 tally=1 +34:43.186 [INFO] TestAgent.server.raft: election won: tally=1 +34:43.186 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25004 [Leader]" +34:43.186 [INFO] TestAgent.server: cluster leadership acquired +34:43.186 [INFO] TestAgent.server: New leader elected: payload=test-node +34:43.186 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25004 +34:43.187 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:43.189 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:43.190 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:43.190 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:43.190 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:43.190 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:43.190 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +34:43.190 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +34:43.191 [INFO] TestAgent.server: federation state anti-entropy synced +34:43.391 [DEBUG] TestAgent.dns: request served from client: name=test-node.node.consul. type=SOA class=IN latency=214.96µs client=127.0.0.1:49643 client_network=udp +34:43.392 [DEBUG] TestAgent.dns: request served from client: name=test-node.node.test-domain. type=SOA class=IN latency=154.78µs client=127.0.0.1:50345 client_network=udp +34:43.392 [INFO] TestAgent: Requesting shutdown +34:43.392 [INFO] TestAgent.server: shutting down server +34:43.392 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:43.392 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:43.392 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:43.392 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:43.392 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:43.392 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:43.392 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:43.392 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:43.392 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:43.392 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:43.392 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:43.392 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:43.392Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:43.392Z [INFO] agent.router.manager: shutting down +34:43.392 [INFO] TestAgent: consul server down +34:43.392 [INFO] TestAgent: shutdown complete +34:43.392 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24992 network=tcp +34:43.393 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24992 network=udp +34:43.393 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25014 network=tcp +34:43.893 [INFO] TestAgent: Waiting for endpoints to shut down +34:43.893 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_AltDomains_SOA (0.80s) +=== CONT TestDNS_ServiceLookup_ServiceAddress_CNAME +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:43.118Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:43.135Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:43.138 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b1e71c58-031f-4f3e-9154-8d6c7005633e Address:127.0.0.1:24638}]" +34:43.138 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24638 [Follower]" leader= +34:43.139 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b1e71c58-031f-4f3e-9154-8d6c7005633e.dc1 127.0.0.1 +34:43.140 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b1e71c58-031f-4f3e-9154-8d6c7005633e 127.0.0.1 +2021-08-13T15:34:43.140Z [INFO] agent.router: Initializing LAN area manager +34:43.140 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25027 network=udp +34:43.140 [INFO] TestAgent.server: Adding LAN server: server="Node-b1e71c58-031f-4f3e-9154-8d6c7005633e (Addr: tcp/127.0.0.1:24638) (DC: dc1)" +34:43.140 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b1e71c58-031f-4f3e-9154-8d6c7005633e.dc1 area=wan +34:43.141 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25027 network=tcp +34:43.141 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24488 network=tcp +34:43.142 [INFO] TestAgent: started state syncer +34:43.142 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24639 network=tcp +34:43.174 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:43.174 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24638 [Candidate]" term=2 +34:43.174 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:43.174 [DEBUG] TestAgent.server.raft: vote granted: from=b1e71c58-031f-4f3e-9154-8d6c7005633e term=2 tally=1 +34:43.174 [INFO] TestAgent.server.raft: election won: tally=1 +34:43.174 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24638 [Leader]" +34:43.174 [INFO] TestAgent.server: cluster leadership acquired +34:43.175 [INFO] TestAgent.server: New leader elected: payload=Node-b1e71c58-031f-4f3e-9154-8d6c7005633e +34:43.175 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24638 +34:43.176 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:43.178 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:43.178 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:43.178 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:43.178 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:43.178 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:43.178 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b1e71c58-031f-4f3e-9154-8d6c7005633e +34:43.178 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b1e71c58-031f-4f3e-9154-8d6c7005633e +34:43.180 [INFO] TestAgent.server: federation state anti-entropy synced +34:43.535 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:43.536 [INFO] TestAgent: Synced node info +34:43.573 [DEBUG] TestAgent.dns: request served from client: name=db.query.consul. type=SRV class=IN latency=247.68µs client=127.0.0.1:60186 client_network=udp +34:43.574 [DEBUG] TestAgent.dns: request served from client: name=db-ttl.query.consul. type=SRV class=IN latency=191.621µs client=127.0.0.1:39043 client_network=udp +34:43.575 [DEBUG] TestAgent.dns: request served from client: name=dblb.query.consul. type=SRV class=IN latency=199.08µs client=127.0.0.1:50016 client_network=udp +34:43.579 [DEBUG] TestAgent.dns: request served from client: name=dblb-ttl.query.consul. type=SRV class=IN latency=237.5µs client=127.0.0.1:43343 client_network=udp +34:43.580 [DEBUG] TestAgent.dns: request served from client: name=dk.query.consul. type=SRV class=IN latency=174.361µs client=127.0.0.1:35371 client_network=udp +34:43.580 [DEBUG] TestAgent.dns: request served from client: name=dk-ttl.query.consul. type=SRV class=IN latency=171.101µs client=127.0.0.1:32854 client_network=udp +34:43.581 [DEBUG] TestAgent.dns: request served from client: name=api.query.consul. type=SRV class=IN latency=164.56µs client=127.0.0.1:56314 client_network=udp +34:43.581 [DEBUG] TestAgent.dns: request served from client: name=api-ttl.query.consul. type=SRV class=IN latency=165.38µs client=127.0.0.1:50824 client_network=udp +34:43.582 [INFO] TestAgent: Requesting shutdown +34:43.582 [INFO] TestAgent.server: shutting down server +34:43.582 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:43.582 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:43.582 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:43.582 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:43.582 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:43.582 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:43.582 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:43.582 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:43.582 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:43.582 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:43.582 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:43.582Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:43.582Z [INFO] agent.router.manager: shutting down +34:43.583 [INFO] TestAgent: consul server down +34:43.583 [INFO] TestAgent: shutdown complete +34:43.583 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25027 network=tcp +34:43.583 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25027 network=udp +34:43.583 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24488 network=tcp +34:44.083 [INFO] TestAgent: Waiting for endpoints to shut down +34:44.083 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_PreparedQuery_TTL (0.98s) + --- PASS: TestDNS_PreparedQuery_TTL/db.query.consul. (0.00s) + --- PASS: TestDNS_PreparedQuery_TTL/db-ttl.query.consul. (0.00s) + --- PASS: TestDNS_PreparedQuery_TTL/dblb.query.consul. (0.00s) + --- PASS: TestDNS_PreparedQuery_TTL/dblb-ttl.query.consul. (0.00s) + --- PASS: TestDNS_PreparedQuery_TTL/dk.query.consul. (0.00s) + --- PASS: TestDNS_PreparedQuery_TTL/dk-ttl.query.consul. (0.00s) + --- PASS: TestDNS_PreparedQuery_TTL/api.query.consul. (0.00s) + --- PASS: TestDNS_PreparedQuery_TTL/api-ttl.query.consul. (0.00s) +=== CONT TestDNS_NonExistentDC_RPC +2021-08-13T15:34:43.461Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:43.485Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:43.496 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:85b4aef1-7fb9-a6d5-1026-afe13116bca4 Address:127.0.0.1:25025}]" +34:43.496 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25025 [Follower]" leader= +34:43.497 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-85b4aef1-7fb9-a6d5-1026-afe13116bca4.dc1 127.0.0.1 +34:43.498 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-85b4aef1-7fb9-a6d5-1026-afe13116bca4 127.0.0.1 +2021-08-13T15:34:43.498Z [INFO] agent.router: Initializing LAN area manager +34:43.498 [INFO] TestAgent.server: Adding LAN server: server="Node-85b4aef1-7fb9-a6d5-1026-afe13116bca4 (Addr: tcp/127.0.0.1:25025) (DC: dc1)" +34:43.498 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-85b4aef1-7fb9-a6d5-1026-afe13116bca4.dc1 area=wan +34:43.498 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25013 network=udp +34:43.498 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25013 network=tcp +34:43.509 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25007 network=tcp +34:43.509 [INFO] TestAgent: started state syncer +34:43.510 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25026 network=tcp +34:43.561 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:43.561 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25025 [Candidate]" term=2 +34:43.561 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:43.561 [DEBUG] TestAgent.server.raft: vote granted: from=85b4aef1-7fb9-a6d5-1026-afe13116bca4 term=2 tally=1 +34:43.561 [INFO] TestAgent.server.raft: election won: tally=1 +34:43.561 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25025 [Leader]" +34:43.562 [INFO] TestAgent.server: cluster leadership acquired +34:43.562 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25025 +34:43.563 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:43.566 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:43.566 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:43.566 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:43.566 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:43.566 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:43.566 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-85b4aef1-7fb9-a6d5-1026-afe13116bca4 +34:43.566 [INFO] TestAgent.server: member joined, marking health alive: member=Node-85b4aef1-7fb9-a6d5-1026-afe13116bca4 +34:43.567 [INFO] TestAgent.server: New leader elected: payload=Node-85b4aef1-7fb9-a6d5-1026-afe13116bca4 +34:43.568 [INFO] TestAgent.server: federation state anti-entropy synced +34:43.603 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:43.604 [INFO] TestAgent: Synced node info +34:43.617 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=342µs client=127.0.0.1:33369 client_network=udp +34:43.619 [DEBUG] TestAgent.dns: request served from client: name=dblb.service.consul. type=SRV class=IN latency=240.941µs client=127.0.0.1:42770 client_network=udp +34:43.620 [DEBUG] TestAgent.dns: request served from client: name=dk.service.consul. type=SRV class=IN latency=183.24µs client=127.0.0.1:41707 client_network=udp +34:43.620 [DEBUG] TestAgent.dns: request served from client: name=api.service.consul. type=SRV class=IN latency=179.92µs client=127.0.0.1:53839 client_network=udp +34:43.620 [INFO] TestAgent: Requesting shutdown +34:43.620 [INFO] TestAgent.server: shutting down server +34:43.620 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:43.620 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:43.620 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:43.620 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:43.620 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:43.620 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:43.620 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:43.620 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:43.620 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:43.620 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:43.620 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:43.621Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:43.621Z [INFO] agent.router.manager: shutting down +34:43.621 [INFO] TestAgent: consul server down +34:43.621 [INFO] TestAgent: shutdown complete +34:43.621 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25013 network=tcp +34:43.621 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25013 network=udp +34:43.621 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25007 network=tcp +34:44.121 [INFO] TestAgent: Waiting for endpoints to shut down +34:44.121 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup_TTL (0.68s) + --- PASS: TestDNS_ServiceLookup_TTL/db.service.consul. (0.00s) + --- PASS: TestDNS_ServiceLookup_TTL/dblb.service.consul. (0.00s) + --- PASS: TestDNS_ServiceLookup_TTL/dk.service.consul. (0.00s) + --- PASS: TestDNS_ServiceLookup_TTL/api.service.consul. (0.00s) +=== CONT TestDNS_ServiceLookup_AnswerLimits +=== RUN TestDNS_ServiceLookup_AnswerLimits/A_lookup_{0_0_0_0_0_0_0_0_0_0_0} +=== PAUSE TestDNS_ServiceLookup_AnswerLimits/A_lookup_{0_0_0_0_0_0_0_0_0_0_0} +=== RUN TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{0_0_0_0_0_0_0_0_0_0_0} +=== PAUSE TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{0_0_0_0_0_0_0_0_0_0_0} +=== RUN TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{0_0_0_0_0_0_0_0_0_0_0} +=== PAUSE TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{0_0_0_0_0_0_0_0_0_0_0} +=== RUN TestDNS_ServiceLookup_AnswerLimits/A_lookup_{1_1_1_1_1_1_1_1_1_1_1} +=== PAUSE TestDNS_ServiceLookup_AnswerLimits/A_lookup_{1_1_1_1_1_1_1_1_1_1_1} +=== RUN TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{1_1_1_1_1_1_1_1_1_1_1} +=== PAUSE TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{1_1_1_1_1_1_1_1_1_1_1} +=== RUN TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{1_1_1_1_1_1_1_1_1_1_1} +=== PAUSE TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{1_1_1_1_1_1_1_1_1_1_1} +=== RUN TestDNS_ServiceLookup_AnswerLimits/A_lookup_{2_2_2_2_2_2_2_2_2_2_2} +=== PAUSE TestDNS_ServiceLookup_AnswerLimits/A_lookup_{2_2_2_2_2_2_2_2_2_2_2} +=== RUN TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{2_2_2_2_2_2_2_2_2_2_2} +=== PAUSE TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{2_2_2_2_2_2_2_2_2_2_2} +=== RUN TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{2_2_2_2_2_2_2_2_2_2_2} +=== PAUSE TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{2_2_2_2_2_2_2_2_2_2_2} +=== RUN TestDNS_ServiceLookup_AnswerLimits/A_lookup_{3_3_3_3_3_3_3_3_3_3_3} +=== PAUSE TestDNS_ServiceLookup_AnswerLimits/A_lookup_{3_3_3_3_3_3_3_3_3_3_3} +=== RUN TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{3_3_3_3_3_3_3_3_3_3_3} +=== PAUSE TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{3_3_3_3_3_3_3_3_3_3_3} +=== RUN TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{3_3_3_3_3_3_3_3_3_3_3} +=== PAUSE TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{3_3_3_3_3_3_3_3_3_3_3} +=== RUN TestDNS_ServiceLookup_AnswerLimits/A_lookup_{4_4_4_4_4_4_4_4_4_4_4} +=== PAUSE TestDNS_ServiceLookup_AnswerLimits/A_lookup_{4_4_4_4_4_4_4_4_4_4_4} +=== RUN TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{4_4_4_4_4_4_4_4_4_4_4} +=== PAUSE TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{4_4_4_4_4_4_4_4_4_4_4} +=== RUN TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{4_4_4_4_4_4_4_4_4_4_4} +=== PAUSE TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{4_4_4_4_4_4_4_4_4_4_4} +=== RUN TestDNS_ServiceLookup_AnswerLimits/A_lookup_{5_5_5_5_5_5_5_5_5_5_5} +=== PAUSE TestDNS_ServiceLookup_AnswerLimits/A_lookup_{5_5_5_5_5_5_5_5_5_5_5} +=== RUN TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{5_5_5_5_5_5_5_5_5_5_5} +=== PAUSE TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{5_5_5_5_5_5_5_5_5_5_5} +=== RUN TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{5_5_5_5_5_5_5_5_5_5_5} +=== PAUSE TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{5_5_5_5_5_5_5_5_5_5_5} +=== RUN TestDNS_ServiceLookup_AnswerLimits/A_lookup_{6_6_6_6_6_6_6_5_6_6_-5} +=== PAUSE TestDNS_ServiceLookup_AnswerLimits/A_lookup_{6_6_6_6_6_6_6_5_6_6_-5} +=== RUN TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{6_6_6_6_6_6_6_5_6_6_-5} +=== PAUSE TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{6_6_6_6_6_6_6_5_6_6_-5} +=== RUN TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{6_6_6_6_6_6_6_5_6_6_-5} +=== PAUSE TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{6_6_6_6_6_6_6_5_6_6_-5} +=== RUN TestDNS_ServiceLookup_AnswerLimits/A_lookup_{7_7_7_7_6_7_7_5_7_7_-5} +=== PAUSE TestDNS_ServiceLookup_AnswerLimits/A_lookup_{7_7_7_7_6_7_7_5_7_7_-5} +=== RUN TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{7_7_7_7_6_7_7_5_7_7_-5} +=== PAUSE TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{7_7_7_7_6_7_7_5_7_7_-5} +=== RUN TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{7_7_7_7_6_7_7_5_7_7_-5} +=== PAUSE TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{7_7_7_7_6_7_7_5_7_7_-5} +=== RUN TestDNS_ServiceLookup_AnswerLimits/A_lookup_{8_8_8_8_6_8_8_5_8_8_-5} +=== PAUSE TestDNS_ServiceLookup_AnswerLimits/A_lookup_{8_8_8_8_6_8_8_5_8_8_-5} +=== RUN TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{8_8_8_8_6_8_8_5_8_8_-5} +=== PAUSE TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{8_8_8_8_6_8_8_5_8_8_-5} +=== RUN TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{8_8_8_8_6_8_8_5_8_8_-5} +=== PAUSE TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{8_8_8_8_6_8_8_5_8_8_-5} +=== RUN TestDNS_ServiceLookup_AnswerLimits/A_lookup_{9_9_8_8_6_8_8_5_8_8_-5} +=== PAUSE TestDNS_ServiceLookup_AnswerLimits/A_lookup_{9_9_8_8_6_8_8_5_8_8_-5} +=== RUN TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{9_9_8_8_6_8_8_5_8_8_-5} +=== PAUSE TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{9_9_8_8_6_8_8_5_8_8_-5} +=== RUN TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{9_9_8_8_6_8_8_5_8_8_-5} +=== PAUSE TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{9_9_8_8_6_8_8_5_8_8_-5} +=== RUN TestDNS_ServiceLookup_AnswerLimits/A_lookup_{20_20_8_8_6_8_8_5_8_-5_-5} +=== PAUSE TestDNS_ServiceLookup_AnswerLimits/A_lookup_{20_20_8_8_6_8_8_5_8_-5_-5} +=== RUN TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{20_20_8_8_6_8_8_5_8_-5_-5} +=== PAUSE TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{20_20_8_8_6_8_8_5_8_-5_-5} +=== RUN TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{20_20_8_8_6_8_8_5_8_-5_-5} +=== PAUSE TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{20_20_8_8_6_8_8_5_8_-5_-5} +=== RUN TestDNS_ServiceLookup_AnswerLimits/A_lookup_{30_30_8_8_6_8_8_5_8_-5_-5} +=== PAUSE TestDNS_ServiceLookup_AnswerLimits/A_lookup_{30_30_8_8_6_8_8_5_8_-5_-5} +=== RUN TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{30_30_8_8_6_8_8_5_8_-5_-5} +=== PAUSE TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{30_30_8_8_6_8_8_5_8_-5_-5} +=== RUN TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{30_30_8_8_6_8_8_5_8_-5_-5} +=== PAUSE TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{30_30_8_8_6_8_8_5_8_-5_-5} +=== CONT TestDNS_NonExistentDC_Server +2021-08-13T15:34:43.235Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:43.251Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:43.254 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d5db532a-ea3c-f36c-a33d-b059f2627f13 Address:127.0.0.1:24983}]" +34:43.254 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24983 [Follower]" leader= +34:43.255 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d5db532a-ea3c-f36c-a33d-b059f2627f13.dc1 127.0.0.1 +34:43.256 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d5db532a-ea3c-f36c-a33d-b059f2627f13 127.0.0.1 +2021-08-13T15:34:43.256Z [INFO] agent.router: Initializing LAN area manager +34:43.256 [INFO] TestAgent.server: Adding LAN server: server="Node-d5db532a-ea3c-f36c-a33d-b059f2627f13 (Addr: tcp/127.0.0.1:24983) (DC: dc1)" +34:43.256 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d5db532a-ea3c-f36c-a33d-b059f2627f13.dc1 area=wan +34:43.256 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24640 network=udp +34:43.256 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24640 network=tcp +34:43.257 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24979 network=tcp +34:43.257 [INFO] TestAgent: started state syncer +34:43.257 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24984 network=tcp +34:43.323 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:43.323 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24983 [Candidate]" term=2 +34:43.324 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:43.324 [DEBUG] TestAgent.server.raft: vote granted: from=d5db532a-ea3c-f36c-a33d-b059f2627f13 term=2 tally=1 +34:43.324 [INFO] TestAgent.server.raft: election won: tally=1 +34:43.324 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24983 [Leader]" +34:43.324 [INFO] TestAgent.server: cluster leadership acquired +34:43.324 [INFO] TestAgent.server: New leader elected: payload=Node-d5db532a-ea3c-f36c-a33d-b059f2627f13 +34:43.324 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24983 +34:43.325 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:43.328 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:43.328 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:43.328 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:43.328 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:43.328 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:43.328 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d5db532a-ea3c-f36c-a33d-b059f2627f13 +34:43.328 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d5db532a-ea3c-f36c-a33d-b059f2627f13 +34:43.329 [INFO] TestAgent.server: federation state anti-entropy synced +34:43.380 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:43.381 [INFO] TestAgent: Synced node info +34:43.381 [DEBUG] TestAgent: Node info in sync +34:43.655 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=287.621µs client=127.0.0.1:36378 client_network=udp +34:43.659 [DEBUG] TestAgent.dns: request served from client: name=db.service.test-domain. type=SRV class=IN latency=263.1µs client=127.0.0.1:36224 client_network=udp +34:43.660 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc1.consul. type=SRV class=IN latency=230.981µs client=127.0.0.1:36112 client_network=udp +34:43.663 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc1.test-domain. type=SRV class=IN latency=215.521µs client=127.0.0.1:51057 client_network=udp +34:43.663 [INFO] TestAgent: Requesting shutdown +34:43.663 [INFO] TestAgent.server: shutting down server +34:43.663 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:43.663 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:43.663 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:43.663 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:43.663 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:43.663 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:43.663 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +34:43.663 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:43.663 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:43.663 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:43.663 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2021-08-13T15:34:43.663Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:43.663Z [INFO] agent.router.manager: shutting down +34:43.663 [INFO] TestAgent: consul server down +34:43.663 [INFO] TestAgent: shutdown complete +34:43.663 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24640 network=tcp +34:43.663 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24640 network=udp +34:43.663 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24979 network=tcp +34:44.166 [INFO] TestAgent: Waiting for endpoints to shut down +34:44.166 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_AltDomains_Service (0.95s) +=== CONT TestDNS_ServiceLookup_LargeResponses +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:43.640Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:43.657Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:43.660 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8614c021-5150-028d-a21b-321b8f79d706 Address:127.0.0.1:25043}]" +34:43.660 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25043 [Follower]" leader= +34:43.661 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8614c021-5150-028d-a21b-321b8f79d706.dc1 127.0.0.1 +34:43.662 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8614c021-5150-028d-a21b-321b8f79d706 127.0.0.1 +2021-08-13T15:34:43.662Z [INFO] agent.router: Initializing LAN area manager +34:43.662 [INFO] TestAgent.server: Adding LAN server: server="Node-8614c021-5150-028d-a21b-321b8f79d706 (Addr: tcp/127.0.0.1:25043) (DC: dc1)" +34:43.662 [DEBUG] TestAgent.dns: recursor enabled +34:43.662 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8614c021-5150-028d-a21b-321b8f79d706.dc1 area=wan +34:43.662 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24978 network=udp +34:43.662 [DEBUG] TestAgent.dns: recursor enabled +34:43.662 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24978 network=tcp +34:43.663 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25011 network=tcp +34:43.663 [INFO] TestAgent: started state syncer +34:43.663 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25030 network=tcp +34:43.720 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:43.720 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25043 [Candidate]" term=2 +34:43.721 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:43.721 [DEBUG] TestAgent.server.raft: vote granted: from=8614c021-5150-028d-a21b-321b8f79d706 term=2 tally=1 +34:43.721 [INFO] TestAgent.server.raft: election won: tally=1 +34:43.721 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25043 [Leader]" +34:43.721 [INFO] TestAgent.server: cluster leadership acquired +34:43.721 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25043 +34:43.721 [INFO] TestAgent.server: New leader elected: payload=Node-8614c021-5150-028d-a21b-321b8f79d706 +34:43.722 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:43.725 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:43.725 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:43.725 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:43.725 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:43.725 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:43.725 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8614c021-5150-028d-a21b-321b8f79d706 +34:43.725 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8614c021-5150-028d-a21b-321b8f79d706 +34:43.726 [INFO] TestAgent.server: federation state anti-entropy synced +34:43.795 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=ANY class=IN latency=151.12µs client=127.0.0.1:51290 client_network=udp +34:43.795 [DEBUG] TestAgent.dns: request served from client: name=bar.node.consul. type=ANY class=IN latency=115.101µs client=127.0.0.1:42506 client_network=udp +34:43.796 [DEBUG] TestAgent.dns: cname recurse RTT for name: name=www.google.com. rtt=162.281µs +34:43.796 [DEBUG] TestAgent.dns: request served from client: name=google.node.consul. type=ANY class=IN latency=387.281µs client=127.0.0.1:48158 client_network=udp +34:43.796 [INFO] TestAgent: Requesting shutdown +34:43.796 [INFO] TestAgent.server: shutting down server +34:43.796 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:43.796 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:43.796 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:43.796 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:43.796 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:43.796 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:43.796 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:43.796 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:43.796 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:43.797 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:43.797 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:43.797 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:43.797Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:43.797Z [INFO] agent.router.manager: shutting down +34:43.797 [INFO] TestAgent: consul server down +34:43.797 [INFO] TestAgent: shutdown complete +34:43.797 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24978 network=tcp +34:43.797 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24978 network=udp +34:43.797 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25011 network=tcp +34:44.297 [INFO] TestAgent: Waiting for endpoints to shut down +34:44.297 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_NodeLookup_TTL (0.68s) +=== CONT TestDNS_ServiceLookup_Truncate +2021-08-13T15:34:43.576Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:43.609Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:43.615 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:eb5299f8-e4dd-0e8e-ef07-4c0c870b46c4 Address:127.0.0.1:25018}]" +34:43.616 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25018 [Follower]" leader= +34:43.616 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-eb5299f8-e4dd-0e8e-ef07-4c0c870b46c4.dc1 127.0.0.1 +34:43.625 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-eb5299f8-e4dd-0e8e-ef07-4c0c870b46c4 127.0.0.1 +2021-08-13T15:34:43.625Z [INFO] agent.router: Initializing LAN area manager +34:43.625 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24487 network=udp +34:43.625 [INFO] TestAgent.server: Adding LAN server: server="Node-eb5299f8-e4dd-0e8e-ef07-4c0c870b46c4 (Addr: tcp/127.0.0.1:25018) (DC: dc1)" +34:43.625 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-eb5299f8-e4dd-0e8e-ef07-4c0c870b46c4.dc1 area=wan +34:43.626 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24487 network=tcp +34:43.627 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25028 network=tcp +34:43.627 [INFO] TestAgent: started state syncer +34:43.627 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25019 network=tcp +34:43.678 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:43.678 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25018 [Candidate]" term=2 +34:43.678 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:43.678 [DEBUG] TestAgent.server.raft: vote granted: from=eb5299f8-e4dd-0e8e-ef07-4c0c870b46c4 term=2 tally=1 +34:43.678 [INFO] TestAgent.server.raft: election won: tally=1 +34:43.678 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25018 [Leader]" +34:43.678 [INFO] TestAgent.server: cluster leadership acquired +34:43.679 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25018 +34:43.679 [INFO] TestAgent.server: New leader elected: payload=Node-eb5299f8-e4dd-0e8e-ef07-4c0c870b46c4 +34:43.680 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:43.682 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:43.682 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:43.682 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:43.682 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:43.682 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:43.682 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-eb5299f8-e4dd-0e8e-ef07-4c0c870b46c4 +34:43.682 [INFO] TestAgent.server: member joined, marking health alive: member=Node-eb5299f8-e4dd-0e8e-ef07-4c0c870b46c4 +34:43.683 [INFO] TestAgent.server: federation state anti-entropy synced +34:43.822 [DEBUG] TestAgent.dns: request served from client: name=webv4.service.consul. type=AAAA class=IN latency=262.261µs client=127.0.0.1:47028 client_network=udp +34:43.822 [DEBUG] TestAgent.dns: request served from client: name=webv4.query.consul. type=AAAA class=IN latency=148.78µs client=127.0.0.1:55872 client_network=udp +34:43.823 [DEBUG] TestAgent.dns: request served from client: name=webv6.service.consul. type=A class=IN latency=175.721µs client=127.0.0.1:53308 client_network=udp +34:43.823 [DEBUG] TestAgent.dns: request served from client: name=webv6.query.consul. type=A class=IN latency=147.681µs client=127.0.0.1:58834 client_network=udp +34:43.823 [INFO] TestAgent: Requesting shutdown +34:43.823 [INFO] TestAgent.server: shutting down server +34:43.823 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:43.823 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:43.823 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:43.823 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:43.823 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:43.823 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:43.823 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:43.823 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:43.823 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:43.823 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:43.823 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:43.823 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:43.824Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:43.824Z [INFO] agent.router.manager: shutting down +34:43.824 [INFO] TestAgent: consul server down +34:43.824 [INFO] TestAgent: shutdown complete +34:43.824 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24487 network=tcp +34:43.824 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24487 network=udp +34:43.824 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25028 network=tcp +34:44.324 [INFO] TestAgent: Waiting for endpoints to shut down +34:44.324 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_NonExistingLookupEmptyAorAAAA (0.83s) +=== CONT TestDNS_TCP_and_UDP_Truncate +2021-08-13T15:34:43.764Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:43.780Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:43.783 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3b192a17-cb79-307c-fe82-6bf40bff39e5 Address:127.0.0.1:25001}]" +34:43.783 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25001 [Follower]" leader= +34:43.784 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3b192a17-cb79-307c-fe82-6bf40bff39e5.dc1 127.0.0.1 +34:43.785 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3b192a17-cb79-307c-fe82-6bf40bff39e5 127.0.0.1 +2021-08-13T15:34:43.785Z [INFO] agent.router: Initializing LAN area manager +34:43.785 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25031 network=udp +34:43.785 [INFO] TestAgent.server: Adding LAN server: server="Node-3b192a17-cb79-307c-fe82-6bf40bff39e5 (Addr: tcp/127.0.0.1:25001) (DC: dc1)" +34:43.785 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3b192a17-cb79-307c-fe82-6bf40bff39e5.dc1 area=wan +34:43.785 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25031 network=tcp +34:43.786 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24508 network=tcp +34:43.786 [INFO] TestAgent: started state syncer +34:43.786 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25009 network=tcp +34:43.841 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:43.841 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25001 [Candidate]" term=2 +34:43.841 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:43.841 [DEBUG] TestAgent.server.raft: vote granted: from=3b192a17-cb79-307c-fe82-6bf40bff39e5 term=2 tally=1 +34:43.841 [INFO] TestAgent.server.raft: election won: tally=1 +34:43.841 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25001 [Leader]" +34:43.841 [INFO] TestAgent.server: cluster leadership acquired +34:43.842 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25001 +34:43.843 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:43.844 [INFO] TestAgent.server: New leader elected: payload=Node-3b192a17-cb79-307c-fe82-6bf40bff39e5 +34:43.845 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:43.845 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:43.845 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:43.845 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:43.845 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:43.845 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3b192a17-cb79-307c-fe82-6bf40bff39e5 +34:43.845 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3b192a17-cb79-307c-fe82-6bf40bff39e5 +34:43.846 [INFO] TestAgent.server: federation state anti-entropy synced +34:43.871 [WARN] TestAgent.dns: QName invalid: qname=nonexisting. +34:43.872 [DEBUG] TestAgent.dns: request served from client: name=nonexisting.consul. type=ANY class=IN latency=116.28µs client=127.0.0.1:55387 client_network=udp +34:43.872 [INFO] TestAgent: Requesting shutdown +34:43.872 [INFO] TestAgent.server: shutting down server +34:43.872 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:43.872 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:43.872 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:43.872 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:43.872 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:43.872 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:43.872 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:43.872 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:43.872 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:43.872 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:43.872 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:43.872 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:43.872Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:43.872Z [INFO] agent.router.manager: shutting down +34:43.872 [INFO] TestAgent: consul server down +34:43.872 [INFO] TestAgent: shutdown complete +34:43.872 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25031 network=tcp +34:43.872 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25031 network=udp +34:43.872 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24508 network=tcp +34:44.373 [INFO] TestAgent: Waiting for endpoints to shut down +34:44.373 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_NonExistingLookup (0.62s) +=== CONT TestDNS_ServiceLookup_Dedup +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=255)_compressed=true +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=255)_compressed=false +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=255)_compressed=true +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=255)_compressed=false +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._tcp_(qType:=255)_compressed=true +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._tcp_(qType:=255)_compressed=false +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._udp_(qType:=255)_compressed=true +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._udp_(qType:=255)_compressed=false +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=1)_compressed=true +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=1)_compressed=false +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=1)_compressed=true +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=1)_compressed=false +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._tcp_(qType:=1)_compressed=true +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._tcp_(qType:=1)_compressed=false +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._udp_(qType:=1)_compressed=true +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._udp_(qType:=1)_compressed=false +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=33)_compressed=true +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=33)_compressed=false +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=33)_compressed=true +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=33)_compressed=false +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._tcp_(qType:=33)_compressed=true +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._tcp_(qType:=33)_compressed=false +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._udp_(qType:=33)_compressed=true +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._udp_(qType:=33)_compressed=false +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=255)_compressed=true#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=255)_compressed=false#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=255)_compressed=true#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=255)_compressed=false#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._tcp_(qType:=255)_compressed=true#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._tcp_(qType:=255)_compressed=false#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._udp_(qType:=255)_compressed=true#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._udp_(qType:=255)_compressed=false#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=1)_compressed=true#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=1)_compressed=false#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=1)_compressed=true#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=1)_compressed=false#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._tcp_(qType:=1)_compressed=true#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._tcp_(qType:=1)_compressed=false#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._udp_(qType:=1)_compressed=true#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._udp_(qType:=1)_compressed=false#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=33)_compressed=true#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=33)_compressed=false#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=33)_compressed=true#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=33)_compressed=false#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._tcp_(qType:=33)_compressed=true#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._tcp_(qType:=33)_compressed=false#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._udp_(qType:=33)_compressed=true#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._udp_(qType:=33)_compressed=false#01 +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:43.341Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:43.341Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:43.357Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:43.357Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:43.360 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6eb2c3ea-c782-4e65-9ed8-9bba873364d2 Address:127.0.0.1:24997}]" +34:43.360 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24997 [Follower]" leader= +34:43.361 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6eb2c3ea-c782-4e65-9ed8-9bba873364d2.dc2 127.0.0.1 +34:43.362 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6eb2c3ea-c782-4e65-9ed8-9bba873364d2 127.0.0.1 +2021-08-13T15:34:43.362Z [INFO] agent.router: Initializing LAN area manager +34:43.362 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6eb2c3ea-c782-4e65-9ed8-9bba873364d2.dc2 area=wan +34:43.362 [INFO] TestAgent.server: Adding LAN server: server="Node-6eb2c3ea-c782-4e65-9ed8-9bba873364d2 (Addr: tcp/127.0.0.1:24997) (DC: dc2)" +34:43.362 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25006 network=udp +34:43.362 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25006 network=tcp +34:43.363 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24993 network=tcp +34:43.363 [INFO] TestAgent: started state syncer +34:43.363 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24998 network=tcp +34:43.412 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:43.427 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:43.427 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24997 [Candidate]" term=2 +34:43.427 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:43.427 [DEBUG] TestAgent.server.raft: vote granted: from=6eb2c3ea-c782-4e65-9ed8-9bba873364d2 term=2 tally=1 +34:43.427 [INFO] TestAgent.server.raft: election won: tally=1 +34:43.427 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24997 [Leader]" +34:43.427 [INFO] TestAgent.server: cluster leadership acquired +34:43.427 [INFO] TestAgent.server: initializing acls +34:43.428 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:43.429 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:43.429 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:43.429 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:43.429 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-6eb2c3ea-c782-4e65-9ed8-9bba873364d2 +34:43.429 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-6eb2c3ea-c782-4e65-9ed8-9bba873364d2.dc2 +34:43.430 [INFO] TestAgent.server: New leader elected: payload=Node-6eb2c3ea-c782-4e65-9ed8-9bba873364d2 +34:43.430 [INFO] TestAgent.server: Updating LAN server: server="Node-6eb2c3ea-c782-4e65-9ed8-9bba873364d2 (Addr: tcp/127.0.0.1:24997) (DC: dc2)" +34:43.430 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:43.430 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-6eb2c3ea-c782-4e65-9ed8-9bba873364d2.dc2 area=wan +34:43.433 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:43.433 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:43.433 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:43.433 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:43.433 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:43.433 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6eb2c3ea-c782-4e65-9ed8-9bba873364d2 +34:43.433 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6eb2c3ea-c782-4e65-9ed8-9bba873364d2 +34:43.434 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6eb2c3ea-c782-4e65-9ed8-9bba873364d2 +34:43.434 [INFO] TestAgent.server: federation state anti-entropy synced +34:43.618 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:43.619 [INFO] TestAgent: Synced node info +34:43.619 [DEBUG] TestAgent: Node info in sync +34:43.765 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:24989] +34:43.766 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:24989 +34:43.767 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a8991bd6-87ab-15ac-bb58-b9fc70837e2f.dc1 127.0.0.1 +34:43.767 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 +34:43.769 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a8991bd6-87ab-15ac-bb58-b9fc70837e2f.dc1 area=wan +34:43.772 [INFO] TestAgent: Requesting shutdown +34:43.772 [INFO] TestAgent.server: shutting down server +34:43.772 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:43.772 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:43.772 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:43.772 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:43.772 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:43.772 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:43.772 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:43.772 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:43.772 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:43.772 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:43.772 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:43.772 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:43.772 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:43.772 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:43.773 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:43.773Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:43.773Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:43.773Z [INFO] agent.router.manager: shutting down +34:43.773 [INFO] TestAgent: consul server down +34:43.773 [INFO] TestAgent: shutdown complete +34:43.773 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25006 network=tcp +34:43.773 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25006 network=udp +34:43.773 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24993 network=tcp +34:44.279 [INFO] TestAgent: Waiting for endpoints to shut down +34:44.279 [INFO] TestAgent: Endpoints down +2021-08-13T15:34:42.980Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:42.980Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:43.007Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:43.007Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:43.010 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a8991bd6-87ab-15ac-bb58-b9fc70837e2f Address:127.0.0.1:24990}]" +34:43.011 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24990 [Follower]" leader= +34:43.011 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a8991bd6-87ab-15ac-bb58-b9fc70837e2f.dc1 127.0.0.1 +34:43.012 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a8991bd6-87ab-15ac-bb58-b9fc70837e2f 127.0.0.1 +2021-08-13T15:34:43.012Z [INFO] agent.router: Initializing LAN area manager +34:43.012 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24971 network=udp +34:43.012 [INFO] TestAgent.server: Adding LAN server: server="Node-a8991bd6-87ab-15ac-bb58-b9fc70837e2f (Addr: tcp/127.0.0.1:24990) (DC: dc1)" +34:43.013 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a8991bd6-87ab-15ac-bb58-b9fc70837e2f.dc1 area=wan +34:43.013 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24971 network=tcp +34:43.014 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24986 network=tcp +34:43.014 [INFO] TestAgent: started state syncer +34:43.014 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24991 network=tcp +34:43.063 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:43.076 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:43.076 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24990 [Candidate]" term=2 +34:43.077 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:43.077 [DEBUG] TestAgent.server.raft: vote granted: from=a8991bd6-87ab-15ac-bb58-b9fc70837e2f term=2 tally=1 +34:43.077 [INFO] TestAgent.server.raft: election won: tally=1 +34:43.077 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24990 [Leader]" +34:43.077 [INFO] TestAgent.server: cluster leadership acquired +34:43.077 [INFO] TestAgent.server: New leader elected: payload=Node-a8991bd6-87ab-15ac-bb58-b9fc70837e2f +34:43.077 [INFO] TestAgent.server: initializing acls +34:43.078 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:43.078 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:43.078 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:43.078 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:43.078 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-a8991bd6-87ab-15ac-bb58-b9fc70837e2f +34:43.079 [INFO] TestAgent.server: Updating LAN server: server="Node-a8991bd6-87ab-15ac-bb58-b9fc70837e2f (Addr: tcp/127.0.0.1:24990) (DC: dc1)" +34:43.079 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-a8991bd6-87ab-15ac-bb58-b9fc70837e2f.dc1 +34:43.079 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-a8991bd6-87ab-15ac-bb58-b9fc70837e2f.dc1 area=wan +34:43.080 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:43.082 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:43.082 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:43.082 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:43.082 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:43.082 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:43.082 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a8991bd6-87ab-15ac-bb58-b9fc70837e2f +34:43.082 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a8991bd6-87ab-15ac-bb58-b9fc70837e2f +34:43.083 [INFO] TestAgent.server: federation state anti-entropy synced +34:43.083 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a8991bd6-87ab-15ac-bb58-b9fc70837e2f +34:43.318 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:43.318 [INFO] TestAgent: Synced node info +34:43.318 [DEBUG] TestAgent: Node info in sync +34:43.766 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:53712 +34:43.767 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6eb2c3ea-c782-4e65-9ed8-9bba873364d2.dc2 127.0.0.1 +34:43.767 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6eb2c3ea-c782-4e65-9ed8-9bba873364d2.dc2 area=wan +34:43.772 [DEBUG] TestAgent.dns: request served from client: name=my-query.query.consul. type=SRV class=IN latency=2.403225ms client=127.0.0.1:58494 client_network=udp +34:44.279 [INFO] TestAgent: Requesting shutdown +34:44.279 [INFO] TestAgent.server: shutting down server +34:44.279 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:44.279 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:44.279 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:44.279 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:44.279 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:44.279 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:44.279 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:44.279 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:44.279 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:44.279 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:44.279 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:44.279 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:44.279 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:44.279 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:44.279 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:44.280Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:44.280Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:44.280Z [INFO] agent.router.manager: shutting down +34:44.280 [INFO] TestAgent: consul server down +34:44.280 [INFO] TestAgent: shutdown complete +34:44.280 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24971 network=tcp +34:44.280 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24971 network=udp +34:44.280 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24986 network=tcp +34:44.781 [INFO] TestAgent: Waiting for endpoints to shut down +34:44.781 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_PreparedQuery_Failover (1.83s) +=== CONT TestBinarySearch +2021-08-13T15:34:43.912Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:43.929Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:43.932 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5c8aad7d-a063-43b1-f607-ff50944e7ef1 Address:127.0.0.1:24631}]" +34:43.932 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24631 [Follower]" leader= +34:43.933 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5c8aad7d-a063-43b1-f607-ff50944e7ef1.dc1 127.0.0.1 +34:43.934 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5c8aad7d-a063-43b1-f607-ff50944e7ef1 127.0.0.1 +2021-08-13T15:34:43.934Z [INFO] agent.router: Initializing LAN area manager +34:43.934 [DEBUG] TestAgent.dns: recursor enabled +34:43.934 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25010 network=udp +34:43.935 [INFO] TestAgent.server: Adding LAN server: server="Node-5c8aad7d-a063-43b1-f607-ff50944e7ef1 (Addr: tcp/127.0.0.1:24631) (DC: dc1)" +34:43.935 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5c8aad7d-a063-43b1-f607-ff50944e7ef1.dc1 area=wan +34:43.935 [DEBUG] TestAgent.dns: recursor enabled +34:43.935 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25010 network=tcp +34:43.936 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24641 network=tcp +34:43.936 [INFO] TestAgent: started state syncer +34:43.936 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24632 network=tcp +34:43.998 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:43.998 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24631 [Candidate]" term=2 +34:43.998 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:43.998 [DEBUG] TestAgent.server.raft: vote granted: from=5c8aad7d-a063-43b1-f607-ff50944e7ef1 term=2 tally=1 +34:43.998 [INFO] TestAgent.server.raft: election won: tally=1 +34:43.998 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24631 [Leader]" +34:43.998 [INFO] TestAgent.server: cluster leadership acquired +34:43.998 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24631 +34:44.000 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:44.002 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:44.002 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:44.002 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:44.002 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:44.002 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:44.002 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5c8aad7d-a063-43b1-f607-ff50944e7ef1 +34:44.002 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5c8aad7d-a063-43b1-f607-ff50944e7ef1 +34:44.003 [INFO] TestAgent.server: New leader elected: payload=Node-5c8aad7d-a063-43b1-f607-ff50944e7ef1 +34:44.003 [INFO] TestAgent.server: federation state anti-entropy synced +34:44.280 [DEBUG] TestAgent.dns: cname recurse RTT for name: name=www.google.com. rtt=361.541µs +34:44.280 [DEBUG] TestAgent.dns: request served from client: name=search.service.consul. type=ANY class=IN latency=782.201µs client=127.0.0.1:59233 client_network=udp +34:44.281 [DEBUG] TestAgent.dns: cname recurse RTT for name: name=www.google.com. rtt=244.16µs +34:44.281 [DEBUG] TestAgent.dns: request served from client: name=b93a27ab-119f-aca1-cc77-567b11d7e4cd.query.consul. type=ANY class=IN latency=592.081µs client=127.0.0.1:34900 client_network=udp +34:44.281 [INFO] TestAgent: Requesting shutdown +34:44.281 [INFO] TestAgent.server: shutting down server +34:44.281 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:44.281 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:44.281 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:44.281 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:44.281 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:44.281 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:44.281 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:44.281 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:44.281 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:44.281 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:44.281 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:44.282 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:44.282Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:44.282Z [INFO] agent.router.manager: shutting down +34:44.282 [INFO] TestAgent: consul server down +34:44.282 [INFO] TestAgent: shutdown complete +34:44.282 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25010 network=tcp +34:44.282 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25010 network=udp +34:44.282 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24641 network=tcp +34:44.782 [INFO] TestAgent: Waiting for endpoints to shut down +34:44.782 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup_ServiceAddress_CNAME (0.89s) +=== CONT TestDNS_ServiceLookup_PreparedQueryNamePeriod +2021-08-13T15:34:44.148Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:44.178Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:44.181 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b86c665d-1702-b218-127d-3fe94e3934c4 Address:127.0.0.1:25071}]" +34:44.181 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25071 [Follower]" leader= +34:44.182 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b86c665d-1702-b218-127d-3fe94e3934c4.dc1 127.0.0.1 +34:44.183 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b86c665d-1702-b218-127d-3fe94e3934c4 127.0.0.1 +2021-08-13T15:34:44.183Z [INFO] agent.router: Initializing LAN area manager +34:44.183 [INFO] TestAgent.server: Adding LAN server: server="Node-b86c665d-1702-b218-127d-3fe94e3934c4 (Addr: tcp/127.0.0.1:25071) (DC: dc1)" +34:44.183 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b86c665d-1702-b218-127d-3fe94e3934c4.dc1 area=wan +34:44.184 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25052 network=tcp +34:44.184 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25052 network=udp +34:44.184 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25053 network=tcp +34:44.185 [INFO] TestAgent: started state syncer +34:44.185 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25058 network=tcp +34:44.223 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:44.223 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25071 [Candidate]" term=2 +34:44.223 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:44.223 [DEBUG] TestAgent.server.raft: vote granted: from=b86c665d-1702-b218-127d-3fe94e3934c4 term=2 tally=1 +34:44.223 [INFO] TestAgent.server.raft: election won: tally=1 +34:44.223 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25071 [Leader]" +34:44.224 [INFO] TestAgent.server: cluster leadership acquired +34:44.224 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25071 +34:44.226 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:44.227 [INFO] TestAgent.server: New leader elected: payload=Node-b86c665d-1702-b218-127d-3fe94e3934c4 +34:44.239 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:44.239 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:44.239 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:44.239 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:44.239 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:44.239 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b86c665d-1702-b218-127d-3fe94e3934c4 +34:44.239 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b86c665d-1702-b218-127d-3fe94e3934c4 +34:44.240 [INFO] TestAgent.server: federation state anti-entropy synced +34:44.290 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc2 method=Health.ServiceNodes +34:44.290 [ERROR] TestAgent.dns: rpc error: error="No path to datacenter" +34:44.290 [DEBUG] TestAgent.dns: request served from client: name=consul.service.dc2.consul. type=ANY class=IN latency=184.081µs client=127.0.0.1:35576 client_network=udp +34:44.290 [INFO] TestAgent: Requesting shutdown +34:44.291 [INFO] TestAgent.server: shutting down server +34:44.291 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:44.291 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:44.291 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:44.291 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:44.291 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:44.291 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:44.291 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:44.291 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:44.291 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:44.291 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:44.291 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +34:44.291 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2021-08-13T15:34:44.291Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:44.291Z [INFO] agent.router.manager: shutting down +34:44.291 [INFO] TestAgent: consul server down +34:44.291 [INFO] TestAgent: shutdown complete +34:44.291 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25052 network=tcp +34:44.291 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25052 network=udp +34:44.291 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25053 network=tcp +34:44.795 [INFO] TestAgent: Waiting for endpoints to shut down +34:44.795 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_NonExistentDC_Server (0.67s) +=== CONT TestDNS_ServiceLookup_Randomize +=== RUN TestBinarySearch/binarySearch_12 +=== RUN TestBinarySearch/binarySearch_256 +=== RUN TestBinarySearch/binarySearch_512 +=== RUN TestBinarySearch/binarySearch_8192 +=== RUN TestBinarySearch/binarySearch_65535 +=== RUN TestBinarySearch/binarySearch_12#01 +=== RUN TestBinarySearch/binarySearch_256#01 +=== RUN TestBinarySearch/binarySearch_512#01 +=== RUN TestBinarySearch/binarySearch_8192#01 +=== RUN TestBinarySearch/binarySearch_65535#01 +--- PASS: TestBinarySearch (0.23s) + --- PASS: TestBinarySearch/binarySearch_12 (0.03s) + --- PASS: TestBinarySearch/binarySearch_256 (0.03s) + --- PASS: TestBinarySearch/binarySearch_512 (0.01s) + --- PASS: TestBinarySearch/binarySearch_8192 (0.01s) + --- PASS: TestBinarySearch/binarySearch_65535 (0.04s) + --- PASS: TestBinarySearch/binarySearch_12#01 (0.03s) + --- PASS: TestBinarySearch/binarySearch_256#01 (0.01s) + --- PASS: TestBinarySearch/binarySearch_512#01 (0.01s) + --- PASS: TestBinarySearch/binarySearch_8192#01 (0.01s) + --- PASS: TestBinarySearch/binarySearch_65535#01 (0.02s) +=== CONT TestDNS_PreparedQueryNearIP +2021-08-13T15:34:44.196Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:44.252Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:44.256 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ebc89ea1-0c64-88ba-c011-e670e8343e94 Address:127.0.0.1:24505}]" +34:44.258 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ebc89ea1-0c64-88ba-c011-e670e8343e94.dc1 127.0.0.1 +34:44.259 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24505 [Follower]" leader= +34:44.261 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ebc89ea1-0c64-88ba-c011-e670e8343e94 127.0.0.1 +2021-08-13T15:34:44.267Z [INFO] agent.router: Initializing LAN area manager +34:44.268 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25059 network=udp +34:44.268 [INFO] TestAgent.server: Adding LAN server: server="Node-ebc89ea1-0c64-88ba-c011-e670e8343e94 (Addr: tcp/127.0.0.1:24505) (DC: dc1)" +34:44.268 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ebc89ea1-0c64-88ba-c011-e670e8343e94.dc1 area=wan +34:44.268 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25059 network=tcp +34:44.270 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24634 network=tcp +34:44.270 [INFO] TestAgent: started state syncer +34:44.270 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24506 network=tcp +34:44.315 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:44.315 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24505 [Candidate]" term=2 +34:44.315 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:44.315 [DEBUG] TestAgent.server.raft: vote granted: from=ebc89ea1-0c64-88ba-c011-e670e8343e94 term=2 tally=1 +34:44.315 [INFO] TestAgent.server.raft: election won: tally=1 +34:44.315 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24505 [Leader]" +34:44.315 [INFO] TestAgent.server: cluster leadership acquired +34:44.316 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24505 +34:44.317 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:44.319 [INFO] TestAgent.server: New leader elected: payload=Node-ebc89ea1-0c64-88ba-c011-e670e8343e94 +34:44.323 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:44.323 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:44.323 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:44.323 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:44.323 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:44.323 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ebc89ea1-0c64-88ba-c011-e670e8343e94 +34:44.323 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ebc89ea1-0c64-88ba-c011-e670e8343e94 +34:44.324 [INFO] TestAgent.server: federation state anti-entropy synced +34:44.521 [DEBUG] TestAgent.dns: request served from client: name=_this-is-a-very-very-very-very-very-long-name-for-a-service._primary.service.consul. type=SRV class=IN latency=274.001µs client=127.0.0.1:46646 client_network=udp +34:44.521 [DEBUG] TestAgent.dns: request served from client: name=this-is-a-very-very-very-very-very-long-name-for-a-service.query.consul. type=SRV class=IN latency=229.841µs client=127.0.0.1:59683 client_network=udp +34:44.521 [INFO] TestAgent: Requesting shutdown +34:44.521 [INFO] TestAgent.server: shutting down server +34:44.521 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:44.521 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:44.521 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:44.521 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:44.521 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:44.521 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:44.521 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:44.521 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:44.521 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:44.521 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:44.521 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:44.521 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:44.522Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:44.522Z [INFO] agent.router.manager: shutting down +34:44.522 [INFO] TestAgent: consul server down +34:44.522 [INFO] TestAgent: shutdown complete +34:44.522 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25059 network=tcp +34:44.522 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25059 network=udp +34:44.522 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24634 network=tcp +34:45.023 [INFO] TestAgent: Waiting for endpoints to shut down +34:45.023 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup_LargeResponses (0.86s) +=== CONT TestDNS_ServiceLookup_OnlyPassing +[WARN] freeport: 10 out of 45 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:44.328Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:44.344Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:44.347 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4ec1fb17-469b-a054-8af3-9da3ae0c94be Address:127.0.0.1:25022}]" +34:44.347 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25022 [Follower]" leader= +34:44.348 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4ec1fb17-469b-a054-8af3-9da3ae0c94be.dc1 127.0.0.1 +34:44.349 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4ec1fb17-469b-a054-8af3-9da3ae0c94be 127.0.0.1 +2021-08-13T15:34:44.349Z [INFO] agent.router: Initializing LAN area manager +34:44.349 [INFO] TestAgent.server: Adding LAN server: server="Node-4ec1fb17-469b-a054-8af3-9da3ae0c94be (Addr: tcp/127.0.0.1:25022) (DC: dc1)" +34:44.349 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4ec1fb17-469b-a054-8af3-9da3ae0c94be.dc1 area=wan +34:44.350 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24507 network=udp +34:44.350 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24507 network=tcp +34:44.350 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25060 network=tcp +34:44.351 [INFO] TestAgent: started state syncer +34:44.351 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25037 network=tcp +34:44.414 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:44.414 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25022 [Candidate]" term=2 +34:44.414 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:44.414 [DEBUG] TestAgent.server.raft: vote granted: from=4ec1fb17-469b-a054-8af3-9da3ae0c94be term=2 tally=1 +34:44.414 [INFO] TestAgent.server.raft: election won: tally=1 +34:44.414 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25022 [Leader]" +34:44.414 [INFO] TestAgent.server: cluster leadership acquired +34:44.414 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25022 +34:44.415 [INFO] TestAgent.server: New leader elected: payload=Node-4ec1fb17-469b-a054-8af3-9da3ae0c94be +34:44.416 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:44.418 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:44.418 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:44.418 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:44.418 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:44.418 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:44.418 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4ec1fb17-469b-a054-8af3-9da3ae0c94be +34:44.418 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4ec1fb17-469b-a054-8af3-9da3ae0c94be +34:44.419 [INFO] TestAgent.server: federation state anti-entropy synced +34:44.612 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:44.614 [INFO] TestAgent: Synced node info +34:44.614 [DEBUG] TestAgent: Node info in sync +34:44.617 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=1.892484ms client=127.0.0.1:43404 client_network=udp +34:44.620 [DEBUG] TestAgent.dns: request served from client: name=4398e219-4f03-2640-bc1e-a62750e6a477.query.consul. type=ANY class=IN latency=1.922304ms client=127.0.0.1:48847 client_network=udp +34:44.620 [INFO] TestAgent: Requesting shutdown +34:44.620 [INFO] TestAgent.server: shutting down server +34:44.620 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:44.620 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:44.620 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:44.620 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:44.620 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:44.620 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:44.620 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:44.620 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:44.620 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:44.620 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:44.620 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:44.620Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:44.620Z [INFO] agent.router.manager: shutting down +34:44.621 [INFO] TestAgent: consul server down +34:44.621 [INFO] TestAgent: shutdown complete +34:44.621 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24507 network=tcp +34:44.621 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24507 network=udp +34:44.621 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25060 network=tcp +34:45.121 [INFO] TestAgent: Waiting for endpoints to shut down +34:45.121 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup_Truncate (0.82s) +=== CONT TestDNS_PreparedQueryNearIPEDNS +2021-08-13T15:34:44.389Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:44.405Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:44.408 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b6604e29-1d7f-6b05-3235-0b55fabba585 Address:127.0.0.1:25057}]" +34:44.408 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25057 [Follower]" leader= +34:44.409 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b6604e29-1d7f-6b05-3235-0b55fabba585.dc1 127.0.0.1 +34:44.410 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b6604e29-1d7f-6b05-3235-0b55fabba585 127.0.0.1 +2021-08-13T15:34:44.410Z [INFO] agent.router: Initializing LAN area manager +34:44.410 [INFO] TestAgent.server: Adding LAN server: server="Node-b6604e29-1d7f-6b05-3235-0b55fabba585 (Addr: tcp/127.0.0.1:25057) (DC: dc1)" +34:44.410 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b6604e29-1d7f-6b05-3235-0b55fabba585.dc1 area=wan +34:44.410 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25080 network=udp +34:44.410 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25080 network=tcp +34:44.411 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25081 network=tcp +34:44.411 [INFO] TestAgent: started state syncer +34:44.411 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25065 network=tcp +34:44.467 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:44.467 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25057 [Candidate]" term=2 +34:44.467 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:44.467 [DEBUG] TestAgent.server.raft: vote granted: from=b6604e29-1d7f-6b05-3235-0b55fabba585 term=2 tally=1 +34:44.467 [INFO] TestAgent.server.raft: election won: tally=1 +34:44.468 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25057 [Leader]" +34:44.468 [INFO] TestAgent.server: cluster leadership acquired +34:44.468 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25057 +34:44.468 [INFO] TestAgent.server: New leader elected: payload=Node-b6604e29-1d7f-6b05-3235-0b55fabba585 +34:44.469 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:44.472 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:44.472 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:44.472 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:44.472 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:44.472 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:44.472 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b6604e29-1d7f-6b05-3235-0b55fabba585 +34:44.472 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b6604e29-1d7f-6b05-3235-0b55fabba585 +34:44.472 [INFO] TestAgent.server: federation state anti-entropy synced +34:44.615 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:44.615 [INFO] TestAgent: Synced node info +34:44.615 [DEBUG] TestAgent: Node info in sync +34:44.700 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=ANY class=IN latency=473.941µs client=127.0.0.1:57268 client_network=udp +34:44.701 [DEBUG] TestAgent.dns: request served from client: name=1fcd3b6e-9f42-bf43-3b3c-bd5752ae0b33.query.consul. type=ANY class=IN latency=263.941µs client=127.0.0.1:54010 client_network=udp +34:44.701 [INFO] TestAgent: Requesting shutdown +34:44.701 [INFO] TestAgent.server: shutting down server +34:44.701 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:44.701 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:44.701 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:44.701 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:44.701 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:44.701 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:44.701 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:44.701 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:44.701 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:44.701 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:44.701 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:44.702Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:44.702Z [INFO] agent.router.manager: shutting down +34:44.702 [INFO] TestAgent: consul server down +34:44.702 [INFO] TestAgent: shutdown complete +34:44.702 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25080 network=tcp +34:44.702 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25080 network=udp +34:44.702 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25081 network=tcp +34:45.202 [INFO] TestAgent: Waiting for endpoints to shut down +34:45.202 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup_Dedup (0.83s) +=== CONT TestDNS_ServiceLookup_TagPeriod +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +Added 3 service nodes +Added 3 service nodes +34:44.483 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: test-client 127.0.0.1 +2021-08-13T15:34:44.483Z [INFO] agent.router: Initializing LAN area manager +34:44.483 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25066 network=tcp +34:44.483 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25066 network=udp +34:44.484 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25112 network=tcp +34:44.484 [INFO] TestAgent: started state syncer +34:44.485 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25044 network=tcp +2021-08-13T15:34:44.485Z [WARN] agent.router.manager: No servers available +34:44.485 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" +34:44.485 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:25048] +34:44.486 [DEBUG] TestAgent.client.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:25048 +34:44.487 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: test-server 127.0.0.1 +34:44.487 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 +34:44.487 [DEBUG] TestAgent: systemd notify failed: error="No socket" +34:44.487 [INFO] TestAgent.client: adding server: server="test-server (Addr: tcp/127.0.0.1:25050) (DC: dc1)" +2021-08-13T15:34:44.487Z [WARN] agent.router.manager: No servers available +34:44.515 [ERROR] TestAgent.client: RPC failed to server: method=Health.ServiceNodes server=127.0.0.1:25050 error="rpc error making call: No path to datacenter" +34:44.515 [ERROR] TestAgent.dns: rpc error: error="rpc error making call: No path to datacenter" +34:44.515 [DEBUG] TestAgent.dns: request served from client: name=consul.service.dc2.consul. type=ANY class=IN latency=533.301µs client=127.0.0.1:34765 client_network=udp +34:44.515 [INFO] TestAgent: Requesting shutdown +34:44.515 [INFO] TestAgent.client: shutting down client +34:44.515 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave +34:44.515 [INFO] TestAgent: consul client down +34:44.515 [INFO] TestAgent: shutdown complete +34:44.515 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25066 network=tcp +34:44.515 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25066 network=udp +34:44.515 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25112 network=tcp +34:45.016 [INFO] TestAgent: Waiting for endpoints to shut down +34:45.016 [INFO] TestAgent: Endpoints down +2021-08-13T15:34:44.110Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:44.132Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:44.135 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:48927865-2c18-185b-977a-04768da9221e Address:127.0.0.1:25050}]" +34:44.136 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-server.dc1 127.0.0.1 +34:44.137 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-server 127.0.0.1 +2021-08-13T15:34:44.137Z [INFO] agent.router: Initializing LAN area manager +34:44.137 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25050 [Follower]" leader= +34:44.138 [INFO] TestAgent.server: Adding LAN server: server="test-server (Addr: tcp/127.0.0.1:25050) (DC: dc1)" +34:44.138 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-server.dc1 area=wan +34:44.138 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24633 network=tcp +34:44.138 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24633 network=udp +34:44.139 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25032 network=tcp +34:44.140 [INFO] TestAgent: started state syncer +34:44.140 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25051 network=tcp +34:44.197 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:44.197 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25050 [Candidate]" term=2 +34:44.198 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:44.198 [DEBUG] TestAgent.server.raft: vote granted: from=48927865-2c18-185b-977a-04768da9221e term=2 tally=1 +34:44.198 [INFO] TestAgent.server.raft: election won: tally=1 +34:44.198 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25050 [Leader]" +34:44.198 [INFO] TestAgent.server: cluster leadership acquired +34:44.198 [INFO] TestAgent.server: New leader elected: payload=test-server +34:44.198 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25050 +34:44.199 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:44.202 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:44.202 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:44.202 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:44.202 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:44.202 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:44.202 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-server +34:44.202 [INFO] TestAgent.server: member joined, marking health alive: member=test-server +34:44.203 [INFO] TestAgent.server: federation state anti-entropy synced +34:44.322 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:44.322 [INFO] TestAgent: Synced node info +34:44.322 [DEBUG] TestAgent: Node info in sync +34:44.486 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:43022 +34:44.486 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-client 127.0.0.1 +34:44.487 [INFO] TestAgent.server: member joined, marking health alive: member=test-client +34:44.515 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc2 method=Health.ServiceNodes +34:44.951 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:44.951 [DEBUG] TestAgent: Node info in sync +34:45.016 [INFO] TestAgent: Requesting shutdown +34:45.016 [INFO] TestAgent.server: shutting down server +34:45.016 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:45.016 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:45.016 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:45.016 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:45.016 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:45.016 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:45.016 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:45.016 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:45.016 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:45.016 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:45.016 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:45.016Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:45.016Z [INFO] agent.router.manager: shutting down +34:45.017 [INFO] TestAgent: consul server down +34:45.017 [INFO] TestAgent: shutdown complete +34:45.017 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24633 network=tcp +34:45.017 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24633 network=udp +34:45.017 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25032 network=tcp +34:45.517 [INFO] TestAgent: Waiting for endpoints to shut down +34:45.517 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_NonExistentDC_RPC (1.43s) +=== CONT TestDNS_CaseInsensitiveServiceLookup +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:44.801Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:44.817Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:44.820 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6b11da2c-bdf4-746c-dc7c-254f03d58e46 Address:127.0.0.1:25092}]" +34:44.820 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25092 [Follower]" leader= +34:44.821 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6b11da2c-bdf4-746c-dc7c-254f03d58e46.dc1 127.0.0.1 +34:44.822 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6b11da2c-bdf4-746c-dc7c-254f03d58e46 127.0.0.1 +2021-08-13T15:34:44.822Z [INFO] agent.router: Initializing LAN area manager +34:44.822 [INFO] TestAgent.server: Adding LAN server: server="Node-6b11da2c-bdf4-746c-dc7c-254f03d58e46 (Addr: tcp/127.0.0.1:25092) (DC: dc1)" +34:44.822 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6b11da2c-bdf4-746c-dc7c-254f03d58e46.dc1 area=wan +34:44.822 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25045 network=udp +34:44.822 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25045 network=tcp +34:44.823 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25067 network=tcp +34:44.823 [INFO] TestAgent: started state syncer +34:44.824 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25109 network=tcp +34:44.866 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:44.866 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25092 [Candidate]" term=2 +34:44.866 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:44.866 [DEBUG] TestAgent.server.raft: vote granted: from=6b11da2c-bdf4-746c-dc7c-254f03d58e46 term=2 tally=1 +34:44.866 [INFO] TestAgent.server.raft: election won: tally=1 +34:44.866 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25092 [Leader]" +34:44.867 [INFO] TestAgent.server: cluster leadership acquired +34:44.867 [INFO] TestAgent.server: New leader elected: payload=Node-6b11da2c-bdf4-746c-dc7c-254f03d58e46 +34:44.867 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25092 +34:44.868 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:44.871 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:44.871 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:44.871 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:44.871 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:44.871 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:44.871 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6b11da2c-bdf4-746c-dc7c-254f03d58e46 +34:44.871 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6b11da2c-bdf4-746c-dc7c-254f03d58e46 +34:44.872 [INFO] TestAgent.server: federation state anti-entropy synced +34:44.879 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:44.880 [INFO] TestAgent: Synced node info +34:44.880 [DEBUG] TestAgent: Node info in sync +34:45.112 [DEBUG] TestAgent.dns: request served from client: name=some.query.we.like.query.consul. type=SRV class=IN latency=213.54µs client=127.0.0.1:35320 client_network=udp +34:45.112 [INFO] TestAgent: Requesting shutdown +34:45.112 [INFO] TestAgent.server: shutting down server +34:45.112 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:45.112 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:45.112 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:45.112 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:45.113 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:45.113 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:45.113 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:45.113 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:45.113 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:45.113 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:45.113 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:45.113Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:45.113Z [INFO] agent.router.manager: shutting down +34:45.113 [INFO] TestAgent: consul server down +34:45.113 [INFO] TestAgent: shutdown complete +34:45.113 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25045 network=tcp +34:45.113 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25045 network=udp +34:45.113 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25067 network=tcp +34:45.613 [INFO] TestAgent: Waiting for endpoints to shut down +34:45.613 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup_PreparedQueryNamePeriod (0.83s) +=== CONT TestDNS_ServiceLookup_OnlyFailing +2021-08-13T15:34:45.040Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:45.057Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:45.059 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:841cad28-f7cc-9c85-d9f8-a941c4015094 Address:127.0.0.1:25131}]" +34:45.059 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25131 [Follower]" leader= +34:45.060 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-841cad28-f7cc-9c85-d9f8-a941c4015094.dc1 127.0.0.1 +34:45.061 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-841cad28-f7cc-9c85-d9f8-a941c4015094 127.0.0.1 +2021-08-13T15:34:45.061Z [INFO] agent.router: Initializing LAN area manager +34:45.061 [INFO] TestAgent.server: Adding LAN server: server="Node-841cad28-f7cc-9c85-d9f8-a941c4015094 (Addr: tcp/127.0.0.1:25131) (DC: dc1)" +34:45.061 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-841cad28-f7cc-9c85-d9f8-a941c4015094.dc1 area=wan +34:45.061 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25087 network=udp +34:45.061 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25087 network=tcp +34:45.062 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25120 network=tcp +34:45.062 [INFO] TestAgent: started state syncer +34:45.062 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25132 network=tcp +34:45.104 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:45.104 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25131 [Candidate]" term=2 +34:45.104 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:45.104 [DEBUG] TestAgent.server.raft: vote granted: from=841cad28-f7cc-9c85-d9f8-a941c4015094 term=2 tally=1 +34:45.104 [INFO] TestAgent.server.raft: election won: tally=1 +34:45.104 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25131 [Leader]" +34:45.105 [INFO] TestAgent.server: cluster leadership acquired +34:45.105 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25131 +34:45.105 [INFO] TestAgent.server: New leader elected: payload=Node-841cad28-f7cc-9c85-d9f8-a941c4015094 +34:45.106 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:45.108 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:45.108 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:45.108 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:45.108 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:45.108 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:45.109 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-841cad28-f7cc-9c85-d9f8-a941c4015094 +34:45.109 [INFO] TestAgent.server: member joined, marking health alive: member=Node-841cad28-f7cc-9c85-d9f8-a941c4015094 +34:45.110 [INFO] TestAgent.server: federation state anti-entropy synced +34:45.258 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=ANY class=IN latency=296.361µs client=127.0.0.1:57686 client_network=udp +34:45.259 [DEBUG] TestAgent.dns: request served from client: name=364691cf-381d-6a3a-02fd-cdd35b10f811.query.consul. type=ANY class=IN latency=188.681µs client=127.0.0.1:47352 client_network=udp +34:45.260 [INFO] TestAgent: Requesting shutdown +34:45.260 [INFO] TestAgent.server: shutting down server +34:45.260 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:45.260 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:45.260 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:45.260 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:45.260Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:45.260Z [INFO] agent.router.manager: shutting down +34:45.260 [INFO] TestAgent: consul server down +34:45.260 [INFO] TestAgent: shutdown complete +34:45.260 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25087 network=tcp +34:45.260 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25087 network=udp +34:45.260 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25120 network=tcp +34:45.761 [INFO] TestAgent: Waiting for endpoints to shut down +34:45.761 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup_OnlyPassing (0.74s) +=== CONT TestDNS_Lookup_TaggedIPAddresses +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:45.155Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:45.189Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:45.192 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6151241d-87c2-737e-168b-61bd4e19e52c Address:127.0.0.1:25103}]" +34:45.192 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25103 [Follower]" leader= +34:45.193 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6151241d-87c2-737e-168b-61bd4e19e52c.dc1 127.0.0.1 +34:45.193 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6151241d-87c2-737e-168b-61bd4e19e52c 127.0.0.1 +2021-08-13T15:34:45.194Z [INFO] agent.router: Initializing LAN area manager +34:45.194 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25133 network=udp +34:45.194 [INFO] TestAgent.server: Adding LAN server: server="Node-6151241d-87c2-737e-168b-61bd4e19e52c (Addr: tcp/127.0.0.1:25103) (DC: dc1)" +34:45.194 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6151241d-87c2-737e-168b-61bd4e19e52c.dc1 area=wan +34:45.194 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25133 network=tcp +34:45.195 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25095 network=tcp +34:45.195 [INFO] TestAgent: started state syncer +34:45.195 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25104 network=tcp +34:45.243 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:45.243 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25103 [Candidate]" term=2 +34:45.243 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:45.243 [DEBUG] TestAgent.server.raft: vote granted: from=6151241d-87c2-737e-168b-61bd4e19e52c term=2 tally=1 +34:45.243 [INFO] TestAgent.server.raft: election won: tally=1 +34:45.243 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25103 [Leader]" +34:45.244 [INFO] TestAgent.server: cluster leadership acquired +34:45.244 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25103 +34:45.244 [INFO] TestAgent.server: New leader elected: payload=Node-6151241d-87c2-737e-168b-61bd4e19e52c +34:45.245 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:45.247 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:45.247 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:45.247 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:45.247 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:45.247 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:45.248 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6151241d-87c2-737e-168b-61bd4e19e52c +34:45.248 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6151241d-87c2-737e-168b-61bd4e19e52c +34:45.249 [INFO] TestAgent.server: federation state anti-entropy synced +34:45.279 [DEBUG] TestAgent.dns: request served from client: name=some.query.we.like.query.consul. type=A class=IN latency=271.921µs client=127.0.0.1:33191 client_network=udp +34:45.305 [DEBUG] TestAgent.dns: request served from client: name=some.query.we.like.query.consul. type=A class=IN latency=272.521µs client=127.0.0.1:53398 client_network=udp +34:45.306 [INFO] TestAgent: Requesting shutdown +34:45.306 [INFO] TestAgent.server: shutting down server +34:45.306 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:45.306 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:45.306 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:45.306 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:45.306 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:45.306 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:45.306 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:45.306 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:45.306 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:45.306 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:45.306 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:45.306 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:45.306Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:45.306Z [INFO] agent.router.manager: shutting down +34:45.307 [INFO] TestAgent: consul server down +34:45.307 [INFO] TestAgent: shutdown complete +34:45.307 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25133 network=tcp +34:45.307 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25133 network=udp +34:45.307 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25095 network=tcp +34:45.807 [INFO] TestAgent: Waiting for endpoints to shut down +34:45.807 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_PreparedQueryNearIPEDNS (0.69s) +=== CONT TestDNS_ServiceLookup_WanTranslation +2021-08-13T15:34:45.220Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:45.237Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:45.240 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d62d3c4a-69a7-68f1-480d-6e632c462ffd Address:127.0.0.1:25117}]" +34:45.241 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25117 [Follower]" leader= +34:45.241 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d62d3c4a-69a7-68f1-480d-6e632c462ffd.dc1 127.0.0.1 +34:45.242 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d62d3c4a-69a7-68f1-480d-6e632c462ffd 127.0.0.1 +2021-08-13T15:34:45.242Z [INFO] agent.router: Initializing LAN area manager +34:45.243 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25105 network=udp +34:45.243 [INFO] TestAgent.server: Adding LAN server: server="Node-d62d3c4a-69a7-68f1-480d-6e632c462ffd (Addr: tcp/127.0.0.1:25117) (DC: dc1)" +34:45.243 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d62d3c4a-69a7-68f1-480d-6e632c462ffd.dc1 area=wan +34:45.243 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25105 network=tcp +34:45.244 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25106 network=tcp +34:45.244 [INFO] TestAgent: started state syncer +34:45.244 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25118 network=tcp +34:45.279 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:45.279 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25117 [Candidate]" term=2 +34:45.279 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:45.279 [DEBUG] TestAgent.server.raft: vote granted: from=d62d3c4a-69a7-68f1-480d-6e632c462ffd term=2 tally=1 +34:45.279 [INFO] TestAgent.server.raft: election won: tally=1 +34:45.279 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25117 [Leader]" +34:45.279 [INFO] TestAgent.server: cluster leadership acquired +34:45.280 [INFO] TestAgent.server: New leader elected: payload=Node-d62d3c4a-69a7-68f1-480d-6e632c462ffd +34:45.280 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25117 +34:45.281 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:45.283 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:45.283 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:45.284 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:45.284 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:45.284 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:45.284 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d62d3c4a-69a7-68f1-480d-6e632c462ffd +34:45.284 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d62d3c4a-69a7-68f1-480d-6e632c462ffd +34:45.285 [INFO] TestAgent.server: federation state anti-entropy synced +34:45.308 [DEBUG] TestAgent.dns: request served from client: name=v1.primary2.db.service.consul. type=SRV class=IN latency=133.601µs client=127.0.0.1:60801 client_network=udp +34:45.309 [DEBUG] TestAgent.dns: request served from client: name=v1.primary.db.service.consul. type=SRV class=IN latency=146.86µs client=127.0.0.1:38517 client_network=udp +34:45.309 [INFO] TestAgent: Requesting shutdown +34:45.309 [INFO] TestAgent.server: shutting down server +34:45.309 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:45.309 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:45.309 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:45.309 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:45.309 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:45.309 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:45.309 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:45.309 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:45.309 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:45.309 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:45.309 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:45.309 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:45.309Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:45.309Z [INFO] agent.router.manager: shutting down +34:45.310 [INFO] TestAgent: consul server down +34:45.310 [INFO] TestAgent: shutdown complete +34:45.310 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25105 network=tcp +34:45.310 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25105 network=udp +34:45.310 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25106 network=tcp +34:45.810 [INFO] TestAgent: Waiting for endpoints to shut down +34:45.810 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup_TagPeriod (0.61s) +=== CONT TestDNS_ServiceLookup_FilterCritical +2021-08-13T15:34:44.822Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:44.859Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:44.862 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6bf679a0-75ef-cd42-20be-11459f91fda6 Address:127.0.0.1:25078}]" +34:44.862 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25078 [Follower]" leader= +34:44.863 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6bf679a0-75ef-cd42-20be-11459f91fda6.dc1 127.0.0.1 +34:44.864 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6bf679a0-75ef-cd42-20be-11459f91fda6 127.0.0.1 +2021-08-13T15:34:44.864Z [INFO] agent.router: Initializing LAN area manager +34:44.864 [INFO] TestAgent.server: Adding LAN server: server="Node-6bf679a0-75ef-cd42-20be-11459f91fda6 (Addr: tcp/127.0.0.1:25078) (DC: dc1)" +34:44.864 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6bf679a0-75ef-cd42-20be-11459f91fda6.dc1 area=wan +34:44.864 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25111 network=udp +34:44.864 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25111 network=tcp +34:44.865 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25088 network=tcp +34:44.865 [INFO] TestAgent: started state syncer +34:44.866 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25093 network=tcp +34:44.931 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:44.931 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25078 [Candidate]" term=2 +34:44.935 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:44.935 [DEBUG] TestAgent.server.raft: vote granted: from=6bf679a0-75ef-cd42-20be-11459f91fda6 term=2 tally=1 +34:44.936 [INFO] TestAgent.server.raft: election won: tally=1 +34:44.936 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25078 [Leader]" +34:44.936 [INFO] TestAgent.server: cluster leadership acquired +34:44.936 [INFO] TestAgent.server: New leader elected: payload=Node-6bf679a0-75ef-cd42-20be-11459f91fda6 +34:44.936 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25078 +34:44.938 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:44.941 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:44.941 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:44.941 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:44.941 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:44.941 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:44.941 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6bf679a0-75ef-cd42-20be-11459f91fda6 +34:44.941 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6bf679a0-75ef-cd42-20be-11459f91fda6 +34:44.943 [INFO] TestAgent.server: federation state anti-entropy synced +34:45.093 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:45.094 [INFO] TestAgent: Synced node info +34:45.269 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=1.902505ms client=127.0.0.1:41556 client_network=udp +34:45.271 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=1.715523ms client=127.0.0.1:60651 client_network=udp +34:45.273 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=1.700864ms client=127.0.0.1:60449 client_network=udp +34:45.275 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=1.750404ms client=127.0.0.1:41199 client_network=udp +34:45.277 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=1.711684ms client=127.0.0.1:46931 client_network=udp +34:45.283 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=5.874534ms client=127.0.0.1:38629 client_network=udp +34:45.286 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=1.881964ms client=127.0.0.1:50674 client_network=udp +34:45.288 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=1.756784ms client=127.0.0.1:59862 client_network=udp +34:45.290 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=1.725984ms client=127.0.0.1:59270 client_network=udp +34:45.292 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=1.831504ms client=127.0.0.1:47755 client_network=udp +34:45.295 [DEBUG] TestAgent.dns: request served from client: name=768c7fdb-5282-ae35-5546-7ffcc5022865.query.consul. type=ANY class=IN latency=1.884585ms client=127.0.0.1:60274 client_network=udp +34:45.297 [DEBUG] TestAgent.dns: request served from client: name=768c7fdb-5282-ae35-5546-7ffcc5022865.query.consul. type=ANY class=IN latency=1.856625ms client=127.0.0.1:59228 client_network=udp +34:45.299 [DEBUG] TestAgent.dns: request served from client: name=768c7fdb-5282-ae35-5546-7ffcc5022865.query.consul. type=ANY class=IN latency=1.867365ms client=127.0.0.1:43637 client_network=udp +34:45.301 [DEBUG] TestAgent.dns: request served from client: name=768c7fdb-5282-ae35-5546-7ffcc5022865.query.consul. type=ANY class=IN latency=1.864425ms client=127.0.0.1:49292 client_network=udp +34:45.305 [DEBUG] TestAgent.dns: request served from client: name=768c7fdb-5282-ae35-5546-7ffcc5022865.query.consul. type=ANY class=IN latency=1.942264ms client=127.0.0.1:54309 client_network=udp +34:45.307 [DEBUG] TestAgent.dns: request served from client: name=768c7fdb-5282-ae35-5546-7ffcc5022865.query.consul. type=ANY class=IN latency=1.927184ms client=127.0.0.1:42936 client_network=udp +34:45.310 [DEBUG] TestAgent.dns: request served from client: name=768c7fdb-5282-ae35-5546-7ffcc5022865.query.consul. type=ANY class=IN latency=2.074724ms client=127.0.0.1:43298 client_network=udp +34:45.312 [DEBUG] TestAgent.dns: request served from client: name=768c7fdb-5282-ae35-5546-7ffcc5022865.query.consul. type=ANY class=IN latency=1.909665ms client=127.0.0.1:53009 client_network=udp +34:45.314 [DEBUG] TestAgent.dns: request served from client: name=768c7fdb-5282-ae35-5546-7ffcc5022865.query.consul. type=ANY class=IN latency=1.932825ms client=127.0.0.1:56457 client_network=udp +34:45.316 [DEBUG] TestAgent.dns: request served from client: name=768c7fdb-5282-ae35-5546-7ffcc5022865.query.consul. type=ANY class=IN latency=1.924865ms client=127.0.0.1:51208 client_network=udp +34:45.316 [INFO] TestAgent: Requesting shutdown +34:45.316 [INFO] TestAgent.server: shutting down server +34:45.316 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:45.316 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:45.316 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:45.316 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:45.316 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:45.316 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:45.316 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:45.316 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:45.316 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:45.317 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +34:45.317 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2021-08-13T15:34:45.317Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:45.317Z [INFO] agent.router.manager: shutting down +34:45.317 [INFO] TestAgent: consul server down +34:45.317 [INFO] TestAgent: shutdown complete +34:45.317 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25111 network=tcp +34:45.317 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25111 network=udp +34:45.317 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25088 network=tcp +34:45.819 [INFO] TestAgent: Waiting for endpoints to shut down +34:45.819 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup_Randomize (1.03s) +=== CONT TestDNS_ServiceLookup_ServiceAddressIPV6 +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestDNS_ServiceLookup_WanTranslation/service-wan-from-dc1 +=== RUN TestDNS_ServiceLookup_WanTranslation/node-addr-from-dc2 +=== RUN TestDNS_ServiceLookup_WanTranslation/node-wan-from-dc2 +=== RUN TestDNS_ServiceLookup_WanTranslation/service-addr-from-dc2 +=== RUN TestDNS_ServiceLookup_WanTranslation/service-wan-from-dc2 +=== RUN TestDNS_ServiceLookup_WanTranslation/node-addr-from-dc1 +=== RUN TestDNS_ServiceLookup_WanTranslation/node-wan-from-dc1 +=== RUN TestDNS_ServiceLookup_WanTranslation/service-addr-from-dc1 +2021-08-13T15:34:45.026Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:45.046Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:45.049 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3c02ac97-7e0c-0220-5783-6114302f73fe Address:127.0.0.1:25099}]" +34:45.050 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25099 [Follower]" leader= +34:45.051 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3c02ac97-7e0c-0220-5783-6114302f73fe.dc1 127.0.0.1 +34:45.052 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3c02ac97-7e0c-0220-5783-6114302f73fe 127.0.0.1 +2021-08-13T15:34:45.052Z [INFO] agent.router: Initializing LAN area manager +34:45.052 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25094 network=udp +34:45.052 [INFO] TestAgent.server: Adding LAN server: server="Node-3c02ac97-7e0c-0220-5783-6114302f73fe (Addr: tcp/127.0.0.1:25099) (DC: dc1)" +34:45.053 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3c02ac97-7e0c-0220-5783-6114302f73fe.dc1 area=wan +34:45.053 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25094 network=tcp +34:45.054 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25046 network=tcp +34:45.054 [INFO] TestAgent: started state syncer +34:45.054 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25086 network=tcp +34:45.093 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:45.093 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25099 [Candidate]" term=2 +34:45.093 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:45.093 [DEBUG] TestAgent.server.raft: vote granted: from=3c02ac97-7e0c-0220-5783-6114302f73fe term=2 tally=1 +34:45.093 [INFO] TestAgent.server.raft: election won: tally=1 +34:45.093 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25099 [Leader]" +34:45.093 [INFO] TestAgent.server: cluster leadership acquired +34:45.093 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25099 +34:45.094 [INFO] TestAgent.server: New leader elected: payload=Node-3c02ac97-7e0c-0220-5783-6114302f73fe +34:45.094 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:45.097 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:45.097 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:45.097 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:45.097 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:45.097 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:45.097 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3c02ac97-7e0c-0220-5783-6114302f73fe +34:45.097 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3c02ac97-7e0c-0220-5783-6114302f73fe +34:45.098 [INFO] TestAgent.server: federation state anti-entropy synced +34:45.363 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:45.364 [INFO] TestAgent: Synced node info +34:45.443 [DEBUG] TestAgent.dns: request served from client: name=some.query.we.like.query.consul. type=A class=IN latency=279.921µs client=127.0.0.1:45968 client_network=udp +34:45.469 [DEBUG] TestAgent.dns: request served from client: name=some.query.we.like.query.consul. type=A class=IN latency=292.34µs client=127.0.0.1:37895 client_network=udp +34:45.477 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:45.477 [DEBUG] TestAgent: Node info in sync +34:45.477 [DEBUG] TestAgent: Node info in sync +34:45.495 [DEBUG] TestAgent.dns: request served from client: name=some.query.we.like.query.consul. type=A class=IN latency=287.38µs client=127.0.0.1:55729 client_network=udp +34:45.521 [DEBUG] TestAgent.dns: request served from client: name=some.query.we.like.query.consul. type=A class=IN latency=290.381µs client=127.0.0.1:39368 client_network=udp +34:45.551 [DEBUG] TestAgent.dns: request served from client: name=some.query.we.like.query.consul. type=A class=IN latency=325.101µs client=127.0.0.1:55511 client_network=udp +34:45.577 [DEBUG] TestAgent.dns: request served from client: name=some.query.we.like.query.consul. type=A class=IN latency=300.961µs client=127.0.0.1:33334 client_network=udp +34:45.604 [DEBUG] TestAgent.dns: request served from client: name=some.query.we.like.query.consul. type=A class=IN latency=310.621µs client=127.0.0.1:45016 client_network=udp +34:45.604 [INFO] TestAgent: Requesting shutdown +34:45.604 [INFO] TestAgent.server: shutting down server +34:45.604 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:45.604 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:45.604 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:45.604 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:45.604 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:45.604 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:45.604 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:45.604 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:45.604 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:45.604 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:45.604 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:45.604Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:45.604Z [INFO] agent.router.manager: shutting down +34:45.605 [INFO] TestAgent: consul server down +34:45.605 [INFO] TestAgent: shutdown complete +34:45.605 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25094 network=tcp +34:45.605 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25094 network=udp +34:45.605 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25046 network=tcp +34:46.105 [INFO] TestAgent: Waiting for endpoints to shut down +34:46.105 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_PreparedQueryNearIP (1.10s) +=== CONT TestDNS_RecursorTimeout +2021-08-13T15:34:45.535Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:45.552Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:45.555 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c7b1e26e-fe4f-bcbb-fa49-8666e709ce5b Address:127.0.0.1:24976}]" +34:45.555 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24976 [Follower]" leader= +34:45.556 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c7b1e26e-fe4f-bcbb-fa49-8666e709ce5b.dc1 127.0.0.1 +34:45.557 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c7b1e26e-fe4f-bcbb-fa49-8666e709ce5b 127.0.0.1 +2021-08-13T15:34:45.557Z [INFO] agent.router: Initializing LAN area manager +34:45.557 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25119 network=udp +34:45.557 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c7b1e26e-fe4f-bcbb-fa49-8666e709ce5b.dc1 area=wan +34:45.557 [INFO] TestAgent.server: Adding LAN server: server="Node-c7b1e26e-fe4f-bcbb-fa49-8666e709ce5b (Addr: tcp/127.0.0.1:24976) (DC: dc1)" +34:45.557 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25119 network=tcp +34:45.558 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24965 network=tcp +34:45.558 [INFO] TestAgent: started state syncer +34:45.558 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24977 network=tcp +34:45.603 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:45.603 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24976 [Candidate]" term=2 +34:45.603 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:45.603 [DEBUG] TestAgent.server.raft: vote granted: from=c7b1e26e-fe4f-bcbb-fa49-8666e709ce5b term=2 tally=1 +34:45.604 [INFO] TestAgent.server.raft: election won: tally=1 +34:45.604 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24976 [Leader]" +34:45.604 [INFO] TestAgent.server: cluster leadership acquired +34:45.604 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24976 +34:45.605 [INFO] TestAgent.server: New leader elected: payload=Node-c7b1e26e-fe4f-bcbb-fa49-8666e709ce5b +34:45.605 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:45.608 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:45.608 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:45.608 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:45.608 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:45.608 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:45.608 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c7b1e26e-fe4f-bcbb-fa49-8666e709ce5b +34:45.608 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c7b1e26e-fe4f-bcbb-fa49-8666e709ce5b +34:45.609 [INFO] TestAgent.server: federation state anti-entropy synced +34:45.627 [DEBUG] TestAgent.dns: request served from client: name=primary.db.service.consul. type=SRV class=IN latency=221.261µs client=127.0.0.1:35791 client_network=udp +34:45.631 [DEBUG] TestAgent.dns: request served from client: name=pRIMARY.dB.service.consul. type=SRV class=IN latency=228.461µs client=127.0.0.1:56116 client_network=udp +34:45.633 [DEBUG] TestAgent.dns: request served from client: name=PRIMARY.dB.service.consul. type=SRV class=IN latency=175.22µs client=127.0.0.1:43924 client_network=udp +34:45.633 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=227.16µs client=127.0.0.1:46530 client_network=udp +34:45.634 [DEBUG] TestAgent.dns: request served from client: name=DB.service.consul. type=SRV class=IN latency=181.68µs client=127.0.0.1:38202 client_network=udp +34:45.634 [DEBUG] TestAgent.dns: request served from client: name=Db.service.consul. type=SRV class=IN latency=178.76µs client=127.0.0.1:57285 client_network=udp +34:45.635 [DEBUG] TestAgent.dns: request served from client: name=somequery.query.consul. type=SRV class=IN latency=147.9µs client=127.0.0.1:60986 client_network=udp +34:45.635 [DEBUG] TestAgent.dns: request served from client: name=SomeQuery.query.consul. type=SRV class=IN latency=141.081µs client=127.0.0.1:50909 client_network=udp +34:45.635 [DEBUG] TestAgent.dns: request served from client: name=SOMEQUERY.query.consul. type=SRV class=IN latency=139.601µs client=127.0.0.1:43686 client_network=udp +34:45.635 [INFO] TestAgent: Requesting shutdown +34:45.635 [INFO] TestAgent.server: shutting down server +34:45.636 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:45.636 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:45.636 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:45.636 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:45.636 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:45.636 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:45.636 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:45.636 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:45.636 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:45.636 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:45.636 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:45.636 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:45.636Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:45.636Z [INFO] agent.router.manager: shutting down +34:45.636 [INFO] TestAgent: consul server down +34:45.636 [INFO] TestAgent: shutdown complete +34:45.636 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25119 network=tcp +34:45.636 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25119 network=udp +34:45.636 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24965 network=tcp +34:46.137 [INFO] TestAgent: Waiting for endpoints to shut down +34:46.137 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_CaseInsensitiveServiceLookup (0.62s) +=== CONT TestDNS_ServiceLookup_ServiceAddress_SRV +=== RUN TestDNS_Lookup_TaggedIPAddresses/simple-ipv4 +=== RUN TestDNS_Lookup_TaggedIPAddresses/simple-ipv6 +=== RUN TestDNS_Lookup_TaggedIPAddresses/ipv4-with-tagged-ipv6 +=== RUN TestDNS_Lookup_TaggedIPAddresses/ipv6-with-tagged-ipv4 +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:45.630Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:45.652Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:45.658 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3f3141d7-e735-cfaa-d548-e9068869fef9 Address:127.0.0.1:24962}]" +34:45.659 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24962 [Follower]" leader= +34:45.662 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3f3141d7-e735-cfaa-d548-e9068869fef9.dc1 127.0.0.1 +34:45.664 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3f3141d7-e735-cfaa-d548-e9068869fef9 127.0.0.1 +2021-08-13T15:34:45.664Z [INFO] agent.router: Initializing LAN area manager +34:45.673 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24985 network=udp +34:45.673 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24985 network=tcp +34:45.674 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3f3141d7-e735-cfaa-d548-e9068869fef9.dc1 area=wan +34:45.681 [INFO] TestAgent.server: Adding LAN server: server="Node-3f3141d7-e735-cfaa-d548-e9068869fef9 (Addr: tcp/127.0.0.1:24962) (DC: dc1)" +34:45.681 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24972 network=tcp +34:45.681 [INFO] TestAgent: started state syncer +34:45.681 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24963 network=tcp +34:45.703 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:45.703 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24962 [Candidate]" term=2 +34:45.703 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:45.703 [DEBUG] TestAgent.server.raft: vote granted: from=3f3141d7-e735-cfaa-d548-e9068869fef9 term=2 tally=1 +34:45.703 [INFO] TestAgent.server.raft: election won: tally=1 +34:45.703 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24962 [Leader]" +34:45.703 [INFO] TestAgent.server: cluster leadership acquired +34:45.704 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24962 +34:45.705 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:45.707 [INFO] TestAgent.server: New leader elected: payload=Node-3f3141d7-e735-cfaa-d548-e9068869fef9 +34:45.707 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:45.707 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:45.707 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:45.707 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:45.707 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:45.707 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3f3141d7-e735-cfaa-d548-e9068869fef9 +34:45.707 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3f3141d7-e735-cfaa-d548-e9068869fef9 +34:45.708 [INFO] TestAgent.server: federation state anti-entropy synced +34:45.803 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=ANY class=IN latency=272.281µs client=127.0.0.1:54316 client_network=udp +34:45.803 [DEBUG] TestAgent.dns: request served from client: name=c606400f-3acf-6741-3745-0ed310f197bb.query.consul. type=ANY class=IN latency=304.941µs client=127.0.0.1:59101 client_network=udp +34:45.803 [INFO] TestAgent: Requesting shutdown +34:45.803 [INFO] TestAgent.server: shutting down server +34:45.803 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:45.803 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:45.803 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:45.803 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:45.804 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:45.804 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:45.804 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:45.804 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:45.804 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:45.804 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:45.804 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:45.804 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:45.804Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:45.804Z [INFO] agent.router.manager: shutting down +34:45.804 [INFO] TestAgent: consul server down +34:45.804 [INFO] TestAgent: shutdown complete +34:45.804 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24985 network=tcp +34:45.804 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24985 network=udp +34:45.804 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24972 network=tcp +34:46.305 [INFO] TestAgent: Waiting for endpoints to shut down +34:46.305 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup_OnlyFailing (0.69s) +=== CONT TestDNS_Recurse_Truncation +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:45.853Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:45.921Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:45.927 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f734502e-a767-3af8-2bc4-79d441ad1a05 Address:127.0.0.1:25159}]" +34:45.928 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f734502e-a767-3af8-2bc4-79d441ad1a05.dc1 127.0.0.1 +34:45.928 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25159 [Follower]" leader= +34:45.929 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f734502e-a767-3af8-2bc4-79d441ad1a05 127.0.0.1 +2021-08-13T15:34:45.929Z [INFO] agent.router: Initializing LAN area manager +34:45.929 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25140 network=udp +34:45.929 [INFO] TestAgent.server: Adding LAN server: server="Node-f734502e-a767-3af8-2bc4-79d441ad1a05 (Addr: tcp/127.0.0.1:25159) (DC: dc1)" +34:45.929 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f734502e-a767-3af8-2bc4-79d441ad1a05.dc1 area=wan +34:45.930 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25140 network=tcp +34:45.931 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25148 network=tcp +34:45.931 [INFO] TestAgent: started state syncer +34:45.931 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25160 network=tcp +34:45.967 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:45.968 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25159 [Candidate]" term=2 +34:45.968 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:45.968 [DEBUG] TestAgent.server.raft: vote granted: from=f734502e-a767-3af8-2bc4-79d441ad1a05 term=2 tally=1 +34:45.968 [INFO] TestAgent.server.raft: election won: tally=1 +34:45.968 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25159 [Leader]" +34:45.968 [INFO] TestAgent.server: cluster leadership acquired +34:45.968 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25159 +34:45.969 [INFO] TestAgent.server: New leader elected: payload=Node-f734502e-a767-3af8-2bc4-79d441ad1a05 +34:45.969 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:45.972 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:45.972 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:45.972 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:45.972 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:45.972 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:45.972 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f734502e-a767-3af8-2bc4-79d441ad1a05 +34:45.972 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f734502e-a767-3af8-2bc4-79d441ad1a05 +34:45.973 [INFO] TestAgent.server: federation state anti-entropy synced +34:46.160 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=371.461µs client=127.0.0.1:50229 client_network=udp +34:46.161 [DEBUG] TestAgent.dns: request served from client: name=97439274-0cd4-5c58-74ef-f94b143b673c.query.consul. type=SRV class=IN latency=223.46µs client=127.0.0.1:49667 client_network=udp +34:46.161 [INFO] TestAgent: Requesting shutdown +34:46.161 [INFO] TestAgent.server: shutting down server +34:46.161 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:46.161 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:46.161 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:46.161 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:46.161 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:46.161 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:46.161 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:46.161 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:46.161 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:46.161 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:46.161 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:46.161 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:46.162Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:46.162Z [INFO] agent.router.manager: shutting down +34:46.162 [INFO] TestAgent: consul server down +34:46.162 [INFO] TestAgent: shutdown complete +34:46.162 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25140 network=tcp +34:46.162 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25140 network=udp +34:46.162 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25148 network=tcp +34:46.662 [INFO] TestAgent: Waiting for endpoints to shut down +34:46.662 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup_ServiceAddressIPV6 (0.84s) +=== CONT TestDNS_Recurse +2021-08-13T15:34:45.778Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:45.793Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:45.796 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:252e1619-9098-e964-b3f0-3d4c257862fd Address:127.0.0.1:25124}]" +34:45.796 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25124 [Follower]" leader= +34:45.797 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-252e1619-9098-e964-b3f0-3d4c257862fd.dc1 127.0.0.1 +34:45.798 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-252e1619-9098-e964-b3f0-3d4c257862fd 127.0.0.1 +2021-08-13T15:34:45.798Z [INFO] agent.router: Initializing LAN area manager +34:45.798 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24964 network=udp +34:45.798 [INFO] TestAgent.server: Adding LAN server: server="Node-252e1619-9098-e964-b3f0-3d4c257862fd (Addr: tcp/127.0.0.1:25124) (DC: dc1)" +34:45.798 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-252e1619-9098-e964-b3f0-3d4c257862fd.dc1 area=wan +34:45.798 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24964 network=tcp +34:45.799 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25134 network=tcp +34:45.799 [INFO] TestAgent: started state syncer +34:45.799 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25125 network=tcp +34:45.835 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:45.835 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25124 [Candidate]" term=2 +34:45.835 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:45.835 [DEBUG] TestAgent.server.raft: vote granted: from=252e1619-9098-e964-b3f0-3d4c257862fd term=2 tally=1 +34:45.835 [INFO] TestAgent.server.raft: election won: tally=1 +34:45.835 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25124 [Leader]" +34:45.835 [INFO] TestAgent.server: cluster leadership acquired +34:45.836 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25124 +34:45.837 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:45.839 [INFO] TestAgent.server: New leader elected: payload=Node-252e1619-9098-e964-b3f0-3d4c257862fd +34:45.843 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:45.843 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:45.843 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:45.843 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:45.843 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:45.843 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-252e1619-9098-e964-b3f0-3d4c257862fd +34:45.843 [INFO] TestAgent.server: member joined, marking health alive: member=Node-252e1619-9098-e964-b3f0-3d4c257862fd +34:45.845 [INFO] TestAgent.server: federation state anti-entropy synced +34:46.087 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:46.088 [INFO] TestAgent: Synced node info +34:46.186 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=A class=IN latency=300.82µs client=127.0.0.1:58889 client_network=udp +34:46.186 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=AAAA class=IN latency=216.781µs client=127.0.0.1:33477 client_network=udp +34:46.195 [DEBUG] TestAgent.dns: request served from client: name=8758b1e8-4d04-71ac-17ea-d216e023119c.query.consul. type=A class=IN latency=8.27026ms client=127.0.0.1:52111 client_network=udp +34:46.196 [DEBUG] TestAgent.dns: request served from client: name=8758b1e8-4d04-71ac-17ea-d216e023119c.query.consul. type=AAAA class=IN latency=255.86µs client=127.0.0.1:34205 client_network=udp +34:46.196 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=A class=IN latency=147.8µs client=127.0.0.1:53680 client_network=udp +34:46.196 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=AAAA class=IN latency=105.32µs client=127.0.0.1:51812 client_network=udp +34:46.198 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=A class=IN latency=270.041µs client=127.0.0.1:33346 client_network=udp +34:46.206 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=AAAA class=IN latency=7.749418ms client=127.0.0.1:60847 client_network=udp +34:46.211 [DEBUG] TestAgent.dns: request served from client: name=8758b1e8-4d04-71ac-17ea-d216e023119c.query.consul. type=A class=IN latency=254.881µs client=127.0.0.1:53787 client_network=udp +34:46.212 [DEBUG] TestAgent.dns: request served from client: name=8758b1e8-4d04-71ac-17ea-d216e023119c.query.consul. type=AAAA class=IN latency=238.12µs client=127.0.0.1:49804 client_network=udp +34:46.214 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=A class=IN latency=160.72µs client=127.0.0.1:57156 client_network=udp +34:46.214 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=AAAA class=IN latency=127.64µs client=127.0.0.1:48040 client_network=udp +34:46.216 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=A class=IN latency=268.681µs client=127.0.0.1:39364 client_network=udp +34:46.217 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=AAAA class=IN latency=225.56µs client=127.0.0.1:44113 client_network=udp +34:46.219 [DEBUG] TestAgent.dns: request served from client: name=8758b1e8-4d04-71ac-17ea-d216e023119c.query.consul. type=A class=IN latency=250.7µs client=127.0.0.1:60937 client_network=udp +34:46.219 [DEBUG] TestAgent.dns: request served from client: name=8758b1e8-4d04-71ac-17ea-d216e023119c.query.consul. type=AAAA class=IN latency=155.401µs client=127.0.0.1:46299 client_network=udp +34:46.220 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=A class=IN latency=108.54µs client=127.0.0.1:47528 client_network=udp +34:46.220 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=AAAA class=IN latency=98.58µs client=127.0.0.1:46906 client_network=udp +34:46.222 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=A class=IN latency=218.22µs client=127.0.0.1:50583 client_network=udp +34:46.222 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=AAAA class=IN latency=186.761µs client=127.0.0.1:34036 client_network=udp +34:46.222 [DEBUG] TestAgent.dns: request served from client: name=8758b1e8-4d04-71ac-17ea-d216e023119c.query.consul. type=A class=IN latency=160.321µs client=127.0.0.1:47699 client_network=udp +34:46.223 [DEBUG] TestAgent.dns: request served from client: name=8758b1e8-4d04-71ac-17ea-d216e023119c.query.consul. type=AAAA class=IN latency=178.401µs client=127.0.0.1:50164 client_network=udp +34:46.227 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=A class=IN latency=166.54µs client=127.0.0.1:47624 client_network=udp +34:46.231 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=AAAA class=IN latency=144.96µs client=127.0.0.1:51835 client_network=udp +34:46.232 [INFO] TestAgent: Requesting shutdown +34:46.232 [INFO] TestAgent.server: shutting down server +34:46.232 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:46.232 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:46.232 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:46.232 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:46.232 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:46.232 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:46.232 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:46.232 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:46.232 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:46.232 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:46.232 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:46.232Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:46.232Z [INFO] agent.router.manager: shutting down +34:46.232 [INFO] TestAgent: consul server down +34:46.232 [INFO] TestAgent: shutdown complete +34:46.232 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24964 network=tcp +34:46.233 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24964 network=udp +34:46.233 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25134 network=tcp +34:46.735 [INFO] TestAgent: Waiting for endpoints to shut down +34:46.735 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_Lookup_TaggedIPAddresses (0.97s) + --- PASS: TestDNS_Lookup_TaggedIPAddresses/simple-ipv4 (0.01s) + --- PASS: TestDNS_Lookup_TaggedIPAddresses/simple-ipv6 (0.02s) + --- PASS: TestDNS_Lookup_TaggedIPAddresses/ipv4-with-tagged-ipv6 (0.01s) + --- PASS: TestDNS_Lookup_TaggedIPAddresses/ipv6-with-tagged-ipv4 (0.01s) +=== CONT TestDNS_ServiceLookup_ServiceAddress_A +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:45.827Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:45.851Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:45.854 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c27a490f-eb6d-1884-eb39-a1a917c169f2 Address:127.0.0.1:25138}]" +34:45.855 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c27a490f-eb6d-1884-eb39-a1a917c169f2.dc1 127.0.0.1 +34:45.855 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25138 [Follower]" leader= +34:45.860 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c27a490f-eb6d-1884-eb39-a1a917c169f2 127.0.0.1 +2021-08-13T15:34:45.860Z [INFO] agent.router: Initializing LAN area manager +34:45.861 [INFO] TestAgent.server: Adding LAN server: server="Node-c27a490f-eb6d-1884-eb39-a1a917c169f2 (Addr: tcp/127.0.0.1:25138) (DC: dc1)" +34:45.862 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c27a490f-eb6d-1884-eb39-a1a917c169f2.dc1 area=wan +34:45.869 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25147 network=udp +34:45.869 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25147 network=tcp +34:45.879 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25127 network=tcp +34:45.879 [INFO] TestAgent: started state syncer +34:45.879 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25139 network=tcp +34:45.907 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:45.907 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25138 [Candidate]" term=2 +34:45.907 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:45.907 [DEBUG] TestAgent.server.raft: vote granted: from=c27a490f-eb6d-1884-eb39-a1a917c169f2 term=2 tally=1 +34:45.907 [INFO] TestAgent.server.raft: election won: tally=1 +34:45.907 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25138 [Leader]" +34:45.908 [INFO] TestAgent.server: cluster leadership acquired +34:45.908 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25138 +34:45.910 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:45.911 [INFO] TestAgent.server: New leader elected: payload=Node-c27a490f-eb6d-1884-eb39-a1a917c169f2 +34:45.918 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:45.918 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:45.918 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:45.918 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:45.918 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:45.918 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c27a490f-eb6d-1884-eb39-a1a917c169f2 +34:45.918 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c27a490f-eb6d-1884-eb39-a1a917c169f2 +34:45.935 [INFO] TestAgent.server: federation state anti-entropy synced +34:46.059 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:46.060 [INFO] TestAgent: Synced node info +34:46.060 [DEBUG] TestAgent: Node info in sync +34:46.271 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=ANY class=IN latency=348.061µs client=127.0.0.1:54878 client_network=udp +34:46.272 [DEBUG] TestAgent.dns: request served from client: name=d47dff05-0a27-1e30-e688-d1ac96147e1d.query.consul. type=ANY class=IN latency=238.941µs client=127.0.0.1:47785 client_network=udp +34:46.272 [INFO] TestAgent: Requesting shutdown +34:46.272 [INFO] TestAgent.server: shutting down server +34:46.272 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:46.272 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:46.272 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:46.272 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:46.272 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:46.272 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:46.272 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:46.272 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:46.272 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:46.272 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:46.272 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:46.273Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:46.273Z [INFO] agent.router.manager: shutting down +34:46.273 [INFO] TestAgent: consul server down +34:46.273 [INFO] TestAgent: shutdown complete +34:46.273 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25147 network=tcp +34:46.273 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25147 network=udp +34:46.273 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25127 network=tcp +34:46.775 [INFO] TestAgent: Waiting for endpoints to shut down +34:46.775 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup_FilterCritical (0.97s) +=== CONT TestDNS_ServiceLookup_Dedup_SRV +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:45.949Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:45.949Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:45.966Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:45.966Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:45.969 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:db630e97-e63e-7983-712b-972c5a4ad0de Address:127.0.0.1:24941}]" +34:45.970 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-db630e97-e63e-7983-712b-972c5a4ad0de.dc2 127.0.0.1 +34:45.971 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24941 [Follower]" leader= +34:45.979 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-db630e97-e63e-7983-712b-972c5a4ad0de 127.0.0.1 +2021-08-13T15:34:45.979Z [INFO] agent.router: Initializing LAN area manager +34:45.980 [INFO] TestAgent.server: Adding LAN server: server="Node-db630e97-e63e-7983-712b-972c5a4ad0de (Addr: tcp/127.0.0.1:24941) (DC: dc2)" +34:45.980 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-db630e97-e63e-7983-712b-972c5a4ad0de.dc2 area=wan +34:45.980 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25161 network=tcp +34:45.981 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25161 network=udp +34:45.982 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24958 network=tcp +34:45.982 [INFO] TestAgent: started state syncer +34:45.982 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24942 network=tcp +34:46.011 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:46.011 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24941 [Candidate]" term=2 +34:46.011 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:46.011 [DEBUG] TestAgent.server.raft: vote granted: from=db630e97-e63e-7983-712b-972c5a4ad0de term=2 tally=1 +34:46.011 [INFO] TestAgent.server.raft: election won: tally=1 +34:46.011 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24941 [Leader]" +34:46.012 [INFO] TestAgent.server: cluster leadership acquired +34:46.012 [INFO] TestAgent.server: initializing acls +34:46.012 [INFO] TestAgent.server: New leader elected: payload=Node-db630e97-e63e-7983-712b-972c5a4ad0de +34:46.012 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:46.013 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:46.013 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:46.013 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:46.013 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-db630e97-e63e-7983-712b-972c5a4ad0de +34:46.013 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-db630e97-e63e-7983-712b-972c5a4ad0de.dc2 +34:46.014 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:46.015 [INFO] TestAgent.server: Updating LAN server: server="Node-db630e97-e63e-7983-712b-972c5a4ad0de (Addr: tcp/127.0.0.1:24941) (DC: dc2)" +34:46.015 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-db630e97-e63e-7983-712b-972c5a4ad0de.dc2 area=wan +34:46.017 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:46.017 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:46.017 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:46.017 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:46.017 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:46.017 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-db630e97-e63e-7983-712b-972c5a4ad0de +34:46.017 [INFO] TestAgent.server: member joined, marking health alive: member=Node-db630e97-e63e-7983-712b-972c5a4ad0de +34:46.018 [INFO] TestAgent.server: federation state anti-entropy synced +34:46.018 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-db630e97-e63e-7983-712b-972c5a4ad0de +34:46.054 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:25144] +34:46.055 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:25144 +34:46.056 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a1abdbad-734d-ecfc-bff5-df759c5c2e92.dc1 127.0.0.1 +34:46.056 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 +34:46.058 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a1abdbad-734d-ecfc-bff5-df759c5c2e92.dc1 area=wan +34:46.068 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=SRV class=IN latency=299.02µs client=127.0.0.1:48064 client_network=udp +34:46.068 [DEBUG] TestAgent.dns: request served from client: name=aaa2f55a-3eb9-aa73-3aa5-34957fa1eaff.query.dc2.consul. type=SRV class=IN latency=195.681µs client=127.0.0.1:43686 client_network=udp +34:46.069 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=A class=IN latency=236.581µs client=127.0.0.1:58286 client_network=udp +34:46.069 [DEBUG] TestAgent.dns: request served from client: name=aaa2f55a-3eb9-aa73-3aa5-34957fa1eaff.query.dc2.consul. type=A class=IN latency=198.2µs client=127.0.0.1:40295 client_network=udp +34:46.070 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=SRV class=IN latency=251.481µs client=127.0.0.1:40782 client_network=udp +34:46.071 [DEBUG] TestAgent.dns: request served from client: name=aaa2f55a-3eb9-aa73-3aa5-34957fa1eaff.query.dc2.consul. type=SRV class=IN latency=476.361µs client=127.0.0.1:46552 client_network=udp +34:46.072 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=A class=IN latency=258.28µs client=127.0.0.1:47936 client_network=udp +34:46.072 [DEBUG] TestAgent.dns: request served from client: name=aaa2f55a-3eb9-aa73-3aa5-34957fa1eaff.query.dc2.consul. type=A class=IN latency=177.681µs client=127.0.0.1:38139 client_network=udp +34:46.074 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=SRV class=IN latency=283.621µs client=127.0.0.1:54718 client_network=udp +34:46.074 [DEBUG] TestAgent.dns: request served from client: name=aaa2f55a-3eb9-aa73-3aa5-34957fa1eaff.query.dc2.consul. type=SRV class=IN latency=199.401µs client=127.0.0.1:35030 client_network=udp +34:46.075 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=A class=IN latency=246.76µs client=127.0.0.1:41507 client_network=udp +34:46.075 [DEBUG] TestAgent.dns: request served from client: name=aaa2f55a-3eb9-aa73-3aa5-34957fa1eaff.query.dc2.consul. type=A class=IN latency=193.72µs client=127.0.0.1:53107 client_network=udp +34:46.077 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=SRV class=IN latency=263.9µs client=127.0.0.1:45191 client_network=udp +34:46.077 [DEBUG] TestAgent.dns: request served from client: name=aaa2f55a-3eb9-aa73-3aa5-34957fa1eaff.query.dc2.consul. type=SRV class=IN latency=202.62µs client=127.0.0.1:49334 client_network=udp +34:46.078 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=A class=IN latency=247.561µs client=127.0.0.1:37201 client_network=udp +34:46.078 [DEBUG] TestAgent.dns: request served from client: name=aaa2f55a-3eb9-aa73-3aa5-34957fa1eaff.query.dc2.consul. type=A class=IN latency=191.201µs client=127.0.0.1:48160 client_network=udp +34:46.095 [INFO] TestAgent: Requesting shutdown +34:46.095 [INFO] TestAgent.server: shutting down server +34:46.095 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:46.095 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:46.095 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:46.095 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:46.095 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:46.095 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:46.095 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:46.095 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:46.095 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:46.095 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:46.095 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:46.095 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:46.095 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:46.095 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +34:46.095 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:46.095 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +2021-08-13T15:34:46.095Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:46.095Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:46.095Z [INFO] agent.router.manager: shutting down +34:46.095 [INFO] TestAgent: consul server down +34:46.095 [INFO] TestAgent: shutdown complete +34:46.095 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25161 network=tcp +34:46.096 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25161 network=udp +34:46.096 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24958 network=tcp +34:46.596 [INFO] TestAgent: Waiting for endpoints to shut down +34:46.596 [INFO] TestAgent: Endpoints down +2021-08-13T15:34:45.825Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:45.825Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:45.842Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:45.842Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:45.845 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a1abdbad-734d-ecfc-bff5-df759c5c2e92 Address:127.0.0.1:25145}]" +34:45.845 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25145 [Follower]" leader= +34:45.845 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a1abdbad-734d-ecfc-bff5-df759c5c2e92.dc1 127.0.0.1 +34:45.846 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a1abdbad-734d-ecfc-bff5-df759c5c2e92 127.0.0.1 +2021-08-13T15:34:45.846Z [INFO] agent.router: Initializing LAN area manager +34:45.847 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25126 network=udp +34:45.847 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a1abdbad-734d-ecfc-bff5-df759c5c2e92.dc1 area=wan +34:45.847 [INFO] TestAgent.server: Adding LAN server: server="Node-a1abdbad-734d-ecfc-bff5-df759c5c2e92 (Addr: tcp/127.0.0.1:25145) (DC: dc1)" +34:45.847 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25126 network=tcp +34:45.848 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25141 network=tcp +34:45.848 [INFO] TestAgent: started state syncer +34:45.848 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25146 network=tcp +34:45.897 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:45.903 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:45.903 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25145 [Candidate]" term=2 +34:45.903 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:45.903 [DEBUG] TestAgent.server.raft: vote granted: from=a1abdbad-734d-ecfc-bff5-df759c5c2e92 term=2 tally=1 +34:45.903 [INFO] TestAgent.server.raft: election won: tally=1 +34:45.903 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25145 [Leader]" +34:45.904 [INFO] TestAgent.server: cluster leadership acquired +34:45.904 [INFO] TestAgent.server: initializing acls +34:45.904 [INFO] TestAgent.server: New leader elected: payload=Node-a1abdbad-734d-ecfc-bff5-df759c5c2e92 +34:45.904 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:45.905 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:45.905 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:45.905 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:45.906 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-a1abdbad-734d-ecfc-bff5-df759c5c2e92 +34:45.906 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-a1abdbad-734d-ecfc-bff5-df759c5c2e92.dc1 +34:45.907 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-a1abdbad-734d-ecfc-bff5-df759c5c2e92.dc1 area=wan +34:45.907 [INFO] TestAgent.server: Updating LAN server: server="Node-a1abdbad-734d-ecfc-bff5-df759c5c2e92 (Addr: tcp/127.0.0.1:25145) (DC: dc1)" +34:45.908 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:45.924 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:45.924 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:45.924 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:45.924 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:45.924 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:45.924 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a1abdbad-734d-ecfc-bff5-df759c5c2e92 +34:45.924 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a1abdbad-734d-ecfc-bff5-df759c5c2e92 +34:45.926 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a1abdbad-734d-ecfc-bff5-df759c5c2e92 +34:45.926 [INFO] TestAgent.server: federation state anti-entropy synced +34:46.055 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:38604 +34:46.056 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-db630e97-e63e-7983-712b-972c5a4ad0de.dc2 127.0.0.1 +34:46.056 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-db630e97-e63e-7983-712b-972c5a4ad0de.dc2 area=wan +34:46.062 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=SRV class=IN latency=3.623489ms client=127.0.0.1:43944 client_network=udp +34:46.063 [DEBUG] TestAgent.dns: request served from client: name=aaa2f55a-3eb9-aa73-3aa5-34957fa1eaff.query.dc2.consul. type=SRV class=IN latency=985.962µs client=127.0.0.1:39324 client_network=udp +34:46.065 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=A class=IN latency=787.742µs client=127.0.0.1:52374 client_network=udp +34:46.066 [DEBUG] TestAgent.dns: request served from client: name=aaa2f55a-3eb9-aa73-3aa5-34957fa1eaff.query.dc2.consul. type=A class=IN latency=776.342µs client=127.0.0.1:51916 client_network=udp +34:46.080 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=SRV class=IN latency=1.026223ms client=127.0.0.1:48417 client_network=udp +34:46.081 [DEBUG] TestAgent.dns: request served from client: name=aaa2f55a-3eb9-aa73-3aa5-34957fa1eaff.query.dc2.consul. type=SRV class=IN latency=708.222µs client=127.0.0.1:52745 client_network=udp +34:46.084 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=A class=IN latency=729.801µs client=127.0.0.1:39792 client_network=udp +34:46.085 [DEBUG] TestAgent.dns: request served from client: name=aaa2f55a-3eb9-aa73-3aa5-34957fa1eaff.query.dc2.consul. type=A class=IN latency=747.582µs client=127.0.0.1:46921 client_network=udp +34:46.087 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=SRV class=IN latency=697.262µs client=127.0.0.1:42378 client_network=udp +34:46.088 [DEBUG] TestAgent.dns: request served from client: name=aaa2f55a-3eb9-aa73-3aa5-34957fa1eaff.query.dc2.consul. type=SRV class=IN latency=711.342µs client=127.0.0.1:40219 client_network=udp +34:46.089 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=A class=IN latency=690.682µs client=127.0.0.1:51938 client_network=udp +34:46.090 [DEBUG] TestAgent.dns: request served from client: name=aaa2f55a-3eb9-aa73-3aa5-34957fa1eaff.query.dc2.consul. type=A class=IN latency=665.022µs client=127.0.0.1:51156 client_network=udp +34:46.092 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=SRV class=IN latency=759.022µs client=127.0.0.1:50508 client_network=udp +34:46.093 [DEBUG] TestAgent.dns: request served from client: name=aaa2f55a-3eb9-aa73-3aa5-34957fa1eaff.query.dc2.consul. type=SRV class=IN latency=684.102µs client=127.0.0.1:35273 client_network=udp +34:46.094 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=A class=IN latency=673.541µs client=127.0.0.1:53887 client_network=udp +34:46.095 [DEBUG] TestAgent.dns: request served from client: name=aaa2f55a-3eb9-aa73-3aa5-34957fa1eaff.query.dc2.consul. type=A class=IN latency=663.841µs client=127.0.0.1:46623 client_network=udp +34:46.129 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:46.130 [INFO] TestAgent: Synced node info +34:46.596 [INFO] TestAgent: Requesting shutdown +34:46.596 [INFO] TestAgent.server: shutting down server +34:46.596 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:46.596 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:46.596 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:46.596 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:46.596 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:46.596 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:46.596 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:46.596 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:46.596 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:46.596 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:46.596 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:46.596 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:46.596 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:46.596 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:46.596 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:46.597Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:46.597Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:46.597Z [INFO] agent.router.manager: shutting down +34:46.597 [INFO] TestAgent: consul server down +34:46.597 [INFO] TestAgent: shutdown complete +34:46.597 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25126 network=tcp +34:46.597 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25126 network=udp +34:46.597 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25141 network=tcp +34:47.097 [INFO] TestAgent: Waiting for endpoints to shut down +34:47.097 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup_WanTranslation (1.29s) + --- PASS: TestDNS_ServiceLookup_WanTranslation/service-wan-from-dc1 (0.01s) + --- PASS: TestDNS_ServiceLookup_WanTranslation/node-addr-from-dc2 (0.00s) + --- PASS: TestDNS_ServiceLookup_WanTranslation/node-wan-from-dc2 (0.00s) + --- PASS: TestDNS_ServiceLookup_WanTranslation/service-addr-from-dc2 (0.00s) + --- PASS: TestDNS_ServiceLookup_WanTranslation/service-wan-from-dc2 (0.00s) + --- PASS: TestDNS_ServiceLookup_WanTranslation/node-addr-from-dc1 (0.01s) + --- PASS: TestDNS_ServiceLookup_WanTranslation/node-wan-from-dc1 (0.00s) + --- PASS: TestDNS_ServiceLookup_WanTranslation/service-addr-from-dc1 (0.00s) +=== CONT TestDNS_ExternalServiceToConsulCNAMENestedLookup +2021-08-13T15:34:46.206Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:46.237Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:46.240 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:42195045-61d1-8903-b58a-eaf8c42f1e57 Address:127.0.0.1:25152}]" +34:46.240 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25152 [Follower]" leader= +34:46.241 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-42195045-61d1-8903-b58a-eaf8c42f1e57.dc1 127.0.0.1 +34:46.242 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-42195045-61d1-8903-b58a-eaf8c42f1e57 127.0.0.1 +2021-08-13T15:34:46.242Z [INFO] agent.router: Initializing LAN area manager +34:46.242 [DEBUG] TestAgent.dns: recursor enabled +34:46.242 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24922 network=udp +34:46.242 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-42195045-61d1-8903-b58a-eaf8c42f1e57.dc1 area=wan +34:46.242 [INFO] TestAgent.server: Adding LAN server: server="Node-42195045-61d1-8903-b58a-eaf8c42f1e57 (Addr: tcp/127.0.0.1:25152) (DC: dc1)" +34:46.243 [DEBUG] TestAgent.dns: recursor enabled +34:46.243 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24922 network=tcp +34:46.244 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25162 network=tcp +34:46.244 [INFO] TestAgent: started state syncer +34:46.244 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25153 network=tcp +34:46.278 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:46.278 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25152 [Candidate]" term=2 +34:46.278 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:46.278 [DEBUG] TestAgent.server.raft: vote granted: from=42195045-61d1-8903-b58a-eaf8c42f1e57 term=2 tally=1 +34:46.278 [INFO] TestAgent.server.raft: election won: tally=1 +34:46.278 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25152 [Leader]" +34:46.279 [INFO] TestAgent.server: cluster leadership acquired +34:46.279 [INFO] TestAgent.server: New leader elected: payload=Node-42195045-61d1-8903-b58a-eaf8c42f1e57 +34:46.279 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25152 +34:46.280 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:46.282 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:46.282 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:46.282 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:46.282 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:46.282 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:46.283 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-42195045-61d1-8903-b58a-eaf8c42f1e57 +34:46.283 [INFO] TestAgent.server: member joined, marking health alive: member=Node-42195045-61d1-8903-b58a-eaf8c42f1e57 +34:46.283 [INFO] TestAgent.server: federation state anti-entropy synced +34:46.460 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:46.461 [INFO] TestAgent: Synced node info +34:46.461 [DEBUG] TestAgent: Node info in sync +34:46.608 [DEBUG] TestAgent.dns: cname recurse RTT for name: name=www.google.com. rtt=180.2µs +34:46.608 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=588.321µs client=127.0.0.1:43076 client_network=udp +34:46.609 [DEBUG] TestAgent.dns: cname recurse RTT for name: name=www.google.com. rtt=137.921µs +34:46.609 [DEBUG] TestAgent.dns: request served from client: name=c715ca7f-bd5a-722d-36de-8790026b776a.query.consul. type=SRV class=IN latency=406.961µs client=127.0.0.1:37165 client_network=udp +34:46.609 [INFO] TestAgent: Requesting shutdown +34:46.609 [INFO] TestAgent.server: shutting down server +34:46.609 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:46.609 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:46.609 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:46.609 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:46.609 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:46.609 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:46.609 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:46.610 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:46.610 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:46.610 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:46.610 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:46.610Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:46.610Z [INFO] agent.router.manager: shutting down +34:46.610 [INFO] TestAgent: consul server down +34:46.610 [INFO] TestAgent: shutdown complete +34:46.610 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24922 network=tcp +34:46.610 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24922 network=udp +34:46.610 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25162 network=tcp +34:47.111 [INFO] TestAgent: Waiting for endpoints to shut down +34:47.111 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup_ServiceAddress_SRV (0.97s) +=== CONT TestDNS_ConnectServiceLookup +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:46.323Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:46.341Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:46.345 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c3c5224f-f352-67d8-cf72-09cb5c7bb3be Address:127.0.0.1:25173}]" +34:46.345 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25173 [Follower]" leader= +34:46.346 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c3c5224f-f352-67d8-cf72-09cb5c7bb3be.dc1 127.0.0.1 +34:46.347 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c3c5224f-f352-67d8-cf72-09cb5c7bb3be 127.0.0.1 +2021-08-13T15:34:46.347Z [INFO] agent.router: Initializing LAN area manager +34:46.347 [INFO] TestAgent.server: Adding LAN server: server="Node-c3c5224f-f352-67d8-cf72-09cb5c7bb3be (Addr: tcp/127.0.0.1:25173) (DC: dc1)" +34:46.347 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c3c5224f-f352-67d8-cf72-09cb5c7bb3be.dc1 area=wan +34:46.347 [DEBUG] TestAgent.dns: recursor enabled +34:46.348 [DEBUG] TestAgent.dns: recursor enabled +34:46.348 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25154 network=udp +34:46.348 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25154 network=tcp +34:46.349 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25155 network=tcp +34:46.349 [INFO] TestAgent: started state syncer +34:46.349 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25174 network=tcp +34:46.415 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:46.415 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25173 [Candidate]" term=2 +34:46.415 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:46.415 [DEBUG] TestAgent.server.raft: vote granted: from=c3c5224f-f352-67d8-cf72-09cb5c7bb3be term=2 tally=1 +34:46.415 [INFO] TestAgent.server.raft: election won: tally=1 +34:46.415 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25173 [Leader]" +34:46.415 [INFO] TestAgent.server: cluster leadership acquired +34:46.416 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25173 +34:46.417 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:46.419 [INFO] TestAgent.server: New leader elected: payload=Node-c3c5224f-f352-67d8-cf72-09cb5c7bb3be +34:46.420 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:46.420 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:46.420 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:46.420 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:46.420 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:46.420 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c3c5224f-f352-67d8-cf72-09cb5c7bb3be +34:46.420 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c3c5224f-f352-67d8-cf72-09cb5c7bb3be +34:46.421 [INFO] TestAgent.server: federation state anti-entropy synced +34:46.567 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:46.568 [INFO] TestAgent: Synced node info +34:46.775 [DEBUG] TestAgent.dns: recurse succeeded for question: question="{apple.com. 255 1}" rtt=3.293168ms recursor=127.0.0.1:58248 +34:46.775 [DEBUG] TestAgent.dns: request served from client: question="{apple.com. 255 1}" network=udp latency=3.525528ms client=127.0.0.1:45321 client_network=udp +34:46.775 [INFO] TestAgent: Requesting shutdown +34:46.775 [INFO] TestAgent.server: shutting down server +34:46.775 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:46.775 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:46.775 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:46.775 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:46.775 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:46.775 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:46.775 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:46.775 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:46.775 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:46.775 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:46.776 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:46.776Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:46.776Z [INFO] agent.router.manager: shutting down +34:46.776 [INFO] TestAgent: consul server down +34:46.776 [INFO] TestAgent: shutdown complete +34:46.776 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25154 network=tcp +34:46.776 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25154 network=udp +34:46.776 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25155 network=tcp +34:47.276 [INFO] TestAgent: Waiting for endpoints to shut down +34:47.276 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_Recurse_Truncation (0.97s) +=== CONT TestDNS_NSRecords_IPV6 +2021-08-13T15:34:46.793Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:46.826Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:46.838 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5790afe5-a379-1c57-87de-f128e5f6c597 Address:127.0.0.1:25194}]" +34:46.839 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25194 [Follower]" leader= +34:46.843 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5790afe5-a379-1c57-87de-f128e5f6c597.dc1 127.0.0.1 +34:46.845 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5790afe5-a379-1c57-87de-f128e5f6c597 127.0.0.1 +2021-08-13T15:34:46.845Z [INFO] agent.router: Initializing LAN area manager +34:46.846 [INFO] TestAgent.server: Adding LAN server: server="Node-5790afe5-a379-1c57-87de-f128e5f6c597 (Addr: tcp/127.0.0.1:25194) (DC: dc1)" +34:46.846 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5790afe5-a379-1c57-87de-f128e5f6c597.dc1 area=wan +34:46.851 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25168 network=tcp +34:46.851 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25168 network=udp +34:46.853 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25190 network=tcp +34:46.853 [INFO] TestAgent: started state syncer +34:46.853 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25195 network=tcp +34:46.897 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:46.897 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25194 [Candidate]" term=2 +34:46.897 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:46.897 [DEBUG] TestAgent.server.raft: vote granted: from=5790afe5-a379-1c57-87de-f128e5f6c597 term=2 tally=1 +34:46.897 [INFO] TestAgent.server.raft: election won: tally=1 +34:46.897 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25194 [Leader]" +34:46.897 [INFO] TestAgent.server: cluster leadership acquired +34:46.897 [INFO] TestAgent.server: New leader elected: payload=Node-5790afe5-a379-1c57-87de-f128e5f6c597 +34:46.897 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25194 +34:46.898 [INFO] TestAgent: Synced node info +34:46.898 [DEBUG] TestAgent: Node info in sync +34:46.899 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:46.901 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:46.901 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:46.901 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:46.901 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:46.901 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:46.901 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5790afe5-a379-1c57-87de-f128e5f6c597 +34:46.901 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5790afe5-a379-1c57-87de-f128e5f6c597 +34:46.902 [INFO] TestAgent.server: federation state anti-entropy synced +34:47.011 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=328.501µs client=127.0.0.1:36279 client_network=udp +34:47.012 [DEBUG] TestAgent.dns: request served from client: name=42622b25-3083-4039-e360-217489413f76.query.consul. type=SRV class=IN latency=196.18µs client=127.0.0.1:43367 client_network=udp +34:47.012 [INFO] TestAgent: Requesting shutdown +34:47.012 [INFO] TestAgent.server: shutting down server +34:47.012 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:47.012 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:47.012 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:47.012 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:47.012 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:47.012 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:47.012 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:47.012 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:47.012 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:47.012 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:47.012 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:47.012Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:47.012Z [INFO] agent.router.manager: shutting down +34:47.012 [INFO] TestAgent: consul server down +34:47.013 [INFO] TestAgent: shutdown complete +34:47.013 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25168 network=tcp +34:47.013 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25168 network=udp +34:47.013 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25190 network=tcp +34:47.513 [INFO] TestAgent: Waiting for endpoints to shut down +34:47.513 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup_Dedup_SRV (0.74s) +=== CONT TestDNS_ServiceLookupWithInternalServiceAddress +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:46.680Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:46.697Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:46.700 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3381f6b2-f1fb-bfff-93df-920de4f4c57c Address:127.0.0.1:25187}]" +34:46.701 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3381f6b2-f1fb-bfff-93df-920de4f4c57c.dc1 127.0.0.1 +34:46.702 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3381f6b2-f1fb-bfff-93df-920de4f4c57c 127.0.0.1 +2021-08-13T15:34:46.702Z [INFO] agent.router: Initializing LAN area manager +34:46.702 [DEBUG] TestAgent.dns: recursor enabled +34:46.702 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25175 network=udp +34:46.702 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25187 [Follower]" leader= +34:46.702 [INFO] TestAgent.server: Adding LAN server: server="Node-3381f6b2-f1fb-bfff-93df-920de4f4c57c (Addr: tcp/127.0.0.1:25187) (DC: dc1)" +34:46.703 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3381f6b2-f1fb-bfff-93df-920de4f4c57c.dc1 area=wan +34:46.703 [DEBUG] TestAgent.dns: recursor enabled +34:46.703 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25175 network=tcp +34:46.704 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25169 network=tcp +34:46.704 [INFO] TestAgent: started state syncer +34:46.704 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25188 network=tcp +34:46.753 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:46.753 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25187 [Candidate]" term=2 +34:46.754 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:46.754 [DEBUG] TestAgent.server.raft: vote granted: from=3381f6b2-f1fb-bfff-93df-920de4f4c57c term=2 tally=1 +34:46.754 [INFO] TestAgent.server.raft: election won: tally=1 +34:46.754 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25187 [Leader]" +34:46.754 [INFO] TestAgent.server: cluster leadership acquired +34:46.754 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25187 +34:46.755 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:46.758 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:46.758 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:46.758 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:46.758 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:46.758 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:46.758 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3381f6b2-f1fb-bfff-93df-920de4f4c57c +34:46.758 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3381f6b2-f1fb-bfff-93df-920de4f4c57c +34:46.759 [INFO] TestAgent.server: New leader elected: payload=Node-3381f6b2-f1fb-bfff-93df-920de4f4c57c +34:46.759 [INFO] TestAgent.server: federation state anti-entropy synced +34:46.859 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:46.859 [INFO] TestAgent: Synced node info +34:46.859 [DEBUG] TestAgent: Node info in sync +34:47.063 [DEBUG] TestAgent.dns: recurse succeeded for question: question="{apple.com. 255 1}" rtt=207.58µs recursor=127.0.0.1:39325 +34:47.063 [DEBUG] TestAgent.dns: request served from client: question="{apple.com. 255 1}" network=udp latency=437.681µs client=127.0.0.1:59720 client_network=udp +34:47.063 [INFO] TestAgent: Requesting shutdown +34:47.064 [INFO] TestAgent.server: shutting down server +34:47.064 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:47.064 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:47.064 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:47.064 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:47.064 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:47.064 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:47.064 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:47.064 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:47.064 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:47.064 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:47.064 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:47.064Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:47.064Z [INFO] agent.router.manager: shutting down +34:47.064 [INFO] TestAgent: consul server down +34:47.064 [INFO] TestAgent: shutdown complete +34:47.064 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25175 network=tcp +34:47.064 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25175 network=udp +34:47.064 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25169 network=tcp +34:47.566 [INFO] TestAgent: Waiting for endpoints to shut down +34:47.566 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_Recurse (0.90s) +=== CONT TestDNS_NSRecords +2021-08-13T15:34:46.755Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:46.772Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:46.775 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6c35fe69-445c-61b3-d792-061abfdcf30d Address:127.0.0.1:25166}]" +34:46.776 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25166 [Follower]" leader= +34:46.776 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6c35fe69-445c-61b3-d792-061abfdcf30d.dc1 127.0.0.1 +34:46.777 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6c35fe69-445c-61b3-d792-061abfdcf30d 127.0.0.1 +2021-08-13T15:34:46.777Z [INFO] agent.router: Initializing LAN area manager +34:46.777 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25189 network=udp +34:46.777 [INFO] TestAgent.server: Adding LAN server: server="Node-6c35fe69-445c-61b3-d792-061abfdcf30d (Addr: tcp/127.0.0.1:25166) (DC: dc1)" +34:46.777 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6c35fe69-445c-61b3-d792-061abfdcf30d.dc1 area=wan +34:46.778 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25189 network=tcp +34:46.779 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25176 network=tcp +34:46.779 [INFO] TestAgent: started state syncer +34:46.779 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25167 network=tcp +34:46.839 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:46.839 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25166 [Candidate]" term=2 +34:46.840 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:46.840 [DEBUG] TestAgent.server.raft: vote granted: from=6c35fe69-445c-61b3-d792-061abfdcf30d term=2 tally=1 +34:46.840 [INFO] TestAgent.server.raft: election won: tally=1 +34:46.840 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25166 [Leader]" +34:46.840 [INFO] TestAgent.server: cluster leadership acquired +34:46.840 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25166 +34:46.842 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:46.843 [INFO] TestAgent.server: New leader elected: payload=Node-6c35fe69-445c-61b3-d792-061abfdcf30d +34:46.857 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:46.857 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:46.857 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:46.857 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:46.857 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:46.858 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6c35fe69-445c-61b3-d792-061abfdcf30d +34:46.858 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6c35fe69-445c-61b3-d792-061abfdcf30d +34:46.858 [INFO] TestAgent.server: federation state anti-entropy synced +34:47.185 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:47.187 [INFO] TestAgent: Synced node info +34:47.187 [DEBUG] TestAgent: Node info in sync +34:47.211 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=410.701µs client=127.0.0.1:47177 client_network=udp +34:47.218 [DEBUG] TestAgent.dns: request served from client: name=25c22993-f143-bea4-729f-8205d886108d.query.consul. type=SRV class=IN latency=325.041µs client=127.0.0.1:57856 client_network=udp +34:47.218 [INFO] TestAgent: Requesting shutdown +34:47.218 [INFO] TestAgent.server: shutting down server +34:47.218 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:47.218 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:47.218 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:47.218 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:47.218 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:47.218 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:47.218 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:47.218 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:47.218 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:47.218 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:47.218 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:47.219Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:47.219Z [INFO] agent.router.manager: shutting down +34:47.219 [INFO] TestAgent: consul server down +34:47.219 [INFO] TestAgent: shutdown complete +34:47.219 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25189 network=tcp +34:47.219 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25189 network=udp +34:47.219 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25176 network=tcp +34:47.719 [INFO] TestAgent: Waiting for endpoints to shut down +34:47.719 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup_ServiceAddress_A (0.98s) +=== CONT TestDNS_ServiceLookup +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:47.302Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:47.321Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:47.324 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4b63a53b-3457-8221-c226-5ad41ae49059 Address:[::1]:25208}]" +34:47.324 [INFO] TestAgent.server.raft: entering follower state: follower="Node at [::1]:25208 [Follower]" leader= +34:47.324 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: server1.dc1 ::1 +34:47.325 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: server1 ::1 +2021-08-13T15:34:47.325Z [INFO] agent.router: Initializing LAN area manager +34:47.326 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=server1.dc1 area=wan +34:47.326 [INFO] TestAgent.server: Adding LAN server: server="server1 (Addr: tcp/[::1]:25208) (DC: dc1)" +34:47.326 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25203 network=udp +34:47.326 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25203 network=tcp +34:47.327 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25218 network=tcp +34:47.327 [INFO] TestAgent: started state syncer +34:47.327 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25209 network=tcp +34:47.377 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:47.377 [INFO] TestAgent.server.raft: entering candidate state: node="Node at [::1]:25208 [Candidate]" term=2 +34:47.377 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:47.378 [DEBUG] TestAgent.server.raft: vote granted: from=4b63a53b-3457-8221-c226-5ad41ae49059 term=2 tally=1 +34:47.378 [INFO] TestAgent.server.raft: election won: tally=1 +34:47.378 [INFO] TestAgent.server.raft: entering leader state: leader="Node at [::1]:25208 [Leader]" +34:47.378 [INFO] TestAgent.server: cluster leadership acquired +34:47.378 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=[::1]:25208 +34:47.379 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:47.382 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:47.382 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:47.382 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:47.382 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:47.382 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:47.382 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=server1 +34:47.382 [INFO] TestAgent.server: member joined, marking health alive: member=server1 +34:47.382 [INFO] TestAgent.server: New leader elected: payload=server1 +34:47.391 [INFO] TestAgent.server: federation state anti-entropy synced +34:47.433 [DEBUG] TestAgent.dns: request served from client: name=server1.node.dc1.consul. type=NS class=IN latency=217.4µs client=127.0.0.1:45247 client_network=udp +34:47.433 [INFO] TestAgent: Requesting shutdown +34:47.433 [INFO] TestAgent.server: shutting down server +34:47.433 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:47.433 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:47.433 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:47.433 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:47.433 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:47.433 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:47.433 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:47.433 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:47.433 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:47.434 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:47.434 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:47.434 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:47.434Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:47.434Z [INFO] agent.router.manager: shutting down +34:47.434 [INFO] TestAgent: consul server down +34:47.434 [INFO] TestAgent: shutdown complete +34:47.434 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25203 network=tcp +34:47.434 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25203 network=udp +34:47.434 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25218 network=tcp +34:47.935 [INFO] TestAgent: Waiting for endpoints to shut down +34:47.935 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_NSRecords_IPV6 (0.66s) +=== CONT TestDNS_ExternalServiceToConsulCNAMELookup +2021-08-13T15:34:47.128Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:47.167Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:47.170 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:926c3c62-7578-8378-9cb2-ad38a303fab9 Address:127.0.0.1:25180}]" +34:47.170 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25180 [Follower]" leader= +34:47.180 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +34:47.184 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2021-08-13T15:34:47.184Z [INFO] agent.router: Initializing LAN area manager +34:47.188 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:25180) (DC: dc1)" +34:47.190 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25196 network=tcp +34:47.192 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +34:47.193 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25196 network=udp +34:47.206 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25197 network=tcp +34:47.206 [INFO] TestAgent: started state syncer +34:47.206 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25181 network=tcp +34:47.223 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:47.223 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25180 [Candidate]" term=2 +34:47.223 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:47.223 [DEBUG] TestAgent.server.raft: vote granted: from=926c3c62-7578-8378-9cb2-ad38a303fab9 term=2 tally=1 +34:47.223 [INFO] TestAgent.server.raft: election won: tally=1 +34:47.223 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25180 [Leader]" +34:47.223 [INFO] TestAgent.server: cluster leadership acquired +34:47.224 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25180 +34:47.226 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:47.228 [INFO] TestAgent.server: New leader elected: payload=test-node +34:47.237 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:47.237 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:47.237 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:47.237 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:47.237 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:47.238 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +34:47.238 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +34:47.238 [INFO] TestAgent.server: federation state anti-entropy synced +34:47.473 [DEBUG] TestAgent.dns: request served from client: name=alias2.service.consul. type=SRV class=IN latency=397.781µs client=127.0.0.1:35070 client_network=udp +34:47.473 [INFO] TestAgent: Requesting shutdown +34:47.473 [INFO] TestAgent.server: shutting down server +34:47.473 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:47.473 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:47.473 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:47.473 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:47.473 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:47.473 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:47.473 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:47.473 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:47.473 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:47.473 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:47.473 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:47.473 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:47.473Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:47.473Z [INFO] agent.router.manager: shutting down +34:47.474 [INFO] TestAgent: consul server down +34:47.474 [INFO] TestAgent: shutdown complete +34:47.474 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25196 network=tcp +34:47.474 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25196 network=udp +34:47.474 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25197 network=tcp +34:47.975 [INFO] TestAgent: Waiting for endpoints to shut down +34:47.975 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ExternalServiceToConsulCNAMENestedLookup (0.88s) +=== CONT TestDNS_ServiceLookupMultiAddrNoCNAME +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:47.532Z [WARN] agent: Node name "my.test-node" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. +2021-08-13T15:34:47.532Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:47.549Z [WARN] agent.auto_config: Node name "my.test-node" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. +2021-08-13T15:34:47.549Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:47.561 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d0ec7d9a-c14d-4e10-2320-bd4dd4a7667e Address:127.0.0.1:25215}]" +34:47.564 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: my.test-node.dc1 127.0.0.1 +34:47.565 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25215 [Follower]" leader= +34:47.584 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: my.test-node 127.0.0.1 +2021-08-13T15:34:47.584Z [INFO] agent.router: Initializing LAN area manager +34:47.585 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25210 network=udp +34:47.585 [INFO] TestAgent.server: Adding LAN server: server="my.test-node (Addr: tcp/127.0.0.1:25215) (DC: dc1)" +34:47.585 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=my.test-node.dc1 area=wan +34:47.585 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25210 network=tcp +34:47.601 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25204 network=tcp +34:47.601 [INFO] TestAgent: started state syncer +34:47.601 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25216 network=tcp +34:47.632 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:47.632 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25215 [Candidate]" term=2 +34:47.632 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:47.632 [DEBUG] TestAgent.server.raft: vote granted: from=d0ec7d9a-c14d-4e10-2320-bd4dd4a7667e term=2 tally=1 +34:47.632 [INFO] TestAgent.server.raft: election won: tally=1 +34:47.632 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25215 [Leader]" +34:47.632 [INFO] TestAgent.server: cluster leadership acquired +34:47.633 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25215 +34:47.634 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:47.635 [INFO] TestAgent.server: New leader elected: payload=my.test-node +34:47.636 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:47.636 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:47.636 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:47.636 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:47.636 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:47.636 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=my.test-node +34:47.636 [INFO] TestAgent.server: member joined, marking health alive: member=my.test-node +34:47.638 [INFO] TestAgent.server: federation state anti-entropy synced +34:47.640 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=311.32µs client=127.0.0.1:48226 client_network=udp +34:47.640 [INFO] TestAgent: Requesting shutdown +34:47.640 [INFO] TestAgent.server: shutting down server +34:47.640 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:47.640 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:47.640 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:47.640 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:47.640 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:47.640 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:47.640 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:47.640 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:47.640 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:47.640 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:47.640 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:47.640 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:47.640Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:47.640Z [INFO] agent.router.manager: shutting down +34:47.640 [INFO] TestAgent: consul server down +34:47.640 [INFO] TestAgent: shutdown complete +34:47.641 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25210 network=tcp +34:47.641 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25210 network=udp +34:47.641 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25204 network=tcp +34:48.141 [INFO] TestAgent: Waiting for endpoints to shut down +34:48.141 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookupWithInternalServiceAddress (0.63s) +=== CONT TestDNS_InifiniteRecursion +2021-08-13T15:34:47.145Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:47.199Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:47.218 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e88d309d-59e7-455f-03ed-49a0d4e8e7c2 Address:127.0.0.1:25201}]" +34:47.219 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e88d309d-59e7-455f-03ed-49a0d4e8e7c2.dc1 127.0.0.1 +34:47.219 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25201 [Follower]" leader= +34:47.238 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e88d309d-59e7-455f-03ed-49a0d4e8e7c2 127.0.0.1 +2021-08-13T15:34:47.238Z [INFO] agent.router: Initializing LAN area manager +34:47.239 [INFO] TestAgent.server: Adding LAN server: server="Node-e88d309d-59e7-455f-03ed-49a0d4e8e7c2 (Addr: tcp/127.0.0.1:25201) (DC: dc1)" +34:47.240 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25182 network=udp +34:47.240 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e88d309d-59e7-455f-03ed-49a0d4e8e7c2.dc1 area=wan +34:47.241 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25182 network=tcp +34:47.242 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25183 network=tcp +34:47.242 [INFO] TestAgent: started state syncer +34:47.242 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25202 network=tcp +34:47.270 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:47.270 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25201 [Candidate]" term=2 +34:47.270 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:47.270 [DEBUG] TestAgent.server.raft: vote granted: from=e88d309d-59e7-455f-03ed-49a0d4e8e7c2 term=2 tally=1 +34:47.270 [INFO] TestAgent.server.raft: election won: tally=1 +34:47.270 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25201 [Leader]" +34:47.270 [INFO] TestAgent.server: cluster leadership acquired +34:47.270 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25201 +34:47.271 [INFO] TestAgent.server: New leader elected: payload=Node-e88d309d-59e7-455f-03ed-49a0d4e8e7c2 +34:47.272 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:47.274 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:47.274 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:47.274 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:47.274 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:47.274 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:47.274 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e88d309d-59e7-455f-03ed-49a0d4e8e7c2 +34:47.274 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e88d309d-59e7-455f-03ed-49a0d4e8e7c2 +34:47.275 [INFO] TestAgent.server: federation state anti-entropy synced +34:47.587 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:47.588 [INFO] TestAgent: Synced node info +34:47.647 [DEBUG] TestAgent.dns: request served from client: name=db.connect.consul. type=SRV class=IN latency=244.281µs client=127.0.0.1:54852 client_network=udp +34:47.647 [INFO] TestAgent: Requesting shutdown +34:47.647 [INFO] TestAgent.server: shutting down server +34:47.647 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:47.647 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:47.647 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:47.647 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:47.647 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:47.647 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:47.647 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:47.647 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:47.647 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:47.647 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:47.647 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:47.647Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:47.647Z [INFO] agent.router.manager: shutting down +34:47.648 [INFO] TestAgent: consul server down +34:47.648 [INFO] TestAgent: shutdown complete +34:47.648 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25182 network=tcp +34:47.648 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25182 network=udp +34:47.648 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25183 network=tcp +34:48.148 [INFO] TestAgent: Waiting for endpoints to shut down +34:48.148 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ConnectServiceLookup (1.04s) +=== CONT TestDNS_ServiceLookupPreferNoCNAME +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:47.625Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:47.658Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:47.661 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:37b4ea3c-3097-5647-54fc-f70e1daecc95 Address:127.0.0.1:25222}]" +34:47.661 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25222 [Follower]" leader= +34:47.661 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: server1.dc1 127.0.0.1 +34:47.662 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: server1 127.0.0.1 +2021-08-13T15:34:47.662Z [INFO] agent.router: Initializing LAN area manager +34:47.662 [INFO] TestAgent.server: Adding LAN server: server="server1 (Addr: tcp/127.0.0.1:25222) (DC: dc1)" +34:47.663 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=server1.dc1 area=wan +34:47.663 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25217 network=udp +34:47.663 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25217 network=tcp +34:47.664 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25211 network=tcp +34:47.664 [INFO] TestAgent: started state syncer +34:47.664 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25223 network=tcp +34:47.725 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:47.725 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25222 [Candidate]" term=2 +34:47.726 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:47.726 [DEBUG] TestAgent.server.raft: vote granted: from=37b4ea3c-3097-5647-54fc-f70e1daecc95 term=2 tally=1 +34:47.726 [INFO] TestAgent.server.raft: election won: tally=1 +34:47.726 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25222 [Leader]" +34:47.726 [INFO] TestAgent.server: cluster leadership acquired +34:47.726 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25222 +34:47.727 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:47.729 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:47.729 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:47.729 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:47.730 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:47.730 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:47.730 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=server1 +34:47.730 [INFO] TestAgent.server: member joined, marking health alive: member=server1 +34:47.730 [INFO] TestAgent.server: New leader elected: payload=server1 +34:47.739 [INFO] TestAgent.server: federation state anti-entropy synced +34:47.880 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:47.880 [INFO] TestAgent: Synced node info +34:47.940 [DEBUG] TestAgent.dns: request served from client: name=something.node.consul. type=NS class=IN latency=246.301µs client=127.0.0.1:57113 client_network=udp +34:47.940 [INFO] TestAgent: Requesting shutdown +34:47.940 [INFO] TestAgent.server: shutting down server +34:47.940 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:47.940 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:47.940 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:47.940 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:47.940 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:47.940 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:47.940 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:47.940 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:47.940 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:47.940 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:47.941 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:47.941Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:47.941Z [INFO] agent.router.manager: shutting down +34:47.941 [INFO] TestAgent: consul server down +34:47.941 [INFO] TestAgent: shutdown complete +34:47.941 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25217 network=tcp +34:47.941 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25217 network=udp +34:47.941 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25211 network=tcp +34:48.441 [INFO] TestAgent: Waiting for endpoints to shut down +34:48.441 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_NSRecords (0.87s) +=== CONT TestDNS_ExternalServiceLookup +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=255)_compressed=true +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:47.737Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:47.753Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:47.755 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5f8b17f3-76d2-ea1c-8448-c69bf61af1ad Address:127.0.0.1:25250}]" +34:47.756 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25250 [Follower]" leader= +34:47.756 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5f8b17f3-76d2-ea1c-8448-c69bf61af1ad.dc1 127.0.0.1 +34:47.757 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5f8b17f3-76d2-ea1c-8448-c69bf61af1ad 127.0.0.1 +2021-08-13T15:34:47.757Z [INFO] agent.router: Initializing LAN area manager +34:47.757 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25224 network=udp +34:47.758 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5f8b17f3-76d2-ea1c-8448-c69bf61af1ad.dc1 area=wan +34:47.758 [INFO] TestAgent.server: Adding LAN server: server="Node-5f8b17f3-76d2-ea1c-8448-c69bf61af1ad (Addr: tcp/127.0.0.1:25250) (DC: dc1)" +34:47.758 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25224 network=tcp +34:47.759 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25225 network=tcp +34:47.759 [INFO] TestAgent: started state syncer +34:47.759 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25251 network=tcp +34:47.802 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:47.802 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25250 [Candidate]" term=2 +34:47.802 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:47.802 [DEBUG] TestAgent.server.raft: vote granted: from=5f8b17f3-76d2-ea1c-8448-c69bf61af1ad term=2 tally=1 +34:47.802 [INFO] TestAgent.server.raft: election won: tally=1 +34:47.802 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25250 [Leader]" +34:47.802 [INFO] TestAgent.server: cluster leadership acquired +34:47.802 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25250 +34:47.803 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:47.806 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:47.806 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:47.806 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:47.806 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:47.806 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:47.806 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5f8b17f3-76d2-ea1c-8448-c69bf61af1ad +34:47.806 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5f8b17f3-76d2-ea1c-8448-c69bf61af1ad +34:47.807 [INFO] TestAgent.server: New leader elected: payload=Node-5f8b17f3-76d2-ea1c-8448-c69bf61af1ad +34:47.807 [INFO] TestAgent.server: federation state anti-entropy synced +34:47.898 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:47.899 [INFO] TestAgent: Synced node info +34:47.899 [DEBUG] TestAgent: Node info in sync +34:48.079 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=304.581µs client=127.0.0.1:52855 client_network=udp +34:48.083 [DEBUG] TestAgent.dns: request served from client: name=99bc42bb-b6d8-1423-1bd9-bfbcc2d40931.query.consul. type=SRV class=IN latency=221.941µs client=127.0.0.1:44224 client_network=udp +34:48.087 [DEBUG] TestAgent.dns: request served from client: name=nodb.service.consul. type=SRV class=IN latency=160.64µs client=127.0.0.1:35969 client_network=udp +34:48.087 [DEBUG] TestAgent.dns: request served from client: name=nope.query.consul. type=SRV class=IN latency=107.3µs client=127.0.0.1:51500 client_network=udp +34:48.088 [INFO] TestAgent: Requesting shutdown +34:48.088 [INFO] TestAgent.server: shutting down server +34:48.088 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:48.088 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:48.088 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:48.088 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:48.088 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:48.088 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:48.088 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:48.088 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:48.088 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:48.088 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:48.088 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:48.088Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:48.088Z [INFO] agent.router.manager: shutting down +34:48.088 [INFO] TestAgent: consul server down +34:48.088 [INFO] TestAgent: shutdown complete +34:48.088 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25224 network=tcp +34:48.088 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25224 network=udp +34:48.088 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25225 network=tcp +34:48.591 [INFO] TestAgent: Waiting for endpoints to shut down +34:48.591 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup (0.88s) +=== CONT TestDNS_ServiceLookupNoMultiCNAME +2021-08-13T15:34:48.009Z [WARN] agent: Node name "test node" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. +2021-08-13T15:34:48.009Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:48.044Z [WARN] agent.auto_config: Node name "test node" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. +2021-08-13T15:34:48.044Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:48.046 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fdf5eee7-cdaf-53ae-deb9-be6a12eb156c Address:127.0.0.1:25236}]" +34:48.047 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25236 [Follower]" leader= +34:48.047 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test node.dc1 127.0.0.1 +34:48.048 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test node 127.0.0.1 +2021-08-13T15:34:48.048Z [INFO] agent.router: Initializing LAN area manager +34:48.048 [INFO] TestAgent.server: Adding LAN server: server="test node (Addr: tcp/127.0.0.1:25236) (DC: dc1)" +34:48.048 [INFO] TestAgent.server: Handled event for server in area: event=member-join server="test node.dc1" area=wan +34:48.049 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25252 network=tcp +34:48.049 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25252 network=udp +34:48.050 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25253 network=tcp +34:48.050 [INFO] TestAgent: started state syncer +34:48.050 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25237 network=tcp +34:48.099 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:48.099 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25236 [Candidate]" term=2 +34:48.100 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:48.100 [DEBUG] TestAgent.server.raft: vote granted: from=fdf5eee7-cdaf-53ae-deb9-be6a12eb156c term=2 tally=1 +34:48.100 [INFO] TestAgent.server.raft: election won: tally=1 +34:48.100 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25236 [Leader]" +34:48.100 [INFO] TestAgent.server: cluster leadership acquired +34:48.100 [INFO] TestAgent.server: New leader elected: payload="test node" +34:48.100 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25236 +34:48.101 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:48.104 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:48.104 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:48.104 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:48.104 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:48.104 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:48.104 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node="test node" +34:48.104 [INFO] TestAgent.server: member joined, marking health alive: member="test node" +34:48.105 [INFO] TestAgent.server: federation state anti-entropy synced +34:48.241 [DEBUG] TestAgent.dns: request served from client: name=alias.service.consul. type=SRV class=IN latency=387.441µs client=127.0.0.1:36403 client_network=udp +34:48.242 [DEBUG] TestAgent.dns: request served from client: name=alias.service.CoNsUl. type=SRV class=IN latency=253.38µs client=127.0.0.1:34290 client_network=udp +34:48.242 [INFO] TestAgent: Requesting shutdown +34:48.242 [INFO] TestAgent.server: shutting down server +34:48.242 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:48.242 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:48.242 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:48.242 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:48.242 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:48.242 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:48.242 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:48.242 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:48.242 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:48.242 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:48.242 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:48.242 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:48.242Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:48.242Z [INFO] agent.router.manager: shutting down +34:48.243 [INFO] TestAgent: consul server down +34:48.243 [INFO] TestAgent: shutdown complete +34:48.243 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25252 network=tcp +34:48.243 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25252 network=udp +34:48.243 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25253 network=tcp +34:48.743 [INFO] TestAgent: Waiting for endpoints to shut down +34:48.744 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ExternalServiceToConsulCNAMELookup (0.81s) +=== CONT TestDNS_IngressServiceLookup +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:48.165Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:48.181Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:48.184 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5249e7ff-b0a4-a671-5947-ff559155c777 Address:127.0.0.1:25264}]" +34:48.184 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25264 [Follower]" leader= +34:48.184 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5249e7ff-b0a4-a671-5947-ff559155c777.dc1 127.0.0.1 +34:48.185 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5249e7ff-b0a4-a671-5947-ff559155c777 127.0.0.1 +2021-08-13T15:34:48.185Z [INFO] agent.router: Initializing LAN area manager +34:48.186 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25280 network=udp +34:48.186 [INFO] TestAgent.server: Adding LAN server: server="Node-5249e7ff-b0a4-a671-5947-ff559155c777 (Addr: tcp/127.0.0.1:25264) (DC: dc1)" +34:48.186 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5249e7ff-b0a4-a671-5947-ff559155c777.dc1 area=wan +34:48.186 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25280 network=tcp +34:48.187 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25246 network=tcp +34:48.187 [INFO] TestAgent: started state syncer +34:48.187 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25265 network=tcp +34:48.237 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:48.237 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25264 [Candidate]" term=2 +34:48.237 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:48.237 [DEBUG] TestAgent.server.raft: vote granted: from=5249e7ff-b0a4-a671-5947-ff559155c777 term=2 tally=1 +34:48.237 [INFO] TestAgent.server.raft: election won: tally=1 +34:48.237 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25264 [Leader]" +34:48.237 [INFO] TestAgent.server: cluster leadership acquired +34:48.238 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25264 +34:48.238 [INFO] TestAgent.server: New leader elected: payload=Node-5249e7ff-b0a4-a671-5947-ff559155c777 +34:48.239 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:48.241 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:48.241 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:48.241 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:48.241 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:48.241 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:48.241 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5249e7ff-b0a4-a671-5947-ff559155c777 +34:48.241 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5249e7ff-b0a4-a671-5947-ff559155c777 +34:48.242 [INFO] TestAgent.server: federation state anti-entropy synced +34:48.306 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=ANY class=IN latency=431.821µs client=127.0.0.1:36934 client_network=udp +34:48.306 [INFO] TestAgent: Requesting shutdown +34:48.306 [INFO] TestAgent.server: shutting down server +34:48.306 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:48.306 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:48.306 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:48.306 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:48.306 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:48.306 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:48.306 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:48.307 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:48.307 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:48.311 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +34:48.311 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:48.311 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +2021-08-13T15:34:48.311Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:48.312Z [INFO] agent.router.manager: shutting down +34:48.312 [INFO] TestAgent: consul server down +34:48.312 [INFO] TestAgent: shutdown complete +34:48.312 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25280 network=tcp +34:48.312 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25280 network=udp +34:48.312 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25246 network=tcp +34:48.815 [INFO] TestAgent: Waiting for endpoints to shut down +34:48.815 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookupPreferNoCNAME (0.67s) +=== CONT TestDNS_ServiceReverseLookupNodeAddress +2021-08-13T15:34:48.023Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:48.059Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:48.062 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:23214f30-cb93-844a-8de4-107c1f2952fc Address:127.0.0.1:25243}]" +34:48.062 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25243 [Follower]" leader= +34:48.063 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-23214f30-cb93-844a-8de4-107c1f2952fc.dc1 127.0.0.1 +34:48.063 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-23214f30-cb93-844a-8de4-107c1f2952fc 127.0.0.1 +2021-08-13T15:34:48.063Z [INFO] agent.router: Initializing LAN area manager +34:48.064 [INFO] TestAgent.server: Adding LAN server: server="Node-23214f30-cb93-844a-8de4-107c1f2952fc (Addr: tcp/127.0.0.1:25243) (DC: dc1)" +34:48.064 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-23214f30-cb93-844a-8de4-107c1f2952fc.dc1 area=wan +34:48.064 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25238 network=udp +34:48.064 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25238 network=tcp +34:48.065 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25239 network=tcp +34:48.065 [INFO] TestAgent: started state syncer +34:48.065 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25244 network=tcp +34:48.127 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:48.127 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25243 [Candidate]" term=2 +34:48.127 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:48.127 [DEBUG] TestAgent.server.raft: vote granted: from=23214f30-cb93-844a-8de4-107c1f2952fc term=2 tally=1 +34:48.127 [INFO] TestAgent.server.raft: election won: tally=1 +34:48.127 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25243 [Leader]" +34:48.127 [INFO] TestAgent.server: cluster leadership acquired +34:48.128 [INFO] TestAgent.server: New leader elected: payload=Node-23214f30-cb93-844a-8de4-107c1f2952fc +34:48.128 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25243 +34:48.129 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:48.131 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:48.131 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:48.131 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:48.131 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:48.131 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:48.132 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-23214f30-cb93-844a-8de4-107c1f2952fc +34:48.132 [INFO] TestAgent.server: member joined, marking health alive: member=Node-23214f30-cb93-844a-8de4-107c1f2952fc +34:48.132 [INFO] TestAgent.server: federation state anti-entropy synced +34:48.343 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=ANY class=IN latency=467.981µs client=127.0.0.1:57744 client_network=udp +34:48.343 [INFO] TestAgent: Requesting shutdown +34:48.350 [INFO] TestAgent.server: shutting down server +34:48.350 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:48.350 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:48.350 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:48.350 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:48.350 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:48.350 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:48.350 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:48.350 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:48.350 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:48.351 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:48.351 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:48.351 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:48.351Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:48.351Z [INFO] agent.router.manager: shutting down +34:48.351 [INFO] TestAgent: consul server down +34:48.351 [INFO] TestAgent: shutdown complete +34:48.351 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25238 network=tcp +34:48.351 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25238 network=udp +34:48.351 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25239 network=tcp +34:48.852 [INFO] TestAgent: Waiting for endpoints to shut down +34:48.853 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookupMultiAddrNoCNAME (0.88s) +=== CONT TestDNS_SOA_Settings +2021-08-13T15:34:48.161Z [WARN] agent: Node name "test node" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. +2021-08-13T15:34:48.161Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:48.178Z [WARN] agent.auto_config: Node name "test node" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. +2021-08-13T15:34:48.178Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:48.181 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:41fecb31-89cb-cef6-f3bd-887510565eb0 Address:127.0.0.1:25278}]" +34:48.182 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test node.dc1 127.0.0.1 +34:48.183 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25278 [Follower]" leader= +34:48.183 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test node 127.0.0.1 +2021-08-13T15:34:48.183Z [INFO] agent.router: Initializing LAN area manager +34:48.183 [INFO] TestAgent.server: Handled event for server in area: event=member-join server="test node.dc1" area=wan +34:48.183 [INFO] TestAgent.server: Adding LAN server: server="test node (Addr: tcp/127.0.0.1:25278) (DC: dc1)" +34:48.183 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25245 network=tcp +34:48.184 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25245 network=udp +34:48.184 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25274 network=tcp +34:48.185 [INFO] TestAgent: started state syncer +34:48.185 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25279 network=tcp +34:48.225 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:48.225 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25278 [Candidate]" term=2 +34:48.225 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:48.225 [DEBUG] TestAgent.server.raft: vote granted: from=41fecb31-89cb-cef6-f3bd-887510565eb0 term=2 tally=1 +34:48.225 [INFO] TestAgent.server.raft: election won: tally=1 +34:48.225 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25278 [Leader]" +34:48.225 [INFO] TestAgent.server: cluster leadership acquired +34:48.226 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25278 +34:48.227 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:48.229 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:48.229 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:48.229 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:48.229 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:48.229 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:48.229 [INFO] TestAgent.server: New leader elected: payload="test node" +34:48.229 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node="test node" +34:48.229 [INFO] TestAgent.server: member joined, marking health alive: member="test node" +34:48.230 [INFO] TestAgent.server: federation state anti-entropy synced +34:48.411 [ERROR] TestAgent.dns: Infinite recursion detected for name, won't perform any CNAME resolution.: name=web.service.consul. +34:48.411 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=A class=IN latency=618.682µs client=127.0.0.1:40340 client_network=udp +34:48.411 [INFO] TestAgent: Requesting shutdown +34:48.412 [INFO] TestAgent.server: shutting down server +34:48.412 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:48.412 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:48.412 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:48.412 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:48.412 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:48.412 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:48.412 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:48.412 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:48.412 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:48.412 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:48.412 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:48.412 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:48.412Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:48.412Z [INFO] agent.router.manager: shutting down +34:48.412 [INFO] TestAgent: consul server down +34:48.412 [INFO] TestAgent: shutdown complete +34:48.412 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25245 network=tcp +34:48.412 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25245 network=udp +34:48.412 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25274 network=tcp +34:48.923 [INFO] TestAgent: Waiting for endpoints to shut down +34:48.923 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_InifiniteRecursion (0.78s) +=== CONT TestDNS_ReverseLookup +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=255)_compressed=false +=== RUN TestDNS_IngressServiceLookup/api.ingress.consul. +=== RUN TestDNS_IngressServiceLookup/api.ingress.dc1.consul. +=== RUN TestDNS_IngressServiceLookup/db.ingress.consul. +=== RUN TestDNS_IngressServiceLookup/db.ingress.dc1.consul. +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:48.459Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:48.476Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:48.479 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8dab1ec9-800d-535a-8562-a49326d58856 Address:127.0.0.1:25271}]" +34:48.479 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25271 [Follower]" leader= +34:48.480 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8dab1ec9-800d-535a-8562-a49326d58856.dc1 127.0.0.1 +34:48.481 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8dab1ec9-800d-535a-8562-a49326d58856 127.0.0.1 +2021-08-13T15:34:48.481Z [INFO] agent.router: Initializing LAN area manager +34:48.481 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25266 network=udp +34:48.481 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8dab1ec9-800d-535a-8562-a49326d58856.dc1 area=wan +34:48.481 [INFO] TestAgent.server: Adding LAN server: server="Node-8dab1ec9-800d-535a-8562-a49326d58856 (Addr: tcp/127.0.0.1:25271) (DC: dc1)" +34:48.481 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25266 network=tcp +34:48.482 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25267 network=tcp +34:48.482 [INFO] TestAgent: started state syncer +34:48.482 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25272 network=tcp +34:48.518 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:48.518 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25271 [Candidate]" term=2 +34:48.518 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:48.518 [DEBUG] TestAgent.server.raft: vote granted: from=8dab1ec9-800d-535a-8562-a49326d58856 term=2 tally=1 +34:48.518 [INFO] TestAgent.server.raft: election won: tally=1 +34:48.518 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25271 [Leader]" +34:48.518 [INFO] TestAgent.server: cluster leadership acquired +34:48.518 [INFO] TestAgent.server: New leader elected: payload=Node-8dab1ec9-800d-535a-8562-a49326d58856 +34:48.519 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25271 +34:48.520 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:48.523 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:48.523 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:48.523 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:48.523 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:48.523 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:48.523 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8dab1ec9-800d-535a-8562-a49326d58856 +34:48.523 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8dab1ec9-800d-535a-8562-a49326d58856 +34:48.524 [INFO] TestAgent.server: federation state anti-entropy synced +34:48.839 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=412.921µs client=127.0.0.1:42212 client_network=udp +34:48.840 [INFO] TestAgent: Requesting shutdown +34:48.840 [INFO] TestAgent.server: shutting down server +34:48.840 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:48.840 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:48.840 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:48.840 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:48.840 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:48.840 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:48.840 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:48.840 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:48.840 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:48.840 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:48.840 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:48.840 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:48.841Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:48.841Z [INFO] agent.router.manager: shutting down +34:48.841 [INFO] TestAgent: consul server down +34:48.841 [INFO] TestAgent: shutdown complete +34:48.841 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25266 network=tcp +34:48.841 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25266 network=udp +34:48.841 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25267 network=tcp +34:49.343 [INFO] TestAgent: Waiting for endpoints to shut down +34:49.343 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ExternalServiceLookup (0.90s) +=== CONT TestDNS_ServiceReverseLookup_CustomDomain +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=255)_compressed=true +2021-08-13T15:34:48.793Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:48.826Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:48.830 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8f8cc92e-2aaf-08cc-f888-3930c1e8bf74 Address:127.0.0.1:25285}]" +34:48.831 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8f8cc92e-2aaf-08cc-f888-3930c1e8bf74.dc1 127.0.0.1 +34:48.833 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8f8cc92e-2aaf-08cc-f888-3930c1e8bf74 127.0.0.1 +2021-08-13T15:34:48.833Z [INFO] agent.router: Initializing LAN area manager +34:48.834 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25301 network=udp +34:48.834 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25285 [Follower]" leader= +34:48.834 [INFO] TestAgent.server: Adding LAN server: server="Node-8f8cc92e-2aaf-08cc-f888-3930c1e8bf74 (Addr: tcp/127.0.0.1:25285) (DC: dc1)" +34:48.834 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8f8cc92e-2aaf-08cc-f888-3930c1e8bf74.dc1 area=wan +34:48.835 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25301 network=tcp +34:48.837 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25281 network=tcp +34:48.837 [INFO] TestAgent: started state syncer +34:48.837 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25286 network=tcp +34:48.899 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:48.899 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25285 [Candidate]" term=2 +34:48.899 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:48.899 [DEBUG] TestAgent.server.raft: vote granted: from=8f8cc92e-2aaf-08cc-f888-3930c1e8bf74 term=2 tally=1 +34:48.899 [INFO] TestAgent.server.raft: election won: tally=1 +34:48.900 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25285 [Leader]" +34:48.901 [INFO] TestAgent.server: cluster leadership acquired +34:48.902 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25285 +34:48.903 [INFO] TestAgent.server: New leader elected: payload=Node-8f8cc92e-2aaf-08cc-f888-3930c1e8bf74 +34:48.905 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:48.918 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:48.918 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:48.919 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:48.919 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:48.919 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:48.919 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8f8cc92e-2aaf-08cc-f888-3930c1e8bf74 +34:48.919 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8f8cc92e-2aaf-08cc-f888-3930c1e8bf74 +34:48.922 [INFO] TestAgent.server: federation state anti-entropy synced +34:48.972 [DEBUG] TestAgent.dns: request served from client: name=api.ingress.consul. type=A class=IN latency=369.341µs client=127.0.0.1:51337 client_network=udp +34:48.973 [DEBUG] TestAgent.dns: request served from client: name=api.ingress.dc1.consul. type=A class=IN latency=274.56µs client=127.0.0.1:53135 client_network=udp +34:48.974 [DEBUG] TestAgent.dns: request served from client: name=db.ingress.consul. type=A class=IN latency=277.421µs client=127.0.0.1:39401 client_network=udp +34:48.975 [DEBUG] TestAgent.dns: request served from client: name=db.ingress.dc1.consul. type=A class=IN latency=273.321µs client=127.0.0.1:33622 client_network=udp +34:48.983 [INFO] TestAgent: Requesting shutdown +34:48.983 [INFO] TestAgent.server: shutting down server +34:48.983 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:48.983 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:48.983 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:48.983 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:48.983 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:48.983 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:48.983 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:48.983 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:48.984 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:48.984 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:48.984 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:48.984 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:48.984Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:48.984Z [INFO] agent.router.manager: shutting down +34:48.984 [INFO] TestAgent: consul server down +34:48.984 [INFO] TestAgent: shutdown complete +34:48.984 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25301 network=tcp +34:48.984 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25301 network=udp +34:48.984 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25281 network=tcp +34:49.485 [INFO] TestAgent: Waiting for endpoints to shut down +34:49.485 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_IngressServiceLookup (0.74s) + --- PASS: TestDNS_IngressServiceLookup/api.ingress.consul. (0.00s) + --- PASS: TestDNS_IngressServiceLookup/api.ingress.dc1.consul. (0.00s) + --- PASS: TestDNS_IngressServiceLookup/db.ingress.consul. (0.00s) + --- PASS: TestDNS_IngressServiceLookup/db.ingress.dc1.consul. (0.01s) +=== CONT TestDNS_EDNS0_ECS +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=255)_compressed=false +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:48.643Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:48.669Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:48.681 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0f486b24-c4f8-ef8f-1eb4-afe9b085074b Address:127.0.0.1:25299}]" +34:48.683 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0f486b24-c4f8-ef8f-1eb4-afe9b085074b.dc1 127.0.0.1 +34:48.684 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0f486b24-c4f8-ef8f-1eb4-afe9b085074b 127.0.0.1 +2021-08-13T15:34:48.684Z [INFO] agent.router: Initializing LAN area manager +34:48.685 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25299 [Follower]" leader= +34:48.688 [INFO] TestAgent.server: Adding LAN server: server="Node-0f486b24-c4f8-ef8f-1eb4-afe9b085074b (Addr: tcp/127.0.0.1:25299) (DC: dc1)" +34:48.688 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0f486b24-c4f8-ef8f-1eb4-afe9b085074b.dc1 area=wan +34:48.688 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25273 network=tcp +34:48.690 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25273 network=udp +34:48.692 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25288 network=tcp +34:48.692 [INFO] TestAgent: started state syncer +34:48.693 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25300 network=tcp +34:48.739 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:48.739 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25299 [Candidate]" term=2 +34:48.740 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:48.740 [DEBUG] TestAgent.server.raft: vote granted: from=0f486b24-c4f8-ef8f-1eb4-afe9b085074b term=2 tally=1 +34:48.740 [INFO] TestAgent.server.raft: election won: tally=1 +34:48.740 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25299 [Leader]" +34:48.740 [INFO] TestAgent.server: cluster leadership acquired +34:48.741 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25299 +34:48.744 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:48.752 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:48.752 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:48.752 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:48.752 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:48.752 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:48.752 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0f486b24-c4f8-ef8f-1eb4-afe9b085074b +34:48.752 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0f486b24-c4f8-ef8f-1eb4-afe9b085074b +34:48.755 [INFO] TestAgent.server: federation state anti-entropy synced +34:48.759 [INFO] TestAgent.server: New leader elected: payload=Node-0f486b24-c4f8-ef8f-1eb4-afe9b085074b +34:49.003 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:49.004 [INFO] TestAgent: Synced node info +34:49.120 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=ANY class=IN latency=611.482µs client=127.0.0.1:53862 client_network=udp +34:49.120 [INFO] TestAgent: Requesting shutdown +34:49.120 [INFO] TestAgent.server: shutting down server +34:49.120 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:49.120 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:49.120 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:49.120 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:49.120 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:49.120 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:49.120 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:49.120 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:49.120 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:49.120 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:49.120 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:49.121Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:49.121Z [INFO] agent.router.manager: shutting down +34:49.121 [INFO] TestAgent: consul server down +34:49.121 [INFO] TestAgent: shutdown complete +34:49.121 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25273 network=tcp +34:49.121 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25273 network=udp +34:49.121 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25288 network=tcp +34:49.622 [INFO] TestAgent: Waiting for endpoints to shut down +34:49.622 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookupNoMultiCNAME (1.03s) +=== CONT TestDNS_ServiceReverseLookup_IPV6 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._tcp_(qType:=255)_compressed=true +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:46.123Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:46.140Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:46.142 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0a552afd-26b6-07f5-23a5-7280a8b9a328 Address:127.0.0.1:24948}]" +34:46.143 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24948 [Follower]" leader= +34:46.148 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0a552afd-26b6-07f5-23a5-7280a8b9a328.dc1 127.0.0.1 +34:46.164 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0a552afd-26b6-07f5-23a5-7280a8b9a328 127.0.0.1 +2021-08-13T15:34:46.164Z [INFO] agent.router: Initializing LAN area manager +34:46.164 [DEBUG] TestAgent.dns: recursor enabled +34:46.164 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24936 network=udp +34:46.165 [INFO] TestAgent.server: Adding LAN server: server="Node-0a552afd-26b6-07f5-23a5-7280a8b9a328 (Addr: tcp/127.0.0.1:24948) (DC: dc1)" +34:46.165 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0a552afd-26b6-07f5-23a5-7280a8b9a328.dc1 area=wan +34:46.165 [DEBUG] TestAgent.dns: recursor enabled +34:46.165 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24936 network=tcp +34:46.166 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24930 network=tcp +34:46.167 [INFO] TestAgent: started state syncer +34:46.167 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24949 network=tcp +34:46.205 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:46.205 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24948 [Candidate]" term=2 +34:46.205 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:46.205 [DEBUG] TestAgent.server.raft: vote granted: from=0a552afd-26b6-07f5-23a5-7280a8b9a328 term=2 tally=1 +34:46.205 [INFO] TestAgent.server.raft: election won: tally=1 +34:46.205 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24948 [Leader]" +34:46.206 [INFO] TestAgent.server: cluster leadership acquired +34:46.210 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24948 +34:46.211 [INFO] TestAgent.server: New leader elected: payload=Node-0a552afd-26b6-07f5-23a5-7280a8b9a328 +34:46.211 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:46.215 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:46.215 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:46.215 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:46.215 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:46.215 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:46.215 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0a552afd-26b6-07f5-23a5-7280a8b9a328 +34:46.215 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0a552afd-26b6-07f5-23a5-7280a8b9a328 +34:46.217 [INFO] TestAgent.server: federation state anti-entropy synced +34:46.347 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:46.348 [INFO] TestAgent: Synced node info +34:46.348 [DEBUG] TestAgent: Node info in sync +34:46.468 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:46.468 [DEBUG] TestAgent: Node info in sync +34:49.310 [ERROR] TestAgent.dns: recurse failed: error="read udp 127.0.0.1:56319->127.0.0.1:39308: i/o timeout" +34:49.311 [ERROR] TestAgent.dns: all resolvers failed for question from client: question="{apple.com. 255 1}" client=127.0.0.1:44501 client_network=udp +34:49.311 [DEBUG] TestAgent.dns: request served from client: question="{apple.com. 255 1}" network=udp latency=3.000935277s client=127.0.0.1:44501 client_network=udp +34:49.311 [INFO] TestAgent: Requesting shutdown +34:49.311 [INFO] TestAgent.server: shutting down server +34:49.311 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:49.311 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:49.311 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:49.311 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:49.311 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:49.311 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:49.311 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:49.311 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:49.311 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:49.311 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:49.311 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:49.312Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:49.312Z [INFO] agent.router.manager: shutting down +34:49.312 [INFO] TestAgent: consul server down +34:49.312 [INFO] TestAgent: shutdown complete +34:49.312 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24936 network=tcp +34:49.312 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24936 network=udp +34:49.312 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24930 network=tcp +34:49.812 [INFO] TestAgent: Waiting for endpoints to shut down +34:49.812 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_RecursorTimeout (3.71s) +=== CONT TestDNS_EDNS0 +=== RUN TestDNS_EDNS0_ECS/global +=== RUN TestDNS_EDNS0_ECS/query +=== RUN TestDNS_EDNS0_ECS/query-subnet +2021-08-13T15:34:48.947Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:48.969Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:48.973 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b3c36e9b-e712-6b84-d0ab-9bc640338856 Address:127.0.0.1:25292}]" +34:48.974 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b3c36e9b-e712-6b84-d0ab-9bc640338856.dc1 127.0.0.1 +34:48.975 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25292 [Follower]" leader= +34:48.975 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b3c36e9b-e712-6b84-d0ab-9bc640338856 127.0.0.1 +2021-08-13T15:34:48.975Z [INFO] agent.router: Initializing LAN area manager +34:48.976 [INFO] TestAgent.server: Adding LAN server: server="Node-b3c36e9b-e712-6b84-d0ab-9bc640338856 (Addr: tcp/127.0.0.1:25292) (DC: dc1)" +34:48.976 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b3c36e9b-e712-6b84-d0ab-9bc640338856.dc1 area=wan +34:48.976 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24309 network=udp +34:48.976 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24309 network=tcp +34:48.977 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25302 network=tcp +34:48.978 [INFO] TestAgent: started state syncer +34:48.978 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25293 network=tcp +34:49.039 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:49.039 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25292 [Candidate]" term=2 +34:49.039 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:49.039 [DEBUG] TestAgent.server.raft: vote granted: from=b3c36e9b-e712-6b84-d0ab-9bc640338856 term=2 tally=1 +34:49.039 [INFO] TestAgent.server.raft: election won: tally=1 +34:49.039 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25292 [Leader]" +34:49.040 [INFO] TestAgent.server: cluster leadership acquired +34:49.040 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25292 +34:49.041 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:49.043 [INFO] TestAgent.server: New leader elected: payload=Node-b3c36e9b-e712-6b84-d0ab-9bc640338856 +34:49.044 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:49.044 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:49.044 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:49.044 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:49.044 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:49.044 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b3c36e9b-e712-6b84-d0ab-9bc640338856 +34:49.044 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b3c36e9b-e712-6b84-d0ab-9bc640338856 +34:49.046 [INFO] TestAgent.server: federation state anti-entropy synced +34:49.226 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:49.227 [INFO] TestAgent: Synced node info +34:49.227 [DEBUG] TestAgent: Node info in sync +34:49.357 [DEBUG] TestAgent.dns: request served from client: question="{2.0.0.127.in-addr.arpa. 255 1}" latency=186.98µs client=127.0.0.1:58434 client_network=udp +34:49.357 [INFO] TestAgent: Requesting shutdown +34:49.357 [INFO] TestAgent.server: shutting down server +34:49.357 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:49.357 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:49.357 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:49.357 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:49.357 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:49.357 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:49.357 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:49.357 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:49.357 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:49.357 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:49.357 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:49.358Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:49.358Z [INFO] agent.router.manager: shutting down +34:49.358 [INFO] TestAgent: consul server down +34:49.358 [INFO] TestAgent: shutdown complete +34:49.358 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24309 network=tcp +34:49.358 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24309 network=udp +34:49.358 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25302 network=tcp +34:49.859 [INFO] TestAgent: Waiting for endpoints to shut down +34:49.859 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ReverseLookup (0.94s) +=== CONT TestDNS_ServiceReverseLookup +2021-08-13T15:34:48.867Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:48.927Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:48.931 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:65d70123-87eb-c849-6f89-107c61e9bde7 Address:127.0.0.1:24300}]" +34:48.931 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24300 [Follower]" leader= +34:48.932 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-65d70123-87eb-c849-6f89-107c61e9bde7.dc1 127.0.0.1 +34:48.934 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-65d70123-87eb-c849-6f89-107c61e9bde7 127.0.0.1 +2021-08-13T15:34:48.934Z [INFO] agent.router: Initializing LAN area manager +34:48.935 [INFO] TestAgent.server: Adding LAN server: server="Node-65d70123-87eb-c849-6f89-107c61e9bde7 (Addr: tcp/127.0.0.1:24300) (DC: dc1)" +34:48.935 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-65d70123-87eb-c849-6f89-107c61e9bde7.dc1 area=wan +34:48.935 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25287 network=udp +34:48.935 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25287 network=tcp +34:48.937 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25309 network=tcp +34:48.937 [INFO] TestAgent: started state syncer +34:48.937 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24301 network=tcp +34:48.995 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:48.995 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24300 [Candidate]" term=2 +34:48.996 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:48.996 [DEBUG] TestAgent.server.raft: vote granted: from=65d70123-87eb-c849-6f89-107c61e9bde7 term=2 tally=1 +34:48.996 [INFO] TestAgent.server.raft: election won: tally=1 +34:48.996 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24300 [Leader]" +34:48.996 [INFO] TestAgent.server: cluster leadership acquired +34:48.996 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24300 +34:48.998 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:49.003 [INFO] TestAgent.server: New leader elected: payload=Node-65d70123-87eb-c849-6f89-107c61e9bde7 +34:49.010 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:49.010 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:49.010 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:49.010 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:49.010 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:49.010 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-65d70123-87eb-c849-6f89-107c61e9bde7 +34:49.010 [INFO] TestAgent.server: member joined, marking health alive: member=Node-65d70123-87eb-c849-6f89-107c61e9bde7 +34:49.012 [INFO] TestAgent.server: federation state anti-entropy synced +34:49.105 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:49.106 [INFO] TestAgent: Synced node info +34:49.106 [DEBUG] TestAgent: Node info in sync +34:49.374 [DEBUG] TestAgent.dns: request served from client: question="{1.0.0.127.in-addr.arpa. 255 1}" latency=230.181µs client=127.0.0.1:53175 client_network=udp +34:49.374 [INFO] TestAgent: Requesting shutdown +34:49.374 [INFO] TestAgent.server: shutting down server +34:49.374 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:49.374 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:49.374 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:49.374 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:49.374 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:49.375 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:49.375 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:49.375 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:49.375 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:49.375 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:49.375 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:49.375Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:49.375Z [INFO] agent.router.manager: shutting down +34:49.375 [INFO] TestAgent: consul server down +34:49.375 [INFO] TestAgent: shutdown complete +34:49.375 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25287 network=tcp +34:49.375 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25287 network=udp +34:49.375 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25309 network=tcp +34:49.877 [INFO] TestAgent: Waiting for endpoints to shut down +34:49.878 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceReverseLookupNodeAddress (1.07s) +=== CONT TestDNS_NodeLookup_CNAME +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:49.385Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:49.406Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:49.409 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4fe608e2-ab97-860b-d505-5ef65716194d Address:127.0.0.1:25306}]" +34:49.410 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25306 [Follower]" leader= +34:49.410 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4fe608e2-ab97-860b-d505-5ef65716194d.dc1 127.0.0.1 +34:49.411 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4fe608e2-ab97-860b-d505-5ef65716194d 127.0.0.1 +2021-08-13T15:34:49.411Z [INFO] agent.router: Initializing LAN area manager +34:49.412 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4fe608e2-ab97-860b-d505-5ef65716194d.dc1 area=wan +34:49.412 [INFO] TestAgent.server: Adding LAN server: server="Node-4fe608e2-ab97-860b-d505-5ef65716194d (Addr: tcp/127.0.0.1:25306) (DC: dc1)" +34:49.412 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25294 network=tcp +34:49.412 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25294 network=udp +34:49.413 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25295 network=tcp +34:49.413 [INFO] TestAgent: started state syncer +34:49.413 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25307 network=tcp +34:49.473 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:49.473 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25306 [Candidate]" term=2 +34:49.473 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:49.473 [DEBUG] TestAgent.server.raft: vote granted: from=4fe608e2-ab97-860b-d505-5ef65716194d term=2 tally=1 +34:49.473 [INFO] TestAgent.server.raft: election won: tally=1 +34:49.473 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25306 [Leader]" +34:49.473 [INFO] TestAgent.server: cluster leadership acquired +34:49.473 [INFO] TestAgent.server: New leader elected: payload=Node-4fe608e2-ab97-860b-d505-5ef65716194d +34:49.474 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25306 +34:49.475 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:49.477 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:49.477 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:49.477 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:49.477 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:49.477 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:49.478 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4fe608e2-ab97-860b-d505-5ef65716194d +34:49.478 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4fe608e2-ab97-860b-d505-5ef65716194d +34:49.478 [INFO] TestAgent.server: federation state anti-entropy synced +34:49.545 [DEBUG] TestAgent.dns: request served from client: question="{2.0.0.127.in-addr.arpa. 255 1}" latency=269.201µs client=127.0.0.1:50271 client_network=udp +34:49.546 [INFO] TestAgent: Requesting shutdown +34:49.546 [INFO] TestAgent.server: shutting down server +34:49.546 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:49.546 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:49.546 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:49.546 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:49.546 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:49.546 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:49.546 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:49.546 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:49.546 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:49.546 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:49.546 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:49.546 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:49.547Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:49.547Z [INFO] agent.router.manager: shutting down +34:49.547 [INFO] TestAgent: consul server down +34:49.547 [INFO] TestAgent: shutdown complete +34:49.547 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25294 network=tcp +34:49.547 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25294 network=udp +34:49.547 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25295 network=tcp +34:50.051 [INFO] TestAgent: Waiting for endpoints to shut down +34:50.051 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceReverseLookup_CustomDomain (0.71s) +=== CONT TestDNS_ReverseLookup_IPV6 +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:49.507Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:49.569Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:49.582 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6ebd4e7a-799d-2721-cb96-2a0bfb62cd8c Address:127.0.0.1:24335}]" +34:49.583 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24335 [Follower]" leader= +34:49.587 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6ebd4e7a-799d-2721-cb96-2a0bfb62cd8c.dc1 127.0.0.1 +34:49.589 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6ebd4e7a-799d-2721-cb96-2a0bfb62cd8c 127.0.0.1 +2021-08-13T15:34:49.589Z [INFO] agent.router: Initializing LAN area manager +34:49.590 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6ebd4e7a-799d-2721-cb96-2a0bfb62cd8c.dc1 area=wan +34:49.590 [INFO] TestAgent.server: Adding LAN server: server="Node-6ebd4e7a-799d-2721-cb96-2a0bfb62cd8c (Addr: tcp/127.0.0.1:24335) (DC: dc1)" +34:49.590 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25308 network=udp +34:49.591 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25308 network=tcp +34:49.592 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24366 network=tcp +34:49.592 [INFO] TestAgent: started state syncer +34:49.593 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24336 network=tcp +34:49.631 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:49.631 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24335 [Candidate]" term=2 +34:49.631 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:49.631 [DEBUG] TestAgent.server.raft: vote granted: from=6ebd4e7a-799d-2721-cb96-2a0bfb62cd8c term=2 tally=1 +34:49.631 [INFO] TestAgent.server.raft: election won: tally=1 +34:49.631 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24335 [Leader]" +34:49.631 [INFO] TestAgent.server: cluster leadership acquired +34:49.632 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24335 +34:49.633 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:49.635 [INFO] TestAgent.server: New leader elected: payload=Node-6ebd4e7a-799d-2721-cb96-2a0bfb62cd8c +34:49.640 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:49.640 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:49.640 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:49.640 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:49.640 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:49.640 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6ebd4e7a-799d-2721-cb96-2a0bfb62cd8c +34:49.640 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6ebd4e7a-799d-2721-cb96-2a0bfb62cd8c +34:49.642 [INFO] TestAgent.server: federation state anti-entropy synced +34:49.642 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:49.642 [INFO] TestAgent: Synced node info +34:49.643 [DEBUG] TestAgent: Node info in sync +34:49.845 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=A class=IN latency=446.301µs client=127.0.0.1:46069 client_network=udp +34:49.848 [DEBUG] TestAgent.dns: request served from client: name=test.query.consul. type=A class=IN latency=263.56µs client=127.0.0.1:48271 client_network=udp +34:49.849 [INFO] TestAgent: Requesting shutdown +34:49.849 [INFO] TestAgent.server: shutting down server +34:49.849 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:49.849 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:49.849 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:49.849 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:49.849 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:49.849 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:49.849 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:49.849 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:49.849 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:49.850 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:49.850 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:49.850Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:49.850Z [INFO] agent.router.manager: shutting down +34:49.850 [INFO] TestAgent: consul server down +34:49.850 [INFO] TestAgent: shutdown complete +34:49.850 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25308 network=tcp +34:49.850 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25308 network=udp +34:49.850 [DEBUG] TestAgent.dns: request served from client: name=test.query.consul. type=A class=IN latency=3.886489ms client=127.0.0.1:51045 client_network=udp +34:49.851 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24366 network=tcp +34:50.351 [INFO] TestAgent: Waiting for endpoints to shut down +34:50.351 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_EDNS0_ECS (0.87s) + --- PASS: TestDNS_EDNS0_ECS/global (0.00s) + --- PASS: TestDNS_EDNS0_ECS/query (0.00s) + --- PASS: TestDNS_EDNS0_ECS/query-subnet (0.00s) +=== CONT TestDNSCycleRecursorCheck +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._tcp_(qType:=255)_compressed=false +2021-08-13T15:34:49.851Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:49.888Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:49.892 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3f72c453-575f-2d9c-b8b3-d69f5f8bb312 Address:127.0.0.1:24314}]" +34:49.893 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3f72c453-575f-2d9c-b8b3-d69f5f8bb312.dc1 127.0.0.1 +34:49.894 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3f72c453-575f-2d9c-b8b3-d69f5f8bb312 127.0.0.1 +2021-08-13T15:34:49.894Z [INFO] agent.router: Initializing LAN area manager +34:49.894 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24344 network=udp +34:49.894 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24314 [Follower]" leader= +34:49.895 [INFO] TestAgent.server: Adding LAN server: server="Node-3f72c453-575f-2d9c-b8b3-d69f5f8bb312 (Addr: tcp/127.0.0.1:24314) (DC: dc1)" +34:49.895 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3f72c453-575f-2d9c-b8b3-d69f5f8bb312.dc1 area=wan +34:49.895 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24344 network=tcp +34:49.896 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24345 network=tcp +34:49.896 [INFO] TestAgent: started state syncer +34:49.896 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24315 network=tcp +34:49.948 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:49.948 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24314 [Candidate]" term=2 +34:49.948 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:49.948 [DEBUG] TestAgent.server.raft: vote granted: from=3f72c453-575f-2d9c-b8b3-d69f5f8bb312 term=2 tally=1 +34:49.948 [INFO] TestAgent.server.raft: election won: tally=1 +34:49.948 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24314 [Leader]" +34:49.948 [INFO] TestAgent.server: cluster leadership acquired +34:49.949 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24314 +34:49.950 [INFO] TestAgent.server: New leader elected: payload=Node-3f72c453-575f-2d9c-b8b3-d69f5f8bb312 +34:49.951 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:49.955 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:49.955 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:49.955 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:49.955 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:49.955 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:49.955 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3f72c453-575f-2d9c-b8b3-d69f5f8bb312 +34:49.955 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3f72c453-575f-2d9c-b8b3-d69f5f8bb312 +34:49.956 [INFO] TestAgent.server: federation state anti-entropy synced +34:49.988 [DEBUG] TestAgent.dns: request served from client: name=foo.node.dc1.consul. type=ANY class=IN latency=203.72µs client=127.0.0.1:44661 client_network=udp +34:49.988 [INFO] TestAgent: Requesting shutdown +34:49.988 [INFO] TestAgent.server: shutting down server +34:49.988 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:49.988 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:49.988 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:49.988 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:49.989 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:49.989 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:49.989 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:49.989 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:49.989 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:49.989 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:49.989 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:49.989 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:49.990Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:49.990Z [INFO] agent.router.manager: shutting down +34:49.990 [INFO] TestAgent: consul server down +34:49.990 [INFO] TestAgent: shutdown complete +34:49.990 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24344 network=tcp +34:49.990 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24344 network=udp +34:49.990 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24345 network=tcp +34:50.491 [INFO] TestAgent: Waiting for endpoints to shut down +34:50.491 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_EDNS0 (0.68s) +=== CONT TestDNS_ReverseLookup_CustomDomain +2021-08-13T15:34:49.641Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:49.659Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:49.662 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b4d47073-e19e-1d42-2d25-f9963606975b Address:127.0.0.1:24349}]" +34:49.663 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24349 [Follower]" leader= +34:49.664 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b4d47073-e19e-1d42-2d25-f9963606975b.dc1 127.0.0.1 +34:49.665 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b4d47073-e19e-1d42-2d25-f9963606975b 127.0.0.1 +2021-08-13T15:34:49.665Z [INFO] agent.router: Initializing LAN area manager +34:49.665 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24337 network=udp +34:49.665 [INFO] TestAgent.server: Adding LAN server: server="Node-b4d47073-e19e-1d42-2d25-f9963606975b (Addr: tcp/127.0.0.1:24349) (DC: dc1)" +34:49.665 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b4d47073-e19e-1d42-2d25-f9963606975b.dc1 area=wan +34:49.665 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24337 network=tcp +34:49.666 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24310 network=tcp +34:49.666 [INFO] TestAgent: started state syncer +34:49.666 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24350 network=tcp +34:49.723 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:49.723 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24349 [Candidate]" term=2 +34:49.723 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:49.723 [DEBUG] TestAgent.server.raft: vote granted: from=b4d47073-e19e-1d42-2d25-f9963606975b term=2 tally=1 +34:49.723 [INFO] TestAgent.server.raft: election won: tally=1 +34:49.723 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24349 [Leader]" +34:49.724 [INFO] TestAgent.server: cluster leadership acquired +34:49.724 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24349 +34:49.725 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:49.726 [INFO] TestAgent.server: New leader elected: payload=Node-b4d47073-e19e-1d42-2d25-f9963606975b +34:49.728 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:49.729 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:49.729 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:49.729 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:49.729 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:49.729 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b4d47073-e19e-1d42-2d25-f9963606975b +34:49.729 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b4d47073-e19e-1d42-2d25-f9963606975b +34:49.730 [INFO] TestAgent.server: federation state anti-entropy synced +34:49.763 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:49.764 [INFO] TestAgent: Synced node info +34:49.764 [DEBUG] TestAgent: Node info in sync +34:49.998 [DEBUG] TestAgent.dns: request served from client: question="{9.2.3.8.2.4.0.0.0.0.f.f.0.0.0.0.0.0.0.0.0.0.0.0.8.b.d.0.1.0.0.2.ip6.arpa. 255 1}" latency=300.801µs client=127.0.0.1:34401 client_network=udp +34:49.999 [INFO] TestAgent: Requesting shutdown +34:49.999 [INFO] TestAgent.server: shutting down server +34:49.999 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:49.999 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:49.999 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:49.999 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:49.999 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:49.999 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:49.999 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:49.999 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:49.999 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:49.999 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:49.999 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:50.000Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:50.000Z [INFO] agent.router.manager: shutting down +34:50.000 [INFO] TestAgent: consul server down +34:50.000 [INFO] TestAgent: shutdown complete +34:50.000 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24337 network=tcp +34:50.000 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24337 network=udp +34:50.000 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24310 network=tcp +34:50.501 [INFO] TestAgent: Waiting for endpoints to shut down +34:50.501 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceReverseLookup_IPV6 (0.88s) +=== CONT TestDNS_NodeLookup_AAAA +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:49.945Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:50.009Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:50.014 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e9b35318-3d05-8a0d-fedd-7aaa0d268d24 Address:127.0.0.1:24370}]" +34:50.014 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24370 [Follower]" leader= +34:50.015 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e9b35318-3d05-8a0d-fedd-7aaa0d268d24.dc1 127.0.0.1 +34:50.018 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e9b35318-3d05-8a0d-fedd-7aaa0d268d24 127.0.0.1 +2021-08-13T15:34:50.018Z [INFO] agent.router: Initializing LAN area manager +34:50.021 [INFO] TestAgent.server: Adding LAN server: server="Node-e9b35318-3d05-8a0d-fedd-7aaa0d268d24 (Addr: tcp/127.0.0.1:24370) (DC: dc1)" +34:50.021 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e9b35318-3d05-8a0d-fedd-7aaa0d268d24.dc1 area=wan +34:50.021 [DEBUG] TestAgent.dns: recursor enabled +34:50.021 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24330 network=tcp +34:50.022 [DEBUG] TestAgent.dns: recursor enabled +34:50.022 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24330 network=udp +34:50.032 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24317 network=tcp +34:50.032 [INFO] TestAgent: started state syncer +34:50.032 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24371 network=tcp +34:50.072 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:50.072 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24370 [Candidate]" term=2 +34:50.072 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:50.072 [DEBUG] TestAgent.server.raft: vote granted: from=e9b35318-3d05-8a0d-fedd-7aaa0d268d24 term=2 tally=1 +34:50.072 [INFO] TestAgent.server.raft: election won: tally=1 +34:50.072 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24370 [Leader]" +34:50.073 [INFO] TestAgent.server: cluster leadership acquired +34:50.073 [INFO] TestAgent.server: New leader elected: payload=Node-e9b35318-3d05-8a0d-fedd-7aaa0d268d24 +34:50.074 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24370 +34:50.082 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:50.085 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:50.085 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:50.085 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:50.085 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:50.085 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:50.085 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e9b35318-3d05-8a0d-fedd-7aaa0d268d24 +34:50.085 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e9b35318-3d05-8a0d-fedd-7aaa0d268d24 +34:50.087 [INFO] TestAgent.server: federation state anti-entropy synced +34:50.179 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:50.181 [INFO] TestAgent: Synced node info +34:50.181 [DEBUG] TestAgent: Node info in sync +34:50.185 [DEBUG] TestAgent.dns: cname recurse RTT for name: name=www.google.com. rtt=625.022µs +34:50.185 [DEBUG] TestAgent.dns: request served from client: name=google.node.consul. type=ANY class=IN latency=1.084342ms client=127.0.0.1:45191 client_network=udp +34:50.185 [INFO] TestAgent: Requesting shutdown +34:50.186 [INFO] TestAgent.server: shutting down server +34:50.186 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:50.186 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:50.186 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:50.186 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:50.186 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:50.186 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:50.186 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:50.186 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:50.186 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:50.186 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:50.187 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:50.204Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:50.204Z [INFO] agent.router.manager: shutting down +34:50.207 [INFO] TestAgent: consul server down +34:50.207 [INFO] TestAgent: shutdown complete +34:50.207 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24330 network=tcp +34:50.207 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24330 network=udp +34:50.207 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24317 network=tcp +34:50.711 [INFO] TestAgent: Waiting for endpoints to shut down +34:50.711 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_NodeLookup_CNAME (0.83s) +=== CONT TestDNS_NodeLookup_PeriodName +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:49.921Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:50.001Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:50.005 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ba8e6d06-e9f5-eda5-b6e5-f8d3effd4f10 Address:127.0.0.1:24328}]" +34:50.005 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24328 [Follower]" leader= +34:50.009 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ba8e6d06-e9f5-eda5-b6e5-f8d3effd4f10.dc1 127.0.0.1 +34:50.011 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ba8e6d06-e9f5-eda5-b6e5-f8d3effd4f10 127.0.0.1 +2021-08-13T15:34:50.011Z [INFO] agent.router: Initializing LAN area manager +34:50.011 [INFO] TestAgent.server: Adding LAN server: server="Node-ba8e6d06-e9f5-eda5-b6e5-f8d3effd4f10 (Addr: tcp/127.0.0.1:24328) (DC: dc1)" +34:50.011 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ba8e6d06-e9f5-eda5-b6e5-f8d3effd4f10.dc1 area=wan +34:50.012 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24316 network=udp +34:50.012 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24316 network=tcp +34:50.013 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24324 network=tcp +34:50.013 [INFO] TestAgent: started state syncer +34:50.014 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24329 network=tcp +34:50.067 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:50.067 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24328 [Candidate]" term=2 +34:50.067 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:50.067 [DEBUG] TestAgent.server.raft: vote granted: from=ba8e6d06-e9f5-eda5-b6e5-f8d3effd4f10 term=2 tally=1 +34:50.067 [INFO] TestAgent.server.raft: election won: tally=1 +34:50.067 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24328 [Leader]" +34:50.068 [INFO] TestAgent.server: cluster leadership acquired +34:50.068 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24328 +34:50.070 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:50.074 [INFO] TestAgent.server: New leader elected: payload=Node-ba8e6d06-e9f5-eda5-b6e5-f8d3effd4f10 +34:50.074 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:50.074 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:50.074 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:50.074 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:50.074 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:50.074 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ba8e6d06-e9f5-eda5-b6e5-f8d3effd4f10 +34:50.074 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ba8e6d06-e9f5-eda5-b6e5-f8d3effd4f10 +34:50.081 [INFO] TestAgent.server: federation state anti-entropy synced +34:50.227 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:50.229 [INFO] TestAgent: Synced node info +34:50.401 [DEBUG] TestAgent.dns: request served from client: question="{2.0.0.127.in-addr.arpa. 255 1}" latency=151.88µs client=127.0.0.1:51259 client_network=udp +34:50.401 [INFO] TestAgent: Requesting shutdown +34:50.402 [INFO] TestAgent.server: shutting down server +34:50.402 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:50.402 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:50.402 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:50.402 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:50.402 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:50.402 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:50.402 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +34:50.402 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:50.402 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:50.402 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:50.402 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2021-08-13T15:34:50.402Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:50.402Z [INFO] agent.router.manager: shutting down +34:50.402 [INFO] TestAgent: consul server down +34:50.402 [INFO] TestAgent: shutdown complete +34:50.402 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24316 network=tcp +34:50.402 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24316 network=udp +34:50.402 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24324 network=tcp +34:50.903 [INFO] TestAgent: Waiting for endpoints to shut down +34:50.914 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceReverseLookup (1.05s) +=== CONT TestRecursorAddr +--- PASS: TestRecursorAddr (0.00s) +=== CONT TestDNS_CaseInsensitiveNodeLookup +2021-08-13T15:34:50.094Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:50.149Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:50.156 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3e642cb3-b0a4-05f5-3820-e72c0a51fd6e Address:127.0.0.1:24321}]" +34:50.156 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24321 [Follower]" leader= +34:50.157 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3e642cb3-b0a4-05f5-3820-e72c0a51fd6e.dc1 127.0.0.1 +34:50.158 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3e642cb3-b0a4-05f5-3820-e72c0a51fd6e 127.0.0.1 +2021-08-13T15:34:50.158Z [INFO] agent.router: Initializing LAN area manager +34:50.159 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24358 network=udp +34:50.159 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3e642cb3-b0a4-05f5-3820-e72c0a51fd6e.dc1 area=wan +34:50.159 [INFO] TestAgent.server: Adding LAN server: server="Node-3e642cb3-b0a4-05f5-3820-e72c0a51fd6e (Addr: tcp/127.0.0.1:24321) (DC: dc1)" +34:50.160 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24358 network=tcp +34:50.161 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24359 network=tcp +34:50.161 [INFO] TestAgent: started state syncer +34:50.161 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24322 network=tcp +34:50.207 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:50.207 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24321 [Candidate]" term=2 +34:50.207 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:50.207 [DEBUG] TestAgent.server.raft: vote granted: from=3e642cb3-b0a4-05f5-3820-e72c0a51fd6e term=2 tally=1 +34:50.208 [INFO] TestAgent.server.raft: election won: tally=1 +34:50.208 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24321 [Leader]" +34:50.208 [INFO] TestAgent.server: cluster leadership acquired +34:50.208 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24321 +34:50.210 [INFO] TestAgent.server: New leader elected: payload=Node-3e642cb3-b0a4-05f5-3820-e72c0a51fd6e +34:50.228 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:50.233 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:50.233 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:50.233 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:50.233 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:50.233 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:50.234 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3e642cb3-b0a4-05f5-3820-e72c0a51fd6e +34:50.234 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3e642cb3-b0a4-05f5-3820-e72c0a51fd6e +34:50.235 [INFO] TestAgent.server: federation state anti-entropy synced +34:50.401 [DEBUG] TestAgent.dns: request served from client: question="{2.4.2.4.2.4.2.4.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa. 255 1}" latency=152µs client=127.0.0.1:39555 client_network=udp +34:50.401 [INFO] TestAgent: Requesting shutdown +34:50.402 [INFO] TestAgent.server: shutting down server +34:50.402 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:50.402 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:50.402 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:50.402 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:50.402 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:50.402 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:50.402 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:50.402 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:50.402 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:50.402 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:50.402 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:50.402 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:50.403Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:50.403Z [INFO] agent.router.manager: shutting down +34:50.403 [INFO] TestAgent: consul server down +34:50.403 [INFO] TestAgent: shutdown complete +34:50.403 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24358 network=tcp +34:50.403 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24358 network=udp +34:50.403 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24359 network=tcp +34:50.905 [INFO] TestAgent: Waiting for endpoints to shut down +34:50.916 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ReverseLookup_IPV6 (0.86s) +=== CONT TestDiscoveryChainRead +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._udp_(qType:=255)_compressed=true +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._udp_(qType:=255)_compressed=false +=== RUN TestDiscoveryChainRead/GET:_error_on_no_service_name +=== RUN TestDiscoveryChainRead/GET:_read_default_chain +=== RUN TestDiscoveryChainRead/GET:_read_default_chain;_evaluate_in_dc2 +=== RUN TestDiscoveryChainRead/GET:_read_default_chain_with_cache +=== RUN TestDiscoveryChainRead/POST:_error_on_no_service_name +=== RUN TestDiscoveryChainRead/POST:_read_default_chain +=== RUN TestDiscoveryChainRead/POST:_read_default_chain;_evaluate_in_dc2 +=== RUN TestDiscoveryChainRead/POST:_read_default_chain_with_cache +=== RUN TestDiscoveryChainRead/GET:_read_modified_chain +=== RUN TestDiscoveryChainRead/POST:_read_modified_chain_with_overrides_(camel_case) +=== RUN TestDiscoveryChainRead/POST:_read_modified_chain_with_overrides_(snake_case) +2021-08-13T15:34:50.381Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:50.399Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:50.402 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:aa4a0bb1-183e-6078-4b08-4da9ded64b0d Address:127.0.0.1:24384}]" +34:50.402 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24384 [Follower]" leader= +34:50.402 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-aa4a0bb1-183e-6078-4b08-4da9ded64b0d.dc1 127.0.0.1 +34:50.404 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-aa4a0bb1-183e-6078-4b08-4da9ded64b0d 127.0.0.1 +2021-08-13T15:34:50.404Z [INFO] agent.router: Initializing LAN area manager +34:50.404 [INFO] TestAgent.server: Adding LAN server: server="Node-aa4a0bb1-183e-6078-4b08-4da9ded64b0d (Addr: tcp/127.0.0.1:24384) (DC: dc1)" +34:50.404 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-aa4a0bb1-183e-6078-4b08-4da9ded64b0d.dc1 area=wan +34:50.404 [DEBUG] TestAgent.dns: recursor enabled +34:50.404 [DEBUG] TestAgent.dns: recursor enabled +34:50.404 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24323 network=tcp +34:50.404 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24323 network=udp +34:50.405 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24380 network=tcp +34:50.405 [INFO] TestAgent: started state syncer +34:50.405 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24385 network=tcp +34:50.457 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:50.457 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24384 [Candidate]" term=2 +34:50.457 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:50.457 [DEBUG] TestAgent.server.raft: vote granted: from=aa4a0bb1-183e-6078-4b08-4da9ded64b0d term=2 tally=1 +34:50.457 [INFO] TestAgent.server.raft: election won: tally=1 +34:50.457 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24384 [Leader]" +34:50.457 [INFO] TestAgent.server: cluster leadership acquired +34:50.458 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24384 +34:50.462 [INFO] TestAgent.server: New leader elected: payload=Node-aa4a0bb1-183e-6078-4b08-4da9ded64b0d +34:50.463 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:50.466 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:50.466 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:50.466 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:50.466 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:50.466 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:50.466 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-aa4a0bb1-183e-6078-4b08-4da9ded64b0d +34:50.466 [INFO] TestAgent.server: member joined, marking health alive: member=Node-aa4a0bb1-183e-6078-4b08-4da9ded64b0d +34:50.473 [INFO] TestAgent.server: federation state anti-entropy synced +34:50.568 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:50.569 [INFO] TestAgent: Synced node info +34:50.569 [DEBUG] TestAgent: Node info in sync +34:50.627 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:50.627 [DEBUG] TestAgent: Node info in sync +34:50.684 [DEBUG] TestAgent.dns: recurse failed for question: question="{google.com. 1 1}" rtt=222.88µs recursor=127.0.0.1:47841 rcode=SERVFAIL +34:50.684 [DEBUG] TestAgent.dns: recurse succeeded for question: question="{google.com. 1 1}" rtt=177.36µs recursor=127.0.0.1:51968 +34:50.684 [DEBUG] TestAgent.dns: request served from client: question="{google.com. 1 1}" network=udp latency=836.781µs client=127.0.0.1:55006 client_network=udp +34:50.684 [INFO] TestAgent: Requesting shutdown +34:50.684 [INFO] TestAgent.server: shutting down server +34:50.684 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:50.684 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:50.684 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:50.684 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:50.685 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:50.685 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:50.685 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:50.685 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:50.685 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:50.685 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:50.685 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:50.685Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:50.685Z [INFO] agent.router.manager: shutting down +34:50.685 [INFO] TestAgent: consul server down +34:50.685 [INFO] TestAgent: shutdown complete +34:50.685 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24323 network=tcp +34:50.685 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24323 network=udp +34:50.686 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24380 network=tcp +34:51.186 [INFO] TestAgent: Waiting for endpoints to shut down +34:51.186 [INFO] TestAgent: Endpoints down +--- PASS: TestDNSCycleRecursorCheck (0.84s) +=== CONT TestDNS_NodeLookup +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=1)_compressed=true +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:50.574Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:50.603Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:50.607 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2c9be206-fe13-fd35-d435-4de245cf5cd9 Address:127.0.0.1:24391}]" +34:50.607 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24391 [Follower]" leader= +34:50.609 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2c9be206-fe13-fd35-d435-4de245cf5cd9.dc1 127.0.0.1 +34:50.613 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2c9be206-fe13-fd35-d435-4de245cf5cd9 127.0.0.1 +2021-08-13T15:34:50.613Z [INFO] agent.router: Initializing LAN area manager +34:50.614 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24379 network=udp +34:50.615 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2c9be206-fe13-fd35-d435-4de245cf5cd9.dc1 area=wan +34:50.616 [INFO] TestAgent.server: Adding LAN server: server="Node-2c9be206-fe13-fd35-d435-4de245cf5cd9 (Addr: tcp/127.0.0.1:24391) (DC: dc1)" +34:50.617 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24379 network=tcp +34:50.619 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24387 network=tcp +34:50.619 [INFO] TestAgent: started state syncer +34:50.619 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24392 network=tcp +34:50.666 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:50.666 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24391 [Candidate]" term=2 +34:50.667 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:50.667 [DEBUG] TestAgent.server.raft: vote granted: from=2c9be206-fe13-fd35-d435-4de245cf5cd9 term=2 tally=1 +34:50.667 [INFO] TestAgent.server.raft: election won: tally=1 +34:50.667 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24391 [Leader]" +34:50.667 [INFO] TestAgent.server: cluster leadership acquired +34:50.667 [INFO] TestAgent.server: New leader elected: payload=Node-2c9be206-fe13-fd35-d435-4de245cf5cd9 +34:50.667 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24391 +34:50.669 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:50.671 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:50.671 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:50.671 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:50.671 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:50.671 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:50.672 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2c9be206-fe13-fd35-d435-4de245cf5cd9 +34:50.672 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2c9be206-fe13-fd35-d435-4de245cf5cd9 +34:50.673 [INFO] TestAgent.server: federation state anti-entropy synced +34:51.011 [DEBUG] TestAgent.dns: request served from client: name=bar.node.consul. type=AAAA class=IN latency=226.761µs client=127.0.0.1:56139 client_network=udp +34:51.011 [INFO] TestAgent: Requesting shutdown +34:51.011 [INFO] TestAgent.server: shutting down server +34:51.011 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:51.011 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:51.011 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:51.011 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:51.011 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:51.011 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:51.012 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:51.012 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:51.012 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:51.012 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:51.012 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:51.012 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:51.012Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:51.012Z [INFO] agent.router.manager: shutting down +34:51.012 [INFO] TestAgent: consul server down +34:51.012 [INFO] TestAgent: shutdown complete +34:51.012 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24379 network=tcp +34:51.012 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24379 network=udp +34:51.012 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24387 network=tcp +34:51.513 [INFO] TestAgent: Waiting for endpoints to shut down +34:51.513 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_NodeLookup_AAAA (1.01s) +=== CONT TestDenylist +=== RUN TestDenylist/nothing_blocked_root +=== RUN TestDenylist/nothing_blocked_path +=== RUN TestDenylist/exact_match_1 +=== RUN TestDenylist/exact_match_2 +=== RUN TestDenylist/subpath +=== RUN TestDenylist/longer_prefix +=== RUN TestDenylist/longer_subpath +=== RUN TestDenylist/partial_prefix +=== RUN TestDenylist/no_match +--- PASS: TestDenylist (0.00s) + --- PASS: TestDenylist/nothing_blocked_root (0.00s) + --- PASS: TestDenylist/nothing_blocked_path (0.00s) + --- PASS: TestDenylist/exact_match_1 (0.00s) + --- PASS: TestDenylist/exact_match_2 (0.00s) + --- PASS: TestDenylist/subpath (0.00s) + --- PASS: TestDenylist/longer_prefix (0.00s) + --- PASS: TestDenylist/longer_subpath (0.00s) + --- PASS: TestDenylist/partial_prefix (0.00s) + --- PASS: TestDenylist/no_match (0.00s) +=== CONT TestDNS_EmptyAltDomain +2021-08-13T15:34:50.536Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:50.589Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:50.593 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cb480960-2227-5c5c-2631-27e799e6e70c Address:127.0.0.1:24363}]" +34:50.593 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24363 [Follower]" leader= +34:50.594 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-cb480960-2227-5c5c-2631-27e799e6e70c.dc1 127.0.0.1 +34:50.595 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-cb480960-2227-5c5c-2631-27e799e6e70c 127.0.0.1 +2021-08-13T15:34:50.595Z [INFO] agent.router: Initializing LAN area manager +34:50.595 [INFO] TestAgent.server: Adding LAN server: server="Node-cb480960-2227-5c5c-2631-27e799e6e70c (Addr: tcp/127.0.0.1:24363) (DC: dc1)" +34:50.595 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-cb480960-2227-5c5c-2631-27e799e6e70c.dc1 area=wan +34:50.596 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24372 network=tcp +34:50.596 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24372 network=udp +34:50.597 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24331 network=tcp +34:50.597 [INFO] TestAgent: started state syncer +34:50.597 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24364 network=tcp +34:50.663 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:50.663 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24363 [Candidate]" term=2 +34:50.663 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:50.663 [DEBUG] TestAgent.server.raft: vote granted: from=cb480960-2227-5c5c-2631-27e799e6e70c term=2 tally=1 +34:50.663 [INFO] TestAgent.server.raft: election won: tally=1 +34:50.663 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24363 [Leader]" +34:50.664 [INFO] TestAgent.server: cluster leadership acquired +34:50.664 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24363 +34:50.665 [INFO] TestAgent.server: New leader elected: payload=Node-cb480960-2227-5c5c-2631-27e799e6e70c +34:50.667 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:50.670 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:50.670 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:50.670 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:50.670 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:50.670 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:50.670 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cb480960-2227-5c5c-2631-27e799e6e70c +34:50.670 [INFO] TestAgent.server: member joined, marking health alive: member=Node-cb480960-2227-5c5c-2631-27e799e6e70c +34:50.671 [INFO] TestAgent.server: federation state anti-entropy synced +34:51.015 [DEBUG] TestAgent.dns: request served from client: question="{2.0.0.127.in-addr.arpa. 255 1}" latency=214.061µs client=127.0.0.1:50687 client_network=udp +34:51.015 [INFO] TestAgent: Requesting shutdown +34:51.016 [INFO] TestAgent.server: shutting down server +34:51.016 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:51.016 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:51.016 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:51.016 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:51.016 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:51.016 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:51.016 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:51.016 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:51.016 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:51.016 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:51.016 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:51.016 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:51.016Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:51.016Z [INFO] agent.router.manager: shutting down +34:51.017 [INFO] TestAgent: consul server down +34:51.017 [INFO] TestAgent: shutdown complete +34:51.017 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24372 network=tcp +34:51.017 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24372 network=udp +34:51.017 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24331 network=tcp +34:51.517 [INFO] TestAgent: Waiting for endpoints to shut down +34:51.517 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ReverseLookup_CustomDomain (1.03s) +=== CONT TestCoordinate_Update_ACLDeny +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:50.752Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:50.797Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:50.808 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:76779fbe-a8a4-8fe2-7495-3b529eae2d7d Address:127.0.0.1:24398}]" +34:50.810 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-76779fbe-a8a4-8fe2-7495-3b529eae2d7d.dc1 127.0.0.1 +34:50.813 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24398 [Follower]" leader= +34:50.816 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-76779fbe-a8a4-8fe2-7495-3b529eae2d7d 127.0.0.1 +2021-08-13T15:34:50.816Z [INFO] agent.router: Initializing LAN area manager +34:50.817 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24386 network=udp +34:50.819 [INFO] TestAgent.server: Adding LAN server: server="Node-76779fbe-a8a4-8fe2-7495-3b529eae2d7d (Addr: tcp/127.0.0.1:24398) (DC: dc1)" +34:50.821 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-76779fbe-a8a4-8fe2-7495-3b529eae2d7d.dc1 area=wan +34:50.821 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24386 network=tcp +34:50.822 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24394 network=tcp +34:50.822 [INFO] TestAgent: started state syncer +34:50.822 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24399 network=tcp +34:50.879 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:50.879 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24398 [Candidate]" term=2 +34:50.879 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:50.879 [DEBUG] TestAgent.server.raft: vote granted: from=76779fbe-a8a4-8fe2-7495-3b529eae2d7d term=2 tally=1 +34:50.879 [INFO] TestAgent.server.raft: election won: tally=1 +34:50.879 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24398 [Leader]" +34:50.880 [INFO] TestAgent.server: cluster leadership acquired +34:50.880 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24398 +34:50.882 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:50.883 [INFO] TestAgent.server: New leader elected: payload=Node-76779fbe-a8a4-8fe2-7495-3b529eae2d7d +34:50.896 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:50.896 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:50.896 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:50.896 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:50.896 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:50.897 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-76779fbe-a8a4-8fe2-7495-3b529eae2d7d +34:50.897 [INFO] TestAgent.server: member joined, marking health alive: member=Node-76779fbe-a8a4-8fe2-7495-3b529eae2d7d +34:50.915 [INFO] TestAgent.server: federation state anti-entropy synced +34:50.916 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:50.917 [INFO] TestAgent: Synced node info +34:51.022 [DEBUG] TestAgent.dns: request served from client: name=foo.bar.node.consul. type=ANY class=IN latency=145.001µs client=127.0.0.1:50295 client_network=udp +34:51.022 [INFO] TestAgent: Requesting shutdown +34:51.022 [INFO] TestAgent.server: shutting down server +34:51.022 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:51.022 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:51.022 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:51.022 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:51.022 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:51.023 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:51.023 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:51.023 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:51.023 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:51.023 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:51.023 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:51.023Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:51.023Z [INFO] agent.router.manager: shutting down +34:51.023 [INFO] TestAgent: consul server down +34:51.023 [INFO] TestAgent: shutdown complete +34:51.023 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24386 network=tcp +34:51.023 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24386 network=udp +34:51.023 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24394 network=tcp +34:51.527 [INFO] TestAgent: Waiting for endpoints to shut down +34:51.527 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_NodeLookup_PeriodName (0.82s) +=== CONT TestDNS_Over_TCP +2021-08-13T15:34:50.936Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:50.952Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:50.955 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cdc9c997-0040-2781-910d-43024ca6b9cd Address:127.0.0.1:24426}]" +34:50.955 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24426 [Follower]" leader= +34:50.957 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-cdc9c997-0040-2781-910d-43024ca6b9cd.dc1 127.0.0.1 +34:50.969 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-cdc9c997-0040-2781-910d-43024ca6b9cd 127.0.0.1 +2021-08-13T15:34:50.969Z [INFO] agent.router: Initializing LAN area manager +34:50.970 [INFO] TestAgent.server: Adding LAN server: server="Node-cdc9c997-0040-2781-910d-43024ca6b9cd (Addr: tcp/127.0.0.1:24426) (DC: dc1)" +34:50.971 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-cdc9c997-0040-2781-910d-43024ca6b9cd.dc1 area=wan +34:50.975 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24400 network=tcp +34:50.975 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24400 network=udp +34:50.977 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24401 network=tcp +34:50.977 [INFO] TestAgent: started state syncer +34:50.977 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24427 network=tcp +34:51.029 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:51.029 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24426 [Candidate]" term=2 +34:51.030 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:51.030 [DEBUG] TestAgent.server.raft: vote granted: from=cdc9c997-0040-2781-910d-43024ca6b9cd term=2 tally=1 +34:51.030 [INFO] TestAgent.server.raft: election won: tally=1 +34:51.030 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24426 [Leader]" +34:51.030 [INFO] TestAgent.server: cluster leadership acquired +34:51.030 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24426 +34:51.032 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:51.035 [INFO] TestAgent.server: New leader elected: payload=Node-cdc9c997-0040-2781-910d-43024ca6b9cd +34:51.038 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:51.038 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:51.038 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:51.038 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:51.038 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:51.039 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cdc9c997-0040-2781-910d-43024ca6b9cd +34:51.039 [INFO] TestAgent.server: member joined, marking health alive: member=Node-cdc9c997-0040-2781-910d-43024ca6b9cd +34:51.088 [DEBUG] TestAgent.dns: request served from client: name=fOO.node.dc1.consul. type=ANY class=IN latency=194.181µs client=127.0.0.1:38059 client_network=udp +34:51.088 [INFO] TestAgent: Requesting shutdown +34:51.088 [INFO] TestAgent.server: shutting down server +34:51.088 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:51.088 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:51.088 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:51.088 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:51.088 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="shutdown waiting for leader" +34:51.088 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:51.088 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:51.088 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:51.088 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:51.088 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:51.088 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:51.089 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:51.089Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:51.089Z [INFO] agent.router.manager: shutting down +34:51.089 [INFO] TestAgent: consul server down +34:51.089 [INFO] TestAgent: shutdown complete +34:51.089 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24400 network=tcp +34:51.089 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24400 network=udp +34:51.089 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24401 network=tcp +34:51.591 [INFO] TestAgent: Waiting for endpoints to shut down +34:51.591 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_CaseInsensitiveNodeLookup (0.68s) +=== CONT TestCoordinate_Update +2021-08-13T15:34:50.936Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:50.952Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:50.955 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a03633ea-2fd9-a00d-2479-40127c487ae8 Address:127.0.0.1:25229}]" +34:50.957 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a03633ea-2fd9-a00d-2479-40127c487ae8.dc1 127.0.0.1 +34:50.959 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25229 [Follower]" leader= +34:50.959 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a03633ea-2fd9-a00d-2479-40127c487ae8 127.0.0.1 +2021-08-13T15:34:50.959Z [INFO] agent.router: Initializing LAN area manager +34:50.960 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24414 network=udp +34:50.963 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24414 network=tcp +34:50.963 [INFO] TestAgent.server: Adding LAN server: server="Node-a03633ea-2fd9-a00d-2479-40127c487ae8 (Addr: tcp/127.0.0.1:25229) (DC: dc1)" +34:50.964 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a03633ea-2fd9-a00d-2479-40127c487ae8.dc1 area=wan +34:50.965 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25260 network=tcp +34:50.965 [INFO] TestAgent: started state syncer +34:50.966 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25230 network=tcp +34:50.999 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:50.999 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25229 [Candidate]" term=2 +34:50.999 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:51.000 [DEBUG] TestAgent.server.raft: vote granted: from=a03633ea-2fd9-a00d-2479-40127c487ae8 term=2 tally=1 +34:51.000 [INFO] TestAgent.server.raft: election won: tally=1 +34:51.000 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25229 [Leader]" +34:51.000 [INFO] TestAgent.server: cluster leadership acquired +34:51.000 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25229 +34:51.002 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:51.003 [INFO] TestAgent.server: New leader elected: payload=Node-a03633ea-2fd9-a00d-2479-40127c487ae8 +34:51.010 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:51.010 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:51.010 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:51.010 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:51.010 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:51.010 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a03633ea-2fd9-a00d-2479-40127c487ae8 +34:51.010 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a03633ea-2fd9-a00d-2479-40127c487ae8 +34:51.011 [INFO] TestAgent.server: federation state anti-entropy synced +34:51.119 [INFO] TestAgent: Requesting shutdown +34:51.119 [INFO] TestAgent.server: shutting down server +34:51.120 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:51.120 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:51.120 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:51.120 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:51.120 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:51.120 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:51.120 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:51.120 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:51.120 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:51.120 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:51.120 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:51.120 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:51.120Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:51.120Z [INFO] agent.router.manager: shutting down +34:51.120 [INFO] TestAgent: consul server down +34:51.120 [INFO] TestAgent: shutdown complete +34:51.120 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24414 network=tcp +34:51.121 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24414 network=udp +34:51.121 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25260 network=tcp +34:51.623 [INFO] TestAgent: Waiting for endpoints to shut down +34:51.623 [INFO] TestAgent: Endpoints down +--- PASS: TestDiscoveryChainRead (0.71s) + --- PASS: TestDiscoveryChainRead/GET:_error_on_no_service_name (0.00s) + --- PASS: TestDiscoveryChainRead/GET:_read_default_chain (0.00s) + --- PASS: TestDiscoveryChainRead/GET:_read_default_chain;_evaluate_in_dc2 (0.00s) + --- PASS: TestDiscoveryChainRead/GET:_read_default_chain_with_cache (0.00s) + --- PASS: TestDiscoveryChainRead/POST:_error_on_no_service_name (0.00s) + --- PASS: TestDiscoveryChainRead/POST:_read_default_chain (0.00s) + --- PASS: TestDiscoveryChainRead/POST:_read_default_chain;_evaluate_in_dc2 (0.00s) + --- PASS: TestDiscoveryChainRead/POST:_read_default_chain_with_cache (0.00s) + --- PASS: TestDiscoveryChainRead/GET:_read_modified_chain (0.00s) + --- PASS: TestDiscoveryChainRead/POST:_read_modified_chain_with_overrides_(camel_case) (0.00s) + --- PASS: TestDiscoveryChainRead/POST:_read_modified_chain_with_overrides_(snake_case) (0.00s) +=== CONT TestCoordinate_Disabled_Response +=== RUN TestCoordinate_Update_ACLDeny/no_token +=== RUN TestCoordinate_Update_ACLDeny/valid_token +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=1)_compressed=false +=== RUN TestCoordinate_Disabled_Response/0 +=== RUN TestCoordinate_Disabled_Response/1 +=== RUN TestCoordinate_Disabled_Response/2 +=== RUN TestCoordinate_Disabled_Response/3 +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:51.204Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:51.220Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:51.223 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a9e7361a-17c4-64a2-382b-d8be60cc8e14 Address:127.0.0.1:24405}]" +34:51.223 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24405 [Follower]" leader= +34:51.224 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a9e7361a-17c4-64a2-382b-d8be60cc8e14.dc1 127.0.0.1 +34:51.224 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a9e7361a-17c4-64a2-382b-d8be60cc8e14 127.0.0.1 +2021-08-13T15:34:51.225Z [INFO] agent.router: Initializing LAN area manager +34:51.225 [INFO] TestAgent.server: Adding LAN server: server="Node-a9e7361a-17c4-64a2-382b-d8be60cc8e14 (Addr: tcp/127.0.0.1:24405) (DC: dc1)" +34:51.225 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a9e7361a-17c4-64a2-382b-d8be60cc8e14.dc1 area=wan +34:51.225 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25231 network=udp +34:51.225 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25231 network=tcp +34:51.226 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25232 network=tcp +34:51.226 [INFO] TestAgent: started state syncer +34:51.226 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24406 network=tcp +34:51.292 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:51.292 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24405 [Candidate]" term=2 +34:51.292 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:51.292 [DEBUG] TestAgent.server.raft: vote granted: from=a9e7361a-17c4-64a2-382b-d8be60cc8e14 term=2 tally=1 +34:51.292 [INFO] TestAgent.server.raft: election won: tally=1 +34:51.292 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24405 [Leader]" +34:51.292 [INFO] TestAgent.server: cluster leadership acquired +34:51.292 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24405 +34:51.293 [INFO] TestAgent.server: New leader elected: payload=Node-a9e7361a-17c4-64a2-382b-d8be60cc8e14 +34:51.294 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:51.296 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:51.296 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:51.296 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:51.296 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:51.296 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:51.297 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a9e7361a-17c4-64a2-382b-d8be60cc8e14 +34:51.297 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a9e7361a-17c4-64a2-382b-d8be60cc8e14 +34:51.298 [INFO] TestAgent.server: federation state anti-entropy synced +34:51.561 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=ANY class=IN latency=247.561µs client=127.0.0.1:33530 client_network=udp +34:51.561 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=A class=IN latency=158.54µs client=127.0.0.1:43183 client_network=udp +34:51.562 [DEBUG] TestAgent.dns: request served from client: name=foo.node.dc1.consul. type=ANY class=IN latency=158.361µs client=127.0.0.1:39899 client_network=udp +34:51.563 [DEBUG] TestAgent.dns: request served from client: name=nofoo.node.dc1.consul. type=ANY class=IN latency=160.761µs client=127.0.0.1:45251 client_network=udp +34:51.563 [INFO] TestAgent: Requesting shutdown +34:51.563 [INFO] TestAgent.server: shutting down server +34:51.563 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:51.563 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:51.563 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:51.563 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:51.563 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:51.563 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:51.563 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:51.564 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:51.564 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:51.564 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:51.564 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:51.565 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:51.565Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:51.565Z [INFO] agent.router.manager: shutting down +34:51.565 [INFO] TestAgent: consul server down +34:51.565 [INFO] TestAgent: shutdown complete +34:51.565 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25231 network=tcp +34:51.565 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25231 network=udp +34:51.566 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25232 network=tcp +34:52.075 [INFO] TestAgent: Waiting for endpoints to shut down +34:52.075 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_NodeLookup (0.89s) +=== CONT TestCoordinate_Node +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=1)_compressed=true +2021-08-13T15:34:51.583Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:51.583Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:51.645Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:51.645Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:51.649 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cf65ae58-9699-d66b-0595-2d3d76b3dc0f Address:127.0.0.1:24934}]" +34:51.650 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-cf65ae58-9699-d66b-0595-2d3d76b3dc0f.dc1 127.0.0.1 +34:51.656 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-cf65ae58-9699-d66b-0595-2d3d76b3dc0f 127.0.0.1 +2021-08-13T15:34:51.656Z [INFO] agent.router: Initializing LAN area manager +34:51.656 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25259 network=udp +34:51.656 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24934 [Follower]" leader= +34:51.682 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-cf65ae58-9699-d66b-0595-2d3d76b3dc0f.dc1 area=wan +34:51.684 [INFO] TestAgent.server: Adding LAN server: server="Node-cf65ae58-9699-d66b-0595-2d3d76b3dc0f (Addr: tcp/127.0.0.1:24934) (DC: dc1)" +34:51.685 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25259 network=tcp +34:51.687 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24951 network=tcp +34:51.687 [INFO] TestAgent: started state syncer +34:51.688 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24935 network=tcp +34:51.696 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:51.696 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24934 [Candidate]" term=2 +34:51.697 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:51.697 [DEBUG] TestAgent.server.raft: vote granted: from=cf65ae58-9699-d66b-0595-2d3d76b3dc0f term=2 tally=1 +34:51.697 [INFO] TestAgent.server.raft: election won: tally=1 +34:51.697 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24934 [Leader]" +34:51.697 [INFO] TestAgent.server: cluster leadership acquired +34:51.698 [INFO] TestAgent.server: initializing acls +34:51.698 [INFO] TestAgent.server: New leader elected: payload=Node-cf65ae58-9699-d66b-0595-2d3d76b3dc0f +34:51.700 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:51.700 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:51.704 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:51.705 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:51.705 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:51.705 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:51.706 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-cf65ae58-9699-d66b-0595-2d3d76b3dc0f +34:51.706 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-cf65ae58-9699-d66b-0595-2d3d76b3dc0f.dc1 +34:51.707 [INFO] TestAgent.server: Updating LAN server: server="Node-cf65ae58-9699-d66b-0595-2d3d76b3dc0f (Addr: tcp/127.0.0.1:24934) (DC: dc1)" +34:51.708 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:51.711 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-cf65ae58-9699-d66b-0595-2d3d76b3dc0f.dc1 area=wan +34:51.711 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:51.711 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:51.711 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:51.711 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:51.711 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:51.711 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cf65ae58-9699-d66b-0595-2d3d76b3dc0f +34:51.711 [INFO] TestAgent.server: member joined, marking health alive: member=Node-cf65ae58-9699-d66b-0595-2d3d76b3dc0f +34:51.712 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cf65ae58-9699-d66b-0595-2d3d76b3dc0f +34:51.713 [INFO] TestAgent.server: federation state anti-entropy synced +34:51.723 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-cf65ae58-9699-d66b-0595-2d3d76b3dc0f +34:51.724 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-cf65ae58-9699-d66b-0595-2d3d76b3dc0f +34:51.726 [INFO] TestAgent: Requesting shutdown +34:51.726 [INFO] TestAgent.server: shutting down server +34:51.726 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:51.726 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:51.726 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:51.726 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:51.726 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:51.726 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:51.726 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:51.726 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:51.726 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:51.727 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:51.727 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:51.727 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:51.727 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:51.727 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:51.727 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:51.731 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:51.731Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:51.731Z [INFO] agent.router.manager: shutting down +34:51.731 [INFO] TestAgent: consul server down +34:51.731 [INFO] TestAgent: shutdown complete +34:51.731 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25259 network=tcp +34:51.732 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25259 network=udp +34:51.732 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24951 network=tcp +34:52.235 [INFO] TestAgent: Waiting for endpoints to shut down +34:52.235 [INFO] TestAgent: Endpoints down +--- PASS: TestCoordinate_Update_ACLDeny (0.72s) + --- PASS: TestCoordinate_Update_ACLDeny/no_token (0.00s) + --- PASS: TestCoordinate_Update_ACLDeny/valid_token (0.00s) +=== CONT TestConnectCAConfig +=== RUN TestConnectCAConfig/basic +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=1)_compressed=false +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._tcp_(qType:=1)_compressed=true +2021-08-13T15:34:51.586Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:51.658Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:51.662 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:be498045-d50f-12b1-2aa8-0d94f7bee4c0 Address:127.0.0.1:24461}]" +34:51.663 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-be498045-d50f-12b1-2aa8-0d94f7bee4c0.dc1 127.0.0.1 +34:51.664 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-be498045-d50f-12b1-2aa8-0d94f7bee4c0 127.0.0.1 +2021-08-13T15:34:51.664Z [INFO] agent.router: Initializing LAN area manager +34:51.665 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24957 network=udp +34:51.668 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24461 [Follower]" leader= +34:51.677 [INFO] TestAgent.server: Adding LAN server: server="Node-be498045-d50f-12b1-2aa8-0d94f7bee4c0 (Addr: tcp/127.0.0.1:24461) (DC: dc1)" +34:51.677 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-be498045-d50f-12b1-2aa8-0d94f7bee4c0.dc1 area=wan +34:51.681 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24957 network=tcp +34:51.683 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24436 network=tcp +34:51.683 [INFO] TestAgent: started state syncer +34:51.684 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24462 network=tcp +34:51.707 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:51.707 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24461 [Candidate]" term=2 +34:51.708 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:51.708 [DEBUG] TestAgent.server.raft: vote granted: from=be498045-d50f-12b1-2aa8-0d94f7bee4c0 term=2 tally=1 +34:51.708 [INFO] TestAgent.server.raft: election won: tally=1 +34:51.708 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24461 [Leader]" +34:51.708 [INFO] TestAgent.server: cluster leadership acquired +34:51.708 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24461 +34:51.710 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:51.710 [INFO] TestAgent.server: New leader elected: payload=Node-be498045-d50f-12b1-2aa8-0d94f7bee4c0 +34:51.713 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:51.713 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:51.713 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:51.713 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:51.713 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:51.713 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-be498045-d50f-12b1-2aa8-0d94f7bee4c0 +34:51.713 [INFO] TestAgent.server: member joined, marking health alive: member=Node-be498045-d50f-12b1-2aa8-0d94f7bee4c0 +34:51.714 [INFO] TestAgent.server: federation state anti-entropy synced +34:51.818 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:51.819 [INFO] TestAgent: Synced node info +34:51.819 [DEBUG] TestAgent: Node info in sync +34:51.930 [DEBUG] TestAgent.dns: request served from client: name=nofoo.node.dc1.consul. type=ANY class=IN latency=238.641µs client=127.0.0.1:35392 client_network=udp +34:51.930 [INFO] TestAgent: Requesting shutdown +34:51.930 [INFO] TestAgent.server: shutting down server +34:51.930 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:51.930 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:51.930 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:51.930 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:51.930 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:51.931 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:51.931 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:51.931 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:51.931 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:51.931 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:51.931 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:51.931Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:51.931Z [INFO] agent.router.manager: shutting down +34:51.932 [INFO] TestAgent: consul server down +34:51.932 [INFO] TestAgent: shutdown complete +34:51.932 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24957 network=tcp +34:51.932 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24957 network=udp +34:51.932 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24436 network=tcp +34:52.433 [INFO] TestAgent: Waiting for endpoints to shut down +34:52.433 [INFO] TestAgent: Endpoints down +2021-08-13T15:34:50.589Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:50.620Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:50.632 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2221f701-eec7-80c4-6914-fa38790902ee Address:127.0.0.1:24377}]" +34:50.635 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24377 [Follower]" leader= +34:50.636 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2221f701-eec7-80c4-6914-fa38790902ee.dc1 127.0.0.1 +34:50.638 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2221f701-eec7-80c4-6914-fa38790902ee 127.0.0.1 +2021-08-13T15:34:50.638Z [INFO] agent.router: Initializing LAN area manager +34:50.638 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24393 network=udp +34:50.639 [INFO] TestAgent.server: Adding LAN server: server="Node-2221f701-eec7-80c4-6914-fa38790902ee (Addr: tcp/127.0.0.1:24377) (DC: dc1)" +34:50.639 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24393 network=tcp +34:50.639 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2221f701-eec7-80c4-6914-fa38790902ee.dc1 area=wan +34:50.641 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24373 network=tcp +34:50.641 [INFO] TestAgent: started state syncer +34:50.642 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24378 network=tcp +34:50.676 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:50.676 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24377 [Candidate]" term=2 +34:50.676 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:50.676 [DEBUG] TestAgent.server.raft: vote granted: from=2221f701-eec7-80c4-6914-fa38790902ee term=2 tally=1 +34:50.676 [INFO] TestAgent.server.raft: election won: tally=1 +34:50.676 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24377 [Leader]" +34:50.676 [INFO] TestAgent.server: cluster leadership acquired +34:50.677 [INFO] TestAgent.server: New leader elected: payload=Node-2221f701-eec7-80c4-6914-fa38790902ee +34:50.677 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24377 +34:50.679 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:50.682 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:50.682 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:50.682 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:50.682 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:50.682 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:50.682 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2221f701-eec7-80c4-6914-fa38790902ee +34:50.682 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2221f701-eec7-80c4-6914-fa38790902ee +34:50.683 [INFO] TestAgent.server: federation state anti-entropy synced +34:51.020 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:51.021 [DEBUG] TestAgent.dns: request served from client: name=nofoo.node.dc1.consul. type=ANY class=IN latency=283.62µs client=127.0.0.1:36484 client_network=udp +34:51.021 [INFO] TestAgent: Synced node info +34:51.021 [DEBUG] TestAgent: Node info in sync +34:51.021 [INFO] TestAgent: Requesting shutdown +34:51.021 [INFO] TestAgent.server: shutting down server +34:51.021 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:51.021 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:51.021 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:51.021 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:51.021 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:51.022 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:51.022 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:51.022 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:51.022 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +34:51.022 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="shutdown waiting for leader" +34:51.022 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +2021-08-13T15:34:51.022Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:51.022Z [INFO] agent.router.manager: shutting down +34:51.022 [INFO] TestAgent: consul server down +34:51.022 [INFO] TestAgent: shutdown complete +34:51.022 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24393 network=tcp +34:51.022 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24393 network=udp +34:51.022 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24373 network=tcp +34:51.523 [INFO] TestAgent: Waiting for endpoints to shut down +34:51.523 [INFO] TestAgent: Endpoints down +2021-08-13T15:34:49.754Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:49.775Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:49.778 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:605cdb90-211e-6656-84a5-51b732c2d4f1 Address:127.0.0.1:24356}]" +34:49.778 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24356 [Follower]" leader= +34:49.779 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-605cdb90-211e-6656-84a5-51b732c2d4f1.dc1 127.0.0.1 +34:49.780 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-605cdb90-211e-6656-84a5-51b732c2d4f1 127.0.0.1 +2021-08-13T15:34:49.780Z [INFO] agent.router: Initializing LAN area manager +34:49.780 [INFO] TestAgent.server: Adding LAN server: server="Node-605cdb90-211e-6656-84a5-51b732c2d4f1 (Addr: tcp/127.0.0.1:24356) (DC: dc1)" +34:49.780 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-605cdb90-211e-6656-84a5-51b732c2d4f1.dc1 area=wan +34:49.780 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24365 network=udp +34:49.781 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24365 network=tcp +34:49.782 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24338 network=tcp +34:49.782 [INFO] TestAgent: started state syncer +34:49.782 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24357 network=tcp +34:49.818 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:49.818 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24356 [Candidate]" term=2 +34:49.818 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:49.818 [DEBUG] TestAgent.server.raft: vote granted: from=605cdb90-211e-6656-84a5-51b732c2d4f1 term=2 tally=1 +34:49.818 [INFO] TestAgent.server.raft: election won: tally=1 +34:49.818 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24356 [Leader]" +34:49.818 [INFO] TestAgent.server: cluster leadership acquired +34:49.819 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24356 +34:49.820 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:49.822 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:49.822 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:49.822 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:49.822 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:49.822 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:49.823 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-605cdb90-211e-6656-84a5-51b732c2d4f1 +34:49.823 [INFO] TestAgent.server: member joined, marking health alive: member=Node-605cdb90-211e-6656-84a5-51b732c2d4f1 +34:49.823 [INFO] TestAgent.server: New leader elected: payload=Node-605cdb90-211e-6656-84a5-51b732c2d4f1 +34:49.827 [INFO] TestAgent.server: federation state anti-entropy synced +34:50.024 [DEBUG] TestAgent.dns: request served from client: name=nofoo.node.dc1.consul. type=ANY class=IN latency=231.381µs client=127.0.0.1:42647 client_network=udp +34:50.024 [INFO] TestAgent: Requesting shutdown +34:50.024 [INFO] TestAgent.server: shutting down server +34:50.024 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:50.024 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:50.024 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:50.024 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:50.024 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:50.024 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:50.024 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:50.024 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:50.025 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:50.025 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:50.025 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:50.025 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:50.025Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:50.025Z [INFO] agent.router.manager: shutting down +34:50.025 [INFO] TestAgent: consul server down +34:50.025 [INFO] TestAgent: shutdown complete +34:50.025 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24365 network=tcp +34:50.025 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24365 network=udp +34:50.025 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24338 network=tcp +34:50.531 [INFO] TestAgent: Waiting for endpoints to shut down +34:50.532 [INFO] TestAgent: Endpoints down +2021-08-13T15:34:48.891Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:48.932Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:48.936 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7f8b9695-17ec-a258-fda2-a3433e99f435 Address:127.0.0.1:24307}]" +34:48.937 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24307 [Follower]" leader= +34:48.937 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7f8b9695-17ec-a258-fda2-a3433e99f435.dc1 127.0.0.1 +34:48.938 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7f8b9695-17ec-a258-fda2-a3433e99f435 127.0.0.1 +2021-08-13T15:34:48.938Z [INFO] agent.router: Initializing LAN area manager +34:48.939 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24302 network=udp +34:48.939 [INFO] TestAgent.server: Adding LAN server: server="Node-7f8b9695-17ec-a258-fda2-a3433e99f435 (Addr: tcp/127.0.0.1:24307) (DC: dc1)" +34:48.939 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7f8b9695-17ec-a258-fda2-a3433e99f435.dc1 area=wan +34:48.939 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24302 network=tcp +34:48.940 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24303 network=tcp +34:48.940 [INFO] TestAgent: started state syncer +34:48.941 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24308 network=tcp +34:48.991 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:48.991 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24307 [Candidate]" term=2 +34:48.991 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:48.991 [DEBUG] TestAgent.server.raft: vote granted: from=7f8b9695-17ec-a258-fda2-a3433e99f435 term=2 tally=1 +34:48.991 [INFO] TestAgent.server.raft: election won: tally=1 +34:48.991 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24307 [Leader]" +34:48.992 [INFO] TestAgent.server: cluster leadership acquired +34:48.992 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24307 +34:48.994 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:48.995 [INFO] TestAgent.server: New leader elected: payload=Node-7f8b9695-17ec-a258-fda2-a3433e99f435 +34:48.997 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:48.997 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:48.997 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:48.997 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:48.997 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:48.997 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7f8b9695-17ec-a258-fda2-a3433e99f435 +34:48.997 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7f8b9695-17ec-a258-fda2-a3433e99f435 +34:49.007 [INFO] TestAgent.server: federation state anti-entropy synced +34:49.187 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:49.189 [INFO] TestAgent: Synced node info +34:49.212 [DEBUG] TestAgent.dns: request served from client: name=nofoo.node.dc1.consul. type=ANY class=IN latency=305.321µs client=127.0.0.1:56708 client_network=udp +34:49.212 [INFO] TestAgent: Requesting shutdown +34:49.212 [INFO] TestAgent.server: shutting down server +34:49.212 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:49.212 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:49.212 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:49.212 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:49.213 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:49.213 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:49.213 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:49.213 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:49.213 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:49.213 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:49.213 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:49.213Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:49.213Z [INFO] agent.router.manager: shutting down +34:49.213 [INFO] TestAgent: consul server down +34:49.213 [INFO] TestAgent: shutdown complete +34:49.213 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24302 network=tcp +34:49.213 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24302 network=udp +34:49.214 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24303 network=tcp +34:49.715 [INFO] TestAgent: Waiting for endpoints to shut down +34:49.715 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_SOA_Settings (3.58s) +=== CONT TestCoordinate_Nodes +2021-08-13T15:34:51.579Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:51.633Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:51.652 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7e609325-0e66-7389-cb3b-0825598e8d02 Address:127.0.0.1:25257}]" +34:51.654 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7e609325-0e66-7389-cb3b-0825598e8d02.dc1 127.0.0.1 +34:51.657 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7e609325-0e66-7389-cb3b-0825598e8d02 127.0.0.1 +2021-08-13T15:34:51.657Z [INFO] agent.router: Initializing LAN area manager +34:51.657 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24407 network=udp +34:51.660 [INFO] TestAgent.server: Adding LAN server: server="Node-7e609325-0e66-7389-cb3b-0825598e8d02 (Addr: tcp/127.0.0.1:25257) (DC: dc1)" +34:51.665 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25257 [Follower]" leader= +34:51.667 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7e609325-0e66-7389-cb3b-0825598e8d02.dc1 area=wan +34:51.668 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24407 network=tcp +34:51.675 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24415 network=tcp +34:51.675 [INFO] TestAgent: started state syncer +34:51.676 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25258 network=tcp +34:51.732 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:51.732 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25257 [Candidate]" term=2 +34:51.732 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:51.732 [DEBUG] TestAgent.server.raft: vote granted: from=7e609325-0e66-7389-cb3b-0825598e8d02 term=2 tally=1 +34:51.732 [INFO] TestAgent.server.raft: election won: tally=1 +34:51.732 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25257 [Leader]" +34:51.732 [INFO] TestAgent.server: cluster leadership acquired +34:51.733 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25257 +34:51.734 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:51.737 [INFO] TestAgent.server: New leader elected: payload=Node-7e609325-0e66-7389-cb3b-0825598e8d02 +34:51.737 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:51.737 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:51.737 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:51.737 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:51.737 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:51.738 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7e609325-0e66-7389-cb3b-0825598e8d02 +34:51.738 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7e609325-0e66-7389-cb3b-0825598e8d02 +34:51.749 [INFO] TestAgent.server: federation state anti-entropy synced +34:51.783 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:51.785 [INFO] TestAgent: Synced node info +34:51.948 [INFO] TestAgent: Requesting shutdown +34:51.949 [INFO] TestAgent.server: shutting down server +34:51.949 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:51.949 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:51.949 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:51.949 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:51.949 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:51.949 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:51.949 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:51.949 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:51.949 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:51.949 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:51.950 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:51.950Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:51.950Z [INFO] agent.router.manager: shutting down +34:51.950 [INFO] TestAgent: consul server down +34:51.950 [INFO] TestAgent: shutdown complete +34:51.950 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24407 network=tcp +34:51.950 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24407 network=udp +34:51.950 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24415 network=tcp +34:52.451 [INFO] TestAgent: Waiting for endpoints to shut down +34:52.451 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_EmptyAltDomain (0.94s) +=== CONT TestConnectCARoots_list +2021-08-13T15:34:51.705Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:51.749Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:51.754 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:308d3f91-bec6-e066-79a2-733405e31132 Address:127.0.0.1:24453}]" +34:51.755 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24453 [Follower]" leader= +34:51.756 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-308d3f91-bec6-e066-79a2-733405e31132.dc1 127.0.0.1 +34:51.757 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-308d3f91-bec6-e066-79a2-733405e31132 127.0.0.1 +2021-08-13T15:34:51.757Z [INFO] agent.router: Initializing LAN area manager +34:51.758 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24477 network=udp +34:51.759 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-308d3f91-bec6-e066-79a2-733405e31132.dc1 area=wan +34:51.760 [INFO] TestAgent.server: Adding LAN server: server="Node-308d3f91-bec6-e066-79a2-733405e31132 (Addr: tcp/127.0.0.1:24453) (DC: dc1)" +34:51.760 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24477 network=tcp +34:51.762 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24485 network=tcp +34:51.762 [INFO] TestAgent: started state syncer +34:51.762 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24454 network=tcp +34:51.823 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:51.823 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24453 [Candidate]" term=2 +34:51.823 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:51.823 [DEBUG] TestAgent.server.raft: vote granted: from=308d3f91-bec6-e066-79a2-733405e31132 term=2 tally=1 +34:51.823 [INFO] TestAgent.server.raft: election won: tally=1 +34:51.823 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24453 [Leader]" +34:51.823 [INFO] TestAgent.server: cluster leadership acquired +34:51.824 [INFO] TestAgent.server: New leader elected: payload=Node-308d3f91-bec6-e066-79a2-733405e31132 +34:51.824 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24453 +34:51.825 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:51.828 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:51.828 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:51.828 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:51.828 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:51.828 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:51.828 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-308d3f91-bec6-e066-79a2-733405e31132 +34:51.828 [INFO] TestAgent.server: member joined, marking health alive: member=Node-308d3f91-bec6-e066-79a2-733405e31132 +34:51.829 [INFO] TestAgent.server: federation state anti-entropy synced +34:51.958 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:51.959 [INFO] TestAgent: Synced node info +34:51.959 [DEBUG] TestAgent: Node info in sync +34:51.988 [INFO] TestAgent: Requesting shutdown +34:51.988 [INFO] TestAgent.server: shutting down server +34:51.988 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:51.988 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:51.988 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:51.988 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:51.988 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:51.989 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:51.989 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:51.989 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:51.989 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:51.989 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:51.989 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:51.990Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:51.990Z [INFO] agent.router.manager: shutting down +34:51.990 [INFO] TestAgent: consul server down +34:51.990 [INFO] TestAgent: shutdown complete +34:51.990 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24477 network=tcp +34:51.990 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24477 network=udp +34:51.990 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24485 network=tcp +34:52.491 [INFO] TestAgent: Waiting for endpoints to shut down +34:52.491 [INFO] TestAgent: Endpoints down +--- PASS: TestCoordinate_Disabled_Response (0.87s) + --- PASS: TestCoordinate_Disabled_Response/0 (0.00s) + --- PASS: TestCoordinate_Disabled_Response/1 (0.00s) + --- PASS: TestCoordinate_Disabled_Response/2 (0.00s) + --- PASS: TestCoordinate_Disabled_Response/3 (0.00s) +=== CONT TestCoordinate_Datacenters +2021-08-13T15:34:51.594Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:51.655Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:51.658 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6a4ac876-aecb-66eb-0ac1-1f1c73b703fe Address:127.0.0.1:24482}]" +34:51.677 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6a4ac876-aecb-66eb-0ac1-1f1c73b703fe.dc1 127.0.0.1 +34:51.688 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6a4ac876-aecb-66eb-0ac1-1f1c73b703fe 127.0.0.1 +2021-08-13T15:34:51.688Z [INFO] agent.router: Initializing LAN area manager +34:51.689 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24463 network=udp +34:51.691 [INFO] TestAgent.server: Adding LAN server: server="Node-6a4ac876-aecb-66eb-0ac1-1f1c73b703fe (Addr: tcp/127.0.0.1:24482) (DC: dc1)" +34:51.692 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24482 [Follower]" leader= +34:51.695 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24463 network=tcp +34:51.697 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6a4ac876-aecb-66eb-0ac1-1f1c73b703fe.dc1 area=wan +34:51.698 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24457 network=tcp +34:51.698 [INFO] TestAgent: started state syncer +34:51.699 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24483 network=tcp +34:51.731 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:51.731 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24482 [Candidate]" term=2 +34:51.731 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:51.731 [DEBUG] TestAgent.server.raft: vote granted: from=6a4ac876-aecb-66eb-0ac1-1f1c73b703fe term=2 tally=1 +34:51.731 [INFO] TestAgent.server.raft: election won: tally=1 +34:51.731 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24482 [Leader]" +34:51.732 [INFO] TestAgent.server: cluster leadership acquired +34:51.732 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24482 +34:51.734 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:51.741 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:51.741 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:51.741 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:51.741 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:51.741 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:51.742 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6a4ac876-aecb-66eb-0ac1-1f1c73b703fe +34:51.742 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6a4ac876-aecb-66eb-0ac1-1f1c73b703fe +34:51.747 [INFO] TestAgent.server: New leader elected: payload=Node-6a4ac876-aecb-66eb-0ac1-1f1c73b703fe +34:51.752 [INFO] TestAgent.server: federation state anti-entropy synced +34:51.999 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:52.000 [INFO] TestAgent: Synced node info +34:52.000 [DEBUG] TestAgent: Node info in sync +34:52.011 [DEBUG] TestAgent.dns: request served from client: name=foo.node.dc1.consul. type=ANY class=IN latency=221.161µs client=127.0.0.1:44266 client_network=tcp +34:52.011 [INFO] TestAgent: Requesting shutdown +34:52.011 [INFO] TestAgent.server: shutting down server +34:52.011 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:52.011 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:52.011 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:52.011 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:52.011 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:52.011 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:52.011 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:52.011 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:52.011 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:52.011 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:52.012 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:52.012Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:52.012Z [INFO] agent.router.manager: shutting down +34:52.012 [INFO] TestAgent: consul server down +34:52.012 [INFO] TestAgent: shutdown complete +34:52.012 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24463 network=tcp +34:52.012 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24463 network=udp +34:52.012 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24457 network=tcp +34:52.515 [INFO] TestAgent: Waiting for endpoints to shut down +34:52.515 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_Over_TCP (0.99s) +=== CONT TestConnectCARoots_empty +[WARN] freeport: 10 out of 59 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:51.625Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:51.681Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:51.690 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c6801a08-614e-1cec-5adc-23853ab1e20a Address:127.0.0.1:24475}]" +34:51.695 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24475 [Follower]" leader= +34:51.699 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c6801a08-614e-1cec-5adc-23853ab1e20a.dc1 127.0.0.1 +34:51.701 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c6801a08-614e-1cec-5adc-23853ab1e20a 127.0.0.1 +2021-08-13T15:34:51.701Z [INFO] agent.router: Initializing LAN area manager +34:51.702 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c6801a08-614e-1cec-5adc-23853ab1e20a.dc1 area=wan +34:51.702 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24491 network=udp +34:51.702 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24491 network=tcp +34:51.703 [INFO] TestAgent.server: Adding LAN server: server="Node-c6801a08-614e-1cec-5adc-23853ab1e20a (Addr: tcp/127.0.0.1:24475) (DC: dc1)" +34:51.705 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24492 network=tcp +34:51.705 [INFO] TestAgent: started state syncer +34:51.705 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24476 network=tcp +34:51.749 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:51.749 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24475 [Candidate]" term=2 +34:51.749 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:51.749 [DEBUG] TestAgent.server.raft: vote granted: from=c6801a08-614e-1cec-5adc-23853ab1e20a term=2 tally=1 +34:51.749 [INFO] TestAgent.server.raft: election won: tally=1 +34:51.749 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24475 [Leader]" +34:51.749 [INFO] TestAgent.server: cluster leadership acquired +34:51.750 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24475 +34:51.751 [INFO] TestAgent.server: New leader elected: payload=Node-c6801a08-614e-1cec-5adc-23853ab1e20a +34:51.751 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:51.755 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:51.755 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:51.755 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:51.755 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:51.755 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:51.755 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c6801a08-614e-1cec-5adc-23853ab1e20a +34:51.755 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c6801a08-614e-1cec-5adc-23853ab1e20a +34:51.756 [INFO] TestAgent.server: federation state anti-entropy synced +34:52.030 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:52.030 [INFO] TestAgent: Synced node info +34:52.311 [INFO] TestAgent: Requesting shutdown +34:52.311 [INFO] TestAgent.server: shutting down server +34:52.311 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:52.311 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:52.311 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:52.311 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:52.311 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:52.311 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:52.311 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:52.311 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:52.311 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:52.311 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:52.311 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:52.311Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:52.311Z [INFO] agent.router.manager: shutting down +34:52.312 [INFO] TestAgent: consul server down +34:52.312 [INFO] TestAgent: shutdown complete +34:52.312 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24491 network=tcp +34:52.312 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24491 network=udp +34:52.312 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24492 network=tcp +34:52.813 [INFO] TestAgent: Waiting for endpoints to shut down +34:52.813 [INFO] TestAgent: Endpoints down +--- PASS: TestCoordinate_Update (1.22s) +=== CONT TestConfig_Apply_CAS +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._tcp_(qType:=1)_compressed=false +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:52.119Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:52.136Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:52.139 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9935bfea-a554-4d43-f158-09eb603110e4 Address:127.0.0.1:24495}]" +34:52.139 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24495 [Follower]" leader= +34:52.140 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9935bfea-a554-4d43-f158-09eb603110e4.dc1 127.0.0.1 +34:52.141 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9935bfea-a554-4d43-f158-09eb603110e4 127.0.0.1 +2021-08-13T15:34:52.141Z [INFO] agent.router: Initializing LAN area manager +34:52.141 [INFO] TestAgent.server: Adding LAN server: server="Node-9935bfea-a554-4d43-f158-09eb603110e4 (Addr: tcp/127.0.0.1:24495) (DC: dc1)" +34:52.141 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9935bfea-a554-4d43-f158-09eb603110e4.dc1 area=wan +34:52.141 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24455 network=udp +34:52.141 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24455 network=tcp +34:52.142 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24456 network=tcp +34:52.142 [INFO] TestAgent: started state syncer +34:52.142 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24496 network=tcp +34:52.198 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:52.198 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24495 [Candidate]" term=2 +34:52.199 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:52.199 [DEBUG] TestAgent.server.raft: vote granted: from=9935bfea-a554-4d43-f158-09eb603110e4 term=2 tally=1 +34:52.199 [INFO] TestAgent.server.raft: election won: tally=1 +34:52.199 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24495 [Leader]" +34:52.199 [INFO] TestAgent.server: cluster leadership acquired +34:52.199 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24495 +34:52.199 [INFO] TestAgent.server: New leader elected: payload=Node-9935bfea-a554-4d43-f158-09eb603110e4 +34:52.201 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:52.203 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:52.203 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:52.203 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:52.203 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:52.203 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:52.204 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9935bfea-a554-4d43-f158-09eb603110e4 +34:52.204 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9935bfea-a554-4d43-f158-09eb603110e4 +34:52.205 [INFO] TestAgent.server: federation state anti-entropy synced +34:52.540 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:52.542 [INFO] TestAgent: Synced node info +34:52.542 [DEBUG] TestAgent: Node info in sync +34:52.739 [INFO] TestAgent: Requesting shutdown +34:52.740 [INFO] TestAgent.server: shutting down server +34:52.740 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:52.740 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:52.740 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:52.740 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:52.740 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:52.740 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:52.740 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:52.740 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:52.740 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:52.740 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:52.740 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:52.740Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:52.740Z [INFO] agent.router.manager: shutting down +34:52.741 [INFO] TestAgent: consul server down +34:52.741 [INFO] TestAgent: shutdown complete +34:52.741 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24455 network=tcp +34:52.741 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24455 network=udp +34:52.741 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24456 network=tcp +34:53.241 [INFO] TestAgent: Waiting for endpoints to shut down +34:53.241 [INFO] TestAgent: Endpoints down +--- PASS: TestCoordinate_Node (1.17s) +=== CONT TestConfig_Apply_ProxyDefaultsExpose +2021-08-13T15:34:52.290Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:52.309Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:52.313 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b5f84a8d-4030-acd0-99ba-1ab20f22aed4 Address:127.0.0.1:24468}]" +34:52.313 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24468 [Follower]" leader= +34:52.314 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b5f84a8d-4030-acd0-99ba-1ab20f22aed4.dc1 127.0.0.1 +34:52.315 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b5f84a8d-4030-acd0-99ba-1ab20f22aed4 127.0.0.1 +2021-08-13T15:34:52.315Z [INFO] agent.router: Initializing LAN area manager +34:52.315 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24497 network=udp +34:52.315 [INFO] TestAgent.server: Adding LAN server: server="Node-b5f84a8d-4030-acd0-99ba-1ab20f22aed4 (Addr: tcp/127.0.0.1:24468) (DC: dc1)" +34:52.315 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b5f84a8d-4030-acd0-99ba-1ab20f22aed4.dc1 area=wan +34:52.316 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24497 network=tcp +34:52.316 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24484 network=tcp +34:52.317 [INFO] TestAgent: started state syncer +34:52.317 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24469 network=tcp +34:52.382 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:52.382 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24468 [Candidate]" term=2 +34:52.382 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:52.382 [DEBUG] TestAgent.server.raft: vote granted: from=b5f84a8d-4030-acd0-99ba-1ab20f22aed4 term=2 tally=1 +34:52.382 [INFO] TestAgent.server.raft: election won: tally=1 +34:52.382 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24468 [Leader]" +34:52.382 [INFO] TestAgent.server: cluster leadership acquired +34:52.383 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24468 +34:52.383 [INFO] TestAgent.server: New leader elected: payload=Node-b5f84a8d-4030-acd0-99ba-1ab20f22aed4 +34:52.384 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:52.387 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:52.387 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:52.387 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:52.387 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:52.387 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:52.387 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b5f84a8d-4030-acd0-99ba-1ab20f22aed4 +34:52.387 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b5f84a8d-4030-acd0-99ba-1ab20f22aed4 +34:52.388 [INFO] TestAgent.server: federation state anti-entropy synced +34:52.740 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:52.741 [INFO] TestAgent: Synced node info +34:52.752 [INFO] TestAgent.server.connect: CA provider config updated +34:52.752 [INFO] TestAgent: Requesting shutdown +34:52.752 [INFO] TestAgent.server: shutting down server +34:52.752 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:52.752 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:52.752 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:52.752 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:52.752 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:52.752 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:52.752 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:52.752 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:52.752 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:52.752 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:52.752 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:52.753Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:52.753Z [INFO] agent.router.manager: shutting down +34:52.753 [INFO] TestAgent: consul server down +34:52.753 [INFO] TestAgent: shutdown complete +34:52.753 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24497 network=tcp +34:52.753 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24497 network=udp +34:52.753 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24484 network=tcp +34:53.254 [INFO] TestAgent: Waiting for endpoints to shut down +34:53.254 [INFO] TestAgent: Endpoints down +=== RUN TestConnectCAConfig/basic_with_IntermediateCertTTL +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:52.582Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:52.626Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:52.631 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:009f1012-e826-2cdf-19fe-63f1e7ee6501 Address:127.0.0.1:24440}]" +34:52.633 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-009f1012-e826-2cdf-19fe-63f1e7ee6501.dc1 127.0.0.1 +34:52.634 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-009f1012-e826-2cdf-19fe-63f1e7ee6501 127.0.0.1 +2021-08-13T15:34:52.634Z [INFO] agent.router: Initializing LAN area manager +34:52.635 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24440 [Follower]" leader= +34:52.636 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-009f1012-e826-2cdf-19fe-63f1e7ee6501.dc1 area=wan +34:52.637 [INFO] TestAgent.server: Adding LAN server: server="Node-009f1012-e826-2cdf-19fe-63f1e7ee6501 (Addr: tcp/127.0.0.1:24440) (DC: dc1)" +34:52.637 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24413 network=tcp +34:52.638 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24413 network=udp +34:52.640 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24421 network=tcp +34:52.640 [INFO] TestAgent: started state syncer +34:52.642 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24441 network=tcp +34:52.699 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:52.699 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24440 [Candidate]" term=2 +34:52.699 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:52.699 [DEBUG] TestAgent.server.raft: vote granted: from=009f1012-e826-2cdf-19fe-63f1e7ee6501 term=2 tally=1 +34:52.700 [INFO] TestAgent.server.raft: election won: tally=1 +34:52.700 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24440 [Leader]" +34:52.700 [INFO] TestAgent.server: cluster leadership acquired +34:52.700 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24440 +34:52.701 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:52.701 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:52.701 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-009f1012-e826-2cdf-19fe-63f1e7ee6501 +34:52.701 [INFO] TestAgent.server: member joined, marking health alive: member=Node-009f1012-e826-2cdf-19fe-63f1e7ee6501 +34:52.703 [INFO] TestAgent.server: New leader elected: payload=Node-009f1012-e826-2cdf-19fe-63f1e7ee6501 +34:52.703 [INFO] TestAgent.server: federation state anti-entropy synced +34:52.740 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:52.741 [INFO] TestAgent: Synced node info +34:52.825 [INFO] TestAgent: Requesting shutdown +34:52.825 [INFO] TestAgent.server: shutting down server +34:52.825 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:52.825 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:52.825 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:52.825 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:52.826 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:52.826 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:52.826 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:52.826Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:52.826Z [INFO] agent.router.manager: shutting down +34:52.826 [INFO] TestAgent: consul server down +34:52.826 [INFO] TestAgent: shutdown complete +34:52.826 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24413 network=tcp +34:52.826 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24413 network=udp +34:52.826 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24421 network=tcp +34:53.327 [INFO] TestAgent: Waiting for endpoints to shut down +34:53.327 [INFO] TestAgent: Endpoints down +--- PASS: TestConnectCARoots_empty (0.81s) +=== CONT TestConfig_Apply_ProxyDefaultsMeshGateway +2021-08-13T15:34:52.468Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:52.485Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:52.488 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e3ee8c95-1f96-e724-5ce8-b98543e410be Address:127.0.0.1:24432}]" +34:52.488 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24432 [Follower]" leader= +34:52.489 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e3ee8c95-1f96-e724-5ce8-b98543e410be.dc1 127.0.0.1 +34:52.490 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e3ee8c95-1f96-e724-5ce8-b98543e410be 127.0.0.1 +2021-08-13T15:34:52.490Z [INFO] agent.router: Initializing LAN area manager +34:52.490 [INFO] TestAgent.server: Adding LAN server: server="Node-e3ee8c95-1f96-e724-5ce8-b98543e410be (Addr: tcp/127.0.0.1:24432) (DC: dc1)" +34:52.490 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e3ee8c95-1f96-e724-5ce8-b98543e410be.dc1 area=wan +34:52.490 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24420 network=udp +34:52.490 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24420 network=tcp +34:52.491 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24428 network=tcp +34:52.491 [INFO] TestAgent: started state syncer +34:52.491 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24433 network=tcp +34:52.555 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:52.555 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24432 [Candidate]" term=2 +34:52.555 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:52.555 [DEBUG] TestAgent.server.raft: vote granted: from=e3ee8c95-1f96-e724-5ce8-b98543e410be term=2 tally=1 +34:52.555 [INFO] TestAgent.server.raft: election won: tally=1 +34:52.555 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24432 [Leader]" +34:52.557 [INFO] TestAgent.server: cluster leadership acquired +34:52.557 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24432 +34:52.559 [INFO] TestAgent.server: New leader elected: payload=Node-e3ee8c95-1f96-e724-5ce8-b98543e410be +34:52.570 [INFO] TestAgent: Synced node info +34:52.570 [DEBUG] TestAgent: Node info in sync +34:52.572 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:52.576 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:52.576 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:52.576 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:52.576 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:52.576 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:52.576 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e3ee8c95-1f96-e724-5ce8-b98543e410be +34:52.576 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e3ee8c95-1f96-e724-5ce8-b98543e410be +34:52.576 [INFO] TestAgent.server: federation state anti-entropy synced +34:52.922 [DEBUG] connect.ca.consul: consul CA provider configured: id=5a:fc:b7:45:ba:83:1d:ee:6a:c0:7f:e6:5c:de:02:67:02:5d:93:52:98:3a:48:a8:ee:5f:7a:0c:0c:f9:f6:a0 is_primary=true +34:52.927 [INFO] TestAgent.server.connect: CA rotated to new root under provider: provider=consul +34:52.927 [INFO] TestAgent: Requesting shutdown +34:52.927 [INFO] TestAgent.server: shutting down server +34:52.927 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:52.927 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:52.927 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:52.927 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:52.927 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:52.928 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:52.928 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:52.928 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:52.928 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:52.928 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:52.928 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:52.928Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:52.928Z [INFO] agent.router.manager: shutting down +34:52.928 [INFO] TestAgent: consul server down +34:52.928 [INFO] TestAgent: shutdown complete +34:52.928 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24420 network=tcp +34:52.928 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24420 network=udp +34:52.928 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24428 network=tcp +34:53.431 [INFO] TestAgent: Waiting for endpoints to shut down +34:53.431 [INFO] TestAgent: Endpoints down +--- PASS: TestConnectCARoots_list (0.98s) +=== CONT TestConfig_Apply_Decoding +2021-08-13T15:34:52.512Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:52.541Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:52.547 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:98e073a5-5ef1-cbb3-a804-0b216deadbc1 Address:127.0.0.1:24411}]" +34:52.550 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-98e073a5-5ef1-cbb3-a804-0b216deadbc1.dc1 127.0.0.1 +34:52.550 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24411 [Follower]" leader= +34:52.556 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-98e073a5-5ef1-cbb3-a804-0b216deadbc1 127.0.0.1 +2021-08-13T15:34:52.556Z [INFO] agent.router: Initializing LAN area manager +34:52.556 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24434 network=udp +34:52.558 [INFO] TestAgent.server: Adding LAN server: server="Node-98e073a5-5ef1-cbb3-a804-0b216deadbc1 (Addr: tcp/127.0.0.1:24411) (DC: dc1)" +34:52.561 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24434 network=tcp +34:52.563 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-98e073a5-5ef1-cbb3-a804-0b216deadbc1.dc1 area=wan +34:52.569 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24435 network=tcp +34:52.569 [INFO] TestAgent: started state syncer +34:52.571 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24412 network=tcp +34:52.608 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:52.608 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24411 [Candidate]" term=2 +34:52.608 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:52.608 [DEBUG] TestAgent.server.raft: vote granted: from=98e073a5-5ef1-cbb3-a804-0b216deadbc1 term=2 tally=1 +34:52.608 [INFO] TestAgent.server.raft: election won: tally=1 +34:52.608 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24411 [Leader]" +34:52.608 [INFO] TestAgent.server: cluster leadership acquired +34:52.609 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24411 +34:52.610 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:52.611 [INFO] TestAgent.server: New leader elected: payload=Node-98e073a5-5ef1-cbb3-a804-0b216deadbc1 +34:52.618 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:52.619 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:52.619 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:52.619 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:52.619 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:52.619 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-98e073a5-5ef1-cbb3-a804-0b216deadbc1 +34:52.619 [INFO] TestAgent.server: member joined, marking health alive: member=Node-98e073a5-5ef1-cbb3-a804-0b216deadbc1 +34:52.629 [INFO] TestAgent.server: federation state anti-entropy synced +34:52.739 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:52.741 [INFO] TestAgent: Synced node info +34:52.742 [DEBUG] TestAgent: Node info in sync +34:52.944 [INFO] TestAgent: Requesting shutdown +34:52.944 [INFO] TestAgent.server: shutting down server +34:52.944 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:52.944 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:52.944 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:52.944 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:52.944 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:52.944 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:52.944 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:52.944 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:52.944 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:52.944 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:52.944 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:52.945Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:52.945Z [INFO] agent.router.manager: shutting down +34:52.945 [INFO] TestAgent: consul server down +34:52.945 [INFO] TestAgent: shutdown complete +34:52.945 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24434 network=tcp +34:52.945 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24434 network=udp +34:52.945 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24435 network=tcp +34:53.447 [INFO] TestAgent: Waiting for endpoints to shut down +34:53.447 [INFO] TestAgent: Endpoints down +--- PASS: TestCoordinate_Datacenters (0.96s) +=== CONT TestConfig_Apply_IngressGateway +2021-08-13T15:34:52.453Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:52.469Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:52.472 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e12dfc7d-747d-f1f0-00d8-ff0cd6aabc9d Address:127.0.0.1:24418}]" +34:52.472 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24418 [Follower]" leader= +34:52.473 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e12dfc7d-747d-f1f0-00d8-ff0cd6aabc9d.dc1 127.0.0.1 +34:52.474 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e12dfc7d-747d-f1f0-00d8-ff0cd6aabc9d 127.0.0.1 +2021-08-13T15:34:52.474Z [INFO] agent.router: Initializing LAN area manager +34:52.474 [INFO] TestAgent.server: Adding LAN server: server="Node-e12dfc7d-747d-f1f0-00d8-ff0cd6aabc9d (Addr: tcp/127.0.0.1:24418) (DC: dc1)" +34:52.474 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e12dfc7d-747d-f1f0-00d8-ff0cd6aabc9d.dc1 area=wan +34:52.474 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24470 network=udp +34:52.474 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24470 network=tcp +34:52.476 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24471 network=tcp +34:52.476 [INFO] TestAgent: started state syncer +34:52.476 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24419 network=tcp +34:52.533 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:52.533 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24418 [Candidate]" term=2 +34:52.534 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:52.534 [DEBUG] TestAgent.server.raft: vote granted: from=e12dfc7d-747d-f1f0-00d8-ff0cd6aabc9d term=2 tally=1 +34:52.534 [INFO] TestAgent.server.raft: election won: tally=1 +34:52.534 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24418 [Leader]" +34:52.534 [INFO] TestAgent.server: cluster leadership acquired +34:52.534 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24418 +34:52.537 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:52.546 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:52.546 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:52.546 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:52.546 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:52.546 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:52.547 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e12dfc7d-747d-f1f0-00d8-ff0cd6aabc9d +34:52.547 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e12dfc7d-747d-f1f0-00d8-ff0cd6aabc9d +34:52.547 [INFO] TestAgent.server: New leader elected: payload=Node-e12dfc7d-747d-f1f0-00d8-ff0cd6aabc9d +34:52.549 [INFO] TestAgent.server: federation state anti-entropy synced +34:52.831 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:52.833 [INFO] TestAgent: Synced node info +34:52.969 [INFO] TestAgent: Requesting shutdown +34:52.969 [INFO] TestAgent.server: shutting down server +34:52.969 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:52.969 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:52.969 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:52.969 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:52.969 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:52.969 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:52.969 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:52.969 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:52.969 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:52.969 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:52.970 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:52.970Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:52.970Z [INFO] agent.router.manager: shutting down +34:52.975 [INFO] TestAgent: consul server down +34:52.975 [INFO] TestAgent: shutdown complete +34:52.975 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24470 network=tcp +34:52.975 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24470 network=udp +34:52.975 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24471 network=tcp +34:53.476 [INFO] TestAgent: Waiting for endpoints to shut down +34:53.476 [INFO] TestAgent: Endpoints down +--- PASS: TestCoordinate_Nodes (1.04s) +=== CONT TestConfig_Apply_TerminatingGateway +2021-08-13T15:34:52.838Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:52.880Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:52.894 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ed763ac6-3635-eea6-fecd-e6f9d44c025a Address:127.0.0.1:24527}]" +34:52.895 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ed763ac6-3635-eea6-fecd-e6f9d44c025a.dc1 127.0.0.1 +34:52.896 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24527 [Follower]" leader= +34:52.898 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ed763ac6-3635-eea6-fecd-e6f9d44c025a 127.0.0.1 +2021-08-13T15:34:52.898Z [INFO] agent.router: Initializing LAN area manager +34:52.899 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ed763ac6-3635-eea6-fecd-e6f9d44c025a.dc1 area=wan +34:52.899 [INFO] TestAgent.server: Adding LAN server: server="Node-ed763ac6-3635-eea6-fecd-e6f9d44c025a (Addr: tcp/127.0.0.1:24527) (DC: dc1)" +34:52.899 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24442 network=udp +34:52.899 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24442 network=tcp +34:52.900 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24443 network=tcp +34:52.900 [INFO] TestAgent: started state syncer +34:52.900 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24528 network=tcp +34:52.955 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:52.955 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24527 [Candidate]" term=2 +34:52.956 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:52.956 [DEBUG] TestAgent.server.raft: vote granted: from=ed763ac6-3635-eea6-fecd-e6f9d44c025a term=2 tally=1 +34:52.956 [INFO] TestAgent.server.raft: election won: tally=1 +34:52.956 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24527 [Leader]" +34:52.956 [INFO] TestAgent.server: cluster leadership acquired +34:52.956 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24527 +34:52.958 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:52.966 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:52.966 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:52.966 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:52.966 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:52.966 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:52.966 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ed763ac6-3635-eea6-fecd-e6f9d44c025a +34:52.966 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ed763ac6-3635-eea6-fecd-e6f9d44c025a +34:52.968 [INFO] TestAgent.server: New leader elected: payload=Node-ed763ac6-3635-eea6-fecd-e6f9d44c025a +34:52.976 [INFO] TestAgent.server: federation state anti-entropy synced +34:53.001 [INFO] TestAgent: Requesting shutdown +34:53.001 [INFO] TestAgent.server: shutting down server +34:53.001 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:53.001 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:53.001 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:53.001 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:53.001 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:53.001 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:53.001 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:53.002 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:53.002 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:53.002 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:53.002 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:53.002 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:53.002Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:53.002Z [INFO] agent.router.manager: shutting down +34:53.002 [INFO] TestAgent: consul server down +34:53.002 [INFO] TestAgent: shutdown complete +34:53.002 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24442 network=tcp +34:53.002 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24442 network=udp +34:53.003 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24443 network=tcp +34:53.519 [INFO] TestAgent: Waiting for endpoints to shut down +34:53.519 [INFO] TestAgent: Endpoints down +--- PASS: TestConfig_Apply_CAS (0.71s) +=== CONT TestConfig_Delete +[WARN] freeport: 10 out of 45 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._udp_(qType:=1)_compressed=true +=== RUN TestConfig_Apply_Decoding/No_Kind +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestConfig_Apply_Decoding/Kind_Not_String +=== RUN TestConfig_Apply_Decoding/Lowercase_kind +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._udp_(qType:=1)_compressed=false +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=33)_compressed=true +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:53.398Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:53.443Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:53.456 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:eb7be789-0630-e83c-7a29-38652684a4ba Address:127.0.0.1:24548}]" +34:53.458 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-eb7be789-0630-e83c-7a29-38652684a4ba.dc1 127.0.0.1 +34:53.463 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24548 [Follower]" leader= +34:53.464 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-eb7be789-0630-e83c-7a29-38652684a4ba 127.0.0.1 +2021-08-13T15:34:53.464Z [INFO] agent.router: Initializing LAN area manager +34:53.465 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24501 network=udp +34:53.469 [INFO] TestAgent.server: Adding LAN server: server="Node-eb7be789-0630-e83c-7a29-38652684a4ba (Addr: tcp/127.0.0.1:24548) (DC: dc1)" +34:53.469 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-eb7be789-0630-e83c-7a29-38652684a4ba.dc1 area=wan +34:53.469 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24501 network=tcp +34:53.471 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24502 network=tcp +34:53.471 [INFO] TestAgent: started state syncer +34:53.471 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24549 network=tcp +34:53.520 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:53.520 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24548 [Candidate]" term=2 +34:53.520 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:53.520 [DEBUG] TestAgent.server.raft: vote granted: from=eb7be789-0630-e83c-7a29-38652684a4ba term=2 tally=1 +34:53.520 [INFO] TestAgent.server.raft: election won: tally=1 +34:53.520 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24548 [Leader]" +34:53.521 [INFO] TestAgent.server: cluster leadership acquired +34:53.521 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24548 +34:53.525 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:53.538 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:53.538 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:53.538 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:53.538 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:53.538 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:53.538 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-eb7be789-0630-e83c-7a29-38652684a4ba +34:53.538 [INFO] TestAgent.server: member joined, marking health alive: member=Node-eb7be789-0630-e83c-7a29-38652684a4ba +34:53.561 [INFO] TestAgent.server: New leader elected: payload=Node-eb7be789-0630-e83c-7a29-38652684a4ba +34:53.563 [INFO] TestAgent: Requesting shutdown +34:53.563 [INFO] TestAgent.server: shutting down server +34:53.563 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:53.563 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:53.563 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:53.563 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:53.563 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:53.563 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:53.564 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:53.564 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:53.564 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:53.564 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:53.564Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:53.564Z [INFO] agent.router.manager: shutting down +34:53.564 [INFO] TestAgent: consul server down +34:53.564 [INFO] TestAgent: shutdown complete +34:53.564 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24501 network=tcp +34:53.564 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24501 network=udp +34:53.564 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24502 network=tcp +34:53.564 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:54.065 [INFO] TestAgent: Waiting for endpoints to shut down +34:54.065 [INFO] TestAgent: Endpoints down +2021-08-13T15:34:53.299Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:53.321Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:53.324 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bf7d5f0f-953b-b681-fdc5-460b6d2815af Address:127.0.0.1:24513}]" +34:53.324 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24513 [Follower]" leader= +34:53.325 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bf7d5f0f-953b-b681-fdc5-460b6d2815af.dc1 127.0.0.1 +34:53.327 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bf7d5f0f-953b-b681-fdc5-460b6d2815af 127.0.0.1 +2021-08-13T15:34:53.327Z [INFO] agent.router: Initializing LAN area manager +34:53.327 [INFO] TestAgent.server: Adding LAN server: server="Node-bf7d5f0f-953b-b681-fdc5-460b6d2815af (Addr: tcp/127.0.0.1:24513) (DC: dc1)" +34:53.328 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bf7d5f0f-953b-b681-fdc5-460b6d2815af.dc1 area=wan +34:53.328 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24529 network=udp +34:53.328 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24529 network=tcp +34:53.337 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24530 network=tcp +34:53.338 [INFO] TestAgent: started state syncer +34:53.355 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24514 network=tcp +34:53.392 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:53.392 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24513 [Candidate]" term=2 +34:53.392 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:53.392 [DEBUG] TestAgent.server.raft: vote granted: from=bf7d5f0f-953b-b681-fdc5-460b6d2815af term=2 tally=1 +34:53.392 [INFO] TestAgent.server.raft: election won: tally=1 +34:53.392 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24513 [Leader]" +34:53.392 [INFO] TestAgent.server: cluster leadership acquired +34:53.393 [INFO] TestAgent.server: New leader elected: payload=Node-bf7d5f0f-953b-b681-fdc5-460b6d2815af +34:53.393 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24513 +34:53.394 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:53.398 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:53.398 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:53.398 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:53.398 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:53.398 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:53.398 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bf7d5f0f-953b-b681-fdc5-460b6d2815af +34:53.398 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bf7d5f0f-953b-b681-fdc5-460b6d2815af +34:53.399 [INFO] TestAgent.server: federation state anti-entropy synced +34:53.431 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:53.440 [INFO] TestAgent: Synced node info +34:53.516 [INFO] TestAgent: Requesting shutdown +34:53.516 [INFO] TestAgent.server: shutting down server +34:53.516 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:53.516 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:53.516 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:53.516 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:53.516 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:53.519 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:53.519 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:53.520 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:53.520 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:53.520 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:53.531 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:53.564Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:53.564Z [INFO] agent.router.manager: shutting down +34:53.564 [INFO] TestAgent: consul server down +34:53.564 [INFO] TestAgent: shutdown complete +34:53.564 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24529 network=tcp +34:53.565 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24529 network=udp +34:53.565 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24530 network=tcp +34:54.065 [INFO] TestAgent: Waiting for endpoints to shut down +34:54.065 [INFO] TestAgent: Endpoints down +--- PASS: TestConfig_Apply_ProxyDefaultsMeshGateway (0.74s) +=== CONT TestConfig_Apply +--- PASS: TestConfig_Apply_ProxyDefaultsExpose (0.82s) +=== CONT TestConfig_Get +2021-08-13T15:34:53.320Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:53.375Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:53.380 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:84b1b88f-a10f-25f6-3290-c10a2a386e93 Address:127.0.0.1:24499}]" +34:53.381 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24499 [Follower]" leader= +34:53.388 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-84b1b88f-a10f-25f6-3290-c10a2a386e93.dc1 127.0.0.1 +34:53.389 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-84b1b88f-a10f-25f6-3290-c10a2a386e93 127.0.0.1 +2021-08-13T15:34:53.390Z [INFO] agent.router: Initializing LAN area manager +34:53.391 [INFO] TestAgent.server: Adding LAN server: server="Node-84b1b88f-a10f-25f6-3290-c10a2a386e93 (Addr: tcp/127.0.0.1:24499) (DC: dc1)" +34:53.391 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24515 network=udp +34:53.393 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24515 network=tcp +34:53.394 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24516 network=tcp +34:53.394 [INFO] TestAgent: started state syncer +34:53.394 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24500 network=tcp +34:53.398 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-84b1b88f-a10f-25f6-3290-c10a2a386e93.dc1 area=wan +34:53.419 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:53.419 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24499 [Candidate]" term=2 +34:53.419 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:53.419 [DEBUG] TestAgent.server.raft: vote granted: from=84b1b88f-a10f-25f6-3290-c10a2a386e93 term=2 tally=1 +34:53.419 [INFO] TestAgent.server.raft: election won: tally=1 +34:53.419 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24499 [Leader]" +34:53.420 [INFO] TestAgent.server: cluster leadership acquired +34:53.421 [INFO] TestAgent: Synced node info +34:53.421 [DEBUG] TestAgent: Node info in sync +34:53.422 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24499 +34:53.423 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:53.427 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:53.427 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:53.427 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:53.427 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:53.427 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:53.427 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-84b1b88f-a10f-25f6-3290-c10a2a386e93 +34:53.427 [INFO] TestAgent.server: member joined, marking health alive: member=Node-84b1b88f-a10f-25f6-3290-c10a2a386e93 +34:53.429 [INFO] TestAgent.server: New leader elected: payload=Node-84b1b88f-a10f-25f6-3290-c10a2a386e93 +34:53.430 [INFO] TestAgent.server: federation state anti-entropy synced +34:53.570 [INFO] TestAgent.server.connect: CA provider config updated +34:53.570 [INFO] TestAgent: Requesting shutdown +34:53.570 [INFO] TestAgent.server: shutting down server +34:53.570 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:53.570 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:53.570 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:53.570 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:53.570 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:53.570 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:53.570 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:53.570 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:53.570 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:53.570 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:53.571 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:53.571Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:53.571Z [INFO] agent.router.manager: shutting down +34:53.571 [INFO] TestAgent: consul server down +34:53.571 [INFO] TestAgent: shutdown complete +34:53.571 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24515 network=tcp +34:53.571 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24515 network=udp +34:53.571 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24516 network=tcp +34:54.072 [INFO] TestAgent: Waiting for endpoints to shut down +34:54.072 [INFO] TestAgent: Endpoints down +=== RUN TestConnectCAConfig/force_without_cross_sign_CamelCase +2021-08-13T15:34:53.494Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:53.547Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:53.554 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4655a003-0767-8593-e8f0-2b785daf4e1d Address:127.0.0.1:24447}]" +34:53.555 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4655a003-0767-8593-e8f0-2b785daf4e1d.dc1 127.0.0.1 +34:53.557 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4655a003-0767-8593-e8f0-2b785daf4e1d 127.0.0.1 +2021-08-13T15:34:53.557Z [INFO] agent.router: Initializing LAN area manager +34:53.563 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24522 network=tcp +34:53.563 [INFO] TestAgent.server: Adding LAN server: server="Node-4655a003-0767-8593-e8f0-2b785daf4e1d (Addr: tcp/127.0.0.1:24447) (DC: dc1)" +34:53.563 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4655a003-0767-8593-e8f0-2b785daf4e1d.dc1 area=wan +34:53.563 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24447 [Follower]" leader= +34:53.563 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24522 network=udp +34:53.564 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24523 network=tcp +34:53.564 [INFO] TestAgent: started state syncer +34:53.564 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24448 network=tcp +34:53.641 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:53.641 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24447 [Candidate]" term=2 +34:53.641 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:53.641 [DEBUG] TestAgent.server.raft: vote granted: from=4655a003-0767-8593-e8f0-2b785daf4e1d term=2 tally=1 +34:53.641 [INFO] TestAgent.server.raft: election won: tally=1 +34:53.641 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24447 [Leader]" +34:53.641 [INFO] TestAgent.server: cluster leadership acquired +34:53.642 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24447 +34:53.647 [INFO] TestAgent.server: New leader elected: payload=Node-4655a003-0767-8593-e8f0-2b785daf4e1d +34:53.648 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:53.651 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:53.651 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:53.651 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:53.651 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:53.651 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:53.651 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4655a003-0767-8593-e8f0-2b785daf4e1d +34:53.651 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4655a003-0767-8593-e8f0-2b785daf4e1d +34:53.652 [INFO] TestAgent.server: federation state anti-entropy synced +34:53.725 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:53.726 [INFO] TestAgent: Synced node info +34:53.726 [DEBUG] TestAgent: Node info in sync +34:53.733 [INFO] TestAgent: Requesting shutdown +34:53.733 [INFO] TestAgent.server: shutting down server +34:53.733 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:53.733 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:53.733 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:53.733 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:53.733 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:53.733 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:53.733 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:53.733 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:53.733 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:53.733 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:53.734 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:53.734Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:53.734Z [INFO] agent.router.manager: shutting down +34:53.734 [INFO] TestAgent: consul server down +34:53.734 [INFO] TestAgent: shutdown complete +34:53.734 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24522 network=tcp +34:53.734 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24522 network=udp +34:53.734 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24523 network=tcp +34:54.235 [INFO] TestAgent: Waiting for endpoints to shut down +34:54.235 [INFO] TestAgent: Endpoints down +--- PASS: TestConfig_Apply_IngressGateway (0.79s) +=== CONT TestCatalog_GatewayServices_Ingress +[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:53.470Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:53.518Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:53.522 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b5273dc2-621f-e755-72ca-152a9361e664 Address:127.0.0.1:24520}]" +34:53.531 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24520 [Follower]" leader= +34:53.532 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b5273dc2-621f-e755-72ca-152a9361e664.dc1 127.0.0.1 +34:53.533 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b5273dc2-621f-e755-72ca-152a9361e664 127.0.0.1 +2021-08-13T15:34:53.533Z [INFO] agent.router: Initializing LAN area manager +34:53.540 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24550 network=udp +34:53.561 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b5273dc2-621f-e755-72ca-152a9361e664.dc1 area=wan +34:53.561 [INFO] TestAgent.server: Adding LAN server: server="Node-b5273dc2-621f-e755-72ca-152a9361e664 (Addr: tcp/127.0.0.1:24520) (DC: dc1)" +34:53.561 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24550 network=tcp +34:53.562 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24551 network=tcp +34:53.562 [INFO] TestAgent: started state syncer +34:53.562 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24521 network=tcp +34:53.611 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:53.611 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24520 [Candidate]" term=2 +34:53.612 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:53.612 [DEBUG] TestAgent.server.raft: vote granted: from=b5273dc2-621f-e755-72ca-152a9361e664 term=2 tally=1 +34:53.612 [INFO] TestAgent.server.raft: election won: tally=1 +34:53.612 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24520 [Leader]" +34:53.612 [INFO] TestAgent.server: cluster leadership acquired +34:53.612 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24520 +34:53.614 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:53.618 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:53.618 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:53.618 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:53.618 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:53.618 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:53.618 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b5273dc2-621f-e755-72ca-152a9361e664 +34:53.618 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b5273dc2-621f-e755-72ca-152a9361e664 +34:53.619 [INFO] TestAgent.server: New leader elected: payload=Node-b5273dc2-621f-e755-72ca-152a9361e664 +34:53.620 [INFO] TestAgent.server: federation state anti-entropy synced +34:53.657 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:53.658 [INFO] TestAgent: Synced node info +34:53.728 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:53.728 [DEBUG] TestAgent: Node info in sync +34:53.728 [DEBUG] TestAgent: Node info in sync +34:53.774 [INFO] TestAgent: Requesting shutdown +34:53.774 [INFO] TestAgent.server: shutting down server +34:53.774 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:53.774 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:53.774 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:53.774 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:53.774 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:53.774 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:53.774 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:53.774 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:53.774 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:53.774 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:53.774 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:53.775Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:53.775Z [INFO] agent.router.manager: shutting down +34:53.775 [INFO] TestAgent: consul server down +34:53.775 [INFO] TestAgent: shutdown complete +34:53.775 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24550 network=tcp +34:53.775 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24550 network=udp +34:53.775 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24551 network=tcp +34:54.277 [INFO] TestAgent: Waiting for endpoints to shut down +34:54.277 [INFO] TestAgent: Endpoints down +--- PASS: TestConfig_Apply_Decoding (0.85s) + --- PASS: TestConfig_Apply_Decoding/No_Kind (0.00s) + --- PASS: TestConfig_Apply_Decoding/Kind_Not_String (0.00s) + --- PASS: TestConfig_Apply_Decoding/Lowercase_kind (0.00s) +=== CONT TestCatalog_GatewayServices_Terminating +2021-08-13T15:34:53.577Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:53.629Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:53.634 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5d09b1a8-2de9-f112-e205-32b169bb5f0a Address:127.0.0.1:24905}]" +34:53.636 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5d09b1a8-2de9-f112-e205-32b169bb5f0a.dc1 127.0.0.1 +34:53.636 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24905 [Follower]" leader= +34:53.643 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5d09b1a8-2de9-f112-e205-32b169bb5f0a 127.0.0.1 +2021-08-13T15:34:53.643Z [INFO] agent.router: Initializing LAN area manager +34:53.644 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24893 network=udp +34:53.647 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5d09b1a8-2de9-f112-e205-32b169bb5f0a.dc1 area=wan +34:53.647 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24893 network=tcp +34:53.647 [INFO] TestAgent.server: Adding LAN server: server="Node-5d09b1a8-2de9-f112-e205-32b169bb5f0a (Addr: tcp/127.0.0.1:24905) (DC: dc1)" +34:53.650 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24915 network=tcp +34:53.650 [INFO] TestAgent: started state syncer +34:53.650 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24906 network=tcp +34:53.683 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:53.683 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24905 [Candidate]" term=2 +34:53.683 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:53.683 [DEBUG] TestAgent.server.raft: vote granted: from=5d09b1a8-2de9-f112-e205-32b169bb5f0a term=2 tally=1 +34:53.683 [INFO] TestAgent.server.raft: election won: tally=1 +34:53.683 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24905 [Leader]" +34:53.684 [INFO] TestAgent.server: cluster leadership acquired +34:53.684 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24905 +34:53.686 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:53.687 [INFO] TestAgent.server: New leader elected: payload=Node-5d09b1a8-2de9-f112-e205-32b169bb5f0a +34:53.693 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:53.693 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:53.693 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:53.693 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:53.693 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:53.693 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5d09b1a8-2de9-f112-e205-32b169bb5f0a +34:53.693 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5d09b1a8-2de9-f112-e205-32b169bb5f0a +34:53.700 [INFO] TestAgent.server: federation state anti-entropy synced +34:53.779 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:53.780 [INFO] TestAgent: Synced node info +34:53.780 [DEBUG] TestAgent: Node info in sync +34:53.822 [INFO] TestAgent: Requesting shutdown +34:53.822 [INFO] TestAgent.server: shutting down server +34:53.822 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:53.822 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:53.822 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:53.822 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:53.822 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:53.822 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:53.822 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:53.822 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:53.822 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:53.822 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:53.822 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:53.822Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:53.822Z [INFO] agent.router.manager: shutting down +34:53.822 [INFO] TestAgent: consul server down +34:53.822 [INFO] TestAgent: shutdown complete +34:53.822 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24893 network=tcp +34:53.822 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24893 network=udp +34:53.823 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24915 network=tcp +34:54.335 [INFO] TestAgent: Waiting for endpoints to shut down +34:54.336 [INFO] TestAgent: Endpoints down +--- PASS: TestConfig_Delete (0.82s) +=== CONT TestCatalogNodeServices_WanTranslation +2021-08-13T15:34:53.522Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:53.586Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:53.590 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f53456f9-7967-5070-133b-b188a82dad03 Address:127.0.0.1:24891}]" +34:53.591 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24891 [Follower]" leader= +34:53.591 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f53456f9-7967-5070-133b-b188a82dad03.dc1 127.0.0.1 +34:53.593 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f53456f9-7967-5070-133b-b188a82dad03 127.0.0.1 +2021-08-13T15:34:53.593Z [INFO] agent.router: Initializing LAN area manager +34:53.594 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24449 network=udp +34:53.596 [INFO] TestAgent.server: Adding LAN server: server="Node-f53456f9-7967-5070-133b-b188a82dad03 (Addr: tcp/127.0.0.1:24891) (DC: dc1)" +34:53.596 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24449 network=tcp +34:53.596 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f53456f9-7967-5070-133b-b188a82dad03.dc1 area=wan +34:53.598 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24450 network=tcp +34:53.598 [INFO] TestAgent: started state syncer +34:53.599 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24892 network=tcp +34:53.631 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:53.631 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24891 [Candidate]" term=2 +34:53.631 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:53.631 [DEBUG] TestAgent.server.raft: vote granted: from=f53456f9-7967-5070-133b-b188a82dad03 term=2 tally=1 +34:53.631 [INFO] TestAgent.server.raft: election won: tally=1 +34:53.631 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24891 [Leader]" +34:53.632 [INFO] TestAgent.server: cluster leadership acquired +34:53.632 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24891 +34:53.634 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:53.637 [INFO] TestAgent.server: New leader elected: payload=Node-f53456f9-7967-5070-133b-b188a82dad03 +34:53.647 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:53.647 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:53.647 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:53.647 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:53.647 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:53.647 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f53456f9-7967-5070-133b-b188a82dad03 +34:53.647 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f53456f9-7967-5070-133b-b188a82dad03 +34:53.648 [INFO] TestAgent.server: federation state anti-entropy synced +34:53.788 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:53.788 [INFO] TestAgent: Synced node info +34:53.883 [INFO] TestAgent: Requesting shutdown +34:53.883 [INFO] TestAgent.server: shutting down server +34:53.884 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:53.884 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:53.884 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:53.884 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:53.884 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:53.884 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:53.884 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:53.884 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:53.884 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:53.884 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:53.884 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:53.884Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:53.884Z [INFO] agent.router.manager: shutting down +34:53.884 [INFO] TestAgent: consul server down +34:53.884 [INFO] TestAgent: shutdown complete +34:53.884 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24449 network=tcp +34:53.884 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24449 network=udp +34:53.884 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24450 network=tcp +34:54.387 [INFO] TestAgent: Waiting for endpoints to shut down +34:54.387 [INFO] TestAgent: Endpoints down +--- PASS: TestConfig_Apply_TerminatingGateway (0.91s) +=== CONT TestCatalogNodeServices_ConnectProxy +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=33)_compressed=false +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestConfig_Get/get_a_single_service_entry +=== RUN TestConfig_Get/list_both_service_entries +=== RUN TestConfig_Get/get_global_proxy_config +=== RUN TestConfig_Get/error_on_no_arguments +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=33)_compressed=true +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=33)_compressed=false +2021-08-13T15:34:54.104Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:54.143Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:54.147 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b3f5f7d7-5b54-dcdd-f197-da0fac67f378 Address:127.0.0.1:24590}]" +34:54.147 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24590 [Follower]" leader= +34:54.148 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b3f5f7d7-5b54-dcdd-f197-da0fac67f378.dc1 127.0.0.1 +34:54.149 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b3f5f7d7-5b54-dcdd-f197-da0fac67f378 127.0.0.1 +2021-08-13T15:34:54.149Z [INFO] agent.router: Initializing LAN area manager +34:54.149 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24578 network=udp +34:54.149 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b3f5f7d7-5b54-dcdd-f197-da0fac67f378.dc1 area=wan +34:54.149 [INFO] TestAgent.server: Adding LAN server: server="Node-b3f5f7d7-5b54-dcdd-f197-da0fac67f378 (Addr: tcp/127.0.0.1:24590) (DC: dc1)" +34:54.150 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24578 network=tcp +34:54.151 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24579 network=tcp +34:54.151 [INFO] TestAgent: started state syncer +34:54.151 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24591 network=tcp +34:54.214 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:54.214 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24590 [Candidate]" term=2 +34:54.215 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:54.215 [DEBUG] TestAgent.server.raft: vote granted: from=b3f5f7d7-5b54-dcdd-f197-da0fac67f378 term=2 tally=1 +34:54.215 [INFO] TestAgent.server.raft: election won: tally=1 +34:54.215 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24590 [Leader]" +34:54.215 [INFO] TestAgent.server: cluster leadership acquired +34:54.215 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24590 +34:54.215 [INFO] TestAgent.server: New leader elected: payload=Node-b3f5f7d7-5b54-dcdd-f197-da0fac67f378 +34:54.217 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:54.221 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:54.221 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:54.221 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:54.221 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:54.221 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:54.221 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b3f5f7d7-5b54-dcdd-f197-da0fac67f378 +34:54.221 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b3f5f7d7-5b54-dcdd-f197-da0fac67f378 +34:54.222 [INFO] TestAgent.server: federation state anti-entropy synced +34:54.345 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:54.347 [INFO] TestAgent: Synced node info +34:54.450 [INFO] TestAgent.server.connect: CA provider config updated +34:54.450 [INFO] TestAgent: Requesting shutdown +34:54.450 [INFO] TestAgent.server: shutting down server +34:54.450 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:54.450 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:54.450 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:54.450 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:54.450 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:54.450 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:54.450 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:54.450 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:54.450 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:54.450 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:54.450 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:54.450Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:54.450Z [INFO] agent.router.manager: shutting down +34:54.451 [INFO] TestAgent: consul server down +34:54.451 [INFO] TestAgent: shutdown complete +34:54.451 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24578 network=tcp +34:54.451 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24578 network=udp +34:54.451 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24579 network=tcp +34:54.951 [INFO] TestAgent: Waiting for endpoints to shut down +34:54.951 [INFO] TestAgent: Endpoints down +=== RUN TestConnectCAConfig/force_without_cross_sign_snake_case +2021-08-13T15:34:54.100Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:54.128Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:54.141 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9a065c68-e8a4-476e-7416-17bd5dc20163 Address:127.0.0.1:24534}]" +34:54.143 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9a065c68-e8a4-476e-7416-17bd5dc20163.dc1 127.0.0.1 +34:54.143 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24534 [Follower]" leader= +34:54.144 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9a065c68-e8a4-476e-7416-17bd5dc20163 127.0.0.1 +2021-08-13T15:34:54.144Z [INFO] agent.router: Initializing LAN area manager +34:54.144 [INFO] TestAgent.server: Adding LAN server: server="Node-9a065c68-e8a4-476e-7416-17bd5dc20163 (Addr: tcp/127.0.0.1:24534) (DC: dc1)" +34:54.144 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24907 network=udp +34:54.144 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9a065c68-e8a4-476e-7416-17bd5dc20163.dc1 area=wan +34:54.144 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24907 network=tcp +34:54.145 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24901 network=tcp +34:54.145 [INFO] TestAgent: started state syncer +34:54.146 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24535 network=tcp +34:54.199 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:54.199 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24534 [Candidate]" term=2 +34:54.200 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:54.200 [DEBUG] TestAgent.server.raft: vote granted: from=9a065c68-e8a4-476e-7416-17bd5dc20163 term=2 tally=1 +34:54.200 [INFO] TestAgent.server.raft: election won: tally=1 +34:54.200 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24534 [Leader]" +34:54.200 [INFO] TestAgent.server: cluster leadership acquired +34:54.201 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24534 +34:54.203 [INFO] TestAgent.server: New leader elected: payload=Node-9a065c68-e8a4-476e-7416-17bd5dc20163 +34:54.206 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:54.212 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:54.212 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:54.212 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:54.212 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:54.212 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:54.212 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9a065c68-e8a4-476e-7416-17bd5dc20163 +34:54.212 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9a065c68-e8a4-476e-7416-17bd5dc20163 +34:54.213 [INFO] TestAgent.server: federation state anti-entropy synced +34:54.431 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:54.432 [INFO] TestAgent: Synced node info +34:54.486 [INFO] TestAgent: Requesting shutdown +34:54.486 [INFO] TestAgent.server: shutting down server +34:54.486 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:54.486 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:54.486 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:54.486 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:54.486 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:54.486 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:54.486 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:54.486 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:54.486 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:54.486 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:54.486 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:54.486Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:54.486Z [INFO] agent.router.manager: shutting down +34:54.487 [INFO] TestAgent: consul server down +34:54.487 [INFO] TestAgent: shutdown complete +34:54.487 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24907 network=tcp +34:54.487 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24907 network=udp +34:54.487 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24901 network=tcp +34:54.987 [INFO] TestAgent: Waiting for endpoints to shut down +34:54.987 [INFO] TestAgent: Endpoints down +--- PASS: TestConfig_Apply (0.92s) +=== CONT TestCatalogNodeServices_Filter +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._tcp_(qType:=33)_compressed=true +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:54.140Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:54.170Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:54.188 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9f187e92-0f0d-6ca4-9446-d3cc24e18e0d Address:127.0.0.1:24576}]" +34:54.202 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9f187e92-0f0d-6ca4-9446-d3cc24e18e0d.dc1 127.0.0.1 +34:54.203 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24576 [Follower]" leader= +34:54.205 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9f187e92-0f0d-6ca4-9446-d3cc24e18e0d 127.0.0.1 +2021-08-13T15:34:54.206Z [INFO] agent.router: Initializing LAN area manager +34:54.208 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9f187e92-0f0d-6ca4-9446-d3cc24e18e0d.dc1 area=wan +34:54.208 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24536 network=udp +34:54.210 [INFO] TestAgent.server: Adding LAN server: server="Node-9f187e92-0f0d-6ca4-9446-d3cc24e18e0d (Addr: tcp/127.0.0.1:24576) (DC: dc1)" +34:54.210 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24536 network=tcp +34:54.211 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24537 network=tcp +34:54.211 [INFO] TestAgent: started state syncer +34:54.211 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24577 network=tcp +34:54.276 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:54.276 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24576 [Candidate]" term=2 +34:54.276 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:54.276 [DEBUG] TestAgent.server.raft: vote granted: from=9f187e92-0f0d-6ca4-9446-d3cc24e18e0d term=2 tally=1 +34:54.276 [INFO] TestAgent.server.raft: election won: tally=1 +34:54.276 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24576 [Leader]" +34:54.277 [INFO] TestAgent.server: cluster leadership acquired +34:54.278 [INFO] TestAgent.server: New leader elected: payload=Node-9f187e92-0f0d-6ca4-9446-d3cc24e18e0d +34:54.279 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24576 +34:54.281 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:54.286 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:54.286 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:54.286 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:54.286 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:54.286 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:54.286 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9f187e92-0f0d-6ca4-9446-d3cc24e18e0d +34:54.286 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9f187e92-0f0d-6ca4-9446-d3cc24e18e0d +34:54.288 [INFO] TestAgent.server: federation state anti-entropy synced +34:54.530 [INFO] TestAgent: Requesting shutdown +34:54.530 [INFO] TestAgent.server: shutting down server +34:54.530 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:54.530 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:54.530 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:54.530 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:54.530 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:54.530 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:54.530 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:54.530 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:54.530 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:54.530 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:54.530 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:54.530 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:54.531Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:54.531Z [INFO] agent.router.manager: shutting down +34:54.535 [INFO] TestAgent: consul server down +34:54.535 [INFO] TestAgent: shutdown complete +34:54.535 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24536 network=tcp +34:54.535 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24536 network=udp +34:54.535 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24537 network=tcp +34:55.036 [INFO] TestAgent: Waiting for endpoints to shut down +34:55.036 [INFO] TestAgent: Endpoints down +--- PASS: TestConfig_Get (0.97s) + --- PASS: TestConfig_Get/get_a_single_service_entry (0.00s) + --- PASS: TestConfig_Get/list_both_service_entries (0.00s) + --- PASS: TestConfig_Get/get_global_proxy_config (0.00s) + --- PASS: TestConfig_Get/error_on_no_arguments (0.00s) +=== CONT TestCatalogNodeServiceList +2021-08-13T15:34:54.433Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:54.452Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:54.455 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7b1ebebe-34f8-598d-54fb-cb964e3846c3 Address:127.0.0.1:24569}]" +34:54.455 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24569 [Follower]" leader= +34:54.456 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7b1ebebe-34f8-598d-54fb-cb964e3846c3.dc1 127.0.0.1 +34:54.457 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7b1ebebe-34f8-598d-54fb-cb964e3846c3 127.0.0.1 +2021-08-13T15:34:54.457Z [INFO] agent.router: Initializing LAN area manager +34:54.457 [INFO] TestAgent.server: Adding LAN server: server="Node-7b1ebebe-34f8-598d-54fb-cb964e3846c3 (Addr: tcp/127.0.0.1:24569) (DC: dc1)" +34:54.457 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7b1ebebe-34f8-598d-54fb-cb964e3846c3.dc1 area=wan +34:54.458 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24613 network=udp +34:54.458 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24613 network=tcp +34:54.459 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24614 network=tcp +34:54.459 [INFO] TestAgent: started state syncer +34:54.459 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24570 network=tcp +34:54.507 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:54.507 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24569 [Candidate]" term=2 +34:54.508 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:54.508 [DEBUG] TestAgent.server.raft: vote granted: from=7b1ebebe-34f8-598d-54fb-cb964e3846c3 term=2 tally=1 +34:54.508 [INFO] TestAgent.server.raft: election won: tally=1 +34:54.508 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24569 [Leader]" +34:54.508 [INFO] TestAgent.server: cluster leadership acquired +34:54.508 [INFO] TestAgent.server: New leader elected: payload=Node-7b1ebebe-34f8-598d-54fb-cb964e3846c3 +34:54.508 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24569 +34:54.510 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:54.512 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:54.512 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:54.512 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:54.512 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:54.512 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:54.512 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7b1ebebe-34f8-598d-54fb-cb964e3846c3 +34:54.512 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7b1ebebe-34f8-598d-54fb-cb964e3846c3 +34:54.513 [INFO] TestAgent.server: federation state anti-entropy synced +34:54.535 [INFO] TestAgent: Requesting shutdown +34:54.536 [INFO] TestAgent.server: shutting down server +34:54.536 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:54.536 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:54.536 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:54.536 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:54.536 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:54.536 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:54.536 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:54.536 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:54.536 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:54.536 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:54.536 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:54.536 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:54.536Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:54.536Z [INFO] agent.router.manager: shutting down +34:54.536 [INFO] TestAgent: consul server down +34:54.536 [INFO] TestAgent: shutdown complete +34:54.536 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24613 network=tcp +34:54.536 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24613 network=udp +34:54.537 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24614 network=tcp +34:55.044 [INFO] TestAgent: Waiting for endpoints to shut down +34:55.044 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogNodeServices_ConnectProxy (0.66s) +=== CONT TestCatalogNodeServices +2021-08-13T15:34:54.286Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:54.322Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:54.336 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ce07cc8b-2405-4c9d-180a-ca7b2895b7bf Address:127.0.0.1:24562}]" +34:54.343 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24562 [Follower]" leader= +34:54.350 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ce07cc8b-2405-4c9d-180a-ca7b2895b7bf.dc1 127.0.0.1 +34:54.353 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ce07cc8b-2405-4c9d-180a-ca7b2895b7bf 127.0.0.1 +2021-08-13T15:34:54.353Z [INFO] agent.router: Initializing LAN area manager +34:54.354 [INFO] TestAgent.server: Adding LAN server: server="Node-ce07cc8b-2405-4c9d-180a-ca7b2895b7bf (Addr: tcp/127.0.0.1:24562) (DC: dc1)" +34:54.354 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ce07cc8b-2405-4c9d-180a-ca7b2895b7bf.dc1 area=wan +34:54.354 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24592 network=udp +34:54.354 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24592 network=tcp +34:54.356 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24593 network=tcp +34:54.356 [INFO] TestAgent: started state syncer +34:54.356 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24563 network=tcp +34:54.403 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:54.403 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24562 [Candidate]" term=2 +34:54.404 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:54.404 [DEBUG] TestAgent.server.raft: vote granted: from=ce07cc8b-2405-4c9d-180a-ca7b2895b7bf term=2 tally=1 +34:54.404 [INFO] TestAgent.server.raft: election won: tally=1 +34:54.404 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24562 [Leader]" +34:54.405 [INFO] TestAgent.server: cluster leadership acquired +34:54.405 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24562 +34:54.406 [INFO] TestAgent.server: New leader elected: payload=Node-ce07cc8b-2405-4c9d-180a-ca7b2895b7bf +34:54.412 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:54.423 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:54.423 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:54.423 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:54.423 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:54.423 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:54.423 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ce07cc8b-2405-4c9d-180a-ca7b2895b7bf +34:54.423 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ce07cc8b-2405-4c9d-180a-ca7b2895b7bf +34:54.425 [INFO] TestAgent.server: federation state anti-entropy synced +34:54.618 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:54.624 [INFO] TestAgent: Synced node info +34:54.624 [DEBUG] TestAgent: Node info in sync +34:54.714 [INFO] TestAgent: Requesting shutdown +34:54.715 [INFO] TestAgent.server: shutting down server +34:54.715 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:54.715 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:54.715 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:54.715 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:54.715 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:54.715 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:54.715 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:54.715 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:54.715 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:54.715 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:54.719 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:54.720Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:54.720Z [INFO] agent.router.manager: shutting down +34:54.720 [INFO] TestAgent: consul server down +34:54.720 [INFO] TestAgent: shutdown complete +34:54.720 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24592 network=tcp +34:54.720 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24592 network=udp +34:54.720 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24593 network=tcp +34:55.221 [INFO] TestAgent: Waiting for endpoints to shut down +34:55.221 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalog_GatewayServices_Ingress (0.99s) +=== CONT TestCatalogConnectServiceNodes_good +2021-08-13T15:34:54.328Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:54.365Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:54.372 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4d03e901-e114-7709-a351-1af9d51ade03 Address:127.0.0.1:24555}]" +34:54.373 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24555 [Follower]" leader= +34:54.373 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4d03e901-e114-7709-a351-1af9d51ade03.dc1 127.0.0.1 +34:54.375 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4d03e901-e114-7709-a351-1af9d51ade03 127.0.0.1 +2021-08-13T15:34:54.375Z [INFO] agent.router: Initializing LAN area manager +34:54.375 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24564 network=udp +34:54.377 [INFO] TestAgent.server: Adding LAN server: server="Node-4d03e901-e114-7709-a351-1af9d51ade03 (Addr: tcp/127.0.0.1:24555) (DC: dc1)" +34:54.377 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24564 network=tcp +34:54.377 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4d03e901-e114-7709-a351-1af9d51ade03.dc1 area=wan +34:54.379 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24565 network=tcp +34:54.379 [INFO] TestAgent: started state syncer +34:54.379 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24556 network=tcp +34:54.425 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:54.425 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24555 [Candidate]" term=2 +34:54.425 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:54.425 [DEBUG] TestAgent.server.raft: vote granted: from=4d03e901-e114-7709-a351-1af9d51ade03 term=2 tally=1 +34:54.425 [INFO] TestAgent.server.raft: election won: tally=1 +34:54.426 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24555 [Leader]" +34:54.426 [INFO] TestAgent.server: cluster leadership acquired +34:54.428 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24555 +34:54.429 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:54.431 [INFO] TestAgent.server: New leader elected: payload=Node-4d03e901-e114-7709-a351-1af9d51ade03 +34:54.432 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:54.432 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:54.432 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:54.432 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:54.432 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:54.432 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4d03e901-e114-7709-a351-1af9d51ade03 +34:54.432 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4d03e901-e114-7709-a351-1af9d51ade03 +34:54.434 [INFO] TestAgent.server: federation state anti-entropy synced +34:54.508 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:54.509 [INFO] TestAgent: Synced node info +34:54.509 [DEBUG] TestAgent: Node info in sync +34:54.734 [INFO] TestAgent: Requesting shutdown +34:54.734 [INFO] TestAgent.server: shutting down server +34:54.734 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:54.734 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:54.734 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:54.734 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:54.734 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:54.735 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:54.735 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:54.735 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:54.735 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:54.735 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:54.735 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:54.739Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:54.739Z [INFO] agent.router.manager: shutting down +34:54.739 [INFO] TestAgent: consul server down +34:54.739 [INFO] TestAgent: shutdown complete +34:54.739 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24564 network=tcp +34:54.739 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24564 network=udp +34:54.739 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24565 network=tcp +34:55.243 [INFO] TestAgent: Waiting for endpoints to shut down +34:55.243 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalog_GatewayServices_Terminating (0.96s) +=== CONT TestAgent_Members +[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:54.975Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:54.990Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:54.993 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1894a24f-fdd9-fa61-8906-9a7bdadf160e Address:127.0.0.1:24625}]" +34:54.993 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24625 [Follower]" leader= +34:54.994 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1894a24f-fdd9-fa61-8906-9a7bdadf160e.dc1 127.0.0.1 +34:54.996 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1894a24f-fdd9-fa61-8906-9a7bdadf160e 127.0.0.1 +2021-08-13T15:34:54.996Z [INFO] agent.router: Initializing LAN area manager +34:54.996 [INFO] TestAgent.server: Adding LAN server: server="Node-1894a24f-fdd9-fa61-8906-9a7bdadf160e (Addr: tcp/127.0.0.1:24625) (DC: dc1)" +34:54.997 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24648 network=udp +34:54.997 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1894a24f-fdd9-fa61-8906-9a7bdadf160e.dc1 area=wan +34:54.997 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24648 network=tcp +34:54.998 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24649 network=tcp +34:54.998 [INFO] TestAgent: started state syncer +34:54.998 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24626 network=tcp +34:55.048 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:55.048 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24625 [Candidate]" term=2 +34:55.048 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:55.048 [DEBUG] TestAgent.server.raft: vote granted: from=1894a24f-fdd9-fa61-8906-9a7bdadf160e term=2 tally=1 +34:55.048 [INFO] TestAgent.server.raft: election won: tally=1 +34:55.048 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24625 [Leader]" +34:55.049 [INFO] TestAgent.server: cluster leadership acquired +34:55.049 [INFO] TestAgent.server: New leader elected: payload=Node-1894a24f-fdd9-fa61-8906-9a7bdadf160e +34:55.049 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24625 +34:55.060 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:55.069 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:55.069 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:55.069 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:55.069 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:55.069 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:55.069 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1894a24f-fdd9-fa61-8906-9a7bdadf160e +34:55.069 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1894a24f-fdd9-fa61-8906-9a7bdadf160e +34:55.079 [INFO] TestAgent.server: federation state anti-entropy synced +34:55.159 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:55.160 [INFO] TestAgent: Synced node info +34:55.160 [DEBUG] TestAgent: Node info in sync +34:55.189 [INFO] TestAgent.server.connect: CA provider config updated +34:55.190 [INFO] TestAgent: Requesting shutdown +34:55.190 [INFO] TestAgent.server: shutting down server +34:55.190 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:55.190 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:55.190 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:55.190 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:55.190 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:55.190 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:55.190 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:55.190 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:55.190 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:55.190 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:55.190 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:55.190Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:55.190Z [INFO] agent.router.manager: shutting down +34:55.190 [INFO] TestAgent: consul server down +34:55.190 [INFO] TestAgent: shutdown complete +34:55.190 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24648 network=tcp +34:55.191 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24648 network=udp +34:55.191 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24649 network=tcp +34:55.695 [INFO] TestAgent: Waiting for endpoints to shut down +34:55.695 [INFO] TestAgent: Endpoints down +=== RUN TestConnectCAConfig/setting_state_fails +2021-08-13T15:34:55.117Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:55.146Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:55.149 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5c04cdaa-5b53-fce5-c957-35a8935e7d78 Address:127.0.0.1:24583}]" +34:55.149 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24583 [Follower]" leader= +34:55.150 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5c04cdaa-5b53-fce5-c957-35a8935e7d78.dc1 127.0.0.1 +34:55.151 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5c04cdaa-5b53-fce5-c957-35a8935e7d78 127.0.0.1 +2021-08-13T15:34:55.151Z [INFO] agent.router: Initializing LAN area manager +34:55.151 [INFO] TestAgent.server: Adding LAN server: server="Node-5c04cdaa-5b53-fce5-c957-35a8935e7d78 (Addr: tcp/127.0.0.1:24583) (DC: dc1)" +34:55.151 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5c04cdaa-5b53-fce5-c957-35a8935e7d78.dc1 area=wan +34:55.151 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24599 network=udp +34:55.151 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24599 network=tcp +34:55.152 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24600 network=tcp +34:55.152 [INFO] TestAgent: started state syncer +34:55.152 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24584 network=tcp +34:55.201 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:55.201 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24583 [Candidate]" term=2 +34:55.201 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:55.201 [DEBUG] TestAgent.server.raft: vote granted: from=5c04cdaa-5b53-fce5-c957-35a8935e7d78 term=2 tally=1 +34:55.201 [INFO] TestAgent.server.raft: election won: tally=1 +34:55.201 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24583 [Leader]" +34:55.202 [INFO] TestAgent.server: cluster leadership acquired +34:55.202 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24583 +34:55.203 [INFO] TestAgent.server: New leader elected: payload=Node-5c04cdaa-5b53-fce5-c957-35a8935e7d78 +34:55.204 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:55.207 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:55.207 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:55.207 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:55.207 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:55.207 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:55.208 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5c04cdaa-5b53-fce5-c957-35a8935e7d78 +34:55.208 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5c04cdaa-5b53-fce5-c957-35a8935e7d78 +34:55.216 [INFO] TestAgent.server: federation state anti-entropy synced +34:55.219 [INFO] TestAgent: Requesting shutdown +34:55.219 [INFO] TestAgent.server: shutting down server +34:55.219 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:55.219 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:55.219 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:55.219 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:55.219 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:55.219 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:55.219 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:55.219 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:55.220 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:55.220 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:55.220 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:55.220 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:55.220Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:55.220Z [INFO] agent.router.manager: shutting down +34:55.220 [INFO] TestAgent: consul server down +34:55.220 [INFO] TestAgent: shutdown complete +34:55.220 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24599 network=tcp +34:55.220 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24599 network=udp +34:55.220 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24600 network=tcp +34:55.721 [INFO] TestAgent: Waiting for endpoints to shut down +34:55.721 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogNodeServices (0.68s) +=== CONT TestUIGatewayServiceNodes_Terminating +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._tcp_(qType:=33)_compressed=false +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:54.534Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:54.534Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:54.551Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:54.551Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:54.554 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f097e9a5-c2a0-f82b-2e44-dc8c8ed2c0e4 Address:127.0.0.1:24646}]" +34:54.554 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24646 [Follower]" leader= +34:54.555 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f097e9a5-c2a0-f82b-2e44-dc8c8ed2c0e4.dc2 127.0.0.1 +34:54.556 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f097e9a5-c2a0-f82b-2e44-dc8c8ed2c0e4 127.0.0.1 +2021-08-13T15:34:54.556Z [INFO] agent.router: Initializing LAN area manager +34:54.556 [INFO] TestAgent.server: Adding LAN server: server="Node-f097e9a5-c2a0-f82b-2e44-dc8c8ed2c0e4 (Addr: tcp/127.0.0.1:24646) (DC: dc2)" +34:54.556 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f097e9a5-c2a0-f82b-2e44-dc8c8ed2c0e4.dc2 area=wan +34:54.556 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24571 network=udp +34:54.557 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24571 network=tcp +34:54.557 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24572 network=tcp +34:54.557 [INFO] TestAgent: started state syncer +34:54.557 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24647 network=tcp +34:54.607 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:54.623 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:54.623 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24646 [Candidate]" term=2 +34:54.624 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:54.624 [DEBUG] TestAgent.server.raft: vote granted: from=f097e9a5-c2a0-f82b-2e44-dc8c8ed2c0e4 term=2 tally=1 +34:54.624 [INFO] TestAgent.server.raft: election won: tally=1 +34:54.624 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24646 [Leader]" +34:54.624 [INFO] TestAgent.server: cluster leadership acquired +34:54.624 [INFO] TestAgent.server: New leader elected: payload=Node-f097e9a5-c2a0-f82b-2e44-dc8c8ed2c0e4 +34:54.624 [INFO] TestAgent.server: initializing acls +34:54.625 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:54.625 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:54.625 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:54.625 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:54.626 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-f097e9a5-c2a0-f82b-2e44-dc8c8ed2c0e4 +34:54.626 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-f097e9a5-c2a0-f82b-2e44-dc8c8ed2c0e4.dc2 +34:54.626 [INFO] TestAgent.server: Updating LAN server: server="Node-f097e9a5-c2a0-f82b-2e44-dc8c8ed2c0e4 (Addr: tcp/127.0.0.1:24646) (DC: dc2)" +34:54.626 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-f097e9a5-c2a0-f82b-2e44-dc8c8ed2c0e4.dc2 area=wan +34:54.628 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:54.631 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:54.631 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:54.631 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:54.631 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:54.631 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:54.631 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f097e9a5-c2a0-f82b-2e44-dc8c8ed2c0e4 +34:54.631 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f097e9a5-c2a0-f82b-2e44-dc8c8ed2c0e4 +34:54.632 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f097e9a5-c2a0-f82b-2e44-dc8c8ed2c0e4 +34:54.632 [INFO] TestAgent.server: federation state anti-entropy synced +34:54.839 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:54.840 [INFO] TestAgent: Synced node info +34:54.840 [DEBUG] TestAgent: Node info in sync +34:54.860 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:24610] +34:54.860 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:24610 +34:54.861 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c9d08d21-fc0a-2f3a-1544-5916e935e2f6.dc1 127.0.0.1 +34:54.861 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 +34:54.862 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c9d08d21-fc0a-2f3a-1544-5916e935e2f6.dc1 area=wan +34:54.864 [INFO] TestAgent: Requesting shutdown +34:54.864 [INFO] TestAgent.server: shutting down server +34:54.864 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:54.864 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:54.864 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:54.864 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:54.864 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:54.864 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:54.864 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:54.864 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:54.864 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:54.864 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:54.864 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:54.864 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:54.864 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:54.864 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:54.864 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:54.865Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:54.865Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:54.865Z [INFO] agent.router.manager: shutting down +34:54.865 [INFO] TestAgent: consul server down +34:54.865 [INFO] TestAgent: shutdown complete +34:54.865 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24571 network=tcp +34:54.865 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24571 network=udp +34:54.865 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24572 network=tcp +34:55.367 [INFO] TestAgent: Waiting for endpoints to shut down +34:55.367 [INFO] TestAgent: Endpoints down +2021-08-13T15:34:54.365Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:54.365Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:54.428Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:34:54.428Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:54.432 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c9d08d21-fc0a-2f3a-1544-5916e935e2f6 Address:127.0.0.1:24611}]" +34:54.433 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c9d08d21-fc0a-2f3a-1544-5916e935e2f6.dc1 127.0.0.1 +34:54.434 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24611 [Follower]" leader= +34:54.434 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c9d08d21-fc0a-2f3a-1544-5916e935e2f6 127.0.0.1 +2021-08-13T15:34:54.434Z [INFO] agent.router: Initializing LAN area manager +34:54.434 [INFO] TestAgent.server: Adding LAN server: server="Node-c9d08d21-fc0a-2f3a-1544-5916e935e2f6 (Addr: tcp/127.0.0.1:24611) (DC: dc1)" +34:54.435 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24557 network=udp +34:54.435 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c9d08d21-fc0a-2f3a-1544-5916e935e2f6.dc1 area=wan +34:54.435 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24557 network=tcp +34:54.439 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24558 network=tcp +34:54.439 [INFO] TestAgent: started state syncer +34:54.439 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24612 network=tcp +34:54.476 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:54.476 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24611 [Candidate]" term=2 +34:54.476 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:54.476 [DEBUG] TestAgent.server.raft: vote granted: from=c9d08d21-fc0a-2f3a-1544-5916e935e2f6 term=2 tally=1 +34:54.476 [INFO] TestAgent.server.raft: election won: tally=1 +34:54.476 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24611 [Leader]" +34:54.476 [INFO] TestAgent.server: cluster leadership acquired +34:54.477 [INFO] TestAgent.server: New leader elected: payload=Node-c9d08d21-fc0a-2f3a-1544-5916e935e2f6 +34:54.477 [INFO] TestAgent.server: initializing acls +34:54.477 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:54.478 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:54.478 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:54.478 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:54.478 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-c9d08d21-fc0a-2f3a-1544-5916e935e2f6 +34:54.478 [INFO] TestAgent.server: Updating LAN server: server="Node-c9d08d21-fc0a-2f3a-1544-5916e935e2f6 (Addr: tcp/127.0.0.1:24611) (DC: dc1)" +34:54.478 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-c9d08d21-fc0a-2f3a-1544-5916e935e2f6.dc1 +34:54.478 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-c9d08d21-fc0a-2f3a-1544-5916e935e2f6.dc1 area=wan +34:54.479 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:54.482 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:54.482 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:54.482 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:54.482 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:54.482 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:54.482 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c9d08d21-fc0a-2f3a-1544-5916e935e2f6 +34:54.482 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c9d08d21-fc0a-2f3a-1544-5916e935e2f6 +34:54.483 [INFO] TestAgent.server: federation state anti-entropy synced +34:54.483 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c9d08d21-fc0a-2f3a-1544-5916e935e2f6 +34:54.840 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:54.840 [INFO] TestAgent: Synced node info +34:54.860 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:57834 +34:54.861 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f097e9a5-c2a0-f82b-2e44-dc8c8ed2c0e4.dc2 127.0.0.1 +34:54.861 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f097e9a5-c2a0-f82b-2e44-dc8c8ed2c0e4.dc2 area=wan +34:55.173 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:55.173 [DEBUG] TestAgent: Node info in sync +34:55.173 [DEBUG] TestAgent: Node info in sync +34:55.367 [INFO] TestAgent: Requesting shutdown +34:55.367 [INFO] TestAgent.server: shutting down server +34:55.367 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:55.367 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:55.367 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:55.367 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:55.368 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:55.368 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:55.368 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:55.368 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:55.368 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:55.368 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:55.368 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:55.368 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:55.368 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:55.368 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:55.368 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:55.369Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:55.369Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:55.369Z [INFO] agent.router.manager: shutting down +34:55.369 [INFO] TestAgent: consul server down +34:55.369 [INFO] TestAgent: shutdown complete +34:55.369 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24557 network=tcp +34:55.369 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24557 network=udp +34:55.369 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24558 network=tcp +34:55.869 [INFO] TestAgent: Waiting for endpoints to shut down +34:55.869 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogNodeServices_WanTranslation (1.53s) === CONT TestValidateUserEventParams --- PASS: TestValidateUserEventParams (0.00s) +=== CONT TestUiNodeInfo +2021-08-13T15:34:55.105Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:55.141Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:55.144 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cd943037-44e1-3ac6-4162-ecbd51ae961e Address:127.0.0.1:24597}]" +34:55.145 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-cd943037-44e1-3ac6-4162-ecbd51ae961e.dc1 127.0.0.1 +34:55.146 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-cd943037-44e1-3ac6-4162-ecbd51ae961e 127.0.0.1 +2021-08-13T15:34:55.146Z [INFO] agent.router: Initializing LAN area manager +34:55.146 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24655 network=udp +34:55.146 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24597 [Follower]" leader= +34:55.147 [INFO] TestAgent.server: Adding LAN server: server="Node-cd943037-44e1-3ac6-4162-ecbd51ae961e (Addr: tcp/127.0.0.1:24597) (DC: dc1)" +34:55.147 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-cd943037-44e1-3ac6-4162-ecbd51ae961e.dc1 area=wan +34:55.147 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24655 network=tcp +34:55.148 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24656 network=tcp +34:55.148 [INFO] TestAgent: started state syncer +34:55.148 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24598 network=tcp +34:55.207 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:55.207 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24597 [Candidate]" term=2 +34:55.207 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:55.207 [DEBUG] TestAgent.server.raft: vote granted: from=cd943037-44e1-3ac6-4162-ecbd51ae961e term=2 tally=1 +34:55.207 [INFO] TestAgent.server.raft: election won: tally=1 +34:55.207 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24597 [Leader]" +34:55.208 [INFO] TestAgent.server: cluster leadership acquired +34:55.208 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24597 +34:55.215 [INFO] TestAgent.server: New leader elected: payload=Node-cd943037-44e1-3ac6-4162-ecbd51ae961e +34:55.218 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:55.221 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:55.221 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:55.221 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:55.221 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:55.221 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:55.221 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cd943037-44e1-3ac6-4162-ecbd51ae961e +34:55.221 [INFO] TestAgent.server: member joined, marking health alive: member=Node-cd943037-44e1-3ac6-4162-ecbd51ae961e +34:55.223 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:55.229 [INFO] TestAgent.server: federation state anti-entropy synced +34:55.229 [INFO] TestAgent: Synced node info +34:55.413 [INFO] TestAgent: Requesting shutdown +34:55.413 [INFO] TestAgent.server: shutting down server +34:55.413 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:55.413 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:55.414 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:55.414 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:55.414 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:55.414 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:55.414 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:55.414 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:55.414 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:55.414 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:55.414 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:55.414Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:55.414Z [INFO] agent.router.manager: shutting down +34:55.414 [INFO] TestAgent: consul server down +34:55.414 [INFO] TestAgent: shutdown complete +34:55.414 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24655 network=tcp +34:55.414 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24655 network=udp +34:55.415 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24656 network=tcp +34:55.915 [INFO] TestAgent: Waiting for endpoints to shut down +34:55.915 [INFO] TestAgent: Endpoints down +2021-08-13T15:34:55.006Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:55.022Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:55.025 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a7cedd10-25b4-efa2-3314-5e84f1fae25e Address:127.0.0.1:24653}]" +34:55.025 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24653 [Follower]" leader= +34:55.026 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a7cedd10-25b4-efa2-3314-5e84f1fae25e.dc1 127.0.0.1 +34:55.026 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a7cedd10-25b4-efa2-3314-5e84f1fae25e 127.0.0.1 +2021-08-13T15:34:55.026Z [INFO] agent.router: Initializing LAN area manager +34:55.027 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a7cedd10-25b4-efa2-3314-5e84f1fae25e.dc1 area=wan +34:55.027 [INFO] TestAgent.server: Adding LAN server: server="Node-a7cedd10-25b4-efa2-3314-5e84f1fae25e (Addr: tcp/127.0.0.1:24653) (DC: dc1)" +34:55.027 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24627 network=udp +34:55.027 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24627 network=tcp +34:55.028 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24628 network=tcp +34:55.028 [INFO] TestAgent: started state syncer +34:55.028 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24654 network=tcp +34:55.096 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:55.096 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24653 [Candidate]" term=2 +34:55.096 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:55.096 [DEBUG] TestAgent.server.raft: vote granted: from=a7cedd10-25b4-efa2-3314-5e84f1fae25e term=2 tally=1 +34:55.096 [INFO] TestAgent.server.raft: election won: tally=1 +34:55.096 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24653 [Leader]" +34:55.097 [INFO] TestAgent.server: cluster leadership acquired +34:55.097 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24653 +34:55.099 [INFO] TestAgent.server: New leader elected: payload=Node-a7cedd10-25b4-efa2-3314-5e84f1fae25e +34:55.099 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:55.102 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:55.102 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:55.102 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:55.102 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:55.102 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:55.103 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a7cedd10-25b4-efa2-3314-5e84f1fae25e +34:55.103 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a7cedd10-25b4-efa2-3314-5e84f1fae25e +34:55.105 [INFO] TestAgent.server: federation state anti-entropy synced +34:55.166 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:55.166 [INFO] TestAgent: Synced node info +34:55.413 [INFO] TestAgent: Requesting shutdown +34:55.413 [INFO] TestAgent.server: shutting down server +34:55.413 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:55.413 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:55.413 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:55.413 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:55.413 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:55.414 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:55.414 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:55.414 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:55.414 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:55.414 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:55.414 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:55.414Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:55.414Z [INFO] agent.router.manager: shutting down +34:55.414 [INFO] TestAgent: consul server down +34:55.414 [INFO] TestAgent: shutdown complete +34:55.414 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24627 network=tcp +34:55.414 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24627 network=udp +34:55.415 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24628 network=tcp +34:55.915 [INFO] TestAgent: Waiting for endpoints to shut down +34:55.915 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogNodeServiceList (0.88s) === CONT TestUIGatewayServiceNodes_Ingress -[WARN] freeport: 4 out of 25 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 4 out of 4 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:13.440Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:13.482Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:13.496 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:dda0b5a3-6123-a338-4207-eb9d7616c3b1 Address:127.0.0.1:14000}]" -43:13.497 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-dda0b5a3-6123-a338-4207-eb9d7616c3b1.dc1 127.0.0.1 -43:13.498 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-dda0b5a3-6123-a338-4207-eb9d7616c3b1 127.0.0.1 -2022-09-15T21:43:13.498Z [INFO] agent.router: Initializing LAN area manager -43:13.498 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14000 [Follower]" leader= -43:13.498 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-dda0b5a3-6123-a338-4207-eb9d7616c3b1.dc1 area=wan -43:13.498 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14051 network=udp -43:13.498 [INFO] TestAgent.server: Adding LAN server: server="Node-dda0b5a3-6123-a338-4207-eb9d7616c3b1 (Addr: tcp/127.0.0.1:14000) (DC: dc1)" -43:13.498 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14051 network=tcp -43:13.499 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14052 network=tcp -43:13.499 [INFO] TestAgent: started state syncer -43:13.499 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14001 network=tcp -43:13.540 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:13.540 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14000 [Candidate]" term=2 -43:13.540 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:13.540 [DEBUG] TestAgent.server.raft: vote granted: from=dda0b5a3-6123-a338-4207-eb9d7616c3b1 term=2 tally=1 -43:13.540 [INFO] TestAgent.server.raft: election won: tally=1 -43:13.540 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14000 [Leader]" -43:13.541 [INFO] TestAgent.server: cluster leadership acquired -43:13.541 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14000 -43:13.542 [INFO] TestAgent.server: New leader elected: payload=Node-dda0b5a3-6123-a338-4207-eb9d7616c3b1 -43:13.542 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:13.545 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:13.545 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:13.545 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:13.545 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:13.545 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:13.545 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-dda0b5a3-6123-a338-4207-eb9d7616c3b1 -43:13.545 [INFO] TestAgent.server: member joined, marking health alive: member=Node-dda0b5a3-6123-a338-4207-eb9d7616c3b1 -43:13.551 [INFO] TestAgent.server: federation state anti-entropy synced -43:13.605 [DEBUG] TestAgent.server: User event: event=deploy -43:13.605 [DEBUG] TestAgent.server: User event: event=deploy -43:13.605 [DEBUG] TestAgent: new event: event_name=deploy event_id=df9d5cbf-4a4d-2a3d-d68b-27e4f5df0fa1 -43:13.630 [INFO] TestAgent: Requesting shutdown -43:13.630 [INFO] TestAgent.server: shutting down server -43:13.630 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:13.630 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:13.630 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:13.630 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:13.630 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:13.630 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:13.630 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:13.630 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:13.630 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:13.630 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:13.630 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:13.630 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:13.631Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:13.631Z [INFO] agent.router.manager: shutting down -43:13.631 [INFO] TestAgent: consul server down -43:13.631 [INFO] TestAgent: shutdown complete -43:13.631 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14051 network=tcp -43:13.631 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14051 network=udp -43:13.631 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14052 network=tcp -43:14.132 [INFO] TestAgent: Waiting for endpoints to shut down -43:14.132 [INFO] TestAgent: Endpoints down ---- PASS: TestFireReceiveEvent (0.74s) -=== CONT TestUIGatewayServiceNodes_Terminating -2022-09-15T21:43:13.596Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:13.649Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:13.697 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c5433241-cbf4-d9af-7d00-c063c753bc03 Address:127.0.0.1:13895}]" -43:13.698 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13895 [Follower]" leader= -43:13.699 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c5433241-cbf4-d9af-7d00-c063c753bc03.dc1 127.0.0.1 -43:13.701 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c5433241-cbf4-d9af-7d00-c063c753bc03 127.0.0.1 -2022-09-15T21:43:13.701Z [INFO] agent.router: Initializing LAN area manager -43:13.702 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13953 network=udp -43:13.743 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:13.743 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13895 [Candidate]" term=2 -43:13.743 [WARN] TestAgent.server.raft: unable to get address for sever, using fallback address: id=c5433241-cbf4-d9af-7d00-c063c753bc03 fallback=127.0.0.1:13895 error="Could not find address for server id c5433241-cbf4-d9af-7d00-c063c753bc03" -43:13.743 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:13.743 [DEBUG] TestAgent.server.raft: vote granted: from=c5433241-cbf4-d9af-7d00-c063c753bc03 term=2 tally=1 -43:13.743 [INFO] TestAgent.server.raft: election won: tally=1 -43:13.743 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13895 [Leader]" -43:13.744 [INFO] TestAgent.server: cluster leadership acquired -43:13.744 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=1 found=false leader=127.0.0.1:13895 -43:13.746 [INFO] TestAgent.server: Adding LAN server: server="Node-c5433241-cbf4-d9af-7d00-c063c753bc03 (Addr: tcp/127.0.0.1:13895) (DC: dc1)" -43:13.746 [INFO] TestAgent.server: New leader elected: payload=Node-c5433241-cbf4-d9af-7d00-c063c753bc03 -43:13.754 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c5433241-cbf4-d9af-7d00-c063c753bc03.dc1 area=wan -43:13.754 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13953 network=tcp -43:13.755 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13954 network=tcp -43:13.755 [INFO] TestAgent: started state syncer -43:13.756 [WARN] TestAgent.server: endpoint injected; this should only be used for testing -43:13.756 [INFO] TestAgent: Requesting shutdown -43:13.756 [INFO] TestAgent.server: shutting down server -43:13.756 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:13.756 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13896 network=tcp -43:13.756 [ERROR] TestAgent: gRPC server failed: error="grpc: the server has been stopped" -43:13.757 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:13.757Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:13.757Z [INFO] agent.router.manager: shutting down -43:13.757 [WARN] TestAgent: Syncing node info failed.: error="raft is already shutdown" -43:13.757 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="raft is already shutdown" -43:13.763 [ERROR] TestAgent.server: failed to establish leadership: error="error configuring provider: leadership lost while committing log" -43:13.763 [ERROR] TestAgent.server: failed to transfer leadership attempt, will retry: attempt=0 retry_limit=3 error="raft is already shutdown" -43:13.763 [INFO] TestAgent: consul server down -43:13.763 [INFO] TestAgent: shutdown complete -43:13.763 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13953 network=tcp -43:13.763 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13953 network=udp -43:13.763 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13954 network=tcp -43:14.265 [INFO] TestAgent: Waiting for endpoints to shut down -43:14.265 [INFO] TestAgent: Endpoints down -2022-09-15T21:43:13.432Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:13.452Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:13.455 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:351303ce-eb32-700f-a9e8-bf0e325a39a9 Address:127.0.0.1:14056}]" -43:13.456 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-351303ce-eb32-700f-a9e8-bf0e325a39a9.dc1 127.0.0.1 -43:13.457 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-351303ce-eb32-700f-a9e8-bf0e325a39a9 127.0.0.1 -2022-09-15T21:43:13.457Z [INFO] agent.router: Initializing LAN area manager -43:13.457 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13883 network=udp -43:13.457 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14056 [Follower]" leader= -43:13.458 [INFO] TestAgent.server: Adding LAN server: server="Node-351303ce-eb32-700f-a9e8-bf0e325a39a9 (Addr: tcp/127.0.0.1:14056) (DC: dc1)" -43:13.458 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-351303ce-eb32-700f-a9e8-bf0e325a39a9.dc1 area=wan -43:13.458 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13883 network=tcp -43:13.459 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13884 network=tcp -43:13.459 [INFO] TestAgent: started state syncer -43:13.459 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14057 network=tcp -43:13.518 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:13.518 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14056 [Candidate]" term=2 -43:13.518 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:13.518 [DEBUG] TestAgent.server.raft: vote granted: from=351303ce-eb32-700f-a9e8-bf0e325a39a9 term=2 tally=1 -43:13.518 [INFO] TestAgent.server.raft: election won: tally=1 -43:13.518 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14056 [Leader]" -43:13.518 [INFO] TestAgent.server: cluster leadership acquired -43:13.519 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14056 -43:13.520 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:13.522 [INFO] TestAgent.server: New leader elected: payload=Node-351303ce-eb32-700f-a9e8-bf0e325a39a9 -43:13.527 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:13.527 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:13.527 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:13.527 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:13.527 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:13.527 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-351303ce-eb32-700f-a9e8-bf0e325a39a9 -43:13.527 [INFO] TestAgent.server: member joined, marking health alive: member=Node-351303ce-eb32-700f-a9e8-bf0e325a39a9 -43:13.529 [INFO] TestAgent.server: federation state anti-entropy synced -43:13.661 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.661 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.661 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.661 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.661 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.661 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.661 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.661 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.661 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.661 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.661 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.661 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.661 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.661 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.661 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.661 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.662 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.662 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.662 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.662 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.662 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.662 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.662 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.662 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.662 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.662 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.662 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.663 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.663 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.663 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.663 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.663 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.663 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.663 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.664 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.664 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.664 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.664 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.664 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.664 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.664 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.664 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.665 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.665 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.665 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.665 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.665 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.665 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.674 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.674 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.674 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.674 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.674 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.674 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.674 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.674 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.674 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.674 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.674 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.674 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.675 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.675 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.675 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.675 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.675 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.675 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.675 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.675 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.675 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.675 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.675 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.675 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.675 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.675 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.675 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.675 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.675 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.675 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.675 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.675 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.675 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.675 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.676 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.676 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.676 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.676 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.676 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.676 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.676 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.676 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.676 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.677 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.677 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.677 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.677 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.677 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.677 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.677 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.677 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.677 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.677 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.677 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.677 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.677 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.677 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.694 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.695 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.696 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.697 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.697 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.697 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.697 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.697 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.697 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.697 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.697 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.697 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.697 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.697 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.697 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.697 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.697 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.697 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.697 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.697 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.697 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.697 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.697 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.697 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.697 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.697 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.697 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.697 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.697 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.697 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.697 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.697 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.697 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.697 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.697 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.697 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.739 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.740 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [DEBUG] TestAgent: new event: event_name=test event_id= -43:13.761 [INFO] TestAgent: Requesting shutdown -43:13.761 [INFO] TestAgent.server: shutting down server -43:13.762 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:13.762 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:13.762 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:13.762 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:13.762 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:13.762 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:13.762 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:13.762 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:13.762 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:13.762 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:13.762 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -43:13.762 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2022-09-15T21:43:13.762Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:13.762Z [INFO] agent.router.manager: shutting down -43:13.763 [INFO] TestAgent: consul server down -43:13.763 [INFO] TestAgent: shutdown complete -43:13.763 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13883 network=tcp -43:13.763 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13883 network=udp -43:13.763 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13884 network=tcp -43:14.265 [INFO] TestAgent: Waiting for endpoints to shut down -43:14.265 [INFO] TestAgent: Endpoints down -=== RUN TestPreparedQuery_Execute/#04 -2022-09-15T21:43:12.967Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:13.005Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:13.012 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bbb7aca7-7294-8ade-b391-1173b7135386 Address:127.0.0.1:13951}]" -43:13.013 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bbb7aca7-7294-8ade-b391-1173b7135386.dc1 127.0.0.1 -43:13.014 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13951 [Follower]" leader= -43:13.018 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bbb7aca7-7294-8ade-b391-1173b7135386 127.0.0.1 -2022-09-15T21:43:13.018Z [INFO] agent.router: Initializing LAN area manager -43:13.018 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14002 network=udp -43:13.018 [INFO] TestAgent.server: Adding LAN server: server="Node-bbb7aca7-7294-8ade-b391-1173b7135386 (Addr: tcp/127.0.0.1:13951) (DC: dc1)" -43:13.019 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bbb7aca7-7294-8ade-b391-1173b7135386.dc1 area=wan -43:13.019 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14002 network=tcp -43:13.020 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14003 network=tcp -43:13.020 [INFO] TestAgent: started state syncer -43:13.020 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13952 network=tcp -43:13.070 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:13.070 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13951 [Candidate]" term=2 -43:13.070 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:13.070 [DEBUG] TestAgent.server.raft: vote granted: from=bbb7aca7-7294-8ade-b391-1173b7135386 term=2 tally=1 -43:13.070 [INFO] TestAgent.server.raft: election won: tally=1 -43:13.070 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13951 [Leader]" -43:13.070 [INFO] TestAgent.server: cluster leadership acquired -43:13.071 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13951 -43:13.071 [INFO] TestAgent.server: New leader elected: payload=Node-bbb7aca7-7294-8ade-b391-1173b7135386 -43:13.072 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:13.075 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:13.075 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:13.075 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:13.075 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:13.075 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:13.075 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bbb7aca7-7294-8ade-b391-1173b7135386 -43:13.075 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bbb7aca7-7294-8ade-b391-1173b7135386 -43:13.076 [INFO] TestAgent.server: federation state anti-entropy synced -43:13.390 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:13.391 [INFO] TestAgent: Synced node info -43:13.766 [INFO] TestAgent: Requesting shutdown -43:13.766 [INFO] TestAgent.server: shutting down server -43:13.766 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:13.766 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:13.766 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:13.766 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:13.766 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:13.766 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:13.766 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:13.766 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:13.766 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:13.766 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:13.767 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:13.767Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:13.767Z [INFO] agent.router.manager: shutting down -43:13.767 [INFO] TestAgent: consul server down -43:13.767 [INFO] TestAgent: shutdown complete -43:13.767 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14002 network=tcp -43:13.767 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14002 network=udp -43:13.767 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14003 network=tcp -43:14.270 [INFO] TestAgent: Waiting for endpoints to shut down -43:14.270 [INFO] TestAgent: Endpoints down -=== RUN TestTxnEndpoint_Bad_Size_Net/exceeds_configured_max_txn_len ---- PASS: TestIngestUserEvent (0.86s) +--- PASS: TestCatalogNodeServices_Filter (0.93s) === CONT TestUiServices -[WARN] freeport: 4 out of 32 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:55.319Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:55.385Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:55.400 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a7ee3add-f4b9-6081-419e-7587d726dd11 Address:127.0.0.1:25313}]" +34:55.400 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25313 [Follower]" leader= +34:55.401 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a7ee3add-f4b9-6081-419e-7587d726dd11.dc1 127.0.0.1 +34:55.403 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a7ee3add-f4b9-6081-419e-7587d726dd11 127.0.0.1 +2021-08-13T15:34:55.403Z [INFO] agent.router: Initializing LAN area manager +34:55.403 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24585 network=udp +34:55.408 [INFO] TestAgent.server: Adding LAN server: server="Node-a7ee3add-f4b9-6081-419e-7587d726dd11 (Addr: tcp/127.0.0.1:25313) (DC: dc1)" +34:55.410 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a7ee3add-f4b9-6081-419e-7587d726dd11.dc1 area=wan +34:55.410 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24585 network=tcp +34:55.411 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24586 network=tcp +34:55.411 [INFO] TestAgent: started state syncer +34:55.411 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25314 network=tcp +34:55.445 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:55.445 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25313 [Candidate]" term=2 +34:55.445 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:55.445 [DEBUG] TestAgent.server.raft: vote granted: from=a7ee3add-f4b9-6081-419e-7587d726dd11 term=2 tally=1 +34:55.445 [INFO] TestAgent.server.raft: election won: tally=1 +34:55.446 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25313 [Leader]" +34:55.446 [INFO] TestAgent.server: cluster leadership acquired +34:55.447 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25313 +34:55.449 [INFO] TestAgent.server: New leader elected: payload=Node-a7ee3add-f4b9-6081-419e-7587d726dd11 +34:55.453 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:55.462 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:55.462 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:55.462 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:55.462 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:55.462 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:55.462 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a7ee3add-f4b9-6081-419e-7587d726dd11 +34:55.462 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a7ee3add-f4b9-6081-419e-7587d726dd11 +34:55.476 [INFO] TestAgent: Synced node info +34:55.476 [DEBUG] TestAgent: Node info in sync +34:55.476 [INFO] TestAgent.server: federation state anti-entropy synced +34:55.693 [INFO] TestAgent: Requesting shutdown +34:55.693 [INFO] TestAgent.server: shutting down server +34:55.693 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:55.693 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:55.693 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:55.693 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:55.693 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:55.694 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:55.694 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:55.694 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:55.694 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:55.694 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:55.694 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:55.694Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:55.694Z [INFO] agent.router.manager: shutting down +34:55.695 [INFO] TestAgent: consul server down +34:55.695 [INFO] TestAgent: shutdown complete +34:55.695 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24585 network=tcp +34:55.695 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24585 network=udp +34:55.695 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24586 network=tcp +34:56.197 [INFO] TestAgent: Waiting for endpoints to shut down +34:56.197 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogConnectServiceNodes_good (0.98s) +=== CONT TestDNS_ServiceLookup_AnswerLimits/A_lookup_{0_0_0_0_0_0_0_0_0_0_0} +2021-08-13T15:34:55.318Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:55.370Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:55.376 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:03486c2f-aa98-741e-33a2-e6f69388723a Address:127.0.0.1:24342}]" +34:55.377 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-03486c2f-aa98-741e-33a2-e6f69388723a.dc1 127.0.0.1 +34:55.378 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-03486c2f-aa98-741e-33a2-e6f69388723a 127.0.0.1 +2021-08-13T15:34:55.378Z [INFO] agent.router: Initializing LAN area manager +34:55.379 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24342 [Follower]" leader= +34:55.379 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24295 network=udp +34:55.380 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-03486c2f-aa98-741e-33a2-e6f69388723a.dc1 area=wan +34:55.380 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24295 network=tcp +34:55.381 [INFO] TestAgent.server: Adding LAN server: server="Node-03486c2f-aa98-741e-33a2-e6f69388723a (Addr: tcp/127.0.0.1:24342) (DC: dc1)" +34:55.382 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24296 network=tcp +34:55.382 [INFO] TestAgent: started state syncer +34:55.382 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24343 network=tcp +34:55.422 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:55.422 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24342 [Candidate]" term=2 +34:55.422 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:55.422 [DEBUG] TestAgent.server.raft: vote granted: from=03486c2f-aa98-741e-33a2-e6f69388723a term=2 tally=1 +34:55.422 [INFO] TestAgent.server.raft: election won: tally=1 +34:55.422 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24342 [Leader]" +34:55.422 [INFO] TestAgent.server: cluster leadership acquired +34:55.423 [INFO] TestAgent.server: New leader elected: payload=Node-03486c2f-aa98-741e-33a2-e6f69388723a +34:55.423 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24342 +34:55.424 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:55.438 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:55.438 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:55.438 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:55.438 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:55.438 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:55.438 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-03486c2f-aa98-741e-33a2-e6f69388723a +34:55.438 [INFO] TestAgent.server: member joined, marking health alive: member=Node-03486c2f-aa98-741e-33a2-e6f69388723a +34:55.449 [INFO] TestAgent.server: federation state anti-entropy synced +34:55.479 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:55.480 [INFO] TestAgent: Synced node info +34:55.480 [DEBUG] TestAgent: Node info in sync +34:55.714 [INFO] TestAgent: Requesting shutdown +34:55.714 [INFO] TestAgent.server: shutting down server +34:55.714 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:55.714 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:55.714 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:55.714 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:55.714 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:55.714 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:55.714 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:55.714 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:55.714 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:55.714 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:55.714 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:55.715Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:55.715Z [INFO] agent.router.manager: shutting down +34:55.715 [INFO] TestAgent: consul server down +34:55.715 [INFO] TestAgent: shutdown complete +34:55.715 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24295 network=tcp +34:55.715 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24295 network=udp +34:55.715 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24296 network=tcp +34:56.219 [INFO] TestAgent: Waiting for endpoints to shut down +34:56.219 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Members (0.98s) +=== CONT TestDNS_ServiceLookup_AnswerLimits/A_lookup_{6_6_6_6_6_6_6_5_6_6_-5} +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! === RUN TestUiServices/No_Filter === PAUSE TestUiServices/No_Filter === RUN TestUiServices/Filtered -2022-09-15T21:43:13.497Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:13.604Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:13.615 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:de8c1b75-767d-6ce9-7bff-71bcc37a847a Address:127.0.0.1:14042}]" -43:13.616 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-de8c1b75-767d-6ce9-7bff-71bcc37a847a.dc1 127.0.0.1 -43:13.616 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14042 [Follower]" leader= -43:13.617 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-de8c1b75-767d-6ce9-7bff-71bcc37a847a 127.0.0.1 -2022-09-15T21:43:13.618Z [INFO] agent.router: Initializing LAN area manager -43:13.618 [INFO] TestAgent.server: Adding LAN server: server="Node-de8c1b75-767d-6ce9-7bff-71bcc37a847a (Addr: tcp/127.0.0.1:14042) (DC: dc1)" -43:13.618 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14058 network=udp -43:13.618 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-de8c1b75-767d-6ce9-7bff-71bcc37a847a.dc1 area=wan -43:13.618 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14058 network=tcp -43:13.619 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14059 network=tcp -43:13.619 [INFO] TestAgent: started state syncer -43:13.619 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14043 network=tcp -43:13.671 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:13.672 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14042 [Candidate]" term=2 -43:13.672 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:13.672 [DEBUG] TestAgent.server.raft: vote granted: from=de8c1b75-767d-6ce9-7bff-71bcc37a847a term=2 tally=1 -43:13.672 [INFO] TestAgent.server.raft: election won: tally=1 -43:13.672 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14042 [Leader]" -43:13.673 [INFO] TestAgent.server: cluster leadership acquired -43:13.682 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14042 -43:13.685 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:13.694 [INFO] TestAgent.server: New leader elected: payload=Node-de8c1b75-767d-6ce9-7bff-71bcc37a847a -43:13.738 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:13.738 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:13.738 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:13.738 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:13.738 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:13.738 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-de8c1b75-767d-6ce9-7bff-71bcc37a847a -43:13.738 [INFO] TestAgent.server: member joined, marking health alive: member=Node-de8c1b75-767d-6ce9-7bff-71bcc37a847a -43:13.743 [INFO] TestAgent.server: federation state anti-entropy synced -43:13.786 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:13.786 [INFO] TestAgent: Synced node info -43:13.786 [DEBUG] TestAgent: Node info in sync -43:13.952 [INFO] TestAgent: Requesting shutdown -43:13.952 [INFO] TestAgent.server: shutting down server -43:13.952 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:13.952 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:13.952 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:13.952 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:13.952 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:13.953 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:13.953 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:13.953 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:13.953 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:13.953 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:13.953 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:13.953Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:13.953Z [INFO] agent.router.manager: shutting down -43:13.953 [INFO] TestAgent: consul server down -43:13.953 [INFO] TestAgent: shutdown complete -43:13.953 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14058 network=tcp -43:13.953 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14058 network=udp -43:13.953 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14059 network=tcp -43:14.462 [INFO] TestAgent: Waiting for endpoints to shut down -43:14.462 [INFO] TestAgent: Endpoints down ---- PASS: TestShouldProcessUserEvent (1.04s) -=== CONT TestUiNodeInfo -2022-09-15T21:43:13.676Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:13.790Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:13.793 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8b3f4230-dfcd-c27a-216c-431e4047434b Address:127.0.0.1:14070}]" -43:13.793 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14070 [Follower]" leader= -43:13.799 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8b3f4230-dfcd-c27a-216c-431e4047434b.dc1 127.0.0.1 -43:13.800 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8b3f4230-dfcd-c27a-216c-431e4047434b 127.0.0.1 -2022-09-15T21:43:13.800Z [INFO] agent.router: Initializing LAN area manager -43:13.800 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14044 network=udp -43:13.801 [INFO] TestAgent.server: Adding LAN server: server="Node-8b3f4230-dfcd-c27a-216c-431e4047434b (Addr: tcp/127.0.0.1:14070) (DC: dc1)" -43:13.801 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8b3f4230-dfcd-c27a-216c-431e4047434b.dc1 area=wan -43:13.801 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14044 network=tcp -43:13.802 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14045 network=tcp -43:13.802 [INFO] TestAgent: started state syncer -43:13.802 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14071 network=tcp -43:13.848 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:13.848 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14070 [Candidate]" term=2 -43:13.849 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:13.849 [DEBUG] TestAgent.server.raft: vote granted: from=8b3f4230-dfcd-c27a-216c-431e4047434b term=2 tally=1 -43:13.849 [INFO] TestAgent.server.raft: election won: tally=1 -43:13.849 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14070 [Leader]" -43:13.849 [INFO] TestAgent.server: cluster leadership acquired -43:13.849 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14070 -43:13.851 [INFO] TestAgent.server: New leader elected: payload=Node-8b3f4230-dfcd-c27a-216c-431e4047434b -43:13.851 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:13.853 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:13.853 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:13.853 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:13.853 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:13.853 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:13.854 [INFO] TestAgent.server: federation state anti-entropy synced -43:13.855 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8b3f4230-dfcd-c27a-216c-431e4047434b -43:13.855 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8b3f4230-dfcd-c27a-216c-431e4047434b -43:13.999 [INFO] TestAgent: Requesting shutdown -43:13.999 [INFO] TestAgent.server: shutting down server -43:13.999 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:13.999 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:13.999 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:13.999 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:13.999 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:13.999 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:13.999 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:13.999 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:13.999 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:13.999 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:13.999 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:14.000 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:14.000Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:14.000Z [INFO] agent.router.manager: shutting down -43:14.000 [INFO] TestAgent: consul server down -43:14.000 [INFO] TestAgent: shutdown complete -43:14.000 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14044 network=tcp -43:14.000 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14044 network=udp -43:14.000 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14045 network=tcp -43:14.518 [INFO] TestAgent: Waiting for endpoints to shut down -43:14.518 [INFO] TestAgent: Endpoints down ---- PASS: TestUIGatewayServiceNodes_Ingress (0.93s) -=== CONT TestUiNodes_Filter -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 4 out of 4 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:12.252Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:12.270Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:12.273 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:da12e1da-61ab-cedd-5d5d-9238ceb22ee0 Address:127.0.0.1:14007}]" -43:12.274 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-da12e1da-61ab-cedd-5d5d-9238ceb22ee0.dc1 127.0.0.1 -43:12.275 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-da12e1da-61ab-cedd-5d5d-9238ceb22ee0 127.0.0.1 -2022-09-15T21:43:12.275Z [INFO] agent.router: Initializing LAN area manager -43:12.276 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13981 network=udp -43:12.276 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14007 [Follower]" leader= -43:12.276 [INFO] TestAgent.server: Adding LAN server: server="Node-da12e1da-61ab-cedd-5d5d-9238ceb22ee0 (Addr: tcp/127.0.0.1:14007) (DC: dc1)" -43:12.276 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-da12e1da-61ab-cedd-5d5d-9238ceb22ee0.dc1 area=wan -43:12.276 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13981 network=tcp -43:12.277 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13982 network=tcp -43:12.277 [INFO] TestAgent: started state syncer -43:12.277 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14008 network=tcp -43:12.335 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:12.336 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14007 [Candidate]" term=2 -43:12.336 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:12.336 [DEBUG] TestAgent.server.raft: vote granted: from=da12e1da-61ab-cedd-5d5d-9238ceb22ee0 term=2 tally=1 -43:12.336 [INFO] TestAgent.server.raft: election won: tally=1 -43:12.336 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14007 [Leader]" -43:12.336 [INFO] TestAgent.server: cluster leadership acquired -43:12.336 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14007 -43:12.337 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:12.338 [INFO] TestAgent.server: New leader elected: payload=Node-da12e1da-61ab-cedd-5d5d-9238ceb22ee0 -43:12.340 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:12.340 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:12.340 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:12.340 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:12.340 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:12.340 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-da12e1da-61ab-cedd-5d5d-9238ceb22ee0 -43:12.340 [INFO] TestAgent.server: member joined, marking health alive: member=Node-da12e1da-61ab-cedd-5d5d-9238ceb22ee0 -43:12.341 [INFO] TestAgent.server: federation state anti-entropy synced -43:12.504 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:12.505 [INFO] TestAgent: Synced node info -43:14.352 [INFO] TestAgent: Requesting shutdown -43:14.352 [INFO] TestAgent.server: shutting down server -43:14.352 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:14.352 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:14.352 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:14.352 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:14.352 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:14.352 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:14.352 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:14.352 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:14.352 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:14.352 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:14.352 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:14.352Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:14.352Z [INFO] agent.router.manager: shutting down -43:14.353 [INFO] TestAgent: consul server down -43:14.353 [INFO] TestAgent: shutdown complete -43:14.353 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13981 network=tcp -43:14.353 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13981 network=udp -43:14.353 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13982 network=tcp -43:14.853 [INFO] TestAgent: Waiting for endpoints to shut down -43:14.853 [INFO] TestAgent: Endpoints down ---- PASS: TestOperator_ServerHealth_Unhealthy (2.63s) -=== CONT TestUiNodes -2022-09-15T21:43:14.179Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:14.209Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:14.212 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bfd2486d-fddd-fcae-2bbc-47d993b6cf98 Address:127.0.0.1:13071}]" -43:14.214 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13071 [Follower]" leader= -43:14.219 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bfd2486d-fddd-fcae-2bbc-47d993b6cf98.dc1 127.0.0.1 -43:14.220 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bfd2486d-fddd-fcae-2bbc-47d993b6cf98 127.0.0.1 -2022-09-15T21:43:14.220Z [INFO] agent.router: Initializing LAN area manager -43:14.221 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13066 network=udp -43:14.221 [INFO] TestAgent.server: Adding LAN server: server="Node-bfd2486d-fddd-fcae-2bbc-47d993b6cf98 (Addr: tcp/127.0.0.1:13071) (DC: dc1)" -43:14.221 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bfd2486d-fddd-fcae-2bbc-47d993b6cf98.dc1 area=wan -43:14.222 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13066 network=tcp -43:14.222 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13067 network=tcp -43:14.223 [INFO] TestAgent: started state syncer -43:14.223 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13072 network=tcp -43:14.258 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:14.258 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13071 [Candidate]" term=2 -43:14.258 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:14.258 [DEBUG] TestAgent.server.raft: vote granted: from=bfd2486d-fddd-fcae-2bbc-47d993b6cf98 term=2 tally=1 -43:14.258 [INFO] TestAgent.server.raft: election won: tally=1 -43:14.258 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13071 [Leader]" -43:14.258 [INFO] TestAgent.server: cluster leadership acquired -43:14.259 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13071 -43:14.259 [INFO] TestAgent.server: New leader elected: payload=Node-bfd2486d-fddd-fcae-2bbc-47d993b6cf98 -43:14.261 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:14.263 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:14.264 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:14.264 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:14.264 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:14.264 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:14.264 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bfd2486d-fddd-fcae-2bbc-47d993b6cf98 -43:14.264 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bfd2486d-fddd-fcae-2bbc-47d993b6cf98 -43:14.265 [INFO] TestAgent.server: federation state anti-entropy synced -43:14.374 [INFO] TestAgent: Requesting shutdown -43:14.374 [INFO] TestAgent.server: shutting down server -43:14.374 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:14.374 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:14.374 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:14.374 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:14.374 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:14.374 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:14.374 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:14.374 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:14.374 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:14.374 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:14.374 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:14.375 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:14.375Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:14.375Z [INFO] agent.router.manager: shutting down -43:14.375 [INFO] TestAgent: consul server down -43:14.375 [INFO] TestAgent: shutdown complete -43:14.375 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13066 network=tcp -43:14.375 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13066 network=udp -43:14.375 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13067 network=tcp -43:14.878 [INFO] TestAgent: Waiting for endpoints to shut down -43:14.878 [INFO] TestAgent: Endpoints down ---- PASS: TestUIGatewayServiceNodes_Terminating (0.75s) -=== CONT TestUiIndex -=== CONT TestTxnEndpoint_UpdateCheck -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:14.300Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:14.319Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:14.331 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a66a077b-d015-68e1-19f1-3dbf00b38c9e Address:127.0.0.1:13113}]" -43:14.331 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13113 [Follower]" leader= -43:14.332 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a66a077b-d015-68e1-19f1-3dbf00b38c9e.dc1 127.0.0.1 -43:14.333 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a66a077b-d015-68e1-19f1-3dbf00b38c9e 127.0.0.1 -2022-09-15T21:43:14.333Z [INFO] agent.router: Initializing LAN area manager -43:14.333 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13080 network=udp -43:14.333 [INFO] TestAgent.server: Adding LAN server: server="Node-a66a077b-d015-68e1-19f1-3dbf00b38c9e (Addr: tcp/127.0.0.1:13113) (DC: dc1)" -43:14.333 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a66a077b-d015-68e1-19f1-3dbf00b38c9e.dc1 area=wan -43:14.334 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13080 network=tcp -43:14.335 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13081 network=tcp -43:14.335 [INFO] TestAgent: started state syncer -43:14.335 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13114 network=tcp -43:14.392 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:14.392 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13113 [Candidate]" term=2 -43:14.393 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:14.393 [DEBUG] TestAgent.server.raft: vote granted: from=a66a077b-d015-68e1-19f1-3dbf00b38c9e term=2 tally=1 -43:14.393 [INFO] TestAgent.server.raft: election won: tally=1 -43:14.393 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13113 [Leader]" -43:14.393 [INFO] TestAgent.server: cluster leadership acquired -43:14.393 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13113 -43:14.394 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:14.395 [INFO] TestAgent.server: New leader elected: payload=Node-a66a077b-d015-68e1-19f1-3dbf00b38c9e -43:14.397 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:14.397 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:14.397 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:14.397 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:14.397 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:14.397 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a66a077b-d015-68e1-19f1-3dbf00b38c9e -43:14.397 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a66a077b-d015-68e1-19f1-3dbf00b38c9e -43:14.398 [INFO] TestAgent.server: federation state anti-entropy synced -43:14.530 [WARN] TestAgent.server: endpoint injected; this should only be used for testing -43:14.530 [INFO] TestAgent: Requesting shutdown -43:14.531 [INFO] TestAgent.server: shutting down server -43:14.531 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:14.531 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:14.531 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:14.531 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:14.531 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:14.531 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:14.531 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:14.531 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:14.531 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:14.531 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:14.531 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:14.531 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:14.531Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:14.531Z [INFO] agent.router.manager: shutting down -43:14.532 [INFO] TestAgent: consul server down -43:14.532 [INFO] TestAgent: shutdown complete -43:14.532 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13080 network=tcp -43:14.532 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13080 network=udp -43:14.532 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13081 network=tcp -43:15.052 [INFO] TestAgent: Waiting for endpoints to shut down -43:15.052 [INFO] TestAgent: Endpoints down -=== RUN TestPreparedQuery_Execute/#05 -2022-09-15T21:43:12.395Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:12.436Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:12.485 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b4f66414-942d-f73d-78ad-f8527f47d641 Address:127.0.0.1:14021}]" -43:12.485 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14021 [Follower]" leader= -43:12.486 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b4f66414-942d-f73d-78ad-f8527f47d641.dc1 127.0.0.1 -43:12.488 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b4f66414-942d-f73d-78ad-f8527f47d641 127.0.0.1 -2022-09-15T21:43:12.488Z [INFO] agent.router: Initializing LAN area manager -43:12.492 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b4f66414-942d-f73d-78ad-f8527f47d641.dc1 area=wan -43:12.492 [INFO] TestAgent.server: Adding LAN server: server="Node-b4f66414-942d-f73d-78ad-f8527f47d641 (Addr: tcp/127.0.0.1:14021) (DC: dc1)" -43:12.493 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14016 network=tcp -43:12.498 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14016 network=udp -43:12.510 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14017 network=tcp -43:12.511 [INFO] TestAgent: started state syncer -43:12.511 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14022 network=tcp -43:12.526 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:12.526 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14021 [Candidate]" term=2 -43:12.526 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:12.526 [DEBUG] TestAgent.server.raft: vote granted: from=b4f66414-942d-f73d-78ad-f8527f47d641 term=2 tally=1 -43:12.526 [INFO] TestAgent.server.raft: election won: tally=1 -43:12.526 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14021 [Leader]" -43:12.526 [INFO] TestAgent.server: cluster leadership acquired -43:12.527 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14021 -43:12.528 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:12.530 [INFO] TestAgent.server: New leader elected: payload=Node-b4f66414-942d-f73d-78ad-f8527f47d641 -43:12.535 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:12.535 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:12.535 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:12.535 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:12.535 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:12.535 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b4f66414-942d-f73d-78ad-f8527f47d641 -43:12.535 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b4f66414-942d-f73d-78ad-f8527f47d641 -43:12.536 [INFO] TestAgent.server: federation state anti-entropy synced -43:12.888 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:12.889 [INFO] TestAgent: Synced node info -43:12.889 [DEBUG] TestAgent: Node info in sync -43:13.756 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:13.756 [DEBUG] TestAgent: Node info in sync -43:14.586 [INFO] TestAgent: Requesting shutdown -43:14.586 [INFO] TestAgent.server: shutting down server -43:14.586 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:14.586 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:14.586 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:14.586 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:14.586 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:14.586 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:14.586 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:14.587 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:14.587 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:14.587 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:14.587 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:14.587Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:14.587Z [INFO] agent.router.manager: shutting down -43:14.587 [INFO] TestAgent: consul server down -43:14.587 [INFO] TestAgent: shutdown complete -43:14.587 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14016 network=tcp -43:14.587 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14016 network=udp -43:14.588 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14017 network=tcp -43:15.088 [INFO] TestAgent: Waiting for endpoints to shut down -43:15.088 [INFO] TestAgent: Endpoints down ---- PASS: TestOperator_ServerHealth (2.73s) -=== CONT TestTxnEndpoint_KV_Actions -=== RUN TestTxnEndpoint_KV_Actions/#00 -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:14.618Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:14.654Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:14.657 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:118e7966-caeb-1bc2-34e2-69ac18237ca9 Address:127.0.0.1:13127}]" -43:14.658 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13127 [Follower]" leader= -43:14.659 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-118e7966-caeb-1bc2-34e2-69ac18237ca9.dc1 127.0.0.1 -43:14.660 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-118e7966-caeb-1bc2-34e2-69ac18237ca9 127.0.0.1 -2022-09-15T21:43:14.660Z [INFO] agent.router: Initializing LAN area manager -43:14.660 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13115 network=udp -43:14.660 [INFO] TestAgent.server: Adding LAN server: server="Node-118e7966-caeb-1bc2-34e2-69ac18237ca9 (Addr: tcp/127.0.0.1:13127) (DC: dc1)" -43:14.660 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-118e7966-caeb-1bc2-34e2-69ac18237ca9.dc1 area=wan -43:14.661 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13115 network=tcp -43:14.662 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13116 network=tcp -43:14.662 [INFO] TestAgent: started state syncer -43:14.662 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13128 network=tcp -43:14.710 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:14.710 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13127 [Candidate]" term=2 -43:14.710 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:14.710 [DEBUG] TestAgent.server.raft: vote granted: from=118e7966-caeb-1bc2-34e2-69ac18237ca9 term=2 tally=1 -43:14.710 [INFO] TestAgent.server.raft: election won: tally=1 -43:14.710 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13127 [Leader]" -43:14.710 [INFO] TestAgent.server: cluster leadership acquired -43:14.710 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13127 -43:14.712 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:14.714 [INFO] TestAgent.server: New leader elected: payload=Node-118e7966-caeb-1bc2-34e2-69ac18237ca9 -43:14.715 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:14.715 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:14.715 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:14.715 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:14.715 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:14.715 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-118e7966-caeb-1bc2-34e2-69ac18237ca9 -43:14.715 [INFO] TestAgent.server: member joined, marking health alive: member=Node-118e7966-caeb-1bc2-34e2-69ac18237ca9 -43:14.716 [INFO] TestAgent.server: federation state anti-entropy synced -43:14.847 [INFO] TestAgent: Requesting shutdown -43:14.848 [INFO] TestAgent.server: shutting down server -43:14.848 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:14.848 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:14.848 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:14.848 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:14.848 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:14.848 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:14.848 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:14.848 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:14.848 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:14.848 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:14.848 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:14.848 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:14.848Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:14.848Z [INFO] agent.router.manager: shutting down -43:14.848 [INFO] TestAgent: consul server down -43:14.848 [INFO] TestAgent: shutdown complete -43:14.848 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13115 network=tcp -43:14.848 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13115 network=udp -43:14.849 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13116 network=tcp -43:15.350 [INFO] TestAgent: Waiting for endpoints to shut down -43:15.350 [INFO] TestAgent: Endpoints down ---- PASS: TestUiNodes_Filter (0.83s) -=== CONT TestTxnEndpoint_Bad_Size_Ops -2022-09-15T21:43:14.315Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:14.346Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:14.349 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:dded3f06-7786-7adb-d034-9bd86242ceef Address:127.0.0.1:13078}]" -43:14.350 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13078 [Follower]" leader= -43:14.350 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-dded3f06-7786-7adb-d034-9bd86242ceef.dc1 127.0.0.1 -43:14.351 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-dded3f06-7786-7adb-d034-9bd86242ceef 127.0.0.1 -2022-09-15T21:43:14.351Z [INFO] agent.router: Initializing LAN area manager -43:14.351 [INFO] TestAgent.server: Adding LAN server: server="Node-dded3f06-7786-7adb-d034-9bd86242ceef (Addr: tcp/127.0.0.1:13078) (DC: dc1)" -43:14.351 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-dded3f06-7786-7adb-d034-9bd86242ceef.dc1 area=wan -43:14.351 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13122 network=udp -43:14.351 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13122 network=tcp -43:14.352 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13123 network=tcp -43:14.353 [INFO] TestAgent: started state syncer -43:14.353 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13079 network=tcp -43:14.400 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:14.400 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13078 [Candidate]" term=2 -43:14.400 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:14.400 [DEBUG] TestAgent.server.raft: vote granted: from=dded3f06-7786-7adb-d034-9bd86242ceef term=2 tally=1 -43:14.400 [INFO] TestAgent.server.raft: election won: tally=1 -43:14.400 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13078 [Leader]" -43:14.401 [INFO] TestAgent.server: cluster leadership acquired -43:14.401 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13078 -43:14.402 [INFO] TestAgent.server: New leader elected: payload=Node-dded3f06-7786-7adb-d034-9bd86242ceef -43:14.402 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:14.405 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:14.405 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:14.405 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:14.405 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:14.405 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:14.405 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-dded3f06-7786-7adb-d034-9bd86242ceef -43:14.405 [INFO] TestAgent.server: member joined, marking health alive: member=Node-dded3f06-7786-7adb-d034-9bd86242ceef -43:14.415 [INFO] TestAgent.server: federation state anti-entropy synced -43:14.588 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:14.589 [INFO] TestAgent: Synced node info -43:14.589 [DEBUG] TestAgent: Node info in sync -43:14.845 [INFO] TestAgent: Requesting shutdown -43:14.853 [INFO] TestAgent.server: shutting down server -43:14.853 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:14.853 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:14.853 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:14.853 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:14.853 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:14.853 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:14.853 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:14.853 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:14.853 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:14.853 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:14.854 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:14.854Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:14.854Z [INFO] agent.router.manager: shutting down -43:14.854 [INFO] TestAgent: consul server down -43:14.854 [INFO] TestAgent: shutdown complete -43:14.854 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13122 network=tcp -43:14.854 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13122 network=udp -43:14.854 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13123 network=tcp -43:15.358 [INFO] TestAgent: Waiting for endpoints to shut down -43:15.358 [INFO] TestAgent: Endpoints down -=== RUN TestTxnEndpoint_Bad_Size_Net/exceeds_default_max_kv_value_size -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:14.585Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:14.642Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:14.645 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fa80f50f-e83a-1e2c-e30f-747de8e28ea5 Address:127.0.0.1:13099}]" -43:14.646 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13099 [Follower]" leader= -43:14.650 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fa80f50f-e83a-1e2c-e30f-747de8e28ea5.dc1 127.0.0.1 -43:14.651 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fa80f50f-e83a-1e2c-e30f-747de8e28ea5 127.0.0.1 -2022-09-15T21:43:14.651Z [INFO] agent.router: Initializing LAN area manager -43:14.651 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13108 network=udp -43:14.651 [INFO] TestAgent.server: Adding LAN server: server="Node-fa80f50f-e83a-1e2c-e30f-747de8e28ea5 (Addr: tcp/127.0.0.1:13099) (DC: dc1)" -43:14.652 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fa80f50f-e83a-1e2c-e30f-747de8e28ea5.dc1 area=wan -43:14.652 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13108 network=tcp -43:14.653 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13109 network=tcp -43:14.653 [INFO] TestAgent: started state syncer -43:14.653 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13100 network=tcp -43:14.694 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:14.694 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13099 [Candidate]" term=2 -43:14.695 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:14.695 [DEBUG] TestAgent.server.raft: vote granted: from=fa80f50f-e83a-1e2c-e30f-747de8e28ea5 term=2 tally=1 -43:14.695 [INFO] TestAgent.server.raft: election won: tally=1 -43:14.695 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13099 [Leader]" -43:14.695 [INFO] TestAgent.server: cluster leadership acquired -43:14.695 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13099 -43:14.696 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:14.698 [INFO] TestAgent.server: New leader elected: payload=Node-fa80f50f-e83a-1e2c-e30f-747de8e28ea5 -43:14.699 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:14.699 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:14.699 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:14.699 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:14.699 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:14.699 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fa80f50f-e83a-1e2c-e30f-747de8e28ea5 -43:14.699 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fa80f50f-e83a-1e2c-e30f-747de8e28ea5 -43:14.700 [INFO] TestAgent.server: federation state anti-entropy synced -43:14.979 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:14.980 [INFO] TestAgent: Synced node info -43:14.980 [DEBUG] TestAgent: Node info in sync -43:15.051 [INFO] TestAgent: Requesting shutdown -43:15.051 [INFO] TestAgent.server: shutting down server -43:15.051 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:15.051 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:15.051 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:15.051 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:15.051 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:15.051 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:15.051 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:15.051 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:15.051 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:15.051 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:15.051 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:15.052Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:15.052Z [INFO] agent.router.manager: shutting down -43:15.052 [INFO] TestAgent: consul server down -43:15.052 [INFO] TestAgent: shutdown complete -43:15.052 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13108 network=tcp -43:15.052 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13108 network=udp -43:15.052 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13109 network=tcp -43:15.556 [INFO] TestAgent: Waiting for endpoints to shut down -43:15.556 [INFO] TestAgent: Endpoints down ---- PASS: TestUiNodeInfo (1.09s) -=== CONT TestSnapshot_Options -=== RUN TestSnapshot_Options/GET -2022-09-15T21:43:15.000Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:15.051Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:15.058 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e82dd685-7062-ecfc-41af-75aa47439b6c Address:127.0.0.1:13106}]" -43:15.059 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13106 [Follower]" leader= -43:15.060 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e82dd685-7062-ecfc-41af-75aa47439b6c.dc1 127.0.0.1 -43:15.060 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e82dd685-7062-ecfc-41af-75aa47439b6c 127.0.0.1 -2022-09-15T21:43:15.061Z [INFO] agent.router: Initializing LAN area manager -43:15.061 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13101 network=udp -43:15.061 [INFO] TestAgent.server: Adding LAN server: server="Node-e82dd685-7062-ecfc-41af-75aa47439b6c (Addr: tcp/127.0.0.1:13106) (DC: dc1)" -43:15.061 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e82dd685-7062-ecfc-41af-75aa47439b6c.dc1 area=wan -43:15.061 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13101 network=tcp -43:15.066 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13102 network=tcp -43:15.066 [INFO] TestAgent: started state syncer -43:15.067 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13107 network=tcp -43:15.130 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:15.130 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13106 [Candidate]" term=2 -43:15.130 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:15.130 [DEBUG] TestAgent.server.raft: vote granted: from=e82dd685-7062-ecfc-41af-75aa47439b6c term=2 tally=1 -43:15.130 [INFO] TestAgent.server.raft: election won: tally=1 -43:15.130 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13106 [Leader]" -43:15.130 [INFO] TestAgent.server: cluster leadership acquired -43:15.131 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13106 -43:15.132 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:15.137 [INFO] TestAgent.server: New leader elected: payload=Node-e82dd685-7062-ecfc-41af-75aa47439b6c -43:15.147 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:15.147 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:15.147 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:15.147 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:15.147 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:15.147 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e82dd685-7062-ecfc-41af-75aa47439b6c -43:15.147 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e82dd685-7062-ecfc-41af-75aa47439b6c -43:15.149 [INFO] TestAgent.server: federation state anti-entropy synced -43:15.208 [INFO] TestAgent: Requesting shutdown -43:15.208 [INFO] TestAgent.server: shutting down server -43:15.208 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:15.208 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:15.208 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:15.208 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:15.208 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:15.208 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:15.208 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:15.208 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:15.208 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:15.208 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:15.208 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:15.208 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:15.209Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:15.209Z [INFO] agent.router.manager: shutting down -43:15.209 [INFO] TestAgent: consul server down -43:15.209 [INFO] TestAgent: shutdown complete -43:15.209 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13101 network=tcp -43:15.209 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13101 network=udp -43:15.209 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13102 network=tcp -43:15.710 [INFO] TestAgent: Waiting for endpoints to shut down -43:15.710 [INFO] TestAgent: Endpoints down ---- PASS: TestTxnEndpoint_UpdateCheck (0.79s) -=== CONT TestTxnEndpoint_Bad_Size_Item -=== RUN TestTxnEndpoint_Bad_Size_Item/exceeds_default_limits -2022-09-15T21:43:14.934Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:15.004Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:15.009 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:96e3ab88-ea0a-2d94-653d-6d01bba3da6e Address:127.0.0.1:14063}]" -43:15.010 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-96e3ab88-ea0a-2d94-653d-6d01bba3da6e.dc1 127.0.0.1 -43:15.011 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14063 [Follower]" leader= -43:15.012 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-96e3ab88-ea0a-2d94-653d-6d01bba3da6e 127.0.0.1 -2022-09-15T21:43:15.012Z [INFO] agent.router: Initializing LAN area manager -43:15.012 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14072 network=udp -43:15.013 [INFO] TestAgent.server: Adding LAN server: server="Node-96e3ab88-ea0a-2d94-653d-6d01bba3da6e (Addr: tcp/127.0.0.1:14063) (DC: dc1)" -43:15.013 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-96e3ab88-ea0a-2d94-653d-6d01bba3da6e.dc1 area=wan -43:15.013 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14072 network=tcp -43:15.015 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14073 network=tcp -43:15.015 [INFO] TestAgent: started state syncer -43:15.015 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14064 network=tcp -43:15.067 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:15.067 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14063 [Candidate]" term=2 -43:15.067 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:15.067 [DEBUG] TestAgent.server.raft: vote granted: from=96e3ab88-ea0a-2d94-653d-6d01bba3da6e term=2 tally=1 -43:15.067 [INFO] TestAgent.server.raft: election won: tally=1 -43:15.067 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14063 [Leader]" -43:15.068 [INFO] TestAgent.server: cluster leadership acquired -43:15.068 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14063 -43:15.069 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:15.071 [INFO] TestAgent.server: New leader elected: payload=Node-96e3ab88-ea0a-2d94-653d-6d01bba3da6e -43:15.072 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:15.072 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:15.072 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:15.072 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:15.072 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:15.072 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-96e3ab88-ea0a-2d94-653d-6d01bba3da6e -43:15.072 [INFO] TestAgent.server: member joined, marking health alive: member=Node-96e3ab88-ea0a-2d94-653d-6d01bba3da6e -43:15.090 [INFO] TestAgent.server: federation state anti-entropy synced -43:15.240 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:15.241 [INFO] TestAgent: Synced node info -43:15.246 [INFO] TestAgent: Requesting shutdown -43:15.246 [INFO] TestAgent.server: shutting down server -43:15.246 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:15.246 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:15.246 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:15.246 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:15.246 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:15.247 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:15.247 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:15.247 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:15.247 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:15.247 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:15.247 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:15.247Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:15.247Z [INFO] agent.router.manager: shutting down -43:15.247 [INFO] TestAgent: consul server down -43:15.247 [INFO] TestAgent: shutdown complete -43:15.247 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14072 network=tcp -43:15.247 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14072 network=udp -43:15.247 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14073 network=tcp -43:15.754 [INFO] TestAgent: Waiting for endpoints to shut down -43:15.754 [INFO] TestAgent: Endpoints down ---- PASS: TestUiNodes (0.90s) -=== CONT TestTxnEndpoint_Bad_JSON -[WARN] freeport: 4 out of 25 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:14.975Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:15.063Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:15.068 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6f50e502-428f-9c61-9c24-b2c2606304c5 Address:127.0.0.1:13120}]" -43:15.069 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6f50e502-428f-9c61-9c24-b2c2606304c5.dc1 127.0.0.1 -43:15.070 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6f50e502-428f-9c61-9c24-b2c2606304c5 127.0.0.1 -2022-09-15T21:43:15.070Z [INFO] agent.router: Initializing LAN area manager -43:15.071 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13129 network=udp -43:15.071 [INFO] TestAgent.server: Adding LAN server: server="Node-6f50e502-428f-9c61-9c24-b2c2606304c5 (Addr: tcp/127.0.0.1:13120) (DC: dc1)" -43:15.071 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6f50e502-428f-9c61-9c24-b2c2606304c5.dc1 area=wan -43:15.078 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13129 network=tcp -43:15.079 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13130 network=tcp -43:15.071 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13120 [Follower]" leader= -43:15.079 [INFO] TestAgent: started state syncer -43:15.079 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13121 network=tcp -43:15.137 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:15.137 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13120 [Candidate]" term=2 -43:15.137 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:15.137 [DEBUG] TestAgent.server.raft: vote granted: from=6f50e502-428f-9c61-9c24-b2c2606304c5 term=2 tally=1 -43:15.137 [INFO] TestAgent.server.raft: election won: tally=1 -43:15.137 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13120 [Leader]" -43:15.137 [INFO] TestAgent.server: cluster leadership acquired -43:15.138 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13120 -43:15.139 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:15.141 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:15.141 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:15.141 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:15.141 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:15.141 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:15.142 [INFO] TestAgent.server: New leader elected: payload=Node-6f50e502-428f-9c61-9c24-b2c2606304c5 -43:15.142 [INFO] TestAgent.server: federation state anti-entropy synced -43:15.142 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6f50e502-428f-9c61-9c24-b2c2606304c5 -43:15.143 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6f50e502-428f-9c61-9c24-b2c2606304c5 -43:15.250 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:15.254 [INFO] TestAgent: Synced node info -43:15.304 [INFO] TestAgent: Requesting shutdown -43:15.304 [INFO] TestAgent.server: shutting down server -43:15.304 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:15.304 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:15.304 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:15.304 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:15.304 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:15.304 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:15.304 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:15.304 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:15.304 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:15.304 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:15.304 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:15.304Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:15.304Z [INFO] agent.router.manager: shutting down -43:15.305 [INFO] TestAgent: consul server down -43:15.305 [INFO] TestAgent: shutdown complete -43:15.305 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13129 network=tcp -43:15.305 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13129 network=udp -43:15.305 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13130 network=tcp -43:15.814 [INFO] TestAgent: Waiting for endpoints to shut down -43:15.814 [INFO] TestAgent: Endpoints down ---- PASS: TestUiIndex (0.94s) -=== CONT TestStatusPeersSecondary -2022-09-15T21:43:15.167Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:15.242Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:15.245 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0fd8cb3b-7f48-ef51-8342-2d12e878f4c7 Address:127.0.0.1:13092}]" -43:15.246 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13092 [Follower]" leader= -43:15.251 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0fd8cb3b-7f48-ef51-8342-2d12e878f4c7.dc1 127.0.0.1 -43:15.252 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0fd8cb3b-7f48-ef51-8342-2d12e878f4c7 127.0.0.1 -2022-09-15T21:43:15.252Z [INFO] agent.router: Initializing LAN area manager -43:15.252 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13087 network=udp -43:15.253 [INFO] TestAgent.server: Adding LAN server: server="Node-0fd8cb3b-7f48-ef51-8342-2d12e878f4c7 (Addr: tcp/127.0.0.1:13092) (DC: dc1)" -43:15.253 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0fd8cb3b-7f48-ef51-8342-2d12e878f4c7.dc1 area=wan -43:15.253 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13087 network=tcp -43:15.255 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13088 network=tcp -43:15.255 [INFO] TestAgent: started state syncer -43:15.255 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13093 network=tcp -43:15.290 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:15.290 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13092 [Candidate]" term=2 -43:15.290 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:15.290 [DEBUG] TestAgent.server.raft: vote granted: from=0fd8cb3b-7f48-ef51-8342-2d12e878f4c7 term=2 tally=1 -43:15.290 [INFO] TestAgent.server.raft: election won: tally=1 -43:15.290 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13092 [Leader]" -43:15.290 [INFO] TestAgent.server: cluster leadership acquired -43:15.291 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13092 -43:15.292 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:15.294 [INFO] TestAgent.server: New leader elected: payload=Node-0fd8cb3b-7f48-ef51-8342-2d12e878f4c7 -43:15.295 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:15.295 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:15.295 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:15.295 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:15.295 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:15.295 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0fd8cb3b-7f48-ef51-8342-2d12e878f4c7 -43:15.295 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0fd8cb3b-7f48-ef51-8342-2d12e878f4c7 -43:15.296 [INFO] TestAgent.server: federation state anti-entropy synced -43:15.426 [WARN] TestAgent.server: endpoint injected; this should only be used for testing -43:15.426 [INFO] TestAgent: Requesting shutdown -43:15.426 [INFO] TestAgent.server: shutting down server -43:15.426 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:15.426 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:15.426 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:15.426 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:15.426 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:15.426 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:15.426 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:15.426 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:15.427 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:15.427 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:15.427 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:15.427 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:15.427Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:15.427Z [INFO] agent.router.manager: shutting down -43:15.427 [INFO] TestAgent: consul server down -43:15.427 [INFO] TestAgent: shutdown complete -43:15.427 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13087 network=tcp -43:15.427 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13087 network=udp -43:15.427 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13088 network=tcp -43:15.934 [INFO] TestAgent: Waiting for endpoints to shut down -43:15.934 [INFO] TestAgent: Endpoints down -=== RUN TestPreparedQuery_Execute/#06 -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:15.116Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:15.167Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:15.170 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:83126af7-1890-5b12-e82b-6f3de00d8504 Address:127.0.0.1:13190}]" -43:15.171 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-83126af7-1890-5b12-e82b-6f3de00d8504.dc1 127.0.0.1 -43:15.172 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-83126af7-1890-5b12-e82b-6f3de00d8504 127.0.0.1 -2022-09-15T21:43:15.172Z [INFO] agent.router: Initializing LAN area manager -43:15.173 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13171 network=udp -43:15.173 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13190 [Follower]" leader= -43:15.173 [INFO] TestAgent.server: Adding LAN server: server="Node-83126af7-1890-5b12-e82b-6f3de00d8504 (Addr: tcp/127.0.0.1:13190) (DC: dc1)" -43:15.173 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-83126af7-1890-5b12-e82b-6f3de00d8504.dc1 area=wan -43:15.184 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13171 network=tcp -43:15.185 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13172 network=tcp -43:15.185 [INFO] TestAgent: started state syncer -43:15.186 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13191 network=tcp -43:15.222 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:15.222 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13190 [Candidate]" term=2 -43:15.222 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:15.222 [DEBUG] TestAgent.server.raft: vote granted: from=83126af7-1890-5b12-e82b-6f3de00d8504 term=2 tally=1 -43:15.222 [INFO] TestAgent.server.raft: election won: tally=1 -43:15.222 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13190 [Leader]" -43:15.222 [INFO] TestAgent.server: cluster leadership acquired -43:15.222 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13190 -43:15.224 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:15.226 [INFO] TestAgent.server: New leader elected: payload=Node-83126af7-1890-5b12-e82b-6f3de00d8504 -43:15.229 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:15.229 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:15.229 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:15.229 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:15.229 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:15.229 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-83126af7-1890-5b12-e82b-6f3de00d8504 -43:15.229 [INFO] TestAgent.server: member joined, marking health alive: member=Node-83126af7-1890-5b12-e82b-6f3de00d8504 -43:15.230 [INFO] TestAgent.server: federation state anti-entropy synced -43:15.589 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:15.590 [INFO] TestAgent: Synced node info -43:15.590 [DEBUG] TestAgent: Node info in sync -43:15.612 [INFO] TestAgent: Requesting shutdown -43:15.612 [INFO] TestAgent.server: shutting down server -43:15.612 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:15.612 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:15.612 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:15.612 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:15.612 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:15.612 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:15.612 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:15.612 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:15.612 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:15.612 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:15.612 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:15.613Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:15.613Z [INFO] agent.router.manager: shutting down -43:15.613 [INFO] TestAgent: consul server down -43:15.613 [INFO] TestAgent: shutdown complete -43:15.613 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13171 network=tcp -43:15.613 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13171 network=udp -43:15.613 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13172 network=tcp -43:16.114 [INFO] TestAgent: Waiting for endpoints to shut down -43:16.114 [INFO] TestAgent: Endpoints down -=== RUN TestTxnEndpoint_KV_Actions/#01 -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:15.415Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:15.439Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:15.442 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b6757be8-5f53-7315-22d6-def08452bffa Address:127.0.0.1:13148}]" -43:15.443 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b6757be8-5f53-7315-22d6-def08452bffa.dc1 127.0.0.1 -43:15.444 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b6757be8-5f53-7315-22d6-def08452bffa 127.0.0.1 -2022-09-15T21:43:15.444Z [INFO] agent.router: Initializing LAN area manager -43:15.445 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13136 network=udp -43:15.445 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13148 [Follower]" leader= -43:15.445 [INFO] TestAgent.server: Adding LAN server: server="Node-b6757be8-5f53-7315-22d6-def08452bffa (Addr: tcp/127.0.0.1:13148) (DC: dc1)" -43:15.445 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b6757be8-5f53-7315-22d6-def08452bffa.dc1 area=wan -43:15.445 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13136 network=tcp -43:15.446 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13137 network=tcp -43:15.447 [INFO] TestAgent: started state syncer -43:15.447 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13149 network=tcp -43:15.519 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:15.519 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13148 [Candidate]" term=2 -43:15.519 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:15.519 [DEBUG] TestAgent.server.raft: vote granted: from=b6757be8-5f53-7315-22d6-def08452bffa term=2 tally=1 -43:15.519 [INFO] TestAgent.server.raft: election won: tally=1 -43:15.519 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13148 [Leader]" -43:15.519 [INFO] TestAgent.server: cluster leadership acquired -43:15.520 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13148 -43:15.521 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:15.522 [INFO] TestAgent.server: New leader elected: payload=Node-b6757be8-5f53-7315-22d6-def08452bffa -43:15.524 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:15.524 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:15.524 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:15.524 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:15.524 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:15.524 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b6757be8-5f53-7315-22d6-def08452bffa -43:15.524 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b6757be8-5f53-7315-22d6-def08452bffa -43:15.525 [INFO] TestAgent.server: federation state anti-entropy synced -43:15.530 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:15.531 [INFO] TestAgent: Synced node info -43:15.531 [DEBUG] TestAgent: Node info in sync -43:15.556 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:15.556 [DEBUG] TestAgent: Node info in sync -43:15.792 [INFO] TestAgent: Requesting shutdown -43:15.792 [INFO] TestAgent.server: shutting down server -43:15.792 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:15.792 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:15.792 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:15.792 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:15.792 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:15.793 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:15.793 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:15.793 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:15.793 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:15.793 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:15.793 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:15.793Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:15.793Z [INFO] agent.router.manager: shutting down -43:15.793 [INFO] TestAgent: consul server down -43:15.793 [INFO] TestAgent: shutdown complete -43:15.793 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13136 network=tcp -43:15.793 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13136 network=udp -43:15.798 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13137 network=tcp -43:16.298 [INFO] TestAgent: Waiting for endpoints to shut down -43:16.298 [INFO] TestAgent: Endpoints down ---- PASS: TestTxnEndpoint_Bad_Size_Ops (0.95s) -=== CONT TestStatusPeers -2022-09-15T21:43:15.608Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:15.608Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:15.660Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:15.660Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:15.671 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:658225d7-83f5-c01f-9400-64b8dc4968e7 Address:127.0.0.1:14049}]" -43:15.672 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-658225d7-83f5-c01f-9400-64b8dc4968e7.dc1 127.0.0.1 -43:15.673 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-658225d7-83f5-c01f-9400-64b8dc4968e7 127.0.0.1 -2022-09-15T21:43:15.673Z [INFO] agent.router: Initializing LAN area manager -43:15.673 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14037 network=udp -43:15.673 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14049 [Follower]" leader= -43:15.678 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14037 network=tcp -43:15.679 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14038 network=tcp -43:15.679 [INFO] TestAgent: started state syncer -43:15.679 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14050 network=tcp -43:15.679 [INFO] TestAgent.server: Adding LAN server: server="Node-658225d7-83f5-c01f-9400-64b8dc4968e7 (Addr: tcp/127.0.0.1:14049) (DC: dc1)" -43:15.687 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-658225d7-83f5-c01f-9400-64b8dc4968e7.dc1 area=wan -43:15.730 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -43:15.734 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:15.734 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14049 [Candidate]" term=2 -43:15.734 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:15.734 [DEBUG] TestAgent.server.raft: vote granted: from=658225d7-83f5-c01f-9400-64b8dc4968e7 term=2 tally=1 -43:15.734 [INFO] TestAgent.server.raft: election won: tally=1 -43:15.734 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14049 [Leader]" -43:15.734 [INFO] TestAgent.server: cluster leadership acquired -43:15.735 [INFO] TestAgent.server: initializing acls -43:15.735 [INFO] TestAgent.server: Created ACL 'global-management' policy -43:15.735 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -43:15.736 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -43:15.736 [INFO] TestAgent.server: Created ACL anonymous token from configuration -43:15.736 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -43:15.736 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -43:15.737 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-658225d7-83f5-c01f-9400-64b8dc4968e7 -43:15.737 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-658225d7-83f5-c01f-9400-64b8dc4968e7.dc1 -43:15.738 [INFO] TestAgent.server: New leader elected: payload=Node-658225d7-83f5-c01f-9400-64b8dc4968e7 -43:15.738 [INFO] TestAgent.server: Updating LAN server: server="Node-658225d7-83f5-c01f-9400-64b8dc4968e7 (Addr: tcp/127.0.0.1:14049) (DC: dc1)" -43:15.738 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-658225d7-83f5-c01f-9400-64b8dc4968e7.dc1 area=wan -43:15.738 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:15.758 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:15.758 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:15.758 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:15.758 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:15.758 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:15.758 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-658225d7-83f5-c01f-9400-64b8dc4968e7 -43:15.758 [INFO] TestAgent.server: member joined, marking health alive: member=Node-658225d7-83f5-c01f-9400-64b8dc4968e7 -43:15.759 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-658225d7-83f5-c01f-9400-64b8dc4968e7 -43:15.760 [INFO] TestAgent.server: federation state anti-entropy synced -43:15.914 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-658225d7-83f5-c01f-9400-64b8dc4968e7 -43:15.914 [INFO] TestAgent: Requesting shutdown -43:15.914 [INFO] TestAgent.server: shutting down server -43:15.914 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -43:15.914 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -43:15.914 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:15.914 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:15.914 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:15.914 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:15.914 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:15.914 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:15.914 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:15.914 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:15.914 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -43:15.914 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -43:15.914 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:15.914 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:15.914 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:15.915 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:15.915Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:15.915Z [INFO] agent.router.manager: shutting down -43:15.915 [INFO] TestAgent: consul server down -43:15.915 [INFO] TestAgent: shutdown complete -43:15.915 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14037 network=tcp -43:15.915 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14037 network=udp -43:15.915 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14038 network=tcp -43:16.418 [INFO] TestAgent: Waiting for endpoints to shut down -43:16.418 [INFO] TestAgent: Endpoints down -=== RUN TestSnapshot_Options/GET#01 -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:15.782Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:15.872Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:15.884 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4aac8ebc-0422-173a-d0d6-0edd7a84f179 Address:127.0.0.1:13986}]" -43:15.885 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13986 [Follower]" leader= -43:15.886 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4aac8ebc-0422-173a-d0d6-0edd7a84f179.dc1 127.0.0.1 -43:15.888 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4aac8ebc-0422-173a-d0d6-0edd7a84f179 127.0.0.1 -2022-09-15T21:43:15.888Z [INFO] agent.router: Initializing LAN area manager -43:15.888 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14009 network=udp -43:15.889 [INFO] TestAgent.server: Adding LAN server: server="Node-4aac8ebc-0422-173a-d0d6-0edd7a84f179 (Addr: tcp/127.0.0.1:13986) (DC: dc1)" -43:15.889 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4aac8ebc-0422-173a-d0d6-0edd7a84f179.dc1 area=wan -43:15.889 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14009 network=tcp -43:15.922 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14010 network=tcp -43:15.922 [INFO] TestAgent: started state syncer -43:15.922 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13987 network=tcp -43:15.926 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:15.926 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13986 [Candidate]" term=2 -43:15.926 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:15.926 [DEBUG] TestAgent.server.raft: vote granted: from=4aac8ebc-0422-173a-d0d6-0edd7a84f179 term=2 tally=1 -43:15.926 [INFO] TestAgent.server.raft: election won: tally=1 -43:15.926 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13986 [Leader]" -43:15.926 [INFO] TestAgent.server: cluster leadership acquired -43:15.927 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13986 -43:15.928 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:15.934 [INFO] TestAgent.server: New leader elected: payload=Node-4aac8ebc-0422-173a-d0d6-0edd7a84f179 -43:15.951 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:15.951 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:15.951 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:15.951 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:15.951 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:15.951 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4aac8ebc-0422-173a-d0d6-0edd7a84f179 -43:15.951 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4aac8ebc-0422-173a-d0d6-0edd7a84f179 -43:15.953 [INFO] TestAgent.server: federation state anti-entropy synced -43:16.143 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:16.144 [INFO] TestAgent: Synced node info -43:16.144 [DEBUG] TestAgent: Node info in sync -43:16.182 [INFO] TestAgent: Requesting shutdown -43:16.182 [INFO] TestAgent.server: shutting down server -43:16.183 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:16.183 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:16.183 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:16.183 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:16.183 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:16.183 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:16.183 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:16.183 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:16.183 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:16.183 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:16.183 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:16.183Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:16.183Z [INFO] agent.router.manager: shutting down -43:16.184 [INFO] TestAgent: consul server down -43:16.184 [INFO] TestAgent: shutdown complete -43:16.184 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14009 network=tcp -43:16.184 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14009 network=udp -43:16.184 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14010 network=tcp -43:16.686 [INFO] TestAgent: Waiting for endpoints to shut down -43:16.686 [INFO] TestAgent: Endpoints down -=== RUN TestTxnEndpoint_Bad_Size_Item/exceeds_configured_max_txn_len -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:15.873Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:15.962Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:15.965 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e2679185-9126-7f07-9a36-2db5acaf8583 Address:127.0.0.1:13176}]" -43:15.966 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13176 [Follower]" leader= -43:15.967 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e2679185-9126-7f07-9a36-2db5acaf8583.dc1 127.0.0.1 -43:15.968 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e2679185-9126-7f07-9a36-2db5acaf8583 127.0.0.1 -2022-09-15T21:43:15.968Z [INFO] agent.router: Initializing LAN area manager -43:15.968 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13157 network=udp -43:15.969 [INFO] TestAgent.server: Adding LAN server: server="Node-e2679185-9126-7f07-9a36-2db5acaf8583 (Addr: tcp/127.0.0.1:13176) (DC: dc1)" -43:15.969 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e2679185-9126-7f07-9a36-2db5acaf8583.dc1 area=wan -43:15.969 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13157 network=tcp -43:15.974 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13158 network=tcp -43:15.974 [INFO] TestAgent: started state syncer -43:15.974 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13177 network=tcp -43:16.031 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:16.031 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13176 [Candidate]" term=2 -43:16.031 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:16.031 [DEBUG] TestAgent.server.raft: vote granted: from=e2679185-9126-7f07-9a36-2db5acaf8583 term=2 tally=1 -43:16.031 [INFO] TestAgent.server.raft: election won: tally=1 -43:16.031 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13176 [Leader]" -43:16.031 [INFO] TestAgent.server: cluster leadership acquired -43:16.032 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13176 -43:16.033 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:16.034 [INFO] TestAgent.server: New leader elected: payload=Node-e2679185-9126-7f07-9a36-2db5acaf8583 -43:16.036 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:16.036 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:16.036 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:16.036 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:16.036 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:16.036 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e2679185-9126-7f07-9a36-2db5acaf8583 -43:16.036 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e2679185-9126-7f07-9a36-2db5acaf8583 -43:16.037 [INFO] TestAgent.server: federation state anti-entropy synced -43:16.278 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:16.279 [INFO] TestAgent: Synced node info -43:16.279 [DEBUG] TestAgent: Node info in sync -43:16.298 [INFO] TestAgent: Requesting shutdown -43:16.298 [INFO] TestAgent.server: shutting down server -43:16.298 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:16.298 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:16.298 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:16.298 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:16.298 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:16.298 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:16.298 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:16.298 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:16.298 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:16.298 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:16.298 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:16.299Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:16.299Z [INFO] agent.router.manager: shutting down -43:16.299 [INFO] TestAgent: consul server down -43:16.299 [INFO] TestAgent: shutdown complete -43:16.299 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13157 network=tcp -43:16.299 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13157 network=udp -43:16.299 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13158 network=tcp -43:16.800 [INFO] TestAgent: Waiting for endpoints to shut down -43:16.800 [INFO] TestAgent: Endpoints down ---- PASS: TestTxnEndpoint_Bad_JSON (1.04s) -=== CONT TestStatusLeaderSecondary -2022-09-15T21:43:15.995Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:16.067Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:16.087 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5c7936d6-e159-088b-3214-f2da87aff946 Address:127.0.0.1:13162}]" -43:16.088 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5c7936d6-e159-088b-3214-f2da87aff946.dc1 127.0.0.1 -43:16.089 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5c7936d6-e159-088b-3214-f2da87aff946 127.0.0.1 -2022-09-15T21:43:16.089Z [INFO] agent.router: Initializing LAN area manager -43:16.089 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13150 network=udp -43:16.089 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13162 [Follower]" leader= -43:16.090 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5c7936d6-e159-088b-3214-f2da87aff946.dc1 area=wan -43:16.090 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13150 network=tcp -43:16.091 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13151 network=tcp -43:16.091 [INFO] TestAgent: started state syncer -43:16.091 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13163 network=tcp -43:16.106 [INFO] TestAgent.server: Adding LAN server: server="Node-5c7936d6-e159-088b-3214-f2da87aff946 (Addr: tcp/127.0.0.1:13162) (DC: dc1)" -43:16.164 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:16.164 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13162 [Candidate]" term=2 -43:16.164 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:16.164 [DEBUG] TestAgent.server.raft: vote granted: from=5c7936d6-e159-088b-3214-f2da87aff946 term=2 tally=1 -43:16.164 [INFO] TestAgent.server.raft: election won: tally=1 -43:16.164 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13162 [Leader]" -43:16.164 [INFO] TestAgent.server: cluster leadership acquired -43:16.165 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13162 -43:16.166 [INFO] TestAgent.server: New leader elected: payload=Node-5c7936d6-e159-088b-3214-f2da87aff946 -43:16.175 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:16.186 [INFO] TestAgent: Synced node info -43:16.186 [DEBUG] TestAgent: Node info in sync -43:16.186 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:16.186 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:16.186 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:16.186 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:16.186 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:16.186 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5c7936d6-e159-088b-3214-f2da87aff946 -43:16.186 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5c7936d6-e159-088b-3214-f2da87aff946 -43:16.187 [INFO] TestAgent.server: federation state anti-entropy synced -43:16.404 [WARN] TestAgent.server: endpoint injected; this should only be used for testing -43:16.404 [INFO] TestAgent: Requesting shutdown -43:16.404 [INFO] TestAgent.server: shutting down server -43:16.404 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:16.404 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:16.404 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:16.404 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:16.404 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:16.405 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:16.405 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:16.405 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:16.405 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:16.405 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:16.405 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:16.405Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:16.405Z [INFO] agent.router.manager: shutting down -43:16.405 [INFO] TestAgent: consul server down -43:16.405 [INFO] TestAgent: shutdown complete -43:16.405 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13150 network=tcp -43:16.405 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13150 network=udp -43:16.405 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13151 network=tcp -43:16.906 [INFO] TestAgent: Waiting for endpoints to shut down -43:16.906 [INFO] TestAgent: Endpoints down -=== RUN TestPreparedQuery_Execute/#07 -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:16.329Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:16.350Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:16.353 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:13bb416c-817a-2d27-42cf-aba06bf66201 Address:127.0.0.1:13197}]" -43:16.360 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-13bb416c-817a-2d27-42cf-aba06bf66201.dc1 127.0.0.1 -43:16.360 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13197 [Follower]" leader= -43:16.361 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-13bb416c-817a-2d27-42cf-aba06bf66201 127.0.0.1 -2022-09-15T21:43:16.361Z [INFO] agent.router: Initializing LAN area manager -43:16.361 [INFO] TestAgent.server: Adding LAN server: server="Node-13bb416c-817a-2d27-42cf-aba06bf66201 (Addr: tcp/127.0.0.1:13197) (DC: dc1)" -43:16.361 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-13bb416c-817a-2d27-42cf-aba06bf66201.dc1 area=wan -43:16.361 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13206 network=udp -43:16.363 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13206 network=tcp -43:16.364 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13207 network=tcp -43:16.364 [INFO] TestAgent: started state syncer -43:16.364 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13198 network=tcp -43:16.402 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:16.402 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13197 [Candidate]" term=2 -43:16.402 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:16.402 [DEBUG] TestAgent.server.raft: vote granted: from=13bb416c-817a-2d27-42cf-aba06bf66201 term=2 tally=1 -43:16.402 [INFO] TestAgent.server.raft: election won: tally=1 -43:16.402 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13197 [Leader]" -43:16.402 [INFO] TestAgent.server: cluster leadership acquired -43:16.403 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13197 -43:16.404 [INFO] TestAgent.server: New leader elected: payload=Node-13bb416c-817a-2d27-42cf-aba06bf66201 -43:16.404 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:16.407 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:16.407 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:16.407 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:16.407 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:16.407 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:16.407 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-13bb416c-817a-2d27-42cf-aba06bf66201 -43:16.407 [INFO] TestAgent.server: member joined, marking health alive: member=Node-13bb416c-817a-2d27-42cf-aba06bf66201 -43:16.408 [INFO] TestAgent.server: federation state anti-entropy synced -43:16.538 [INFO] TestAgent: Requesting shutdown -43:16.538 [INFO] TestAgent.server: shutting down server -43:16.538 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:16.538 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:16.538 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:16.538 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:16.538 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:16.538 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:16.538 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:16.538 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:16.538 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:16.538 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:16.538 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:16.538 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:16.539Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:16.539Z [INFO] agent.router.manager: shutting down -43:16.546 [INFO] TestAgent: consul server down -43:16.546 [INFO] TestAgent: shutdown complete -43:16.546 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13206 network=tcp -43:16.546 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13206 network=udp -43:16.546 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13207 network=tcp -43:17.047 [INFO] TestAgent: Waiting for endpoints to shut down -43:17.047 [INFO] TestAgent: Endpoints down ---- PASS: TestStatusPeers (0.75s) -=== CONT TestStatusLeader -2022-09-15T21:43:16.185Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:16.248Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:16.251 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:64e5945a-aea4-b5ec-e53c-9cbe122cfe6b Address:127.0.0.1:13141}]" -43:16.252 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-64e5945a-aea4-b5ec-e53c-9cbe122cfe6b.dc1 127.0.0.1 -43:16.253 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-64e5945a-aea4-b5ec-e53c-9cbe122cfe6b 127.0.0.1 -2022-09-15T21:43:16.253Z [INFO] agent.router: Initializing LAN area manager -43:16.254 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13141 [Follower]" leader= -43:16.254 [INFO] TestAgent.server: Adding LAN server: server="Node-64e5945a-aea4-b5ec-e53c-9cbe122cfe6b (Addr: tcp/127.0.0.1:13141) (DC: dc1)" -43:16.254 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-64e5945a-aea4-b5ec-e53c-9cbe122cfe6b.dc1 area=wan -43:16.254 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13164 network=tcp -43:16.254 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13164 network=udp -43:16.255 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13165 network=tcp -43:16.255 [INFO] TestAgent: started state syncer -43:16.255 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13142 network=tcp -43:16.298 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:16.298 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13141 [Candidate]" term=2 -43:16.298 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:16.298 [DEBUG] TestAgent.server.raft: vote granted: from=64e5945a-aea4-b5ec-e53c-9cbe122cfe6b term=2 tally=1 -43:16.298 [INFO] TestAgent.server.raft: election won: tally=1 -43:16.298 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13141 [Leader]" -43:16.298 [INFO] TestAgent.server: cluster leadership acquired -43:16.298 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13141 -43:16.299 [INFO] TestAgent.server: New leader elected: payload=Node-64e5945a-aea4-b5ec-e53c-9cbe122cfe6b -43:16.300 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:16.302 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:16.302 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:16.302 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:16.302 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:16.302 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:16.302 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-64e5945a-aea4-b5ec-e53c-9cbe122cfe6b -43:16.302 [INFO] TestAgent.server: member joined, marking health alive: member=Node-64e5945a-aea4-b5ec-e53c-9cbe122cfe6b -43:16.304 [INFO] TestAgent.server: federation state anti-entropy synced -43:16.555 [INFO] TestAgent: Requesting shutdown -43:16.555 [INFO] TestAgent.server: shutting down server -43:16.555 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:16.555 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:16.555 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:16.555 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:16.555 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:16.555 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:16.555 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:16.555 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:16.555 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:16.555 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:16.555 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:16.555 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:16.556Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:16.556Z [INFO] agent.router.manager: shutting down -43:16.556 [INFO] TestAgent: consul server down -43:16.556 [INFO] TestAgent: shutdown complete -43:16.556 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13164 network=tcp -43:16.556 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13164 network=udp -43:16.556 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13165 network=tcp -43:17.064 [INFO] TestAgent: Waiting for endpoints to shut down -43:17.064 [INFO] TestAgent: Endpoints down ---- PASS: TestTxnEndpoint_KV_Actions (1.98s) - --- PASS: TestTxnEndpoint_KV_Actions/#00 (1.03s) - --- PASS: TestTxnEndpoint_KV_Actions/#01 (0.95s) -=== CONT TestIntentionsSpecificUpdate_good -2022-09-15T21:43:16.478Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:16.478Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:16.541Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:16.541Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:16.544 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f29f8d15-5bc7-6d86-8349-a8dc5e5805b2 Address:127.0.0.1:13211}]" -43:16.545 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f29f8d15-5bc7-6d86-8349-a8dc5e5805b2.dc1 127.0.0.1 -43:16.546 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13211 [Follower]" leader= -43:16.546 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f29f8d15-5bc7-6d86-8349-a8dc5e5805b2 127.0.0.1 -2022-09-15T21:43:16.546Z [INFO] agent.router: Initializing LAN area manager -43:16.547 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13192 network=udp -43:16.547 [INFO] TestAgent.server: Adding LAN server: server="Node-f29f8d15-5bc7-6d86-8349-a8dc5e5805b2 (Addr: tcp/127.0.0.1:13211) (DC: dc1)" -43:16.547 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f29f8d15-5bc7-6d86-8349-a8dc5e5805b2.dc1 area=wan -43:16.547 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13192 network=tcp -43:16.548 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13193 network=tcp -43:16.548 [INFO] TestAgent: started state syncer -43:16.548 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13212 network=tcp -43:16.591 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:16.591 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13211 [Candidate]" term=2 -43:16.591 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:16.591 [DEBUG] TestAgent.server.raft: vote granted: from=f29f8d15-5bc7-6d86-8349-a8dc5e5805b2 term=2 tally=1 -43:16.591 [INFO] TestAgent.server.raft: election won: tally=1 -43:16.591 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13211 [Leader]" -43:16.592 [INFO] TestAgent.server: cluster leadership acquired -43:16.592 [INFO] TestAgent.server: initializing acls -43:16.592 [INFO] TestAgent.server: Created ACL 'global-management' policy -43:16.592 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -43:16.593 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -43:16.597 [INFO] TestAgent.server: New leader elected: payload=Node-f29f8d15-5bc7-6d86-8349-a8dc5e5805b2 -43:16.597 [INFO] TestAgent.server: initializing acls -43:16.597 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -43:16.598 [INFO] TestAgent.server: Created ACL anonymous token from configuration -43:16.598 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -43:16.598 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -43:16.598 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode -43:16.599 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-f29f8d15-5bc7-6d86-8349-a8dc5e5805b2 -43:16.599 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-f29f8d15-5bc7-6d86-8349-a8dc5e5805b2.dc1 -43:16.599 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-f29f8d15-5bc7-6d86-8349-a8dc5e5805b2.dc1 area=wan -43:16.599 [INFO] TestAgent.server: Created ACL anonymous token from configuration -43:16.599 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-f29f8d15-5bc7-6d86-8349-a8dc5e5805b2 -43:16.599 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-f29f8d15-5bc7-6d86-8349-a8dc5e5805b2.dc1 -43:16.601 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:16.606 [INFO] TestAgent.server: Updating LAN server: server="Node-f29f8d15-5bc7-6d86-8349-a8dc5e5805b2 (Addr: tcp/127.0.0.1:13211) (DC: dc1)" -43:16.606 [INFO] TestAgent.server: Updating LAN server: server="Node-f29f8d15-5bc7-6d86-8349-a8dc5e5805b2 (Addr: tcp/127.0.0.1:13211) (DC: dc1)" -43:16.606 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-f29f8d15-5bc7-6d86-8349-a8dc5e5805b2.dc1 area=wan -43:16.608 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:16.608 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:16.608 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:16.608 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:16.608 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:16.608 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f29f8d15-5bc7-6d86-8349-a8dc5e5805b2 -43:16.608 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f29f8d15-5bc7-6d86-8349-a8dc5e5805b2 -43:16.609 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f29f8d15-5bc7-6d86-8349-a8dc5e5805b2 -43:16.609 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f29f8d15-5bc7-6d86-8349-a8dc5e5805b2 -43:16.610 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-f29f8d15-5bc7-6d86-8349-a8dc5e5805b2 -43:16.610 [INFO] TestAgent: Requesting shutdown -43:16.610 [INFO] TestAgent.server: shutting down server -43:16.610 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -43:16.610 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:16.610 [INFO] TestAgent.server: federation state anti-entropy synced -43:16.610 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -43:16.610 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:16.610 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:16.610 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:16.610 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:16.610 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -43:16.610 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:16.610 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:16.610 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -43:16.610 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:16.610 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:16.610 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:16.610 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:16.610Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:16.610Z [INFO] agent.router.manager: shutting down -43:16.611 [INFO] TestAgent: consul server down -43:16.611 [INFO] TestAgent: shutdown complete -43:16.611 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13192 network=tcp -43:16.611 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13192 network=udp -43:16.611 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13193 network=tcp -43:17.117 [INFO] TestAgent: Waiting for endpoints to shut down -43:17.117 [INFO] TestAgent: Endpoints down -=== RUN TestSnapshot_Options/GET#02 -2022-09-15T21:43:15.421Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:15.475Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:15.490 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ed1f6f80-6c1d-ed7b-bd1c-172690782c83 Address:127.0.0.1:13155}]" -43:15.491 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ed1f6f80-6c1d-ed7b-bd1c-172690782c83.dc1 127.0.0.1 -43:15.492 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ed1f6f80-6c1d-ed7b-bd1c-172690782c83 127.0.0.1 -2022-09-15T21:43:15.492Z [INFO] agent.router: Initializing LAN area manager -43:15.493 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13143 network=udp -43:15.493 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13155 [Follower]" leader= -43:15.493 [INFO] TestAgent.server: Adding LAN server: server="Node-ed1f6f80-6c1d-ed7b-bd1c-172690782c83 (Addr: tcp/127.0.0.1:13155) (DC: dc1)" -43:15.493 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ed1f6f80-6c1d-ed7b-bd1c-172690782c83.dc1 area=wan -43:15.493 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13143 network=tcp -43:15.499 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13144 network=tcp -43:15.499 [INFO] TestAgent: started state syncer -43:15.499 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13156 network=tcp -43:15.544 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:15.544 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13155 [Candidate]" term=2 -43:15.544 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:15.545 [DEBUG] TestAgent.server.raft: vote granted: from=ed1f6f80-6c1d-ed7b-bd1c-172690782c83 term=2 tally=1 -43:15.545 [INFO] TestAgent.server.raft: election won: tally=1 -43:15.545 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13155 [Leader]" -43:15.545 [INFO] TestAgent.server: cluster leadership acquired -43:15.545 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13155 -43:15.546 [INFO] TestAgent.server: New leader elected: payload=Node-ed1f6f80-6c1d-ed7b-bd1c-172690782c83 -43:15.546 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:15.549 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:15.549 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:15.549 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:15.549 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:15.549 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:15.549 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ed1f6f80-6c1d-ed7b-bd1c-172690782c83 -43:15.549 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ed1f6f80-6c1d-ed7b-bd1c-172690782c83 -43:15.551 [INFO] TestAgent.server: federation state anti-entropy synced -43:15.578 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:15.579 [INFO] TestAgent: Synced node info -43:15.579 [DEBUG] TestAgent: Node info in sync -43:16.404 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:16.404 [DEBUG] TestAgent: Node info in sync -43:16.643 [INFO] TestAgent: Requesting shutdown -43:16.644 [INFO] TestAgent.server: shutting down server -43:16.644 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:16.644 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:16.644 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:16.644 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:16.644 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:16.644 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:16.644 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -43:16.644 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:16.644 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:16.644 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:16.644 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2022-09-15T21:43:16.644Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:16.644Z [INFO] agent.router.manager: shutting down -43:16.644 [INFO] TestAgent: consul server down -43:16.644 [INFO] TestAgent: shutdown complete -43:16.644 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13143 network=tcp -43:16.644 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13143 network=udp -43:16.644 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13144 network=tcp -43:17.146 [INFO] TestAgent: Waiting for endpoints to shut down -43:17.146 [INFO] TestAgent: Endpoints down -=== RUN TestTxnEndpoint_Bad_Size_Net/allowed -[WARN] freeport: 4 out of 32 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:16.269Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:16.286Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:16.289 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d973ff1a-ccf0-3ad7-de02-23f9f10b6832 Address:127.0.0.1:13232}]" -43:16.290 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13232 [Follower]" leader= -43:16.290 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d973ff1a-ccf0-3ad7-de02-23f9f10b6832.secondary 127.0.0.1 -43:16.291 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d973ff1a-ccf0-3ad7-de02-23f9f10b6832 127.0.0.1 -2022-09-15T21:43:16.291Z [INFO] agent.router: Initializing LAN area manager -43:16.292 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13227 network=udp -43:16.292 [INFO] TestAgent.server: Adding LAN server: server="Node-d973ff1a-ccf0-3ad7-de02-23f9f10b6832 (Addr: tcp/127.0.0.1:13232) (DC: secondary)" -43:16.292 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d973ff1a-ccf0-3ad7-de02-23f9f10b6832.secondary area=wan -43:16.292 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13227 network=tcp -43:16.293 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13228 network=tcp -43:16.293 [INFO] TestAgent: started state syncer -43:16.293 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13233 network=tcp -43:16.342 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:16.342 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13232 [Candidate]" term=2 -43:16.342 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:16.342 [DEBUG] TestAgent.server.raft: vote granted: from=d973ff1a-ccf0-3ad7-de02-23f9f10b6832 term=2 tally=1 -43:16.342 [INFO] TestAgent.server.raft: election won: tally=1 -43:16.342 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13232 [Leader]" -43:16.342 [INFO] TestAgent.server: cluster leadership acquired -43:16.343 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13232 -43:16.344 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:16.344 [INFO] TestAgent.server: New leader elected: payload=Node-d973ff1a-ccf0-3ad7-de02-23f9f10b6832 -43:16.346 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:16.346 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:16.346 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:16.346 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:16.346 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:16.347 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d973ff1a-ccf0-3ad7-de02-23f9f10b6832 -43:16.347 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d973ff1a-ccf0-3ad7-de02-23f9f10b6832 -43:16.348 [INFO] TestAgent.server: federation state anti-entropy synced -43:16.422 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:13182] -43:16.428 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:13182 -43:16.430 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5fb0b458-289c-2fad-a858-8f577daa1297.primary 127.0.0.1 -43:16.430 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 -43:16.431 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5fb0b458-289c-2fad-a858-8f577daa1297.primary area=wan -43:16.464 [INFO] TestAgent: Requesting shutdown -43:16.465 [INFO] TestAgent.server: shutting down server -43:16.465 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:16.465 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:16.465 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:16.465 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:16.465 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:16.465 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:16.465 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:16.465 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:16.465 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:16.465 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:16.465 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:16.465 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:16.465Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:16.465Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:16.465Z [INFO] agent.router.manager: shutting down -43:16.470 [INFO] TestAgent: consul server down -43:16.470 [INFO] TestAgent: shutdown complete -43:16.470 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13227 network=tcp -43:16.470 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13227 network=udp -43:16.470 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13228 network=tcp -43:16.971 [INFO] TestAgent: Waiting for endpoints to shut down -43:16.971 [INFO] TestAgent: Endpoints down -2022-09-15T21:43:15.942Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:16.009Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:16.012 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5fb0b458-289c-2fad-a858-8f577daa1297 Address:127.0.0.1:13169}]" -43:16.013 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5fb0b458-289c-2fad-a858-8f577daa1297.primary 127.0.0.1 -43:16.014 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13169 [Follower]" leader= -43:16.019 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5fb0b458-289c-2fad-a858-8f577daa1297 127.0.0.1 -2022-09-15T21:43:16.019Z [INFO] agent.router: Initializing LAN area manager -43:16.019 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13178 network=udp -43:16.019 [INFO] TestAgent.server: Adding LAN server: server="Node-5fb0b458-289c-2fad-a858-8f577daa1297 (Addr: tcp/127.0.0.1:13169) (DC: primary)" -43:16.019 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5fb0b458-289c-2fad-a858-8f577daa1297.primary area=wan -43:16.020 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13178 network=tcp -43:16.020 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13179 network=tcp -43:16.021 [INFO] TestAgent: started state syncer -43:16.021 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13170 network=tcp -43:16.082 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:16.082 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13169 [Candidate]" term=2 -43:16.082 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:16.082 [DEBUG] TestAgent.server.raft: vote granted: from=5fb0b458-289c-2fad-a858-8f577daa1297 term=2 tally=1 -43:16.082 [INFO] TestAgent.server.raft: election won: tally=1 -43:16.082 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13169 [Leader]" -43:16.082 [INFO] TestAgent.server: cluster leadership acquired -43:16.083 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13169 -43:16.084 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:16.086 [INFO] TestAgent.server: New leader elected: payload=Node-5fb0b458-289c-2fad-a858-8f577daa1297 -43:16.095 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:16.095 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:16.095 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:16.095 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:16.095 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:16.095 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5fb0b458-289c-2fad-a858-8f577daa1297 -43:16.095 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5fb0b458-289c-2fad-a858-8f577daa1297 -43:16.096 [INFO] TestAgent.server: federation state anti-entropy synced -43:16.394 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:16.395 [INFO] TestAgent: Synced node info -43:16.395 [DEBUG] TestAgent: Node info in sync -43:16.428 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:45804 -43:16.429 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d973ff1a-ccf0-3ad7-de02-23f9f10b6832.secondary 127.0.0.1 -43:16.430 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d973ff1a-ccf0-3ad7-de02-23f9f10b6832.secondary area=wan -43:16.971 [INFO] TestAgent: Requesting shutdown -43:16.971 [INFO] TestAgent.server: shutting down server -43:16.971 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:16.971 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:16.971 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:16.971 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:16.971 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:16.971 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:16.971 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:16.971 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:16.971 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:16.971 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:16.972 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:16.972Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:16.972Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:16.972Z [INFO] agent.router.manager: shutting down -43:16.972 [INFO] TestAgent: consul server down -43:16.972 [INFO] TestAgent: shutdown complete -43:16.972 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13178 network=tcp -43:16.972 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13178 network=udp -43:16.972 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13179 network=tcp -43:17.473 [INFO] TestAgent: Waiting for endpoints to shut down -43:17.473 [INFO] TestAgent: Endpoints down ---- PASS: TestStatusPeersSecondary (1.66s) -=== CONT TestAgent_LoadKeyrings -=== RUN TestAgent_LoadKeyrings/no_keys -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:16.931Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:16.969Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:16.972 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:24ea84d4-3eba-b8fd-b984-82b2b190bca7 Address:127.0.0.1:13085}]" -43:16.973 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13085 [Follower]" leader= -43:16.973 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-24ea84d4-3eba-b8fd-b984-82b2b190bca7.dc1 127.0.0.1 -43:16.974 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-24ea84d4-3eba-b8fd-b984-82b2b190bca7 127.0.0.1 -2022-09-15T21:43:16.974Z [INFO] agent.router: Initializing LAN area manager -43:16.975 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13073 network=udp -43:16.975 [INFO] TestAgent.server: Adding LAN server: server="Node-24ea84d4-3eba-b8fd-b984-82b2b190bca7 (Addr: tcp/127.0.0.1:13085) (DC: dc1)" -43:16.975 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-24ea84d4-3eba-b8fd-b984-82b2b190bca7.dc1 area=wan -43:16.975 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13073 network=tcp -43:16.976 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13074 network=tcp -43:16.976 [INFO] TestAgent: started state syncer -43:16.976 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13086 network=tcp -43:17.018 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:17.018 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13085 [Candidate]" term=2 -43:17.018 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:17.018 [DEBUG] TestAgent.server.raft: vote granted: from=24ea84d4-3eba-b8fd-b984-82b2b190bca7 term=2 tally=1 -43:17.018 [INFO] TestAgent.server.raft: election won: tally=1 -43:17.018 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13085 [Leader]" -43:17.018 [INFO] TestAgent.server: cluster leadership acquired -43:17.019 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13085 -43:17.020 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:17.022 [INFO] TestAgent.server: New leader elected: payload=Node-24ea84d4-3eba-b8fd-b984-82b2b190bca7 -43:17.027 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:17.027 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:17.027 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:17.027 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:17.027 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:17.027 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-24ea84d4-3eba-b8fd-b984-82b2b190bca7 -43:17.027 [INFO] TestAgent.server: member joined, marking health alive: member=Node-24ea84d4-3eba-b8fd-b984-82b2b190bca7 -43:17.035 [INFO] TestAgent.server: federation state anti-entropy synced -43:17.090 [INFO] TestAgent: Requesting shutdown -43:17.091 [INFO] TestAgent.server: shutting down server -43:17.091 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:17.091 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:17.091 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:17.091 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:17.091 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:17.091 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:17.091 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:17.091 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:17.098 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:17.098 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:17.098 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:17.098 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:17.098Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:17.098Z [INFO] agent.router.manager: shutting down -43:17.098 [INFO] TestAgent: consul server down -43:17.098 [INFO] TestAgent: shutdown complete -43:17.098 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13073 network=tcp -43:17.098 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13073 network=udp -43:17.099 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13074 network=tcp -43:17.602 [INFO] TestAgent: Waiting for endpoints to shut down -43:17.602 [INFO] TestAgent: Endpoints down ---- PASS: TestPreparedQuery_Execute (6.26s) - --- PASS: TestPreparedQuery_Execute/#00 (0.71s) - --- PASS: TestPreparedQuery_Execute/#01 (0.83s) - --- PASS: TestPreparedQuery_Execute/#02 (0.67s) - --- PASS: TestPreparedQuery_Execute/#03 (0.71s) - --- PASS: TestPreparedQuery_Execute/#04 (0.79s) - --- PASS: TestPreparedQuery_Execute/#05 (0.88s) - --- PASS: TestPreparedQuery_Execute/#06 (0.97s) - --- PASS: TestPreparedQuery_Execute/#07 (0.70s) -=== CONT TestIntentionsSpecificDelete_good -2022-09-15T21:43:16.705Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:16.734Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:16.737 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7a4e138b-a8d2-47e8-fc74-36ea406d39cf Address:127.0.0.1:13239}]" -43:16.738 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13239 [Follower]" leader= -43:16.743 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7a4e138b-a8d2-47e8-fc74-36ea406d39cf.dc1 127.0.0.1 -43:16.744 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7a4e138b-a8d2-47e8-fc74-36ea406d39cf 127.0.0.1 -2022-09-15T21:43:16.744Z [INFO] agent.router: Initializing LAN area manager -43:16.745 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13234 network=udp -43:16.745 [INFO] TestAgent.server: Adding LAN server: server="Node-7a4e138b-a8d2-47e8-fc74-36ea406d39cf (Addr: tcp/127.0.0.1:13239) (DC: dc1)" -43:16.745 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7a4e138b-a8d2-47e8-fc74-36ea406d39cf.dc1 area=wan -43:16.745 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13234 network=tcp -43:16.746 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13235 network=tcp -43:16.747 [INFO] TestAgent: started state syncer -43:16.747 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13240 network=tcp -43:16.798 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:16.798 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13239 [Candidate]" term=2 -43:16.798 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:16.798 [DEBUG] TestAgent.server.raft: vote granted: from=7a4e138b-a8d2-47e8-fc74-36ea406d39cf term=2 tally=1 -43:16.798 [INFO] TestAgent.server.raft: election won: tally=1 -43:16.798 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13239 [Leader]" -43:16.798 [INFO] TestAgent.server: cluster leadership acquired -43:16.799 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13239 -43:16.800 [INFO] TestAgent.server: New leader elected: payload=Node-7a4e138b-a8d2-47e8-fc74-36ea406d39cf -43:16.806 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:16.809 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:16.809 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:16.809 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:16.809 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:16.809 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:16.809 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7a4e138b-a8d2-47e8-fc74-36ea406d39cf -43:16.809 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7a4e138b-a8d2-47e8-fc74-36ea406d39cf -43:16.814 [INFO] TestAgent.server: federation state anti-entropy synced -43:17.022 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:17.022 [INFO] TestAgent: Synced node info -43:17.140 [INFO] TestAgent: Requesting shutdown -43:17.141 [INFO] TestAgent.server: shutting down server -43:17.141 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:17.141 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:17.141 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:17.141 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:17.141 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:17.141 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:17.141 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:17.141 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:17.141 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:17.141 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:17.141 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:17.150Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:17.150Z [INFO] agent.router.manager: shutting down -43:17.150 [INFO] TestAgent: consul server down -43:17.150 [INFO] TestAgent: shutdown complete -43:17.150 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13234 network=tcp -43:17.150 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13234 network=udp -43:17.150 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13235 network=tcp -43:17.653 [INFO] TestAgent: Waiting for endpoints to shut down -43:17.653 [INFO] TestAgent: Endpoints down -=== RUN TestTxnEndpoint_Bad_Size_Item/exceeds_default_max_kv_value_size -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:17.150Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:17.206Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:17.209 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9f17fc54-a41f-4c68-297a-c698c0c69064 Address:127.0.0.1:13225}]" -43:17.209 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13225 [Follower]" leader= -43:17.209 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9f17fc54-a41f-4c68-297a-c698c0c69064.dc1 127.0.0.1 -43:17.210 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9f17fc54-a41f-4c68-297a-c698c0c69064 127.0.0.1 -2022-09-15T21:43:17.210Z [INFO] agent.router: Initializing LAN area manager -43:17.211 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13241 network=udp -43:17.211 [INFO] TestAgent.server: Adding LAN server: server="Node-9f17fc54-a41f-4c68-297a-c698c0c69064 (Addr: tcp/127.0.0.1:13225) (DC: dc1)" -43:17.211 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9f17fc54-a41f-4c68-297a-c698c0c69064.dc1 area=wan -43:17.211 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13241 network=tcp -43:17.212 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13242 network=tcp -43:17.212 [INFO] TestAgent: started state syncer -43:17.212 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13226 network=tcp -43:17.253 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:17.253 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13225 [Candidate]" term=2 -43:17.253 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:17.253 [DEBUG] TestAgent.server.raft: vote granted: from=9f17fc54-a41f-4c68-297a-c698c0c69064 term=2 tally=1 -43:17.253 [INFO] TestAgent.server.raft: election won: tally=1 -43:17.253 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13225 [Leader]" -43:17.253 [INFO] TestAgent.server: cluster leadership acquired -43:17.253 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13225 -43:17.255 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:17.256 [INFO] TestAgent.server: New leader elected: payload=Node-9f17fc54-a41f-4c68-297a-c698c0c69064 -43:17.257 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:17.257 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:17.257 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:17.257 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:17.257 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:17.258 [INFO] TestAgent.server: federation state anti-entropy synced -43:17.258 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9f17fc54-a41f-4c68-297a-c698c0c69064 -43:17.258 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9f17fc54-a41f-4c68-297a-c698c0c69064 -43:17.390 [INFO] TestAgent: Requesting shutdown -43:17.390 [INFO] TestAgent.server: shutting down server -43:17.390 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:17.390 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:17.390 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:17.390 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:17.390 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:17.390 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:17.390 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:17.390 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:17.390 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:17.390 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:17.390 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:17.391 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:17.391Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:17.391Z [INFO] agent.router.manager: shutting down -43:17.394 [INFO] TestAgent: consul server down -43:17.394 [INFO] TestAgent: shutdown complete -43:17.394 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13241 network=tcp -43:17.394 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13241 network=udp -43:17.394 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13242 network=tcp -43:17.898 [INFO] TestAgent: Waiting for endpoints to shut down -43:17.898 [INFO] TestAgent: Endpoints down ---- PASS: TestStatusLeader (0.85s) -=== CONT TestPProfHandlers_DisableDebugNoACLs -2022-09-15T21:43:17.150Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:17.199Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:17.202 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c43e70d1-b798-6a54-cc4f-55853c75dfa5 Address:127.0.0.1:13218}]" -43:17.203 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13218 [Follower]" leader= -43:17.206 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c43e70d1-b798-6a54-cc4f-55853c75dfa5.dc1 127.0.0.1 -43:17.207 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c43e70d1-b798-6a54-cc4f-55853c75dfa5 127.0.0.1 -2022-09-15T21:43:17.207Z [INFO] agent.router: Initializing LAN area manager -43:17.207 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13199 network=udp -43:17.208 [INFO] TestAgent.server: Adding LAN server: server="Node-c43e70d1-b798-6a54-cc4f-55853c75dfa5 (Addr: tcp/127.0.0.1:13218) (DC: dc1)" -43:17.208 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c43e70d1-b798-6a54-cc4f-55853c75dfa5.dc1 area=wan -43:17.208 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13199 network=tcp -43:17.209 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13200 network=tcp -43:17.209 [INFO] TestAgent: started state syncer -43:17.209 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13219 network=tcp -43:17.244 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:17.244 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13218 [Candidate]" term=2 -43:17.244 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:17.244 [DEBUG] TestAgent.server.raft: vote granted: from=c43e70d1-b798-6a54-cc4f-55853c75dfa5 term=2 tally=1 -43:17.244 [INFO] TestAgent.server.raft: election won: tally=1 -43:17.244 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13218 [Leader]" -43:17.244 [INFO] TestAgent.server: cluster leadership acquired -43:17.245 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13218 -43:17.245 [INFO] TestAgent.server: New leader elected: payload=Node-c43e70d1-b798-6a54-cc4f-55853c75dfa5 -43:17.246 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:17.249 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:17.249 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:17.249 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:17.249 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:17.249 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:17.249 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c43e70d1-b798-6a54-cc4f-55853c75dfa5 -43:17.249 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c43e70d1-b798-6a54-cc4f-55853c75dfa5 -43:17.250 [INFO] TestAgent.server: federation state anti-entropy synced -43:17.390 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:17.391 [INFO] TestAgent: Synced node info -43:17.491 [INFO] TestAgent: Requesting shutdown -43:17.491 [INFO] TestAgent.server: shutting down server -43:17.491 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:17.491 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:17.491 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:17.491 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:17.491 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:17.492 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:17.492 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:17.492 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:17.492 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:17.492 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:17.492 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:17.492Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:17.492Z [INFO] agent.router.manager: shutting down -43:17.492 [INFO] TestAgent: consul server down -43:17.492 [INFO] TestAgent: shutdown complete -43:17.492 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13199 network=tcp -43:17.492 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13199 network=udp -43:17.492 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13200 network=tcp -43:18.002 [INFO] TestAgent: Waiting for endpoints to shut down -43:18.002 [INFO] TestAgent: Endpoints down ---- PASS: TestIntentionsSpecificUpdate_good (0.94s) -=== CONT TestPProfHandlers_EnableDebug -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:17.168Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:17.168Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:17.200Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:17.200Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:17.203 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c259df80-2877-e336-f146-f97c8ae51a0a Address:127.0.0.1:13246}]" -43:17.203 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13246 [Follower]" leader= -43:17.205 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c259df80-2877-e336-f146-f97c8ae51a0a.dc1 127.0.0.1 -43:17.206 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c259df80-2877-e336-f146-f97c8ae51a0a 127.0.0.1 -2022-09-15T21:43:17.206Z [INFO] agent.router: Initializing LAN area manager -43:17.206 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13220 network=udp -43:17.206 [INFO] TestAgent.server: Adding LAN server: server="Node-c259df80-2877-e336-f146-f97c8ae51a0a (Addr: tcp/127.0.0.1:13246) (DC: dc1)" -43:17.206 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c259df80-2877-e336-f146-f97c8ae51a0a.dc1 area=wan -43:17.207 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13220 network=tcp -43:17.207 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13221 network=tcp -43:17.208 [INFO] TestAgent: started state syncer -43:17.208 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13247 network=tcp -43:17.256 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:17.256 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13246 [Candidate]" term=2 -43:17.256 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:17.256 [DEBUG] TestAgent.server.raft: vote granted: from=c259df80-2877-e336-f146-f97c8ae51a0a term=2 tally=1 -43:17.256 [INFO] TestAgent.server.raft: election won: tally=1 -43:17.256 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13246 [Leader]" -43:17.257 [INFO] TestAgent.server: cluster leadership acquired -43:17.257 [INFO] TestAgent.server: initializing acls -43:17.257 [INFO] TestAgent.server: Created ACL 'global-management' policy -43:17.257 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -43:17.258 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -43:17.258 [INFO] TestAgent.server: Created ACL anonymous token from configuration -43:17.258 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -43:17.258 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -43:17.258 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode -43:17.258 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-c259df80-2877-e336-f146-f97c8ae51a0a -43:17.258 [INFO] TestAgent.server: initializing acls -43:17.258 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -43:17.259 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-c259df80-2877-e336-f146-f97c8ae51a0a.dc1 -43:17.259 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-c259df80-2877-e336-f146-f97c8ae51a0a.dc1 area=wan -43:17.259 [INFO] TestAgent.server: New leader elected: payload=Node-c259df80-2877-e336-f146-f97c8ae51a0a -43:17.259 [INFO] TestAgent.server: Updating LAN server: server="Node-c259df80-2877-e336-f146-f97c8ae51a0a (Addr: tcp/127.0.0.1:13246) (DC: dc1)" -43:17.259 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:17.262 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:17.262 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:17.262 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:17.262 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:17.262 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:17.263 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c259df80-2877-e336-f146-f97c8ae51a0a -43:17.263 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c259df80-2877-e336-f146-f97c8ae51a0a -43:17.264 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c259df80-2877-e336-f146-f97c8ae51a0a -43:17.264 [INFO] TestAgent.server: federation state anti-entropy synced -43:17.634 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-c259df80-2877-e336-f146-f97c8ae51a0a -43:17.635 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:17.636 [INFO] TestAgent: Synced node info -43:17.636 [DEBUG] TestAgent: Node info in sync -43:17.636 [INFO] TestAgent.server.fsm: snapshot created: duration=79.66µs -43:17.636 [INFO] TestAgent.server.raft: starting snapshot up to: index=15 -43:17.636 [INFO] TestAgent.server.snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshot_Options_GET#02-TestAgent-agent616105143/raft/snapshots/2-15-1663278197636.tmp -43:17.638 [INFO] TestAgent.server.raft: snapshot complete up to: index=15 -43:17.640 [INFO] TestAgent: Requesting shutdown -43:17.640 [INFO] TestAgent.server: shutting down server -43:17.640 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:17.640 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:17.640 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:17.640 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -43:17.640 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -43:17.640 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:17.640 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:17.640 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:17.641 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:17.641 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:17.641 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:17.641 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:17.641 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -43:17.641 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -43:17.641 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:17.641Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:17.641Z [INFO] agent.router.manager: shutting down -43:17.641 [INFO] TestAgent: consul server down -43:17.641 [INFO] TestAgent: shutdown complete -43:17.641 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13220 network=tcp -43:17.641 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13220 network=udp -43:17.641 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13221 network=tcp -43:18.142 [INFO] TestAgent: Waiting for endpoints to shut down -43:18.142 [INFO] TestAgent: Endpoints down -=== RUN TestSnapshot_Options/PUT -2022-09-15T21:43:17.514Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:17.540Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:17.544 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d609e900-f84f-8c1b-e4e9-4d12851479e7 Address:127.0.0.1:13267}]" -43:17.544 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13267 [Follower]" leader= -43:17.545 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d609e900-f84f-8c1b-e4e9-4d12851479e7.dc1 127.0.0.1 -43:17.550 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d609e900-f84f-8c1b-e4e9-4d12851479e7 127.0.0.1 -2022-09-15T21:43:17.550Z [INFO] agent.router: Initializing LAN area manager -43:17.551 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13262 network=udp -43:17.551 [INFO] TestAgent.server: Adding LAN server: server="Node-d609e900-f84f-8c1b-e4e9-4d12851479e7 (Addr: tcp/127.0.0.1:13267) (DC: dc1)" -43:17.551 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d609e900-f84f-8c1b-e4e9-4d12851479e7.dc1 area=wan -43:17.551 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13262 network=tcp -43:17.552 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13263 network=tcp -43:17.552 [INFO] TestAgent: started state syncer -43:17.553 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13268 network=tcp -43:17.590 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:17.590 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13267 [Candidate]" term=2 -43:17.590 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:17.590 [DEBUG] TestAgent.server.raft: vote granted: from=d609e900-f84f-8c1b-e4e9-4d12851479e7 term=2 tally=1 -43:17.590 [INFO] TestAgent.server.raft: election won: tally=1 -43:17.590 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13267 [Leader]" -43:17.591 [INFO] TestAgent.server: cluster leadership acquired -43:17.591 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13267 -43:17.592 [INFO] TestAgent.server: New leader elected: payload=Node-d609e900-f84f-8c1b-e4e9-4d12851479e7 -43:17.593 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:17.603 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:17.603 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:17.603 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:17.603 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:17.603 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:17.603 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d609e900-f84f-8c1b-e4e9-4d12851479e7 -43:17.603 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d609e900-f84f-8c1b-e4e9-4d12851479e7 -43:17.605 [INFO] TestAgent.server: federation state anti-entropy synced -43:17.702 [INFO] TestAgent: Requesting shutdown -43:17.702 [INFO] TestAgent.server: shutting down server -43:17.702 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:17.702 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:17.702 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:17.702 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:17.702 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:17.703 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:17.703 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:17.703 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:17.703 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:17.703 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:17.703 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:17.703 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:17.704Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:17.704Z [INFO] agent.router.manager: shutting down -43:17.704 [INFO] TestAgent: consul server down -43:17.704 [INFO] TestAgent: shutdown complete -43:17.704 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13262 network=tcp -43:17.704 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13262 network=udp -43:17.704 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13263 network=tcp -43:18.204 [INFO] TestAgent: Waiting for endpoints to shut down -43:18.204 [INFO] TestAgent: Endpoints down -=== RUN TestAgent_LoadKeyrings/server_with_keys -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:17.194Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:17.224Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:17.228 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2f63895f-161f-5348-ae34-72aebf6fbf3e Address:127.0.0.1:13253}]" -43:17.229 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2f63895f-161f-5348-ae34-72aebf6fbf3e.dc1 127.0.0.1 -43:17.230 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13253 [Follower]" leader= -43:17.231 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2f63895f-161f-5348-ae34-72aebf6fbf3e 127.0.0.1 -2022-09-15T21:43:17.231Z [INFO] agent.router: Initializing LAN area manager -43:17.231 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13248 network=udp -43:17.232 [INFO] TestAgent.server: Adding LAN server: server="Node-2f63895f-161f-5348-ae34-72aebf6fbf3e (Addr: tcp/127.0.0.1:13253) (DC: dc1)" -43:17.232 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2f63895f-161f-5348-ae34-72aebf6fbf3e.dc1 area=wan -43:17.232 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13248 network=tcp -43:17.233 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13249 network=tcp -43:17.233 [INFO] TestAgent: started state syncer -43:17.233 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13254 network=tcp -43:17.298 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:17.298 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13253 [Candidate]" term=2 -43:17.299 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:17.299 [DEBUG] TestAgent.server.raft: vote granted: from=2f63895f-161f-5348-ae34-72aebf6fbf3e term=2 tally=1 -43:17.299 [INFO] TestAgent.server.raft: election won: tally=1 -43:17.299 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13253 [Leader]" -43:17.299 [INFO] TestAgent.server: cluster leadership acquired -43:17.299 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13253 -43:17.300 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:17.302 [INFO] TestAgent.server: New leader elected: payload=Node-2f63895f-161f-5348-ae34-72aebf6fbf3e -43:17.303 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:17.303 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:17.303 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:17.303 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:17.303 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:17.303 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2f63895f-161f-5348-ae34-72aebf6fbf3e -43:17.303 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2f63895f-161f-5348-ae34-72aebf6fbf3e -43:17.304 [INFO] TestAgent.server: federation state anti-entropy synced -43:17.327 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:17.328 [INFO] TestAgent: Synced node info -43:17.328 [DEBUG] TestAgent: Node info in sync -43:17.919 [WARN] TestAgent.server.rpc: Attempting to apply large raft entry: size_in_bytes=4719032 -43:17.933 [INFO] TestAgent: Requesting shutdown -43:17.933 [INFO] TestAgent.server: shutting down server -43:17.933 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:17.933 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:17.933 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:17.933 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:17.933 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:17.934 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:17.934 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:17.934 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:17.934 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:17.934 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:17.935 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:17.936Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:17.936Z [INFO] agent.router.manager: shutting down -43:17.936 [INFO] TestAgent: consul server down -43:17.936 [INFO] TestAgent: shutdown complete -43:17.936 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13248 network=tcp -43:17.936 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13248 network=udp -43:17.936 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13249 network=tcp -43:18.436 [INFO] TestAgent: Waiting for endpoints to shut down -43:18.436 [INFO] TestAgent: Endpoints down -=== RUN TestTxnEndpoint_Bad_Size_Net/allowed_kv_max_backward_compatible -2022-09-15T21:43:17.640Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:17.666Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:17.669 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d6172ca3-d60a-855b-4c99-05a5d8f53691 Address:127.0.0.1:13281}]" -43:17.670 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13281 [Follower]" leader= -43:17.671 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d6172ca3-d60a-855b-4c99-05a5d8f53691.dc1 127.0.0.1 -43:17.671 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d6172ca3-d60a-855b-4c99-05a5d8f53691 127.0.0.1 -2022-09-15T21:43:17.672Z [INFO] agent.router: Initializing LAN area manager -43:17.672 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13290 network=udp -43:17.672 [INFO] TestAgent.server: Adding LAN server: server="Node-d6172ca3-d60a-855b-4c99-05a5d8f53691 (Addr: tcp/127.0.0.1:13281) (DC: dc1)" -43:17.672 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d6172ca3-d60a-855b-4c99-05a5d8f53691.dc1 area=wan -43:17.672 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13290 network=tcp -43:17.673 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13291 network=tcp -43:17.673 [INFO] TestAgent: started state syncer -43:17.674 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13282 network=tcp -43:17.730 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:17.730 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13281 [Candidate]" term=2 -43:17.730 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:17.730 [DEBUG] TestAgent.server.raft: vote granted: from=d6172ca3-d60a-855b-4c99-05a5d8f53691 term=2 tally=1 -43:17.730 [INFO] TestAgent.server.raft: election won: tally=1 -43:17.730 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13281 [Leader]" -43:17.730 [INFO] TestAgent.server: cluster leadership acquired -43:17.731 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13281 -43:17.731 [INFO] TestAgent.server: New leader elected: payload=Node-d6172ca3-d60a-855b-4c99-05a5d8f53691 -43:17.732 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:17.735 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:17.735 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:17.735 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:17.735 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:17.735 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:17.736 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d6172ca3-d60a-855b-4c99-05a5d8f53691 -43:17.736 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d6172ca3-d60a-855b-4c99-05a5d8f53691 -43:17.737 [INFO] TestAgent.server: federation state anti-entropy synced -43:17.818 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:17.819 [INFO] TestAgent: Synced node info -43:17.819 [DEBUG] TestAgent: Node info in sync -43:18.014 [INFO] TestAgent: Requesting shutdown -43:18.015 [INFO] TestAgent.server: shutting down server -43:18.015 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:18.015 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:18.015 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:18.015 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:18.015 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:18.015 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:18.015 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:18.015 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:18.015 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:18.015 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:18.015 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:18.015Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:18.015Z [INFO] agent.router.manager: shutting down -43:18.017 [INFO] TestAgent: consul server down -43:18.017 [INFO] TestAgent: shutdown complete -43:18.017 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13290 network=tcp -43:18.017 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13290 network=udp -43:18.017 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13291 network=tcp -43:18.518 [INFO] TestAgent: Waiting for endpoints to shut down -43:18.518 [INFO] TestAgent: Endpoints down ---- PASS: TestIntentionsSpecificDelete_good (0.92s) -=== CONT TestSessionDestroy -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:17.675Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:17.696Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:17.699 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:088bbac3-44c9-91b1-1297-8055e384dcf7 Address:127.0.0.1:13302}]" -43:17.700 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-088bbac3-44c9-91b1-1297-8055e384dcf7.dc1 127.0.0.1 -43:17.701 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-088bbac3-44c9-91b1-1297-8055e384dcf7 127.0.0.1 -2022-09-15T21:43:17.701Z [INFO] agent.router: Initializing LAN area manager -43:17.702 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13302 [Follower]" leader= -43:17.702 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13276 network=udp -43:17.702 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13276 network=tcp -43:17.702 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-088bbac3-44c9-91b1-1297-8055e384dcf7.dc1 area=wan -43:17.703 [INFO] TestAgent.server: Adding LAN server: server="Node-088bbac3-44c9-91b1-1297-8055e384dcf7 (Addr: tcp/127.0.0.1:13302) (DC: dc1)" -43:17.703 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13277 network=tcp -43:17.703 [INFO] TestAgent: started state syncer -43:17.703 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13303 network=tcp -43:17.752 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:17.752 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13302 [Candidate]" term=2 -43:17.752 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:17.752 [DEBUG] TestAgent.server.raft: vote granted: from=088bbac3-44c9-91b1-1297-8055e384dcf7 term=2 tally=1 -43:17.752 [INFO] TestAgent.server.raft: election won: tally=1 -43:17.752 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13302 [Leader]" -43:17.752 [INFO] TestAgent.server: cluster leadership acquired -43:17.752 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13302 -43:17.753 [INFO] TestAgent.server: New leader elected: payload=Node-088bbac3-44c9-91b1-1297-8055e384dcf7 -43:17.754 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:17.757 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:17.757 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:17.757 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:17.757 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:17.757 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:17.757 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-088bbac3-44c9-91b1-1297-8055e384dcf7 -43:17.757 [INFO] TestAgent.server: member joined, marking health alive: member=Node-088bbac3-44c9-91b1-1297-8055e384dcf7 -43:17.759 [INFO] TestAgent.server: federation state anti-entropy synced -43:17.855 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:17.856 [INFO] TestAgent: Synced node info -43:17.856 [DEBUG] TestAgent: Node info in sync -43:18.061 [INFO] TestAgent: Requesting shutdown -43:18.061 [INFO] TestAgent.server: shutting down server -43:18.061 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:18.061 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:18.061 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:18.061 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:18.061 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:18.061 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:18.061 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:18.061 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:18.061 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:18.061 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:18.061 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:18.061Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:18.061Z [INFO] agent.router.manager: shutting down -43:18.061 [INFO] TestAgent: consul server down -43:18.061 [INFO] TestAgent: shutdown complete -43:18.061 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13276 network=tcp -43:18.061 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13276 network=udp -43:18.062 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13277 network=tcp -43:18.562 [INFO] TestAgent: Waiting for endpoints to shut down -43:18.562 [INFO] TestAgent: Endpoints down -=== RUN TestTxnEndpoint_Bad_Size_Item/allowed -2022-09-15T21:43:17.917Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:17.936Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:17.940 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8f77dd49-240d-5b87-f5b7-fbfdddb018e7 Address:127.0.0.1:13274}]" -43:17.940 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13274 [Follower]" leader= -43:17.941 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8f77dd49-240d-5b87-f5b7-fbfdddb018e7.dc1 127.0.0.1 -43:17.942 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8f77dd49-240d-5b87-f5b7-fbfdddb018e7 127.0.0.1 -2022-09-15T21:43:17.942Z [INFO] agent.router: Initializing LAN area manager -43:17.943 [INFO] TestAgent.server: Adding LAN server: server="Node-8f77dd49-240d-5b87-f5b7-fbfdddb018e7 (Addr: tcp/127.0.0.1:13274) (DC: dc1)" -43:17.943 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8f77dd49-240d-5b87-f5b7-fbfdddb018e7.dc1 area=wan -43:17.943 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13297 network=udp -43:17.943 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13297 network=tcp -43:17.944 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13298 network=tcp -43:17.944 [INFO] TestAgent: started state syncer -43:17.944 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13275 network=tcp -43:18.007 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:18.007 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13274 [Candidate]" term=2 -43:18.007 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:18.007 [DEBUG] TestAgent.server.raft: vote granted: from=8f77dd49-240d-5b87-f5b7-fbfdddb018e7 term=2 tally=1 -43:18.007 [INFO] TestAgent.server.raft: election won: tally=1 -43:18.007 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13274 [Leader]" -43:18.008 [INFO] TestAgent.server: cluster leadership acquired -43:18.008 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13274 -43:18.010 [INFO] TestAgent.server: New leader elected: payload=Node-8f77dd49-240d-5b87-f5b7-fbfdddb018e7 -43:18.010 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:18.020 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:18.020 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:18.020 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:18.020 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:18.020 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:18.020 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8f77dd49-240d-5b87-f5b7-fbfdddb018e7 -43:18.020 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8f77dd49-240d-5b87-f5b7-fbfdddb018e7 -43:18.022 [INFO] TestAgent.server: federation state anti-entropy synced -43:18.164 [INFO] TestAgent: Requesting shutdown -43:18.164 [INFO] TestAgent.server: shutting down server -43:18.164 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:18.164 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:18.164 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:18.164 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:18.164 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:18.164 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:18.164 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:18.164 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:18.164 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:18.164 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:18.164 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:18.164 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:18.165Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:18.165Z [INFO] agent.router.manager: shutting down -43:18.165 [INFO] TestAgent: consul server down -43:18.165 [INFO] TestAgent: shutdown complete -43:18.165 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13297 network=tcp -43:18.165 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13297 network=udp -43:18.165 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13298 network=tcp -43:18.665 [INFO] TestAgent: Waiting for endpoints to shut down -43:18.665 [INFO] TestAgent: Endpoints down ---- PASS: TestPProfHandlers_DisableDebugNoACLs (0.77s) -=== CONT TestSessionDeleteDestroy -2022-09-15T21:43:17.276Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:17.296Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:17.300 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4a84ba8d-efef-2585-8b56-c6063e0c972a Address:127.0.0.1:13183}]" -43:17.301 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4a84ba8d-efef-2585-8b56-c6063e0c972a.secondary 127.0.0.1 -43:17.302 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13183 [Follower]" leader= -43:17.302 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4a84ba8d-efef-2585-8b56-c6063e0c972a 127.0.0.1 -2022-09-15T21:43:17.302Z [INFO] agent.router: Initializing LAN area manager -43:17.303 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13185 network=udp -43:17.303 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4a84ba8d-efef-2585-8b56-c6063e0c972a.secondary area=wan -43:17.303 [INFO] TestAgent.server: Adding LAN server: server="Node-4a84ba8d-efef-2585-8b56-c6063e0c972a (Addr: tcp/127.0.0.1:13183) (DC: secondary)" -43:17.303 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13185 network=tcp -43:17.304 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13186 network=tcp -43:17.304 [INFO] TestAgent: started state syncer -43:17.304 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13184 network=tcp -43:17.357 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:17.357 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13183 [Candidate]" term=2 -43:17.358 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:17.358 [DEBUG] TestAgent.server.raft: vote granted: from=4a84ba8d-efef-2585-8b56-c6063e0c972a term=2 tally=1 -43:17.358 [INFO] TestAgent.server.raft: election won: tally=1 -43:17.358 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13183 [Leader]" -43:17.358 [INFO] TestAgent.server: cluster leadership acquired -43:17.358 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13183 -43:17.361 [INFO] TestAgent.server: New leader elected: payload=Node-4a84ba8d-efef-2585-8b56-c6063e0c972a -43:17.363 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:17.366 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:17.366 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:17.366 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:17.366 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:17.366 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:17.366 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4a84ba8d-efef-2585-8b56-c6063e0c972a -43:17.366 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4a84ba8d-efef-2585-8b56-c6063e0c972a -43:17.366 [INFO] TestAgent: Synced node info -43:17.367 [INFO] TestAgent.server: federation state anti-entropy synced -43:17.693 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:13217] -43:17.694 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:13217 -43:17.696 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4728f40a-4398-bcab-b4ff-c35346ab00bf.primary 127.0.0.1 -43:17.696 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 -43:17.697 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4728f40a-4398-bcab-b4ff-c35346ab00bf.primary area=wan -43:17.701 [INFO] TestAgent: Requesting shutdown -43:17.701 [INFO] TestAgent.server: shutting down server -43:17.701 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:17.701 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:17.702 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:17.702 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:17.704 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:17.704 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:17.703 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:17.704 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:17.703 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:17.703 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:17.704 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:17.704Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:17.704Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:17.704Z [INFO] agent.router.manager: shutting down -43:17.704 [INFO] TestAgent: consul server down -43:17.704 [INFO] TestAgent: shutdown complete -43:17.704 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13185 network=tcp -43:17.704 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13185 network=udp -43:17.704 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13186 network=tcp -43:18.205 [INFO] TestAgent: Waiting for endpoints to shut down -43:18.205 [INFO] TestAgent: Endpoints down -2022-09-15T21:43:16.834Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:16.857Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:16.867 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4728f40a-4398-bcab-b4ff-c35346ab00bf Address:127.0.0.1:13204}]" -43:16.867 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13204 [Follower]" leader= -43:16.868 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4728f40a-4398-bcab-b4ff-c35346ab00bf.primary 127.0.0.1 -43:16.869 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4728f40a-4398-bcab-b4ff-c35346ab00bf 127.0.0.1 -2022-09-15T21:43:16.869Z [INFO] agent.router: Initializing LAN area manager -43:16.869 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13213 network=udp -43:16.869 [INFO] TestAgent.server: Adding LAN server: server="Node-4728f40a-4398-bcab-b4ff-c35346ab00bf (Addr: tcp/127.0.0.1:13204) (DC: primary)" -43:16.869 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4728f40a-4398-bcab-b4ff-c35346ab00bf.primary area=wan -43:16.874 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13213 network=tcp -43:16.875 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13214 network=tcp -43:16.875 [INFO] TestAgent: started state syncer -43:16.875 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13205 network=tcp -43:16.938 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:16.938 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13204 [Candidate]" term=2 -43:16.938 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:16.938 [DEBUG] TestAgent.server.raft: vote granted: from=4728f40a-4398-bcab-b4ff-c35346ab00bf term=2 tally=1 -43:16.938 [INFO] TestAgent.server.raft: election won: tally=1 -43:16.938 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13204 [Leader]" -43:16.938 [INFO] TestAgent.server: cluster leadership acquired -43:16.939 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13204 -43:16.940 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:16.942 [INFO] TestAgent.server: New leader elected: payload=Node-4728f40a-4398-bcab-b4ff-c35346ab00bf -43:16.951 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:16.951 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:16.951 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:16.951 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:16.951 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:16.951 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4728f40a-4398-bcab-b4ff-c35346ab00bf -43:16.951 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4728f40a-4398-bcab-b4ff-c35346ab00bf -43:16.952 [INFO] TestAgent.server: federation state anti-entropy synced -43:17.074 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:17.075 [INFO] TestAgent: Synced node info -43:17.075 [DEBUG] TestAgent: Node info in sync -43:17.695 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:40494 -43:17.695 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4a84ba8d-efef-2585-8b56-c6063e0c972a.secondary 127.0.0.1 -43:17.696 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4a84ba8d-efef-2585-8b56-c6063e0c972a.secondary area=wan -43:18.205 [INFO] TestAgent: Requesting shutdown -43:18.205 [INFO] TestAgent.server: shutting down server -43:18.205 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:18.205 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:18.205 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:18.205 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:18.205 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:18.205 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:18.205 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:18.205 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:18.205 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:18.205 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:18.205 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:18.206Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:18.206Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:18.206Z [INFO] agent.router.manager: shutting down -43:18.206 [INFO] TestAgent: consul server down -43:18.206 [INFO] TestAgent: shutdown complete -43:18.206 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13213 network=tcp -43:18.206 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13213 network=udp -43:18.206 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13214 network=tcp -43:18.707 [INFO] TestAgent: Waiting for endpoints to shut down -43:18.707 [INFO] TestAgent: Endpoints down ---- PASS: TestStatusLeaderSecondary (1.91s) -=== CONT TestSessionGet -=== RUN TestSessionGet/#00 -2022-09-15T21:43:18.172Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:18.172Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:18.190Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:18.190Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:18.193 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2ed43002-bcd6-40db-4bf9-8cac6e99f9c9 Address:127.0.0.1:13316}]" -43:18.194 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13316 [Follower]" leader= -43:18.195 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2ed43002-bcd6-40db-4bf9-8cac6e99f9c9.dc1 127.0.0.1 -43:18.196 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2ed43002-bcd6-40db-4bf9-8cac6e99f9c9 127.0.0.1 -2022-09-15T21:43:18.196Z [INFO] agent.router: Initializing LAN area manager -43:18.196 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13283 network=udp -43:18.196 [INFO] TestAgent.server: Adding LAN server: server="Node-2ed43002-bcd6-40db-4bf9-8cac6e99f9c9 (Addr: tcp/127.0.0.1:13316) (DC: dc1)" -43:18.196 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2ed43002-bcd6-40db-4bf9-8cac6e99f9c9.dc1 area=wan -43:18.197 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13283 network=tcp -43:18.198 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13284 network=tcp -43:18.198 [INFO] TestAgent: started state syncer -43:18.198 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13317 network=tcp -43:18.247 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -43:18.264 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:18.264 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13316 [Candidate]" term=2 -43:18.264 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:18.264 [DEBUG] TestAgent.server.raft: vote granted: from=2ed43002-bcd6-40db-4bf9-8cac6e99f9c9 term=2 tally=1 -43:18.264 [INFO] TestAgent.server.raft: election won: tally=1 -43:18.264 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13316 [Leader]" -43:18.264 [INFO] TestAgent.server: cluster leadership acquired -43:18.264 [INFO] TestAgent.server: initializing acls -43:18.265 [INFO] TestAgent.server: New leader elected: payload=Node-2ed43002-bcd6-40db-4bf9-8cac6e99f9c9 -43:18.265 [INFO] TestAgent.server: Created ACL 'global-management' policy -43:18.265 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -43:18.266 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -43:18.266 [INFO] TestAgent.server: Created ACL anonymous token from configuration -43:18.266 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -43:18.267 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -43:18.267 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-2ed43002-bcd6-40db-4bf9-8cac6e99f9c9 -43:18.267 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-2ed43002-bcd6-40db-4bf9-8cac6e99f9c9.dc1 -43:18.267 [INFO] TestAgent.server: Updating LAN server: server="Node-2ed43002-bcd6-40db-4bf9-8cac6e99f9c9 (Addr: tcp/127.0.0.1:13316) (DC: dc1)" -43:18.267 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-2ed43002-bcd6-40db-4bf9-8cac6e99f9c9.dc1 area=wan -43:18.268 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:18.271 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:18.271 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:18.271 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:18.271 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:18.271 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:18.271 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2ed43002-bcd6-40db-4bf9-8cac6e99f9c9 -43:18.271 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2ed43002-bcd6-40db-4bf9-8cac6e99f9c9 -43:18.272 [INFO] TestAgent.server: federation state anti-entropy synced -43:18.272 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2ed43002-bcd6-40db-4bf9-8cac6e99f9c9 -43:18.276 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-2ed43002-bcd6-40db-4bf9-8cac6e99f9c9 -43:18.276 [INFO] TestAgent: Requesting shutdown -43:18.276 [INFO] TestAgent.server: shutting down server -43:18.276 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:18.276 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -43:18.276 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -43:18.276 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:18.276 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:18.276 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:18.276 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:18.276 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:18.276 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:18.276 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:18.276 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:18.276 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -43:18.276 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -43:18.276 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -43:18.276 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:18.277 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -2022-09-15T21:43:18.277Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:18.277Z [INFO] agent.router.manager: shutting down -43:18.277 [INFO] TestAgent: consul server down -43:18.277 [INFO] TestAgent: shutdown complete -43:18.277 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13283 network=tcp -43:18.277 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13283 network=udp -43:18.277 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13284 network=tcp -43:18.778 [INFO] TestAgent: Waiting for endpoints to shut down -43:18.778 [INFO] TestAgent: Endpoints down -=== RUN TestSnapshot_Options/PUT#01 -[WARN] freeport: 4 out of 39 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:18.224Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:18.242Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:18.248 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5531cf6b-22d6-8da8-da11-af05b389974e Address:127.0.0.1:13323}]" -43:18.248 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13323 [Follower]" leader= -43:18.249 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5531cf6b-22d6-8da8-da11-af05b389974e.dc1 127.0.0.1 -43:18.250 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5531cf6b-22d6-8da8-da11-af05b389974e 127.0.0.1 -2022-09-15T21:43:18.251Z [INFO] agent.router: Initializing LAN area manager -43:18.251 [INFO] TestAgent.server: Adding LAN server: server="Node-5531cf6b-22d6-8da8-da11-af05b389974e (Addr: tcp/127.0.0.1:13323) (DC: dc1)" -43:18.251 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5531cf6b-22d6-8da8-da11-af05b389974e.dc1 area=wan -43:18.251 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13255 network=udp -43:18.251 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13255 network=tcp -43:18.252 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13256 network=tcp -43:18.252 [INFO] TestAgent: started state syncer -43:18.252 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13324 network=tcp -43:18.314 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:18.314 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13323 [Candidate]" term=2 -43:18.314 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:18.315 [DEBUG] TestAgent.server.raft: vote granted: from=5531cf6b-22d6-8da8-da11-af05b389974e term=2 tally=1 -43:18.315 [INFO] TestAgent.server.raft: election won: tally=1 -43:18.315 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13323 [Leader]" -43:18.315 [INFO] TestAgent.server: cluster leadership acquired -43:18.315 [INFO] TestAgent.server: New leader elected: payload=Node-5531cf6b-22d6-8da8-da11-af05b389974e -43:18.315 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13323 -43:18.316 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:18.319 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:18.319 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:18.319 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:18.319 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:18.319 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:18.319 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5531cf6b-22d6-8da8-da11-af05b389974e -43:18.319 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5531cf6b-22d6-8da8-da11-af05b389974e -43:18.320 [INFO] TestAgent.server: federation state anti-entropy synced -43:18.363 [INFO] TestAgent: Requesting shutdown -43:18.363 [INFO] TestAgent.server: shutting down server -43:18.363 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:18.363 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:18.363 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:18.363 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:18.363 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:18.363 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:18.363 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:18.363 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:18.363 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:18.363 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:18.363 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:18.363 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:18.364Z [INFO] agent.router.manager: shutting down -43:18.364 [INFO] TestAgent: consul server down -43:18.364 [INFO] TestAgent: shutdown complete -43:18.364 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13255 network=tcp -43:18.364 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13255 network=udp -43:18.364 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13256 network=tcp -2022-09-15T21:43:18.364Z [INFO] agent.router.manager: shutting down -43:18.870 [INFO] TestAgent: Waiting for endpoints to shut down -43:18.870 [INFO] TestAgent: Endpoints down -=== RUN TestAgent_LoadKeyrings/client_with_keys -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:18.537Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:18.554Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:18.557 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cfc2e5a5-48b8-5437-778f-d570c2ee5d42 Address:127.0.0.1:13330}]" -43:18.558 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13330 [Follower]" leader= -43:18.558 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-cfc2e5a5-48b8-5437-778f-d570c2ee5d42.dc1 127.0.0.1 -43:18.559 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-cfc2e5a5-48b8-5437-778f-d570c2ee5d42 127.0.0.1 -2022-09-15T21:43:18.559Z [INFO] agent.router: Initializing LAN area manager -43:18.560 [INFO] TestAgent.server: Adding LAN server: server="Node-cfc2e5a5-48b8-5437-778f-d570c2ee5d42 (Addr: tcp/127.0.0.1:13330) (DC: dc1)" -43:18.560 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-cfc2e5a5-48b8-5437-778f-d570c2ee5d42.dc1 area=wan -43:18.560 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13325 network=tcp -43:18.560 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13325 network=udp -43:18.561 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13326 network=tcp -43:18.561 [INFO] TestAgent: started state syncer -43:18.561 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13331 network=tcp -43:18.620 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:18.620 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13330 [Candidate]" term=2 -43:18.621 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:18.621 [DEBUG] TestAgent.server.raft: vote granted: from=cfc2e5a5-48b8-5437-778f-d570c2ee5d42 term=2 tally=1 -43:18.621 [INFO] TestAgent.server.raft: election won: tally=1 -43:18.621 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13330 [Leader]" -43:18.621 [INFO] TestAgent.server: cluster leadership acquired -43:18.621 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13330 -43:18.622 [INFO] TestAgent.server: New leader elected: payload=Node-cfc2e5a5-48b8-5437-778f-d570c2ee5d42 -43:18.623 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:18.625 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:18.625 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:18.625 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:18.625 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:18.625 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:18.625 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cfc2e5a5-48b8-5437-778f-d570c2ee5d42 -43:18.625 [INFO] TestAgent.server: member joined, marking health alive: member=Node-cfc2e5a5-48b8-5437-778f-d570c2ee5d42 -43:18.626 [INFO] TestAgent.server: federation state anti-entropy synced -43:18.632 [INFO] TestAgent: Requesting shutdown -43:18.632 [INFO] TestAgent.server: shutting down server -43:18.632 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:18.632 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:18.632 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:18.632 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:18.632 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:18.632 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:18.632 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:18.632 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:18.632 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:18.632 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:18.632 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:18.632 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:18.633Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:18.633Z [INFO] agent.router.manager: shutting down -43:18.633 [INFO] TestAgent: consul server down -43:18.633 [INFO] TestAgent: shutdown complete -43:18.633 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13325 network=tcp -43:18.633 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13325 network=udp -43:18.633 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13326 network=tcp -43:19.134 [INFO] TestAgent: Waiting for endpoints to shut down -43:19.134 [INFO] TestAgent: Endpoints down ---- PASS: TestSessionDestroy (0.62s) -=== CONT TestSessionCustomTTL -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -43:18.917 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-cdc9bd0c-c6b8-5346-d132-e57a18ef060c 127.0.0.1 -2022-09-15T21:43:18.917Z [INFO] agent.router: Initializing LAN area manager -43:18.918 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13367 network=udp -43:18.918 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13367 network=tcp -43:18.919 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13368 network=tcp -43:18.919 [INFO] TestAgent: started state syncer -43:18.920 [INFO] TestAgent: Requesting shutdown -43:18.920 [INFO] TestAgent.client: shutting down client -43:18.920 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave -43:18.920 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13401 network=tcp -43:18.920 [ERROR] TestAgent: gRPC server failed: error="grpc: the server has been stopped" -2022-09-15T21:43:18.920Z [WARN] agent.router.manager: No servers available -43:18.920 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" -43:18.920 [INFO] TestAgent: consul client down -43:18.920 [INFO] TestAgent: shutdown complete -43:18.920 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13367 network=tcp -43:18.920 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13367 network=udp -43:18.921 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13368 network=tcp -43:19.421 [INFO] TestAgent: Waiting for endpoints to shut down -43:19.421 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_LoadKeyrings (1.95s) - --- PASS: TestAgent_LoadKeyrings/no_keys (0.73s) - --- PASS: TestAgent_LoadKeyrings/server_with_keys (0.67s) - --- PASS: TestAgent_LoadKeyrings/client_with_keys (0.55s) -=== CONT TestSessionCreate_DefaultCheck -2022-09-15T21:43:18.726Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:18.753Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:18.757 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:99f5123f-807c-0214-b67b-95466e51a5a6 Address:127.0.0.1:13337}]" -43:18.758 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13337 [Follower]" leader= -43:18.758 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-99f5123f-807c-0214-b67b-95466e51a5a6.dc1 127.0.0.1 -43:18.760 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-99f5123f-807c-0214-b67b-95466e51a5a6 127.0.0.1 -2022-09-15T21:43:18.760Z [INFO] agent.router: Initializing LAN area manager -43:18.761 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13332 network=udp -43:18.761 [INFO] TestAgent.server: Adding LAN server: server="Node-99f5123f-807c-0214-b67b-95466e51a5a6 (Addr: tcp/127.0.0.1:13337) (DC: dc1)" -43:18.761 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-99f5123f-807c-0214-b67b-95466e51a5a6.dc1 area=wan -43:18.762 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13332 network=tcp -43:18.764 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13333 network=tcp -43:18.764 [INFO] TestAgent: started state syncer -43:18.764 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13338 network=tcp -43:18.806 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:18.806 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13337 [Candidate]" term=2 -43:18.806 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:18.806 [DEBUG] TestAgent.server.raft: vote granted: from=99f5123f-807c-0214-b67b-95466e51a5a6 term=2 tally=1 -43:18.806 [INFO] TestAgent.server.raft: election won: tally=1 -43:18.806 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13337 [Leader]" -43:18.807 [INFO] TestAgent.server: cluster leadership acquired -43:18.807 [INFO] TestAgent.server: New leader elected: payload=Node-99f5123f-807c-0214-b67b-95466e51a5a6 -43:18.807 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13337 -43:18.809 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:18.812 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:18.812 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:18.812 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:18.812 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:18.812 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:18.813 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-99f5123f-807c-0214-b67b-95466e51a5a6 -43:18.813 [INFO] TestAgent.server: member joined, marking health alive: member=Node-99f5123f-807c-0214-b67b-95466e51a5a6 -43:18.818 [INFO] TestAgent.server: federation state anti-entropy synced -43:18.862 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:18.863 [INFO] TestAgent: Synced node info -43:18.863 [DEBUG] TestAgent: Node info in sync -43:18.929 [INFO] TestAgent: Requesting shutdown -43:18.930 [INFO] TestAgent.server: shutting down server -43:18.930 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:18.930 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:18.930 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:18.930 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:18.930 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:18.930 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:18.930 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:18.930 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:18.930 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:18.930 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:18.930 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:18.931Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:18.931Z [INFO] agent.router.manager: shutting down -43:18.931 [INFO] TestAgent: consul server down -43:18.931 [INFO] TestAgent: shutdown complete -43:18.931 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13332 network=tcp -43:18.931 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13332 network=udp -43:18.931 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13333 network=tcp -43:19.434 [INFO] TestAgent: Waiting for endpoints to shut down -43:19.434 [INFO] TestAgent: Endpoints down -=== RUN TestSessionGet/#01 -2022-09-15T21:43:18.682Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:18.698Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:18.701 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c3622cd0-ae64-9d25-3e91-11cd8101188f Address:127.0.0.1:13260}]" -43:18.701 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13260 [Follower]" leader= -43:18.701 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c3622cd0-ae64-9d25-3e91-11cd8101188f.dc1 127.0.0.1 -43:18.702 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c3622cd0-ae64-9d25-3e91-11cd8101188f 127.0.0.1 -2022-09-15T21:43:18.702Z [INFO] agent.router: Initializing LAN area manager -43:18.703 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13269 network=udp -43:18.703 [INFO] TestAgent.server: Adding LAN server: server="Node-c3622cd0-ae64-9d25-3e91-11cd8101188f (Addr: tcp/127.0.0.1:13260) (DC: dc1)" -43:18.703 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c3622cd0-ae64-9d25-3e91-11cd8101188f.dc1 area=wan -43:18.703 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13269 network=tcp -43:18.704 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13270 network=tcp -43:18.704 [INFO] TestAgent: started state syncer -43:18.705 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13261 network=tcp -43:18.770 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:18.770 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13260 [Candidate]" term=2 -43:18.770 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:18.770 [DEBUG] TestAgent.server.raft: vote granted: from=c3622cd0-ae64-9d25-3e91-11cd8101188f term=2 tally=1 -43:18.770 [INFO] TestAgent.server.raft: election won: tally=1 -43:18.770 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13260 [Leader]" -43:18.771 [INFO] TestAgent.server: cluster leadership acquired -43:18.771 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13260 -43:18.773 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:18.782 [INFO] TestAgent.server: New leader elected: payload=Node-c3622cd0-ae64-9d25-3e91-11cd8101188f -43:18.793 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:18.793 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:18.793 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:18.793 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:18.793 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:18.793 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c3622cd0-ae64-9d25-3e91-11cd8101188f -43:18.793 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c3622cd0-ae64-9d25-3e91-11cd8101188f -43:18.804 [INFO] TestAgent.server: federation state anti-entropy synced -43:18.971 [INFO] TestAgent: Requesting shutdown -43:18.971 [INFO] TestAgent.server: shutting down server -43:18.971 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:18.971 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:18.971 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:18.971 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:18.971 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:18.972 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:18.972 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:18.972 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:18.972 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:18.972 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:18.972 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:18.972 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:18.972Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:18.972Z [INFO] agent.router.manager: shutting down -43:18.972 [INFO] TestAgent: consul server down -43:18.972 [INFO] TestAgent: shutdown complete -43:18.972 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13269 network=tcp -43:18.972 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13269 network=udp -43:18.972 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13270 network=tcp -43:19.478 [INFO] TestAgent: Waiting for endpoints to shut down -43:19.478 [INFO] TestAgent: Endpoints down ---- PASS: TestSessionDeleteDestroy (0.81s) -=== CONT TestSessionCreate_NoCheck -[WARN] freeport: 4 out of 25 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:18.846Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:18.846Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:18.883Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:18.883Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:18.887 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b283f5fb-a5ca-37ea-fdba-7439f5288844 Address:127.0.0.1:13358}]" -43:18.888 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b283f5fb-a5ca-37ea-fdba-7439f5288844.dc1 127.0.0.1 -43:18.888 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b283f5fb-a5ca-37ea-fdba-7439f5288844 127.0.0.1 -2022-09-15T21:43:18.889Z [INFO] agent.router: Initializing LAN area manager -43:18.889 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13346 network=udp -43:18.889 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13358 [Follower]" leader= -43:18.890 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b283f5fb-a5ca-37ea-fdba-7439f5288844.dc1 area=wan -43:18.890 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13346 network=tcp -43:18.891 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13347 network=tcp -43:18.892 [INFO] TestAgent: started state syncer -43:18.891 [INFO] TestAgent.server: Adding LAN server: server="Node-b283f5fb-a5ca-37ea-fdba-7439f5288844 (Addr: tcp/127.0.0.1:13358) (DC: dc1)" -43:18.892 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13359 network=tcp -43:18.940 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -43:18.941 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:18.941 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13358 [Candidate]" term=2 -43:18.941 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:18.941 [DEBUG] TestAgent.server.raft: vote granted: from=b283f5fb-a5ca-37ea-fdba-7439f5288844 term=2 tally=1 -43:18.941 [INFO] TestAgent.server.raft: election won: tally=1 -43:18.941 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13358 [Leader]" -43:18.941 [INFO] TestAgent.server: cluster leadership acquired -43:18.941 [INFO] TestAgent.server: initializing acls -43:18.942 [INFO] TestAgent.server: Created ACL 'global-management' policy -43:18.942 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -43:18.943 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -43:18.943 [INFO] TestAgent.server: Created ACL anonymous token from configuration -43:18.943 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -43:18.943 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -43:18.944 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-b283f5fb-a5ca-37ea-fdba-7439f5288844 -43:18.944 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-b283f5fb-a5ca-37ea-fdba-7439f5288844.dc1 -43:18.945 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:18.946 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-b283f5fb-a5ca-37ea-fdba-7439f5288844.dc1 area=wan -43:18.946 [INFO] TestAgent.server: New leader elected: payload=Node-b283f5fb-a5ca-37ea-fdba-7439f5288844 -43:18.946 [INFO] TestAgent.server: Updating LAN server: server="Node-b283f5fb-a5ca-37ea-fdba-7439f5288844 (Addr: tcp/127.0.0.1:13358) (DC: dc1)" -43:18.948 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:18.948 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:18.948 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:18.948 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:18.948 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:18.948 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b283f5fb-a5ca-37ea-fdba-7439f5288844 -43:18.948 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b283f5fb-a5ca-37ea-fdba-7439f5288844 -43:18.949 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b283f5fb-a5ca-37ea-fdba-7439f5288844 -43:18.950 [INFO] TestAgent.server: federation state anti-entropy synced -43:19.174 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-b283f5fb-a5ca-37ea-fdba-7439f5288844 -43:19.174 [INFO] TestAgent: Requesting shutdown -43:19.174 [INFO] TestAgent.server: shutting down server -43:19.174 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:19.174 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:19.174 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:19.174 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:19.174 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -43:19.174 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -43:19.174 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:19.174 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:19.174 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:19.174 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:19.174 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:19.174 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:19.174 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:19.175 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -43:19.175 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -43:19.175 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:19.175Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:19.175Z [INFO] agent.router.manager: shutting down -43:19.175 [INFO] TestAgent: consul server down -43:19.175 [INFO] TestAgent: shutdown complete -43:19.175 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13346 network=tcp -43:19.175 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13346 network=udp -43:19.175 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13347 network=tcp -43:19.678 [INFO] TestAgent: Waiting for endpoints to shut down -43:19.678 [INFO] TestAgent: Endpoints down -=== RUN TestSnapshot_Options/PUT#02 -=== RUN TestSessionCreate_NoCheck/no_check_fields_should_yield_default_serfHealth -=== RUN TestSessionCreate_NoCheck/overwrite_nodechecks_to_associate_with_no_checks -=== RUN TestSessionCreate_NoCheck/overwrite_checks_to_associate_with_no_checks -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:18.581Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:18.599Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:18.603 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0525e15d-1a11-7009-7d39-9158d5a1f65b Address:127.0.0.1:13295}]" -43:18.603 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13295 [Follower]" leader= -43:18.604 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0525e15d-1a11-7009-7d39-9158d5a1f65b.dc1 127.0.0.1 -43:18.605 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0525e15d-1a11-7009-7d39-9158d5a1f65b 127.0.0.1 -2022-09-15T21:43:18.605Z [INFO] agent.router: Initializing LAN area manager -43:18.605 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13304 network=udp -43:18.605 [INFO] TestAgent.server: Adding LAN server: server="Node-0525e15d-1a11-7009-7d39-9158d5a1f65b (Addr: tcp/127.0.0.1:13295) (DC: dc1)" -43:18.605 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0525e15d-1a11-7009-7d39-9158d5a1f65b.dc1 area=wan -43:18.606 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13304 network=tcp -43:18.606 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13305 network=tcp -43:18.607 [INFO] TestAgent: started state syncer -43:18.607 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13296 network=tcp -43:18.656 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:18.657 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13295 [Candidate]" term=2 -43:18.657 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:18.657 [DEBUG] TestAgent.server.raft: vote granted: from=0525e15d-1a11-7009-7d39-9158d5a1f65b term=2 tally=1 -43:18.657 [INFO] TestAgent.server.raft: election won: tally=1 -43:18.657 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13295 [Leader]" -43:18.657 [INFO] TestAgent.server: cluster leadership acquired -43:18.657 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13295 -43:18.658 [INFO] TestAgent.server: New leader elected: payload=Node-0525e15d-1a11-7009-7d39-9158d5a1f65b -43:18.659 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:18.661 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:18.661 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:18.661 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:18.661 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:18.661 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:18.661 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0525e15d-1a11-7009-7d39-9158d5a1f65b -43:18.661 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0525e15d-1a11-7009-7d39-9158d5a1f65b -43:18.663 [INFO] TestAgent.server: federation state anti-entropy synced -43:18.960 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:18.960 [INFO] TestAgent: Synced node info -43:19.319 [WARN] TestAgent.server.rpc: Attempting to apply large raft entry: size_in_bytes=1573029 -43:19.336 [INFO] TestAgent: Requesting shutdown -43:19.336 [INFO] TestAgent.server: shutting down server -43:19.336 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:19.336 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:19.336 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:19.336 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:19.336 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:19.336 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:19.336 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:19.337 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:19.337 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:19.337 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:19.337 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:19.337Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:19.337Z [INFO] agent.router.manager: shutting down -43:19.337 [INFO] TestAgent: consul server down -43:19.337 [INFO] TestAgent: shutdown complete -43:19.337 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13304 network=tcp -43:19.337 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13304 network=udp -43:19.337 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13305 network=tcp -43:19.838 [INFO] TestAgent: Waiting for endpoints to shut down -43:19.838 [INFO] TestAgent: Endpoints down ---- PASS: TestTxnEndpoint_Bad_Size_Item (4.13s) - --- PASS: TestTxnEndpoint_Bad_Size_Item/exceeds_default_limits (0.98s) - --- PASS: TestTxnEndpoint_Bad_Size_Item/exceeds_configured_max_txn_len (0.97s) - --- PASS: TestTxnEndpoint_Bad_Size_Item/exceeds_default_max_kv_value_size (0.91s) - --- PASS: TestTxnEndpoint_Bad_Size_Item/allowed (1.28s) -=== CONT TestOperator_Keyring_LocalOnly -2022-09-15T21:43:18.035Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:18.053Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:18.057 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:737dc500-6b6f-c6b7-089d-7945bc7773b6 Address:127.0.0.1:13288}]" -43:18.057 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13288 [Follower]" leader= -43:18.058 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-737dc500-6b6f-c6b7-089d-7945bc7773b6.dc1 127.0.0.1 -43:18.059 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-737dc500-6b6f-c6b7-089d-7945bc7773b6 127.0.0.1 -2022-09-15T21:43:18.059Z [INFO] agent.router: Initializing LAN area manager -43:18.059 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-737dc500-6b6f-c6b7-089d-7945bc7773b6.dc1 area=wan -43:18.059 [INFO] TestAgent.server: Adding LAN server: server="Node-737dc500-6b6f-c6b7-089d-7945bc7773b6 (Addr: tcp/127.0.0.1:13288) (DC: dc1)" -43:18.059 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13318 network=udp -43:18.059 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13318 network=tcp -43:18.060 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13319 network=tcp -43:18.060 [INFO] TestAgent: started state syncer -43:18.060 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13289 network=tcp -43:18.121 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:18.121 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13288 [Candidate]" term=2 -43:18.121 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:18.121 [DEBUG] TestAgent.server.raft: vote granted: from=737dc500-6b6f-c6b7-089d-7945bc7773b6 term=2 tally=1 -43:18.121 [INFO] TestAgent.server.raft: election won: tally=1 -43:18.121 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13288 [Leader]" -43:18.121 [INFO] TestAgent.server: cluster leadership acquired -43:18.121 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13288 -43:18.123 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:18.125 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:18.126 [INFO] TestAgent.server: New leader elected: payload=Node-737dc500-6b6f-c6b7-089d-7945bc7773b6 -43:18.126 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:18.126 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:18.126 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:18.126 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:18.126 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-737dc500-6b6f-c6b7-089d-7945bc7773b6 -43:18.126 [INFO] TestAgent.server: member joined, marking health alive: member=Node-737dc500-6b6f-c6b7-089d-7945bc7773b6 -43:18.128 [INFO] TestAgent.server: federation state anti-entropy synced -43:18.165 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:18.166 [INFO] TestAgent: Synced node info -43:19.337 [INFO] TestAgent: Requesting shutdown -43:19.337 [INFO] TestAgent.server: shutting down server -43:19.337 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:19.337 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:19.337 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:19.337 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:19.338 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:19.338 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:19.338 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:19.338 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:19.338 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:19.338 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:19.342 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:19.342Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:19.342Z [INFO] agent.router.manager: shutting down -43:19.343 [INFO] TestAgent: consul server down -43:19.343 [INFO] TestAgent: shutdown complete -43:19.343 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13318 network=tcp -43:19.343 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13318 network=udp -43:19.343 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13319 network=tcp -43:19.843 [INFO] TestAgent: Waiting for endpoints to shut down -43:19.843 [INFO] TestAgent: Endpoints down ---- PASS: TestPProfHandlers_EnableDebug (1.84s) -=== CONT TestACL_Legacy_UpdateUpsert -2022-09-15T21:43:18.462Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:18.485Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:18.488 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7e2e245f-d42e-4be1-38d0-b9d314b1855a Address:127.0.0.1:13309}]" -43:18.488 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13309 [Follower]" leader= -43:18.489 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7e2e245f-d42e-4be1-38d0-b9d314b1855a.dc1 127.0.0.1 -43:18.490 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7e2e245f-d42e-4be1-38d0-b9d314b1855a 127.0.0.1 -2022-09-15T21:43:18.490Z [INFO] agent.router: Initializing LAN area manager -43:18.491 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13311 network=udp -43:18.491 [INFO] TestAgent.server: Adding LAN server: server="Node-7e2e245f-d42e-4be1-38d0-b9d314b1855a (Addr: tcp/127.0.0.1:13309) (DC: dc1)" -43:18.491 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7e2e245f-d42e-4be1-38d0-b9d314b1855a.dc1 area=wan -43:18.491 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13311 network=tcp -43:18.492 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13312 network=tcp -43:18.492 [INFO] TestAgent: started state syncer -43:18.492 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13310 network=tcp -43:18.553 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:18.553 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13309 [Candidate]" term=2 -43:18.554 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:18.554 [DEBUG] TestAgent.server.raft: vote granted: from=7e2e245f-d42e-4be1-38d0-b9d314b1855a term=2 tally=1 -43:18.554 [INFO] TestAgent.server.raft: election won: tally=1 -43:18.554 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13309 [Leader]" -43:18.554 [INFO] TestAgent.server: cluster leadership acquired -43:18.554 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13309 -43:18.555 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:18.558 [INFO] TestAgent.server: New leader elected: payload=Node-7e2e245f-d42e-4be1-38d0-b9d314b1855a -43:18.558 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:18.558 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:18.558 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:18.558 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:18.558 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:18.558 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7e2e245f-d42e-4be1-38d0-b9d314b1855a -43:18.558 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7e2e245f-d42e-4be1-38d0-b9d314b1855a -43:18.560 [INFO] TestAgent.server: federation state anti-entropy synced -43:18.842 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:18.843 [INFO] TestAgent: Synced node info -43:19.421 [WARN] TestAgent.server.rpc: Attempting to apply large raft entry: size_in_bytes=4719032 -43:19.436 [INFO] TestAgent: Requesting shutdown -43:19.436 [INFO] TestAgent.server: shutting down server -43:19.436 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:19.436 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:19.436 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:19.436 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:19.436 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:19.436 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:19.436 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:19.436 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:19.436 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:19.436 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:19.436 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:19.437Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:19.437Z [INFO] agent.router.manager: shutting down -43:19.437 [INFO] TestAgent: consul server down -43:19.437 [INFO] TestAgent: shutdown complete -43:19.437 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13311 network=tcp -43:19.437 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13311 network=udp -43:19.437 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13312 network=tcp -43:19.938 [INFO] TestAgent: Waiting for endpoints to shut down -43:19.938 [INFO] TestAgent: Endpoints down ---- PASS: TestTxnEndpoint_Bad_Size_Net (7.04s) - --- PASS: TestTxnEndpoint_Bad_Size_Net/exceeds_default_limits (1.37s) - --- PASS: TestTxnEndpoint_Bad_Size_Net/exceeds_configured_max_txn_len (1.09s) - --- PASS: TestTxnEndpoint_Bad_Size_Net/exceeds_default_max_kv_value_size (1.79s) - --- PASS: TestTxnEndpoint_Bad_Size_Net/allowed (1.29s) - --- PASS: TestTxnEndpoint_Bad_Size_Net/allowed_kv_max_backward_compatible (1.50s) -=== CONT TestSessionCreate_Delete -[WARN] freeport: 4 out of 25 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:19.454Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:19.472Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:19.475 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3182a8f7-171e-0231-555b-0037e7257d48 Address:127.0.0.1:13064}]" -43:19.476 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3182a8f7-171e-0231-555b-0037e7257d48.dc1 127.0.0.1 -43:19.476 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3182a8f7-171e-0231-555b-0037e7257d48 127.0.0.1 -2022-09-15T21:43:19.476Z [INFO] agent.router: Initializing LAN area manager -43:19.477 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14079 network=udp -43:19.477 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13064 [Follower]" leader= -43:19.477 [INFO] TestAgent.server: Adding LAN server: server="Node-3182a8f7-171e-0231-555b-0037e7257d48 (Addr: tcp/127.0.0.1:13064) (DC: dc1)" -43:19.477 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3182a8f7-171e-0231-555b-0037e7257d48.dc1 area=wan -43:19.477 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14079 network=tcp -43:19.478 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14080 network=tcp -43:19.479 [INFO] TestAgent: started state syncer -43:19.479 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13065 network=tcp -43:19.551 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:19.551 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13064 [Candidate]" term=2 -43:19.551 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:19.551 [DEBUG] TestAgent.server.raft: vote granted: from=3182a8f7-171e-0231-555b-0037e7257d48 term=2 tally=1 -43:19.551 [INFO] TestAgent.server.raft: election won: tally=1 -43:19.551 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13064 [Leader]" -43:19.552 [INFO] TestAgent.server: cluster leadership acquired -43:19.552 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13064 -43:19.554 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:19.557 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:19.557 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:19.557 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:19.557 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:19.557 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:19.557 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3182a8f7-171e-0231-555b-0037e7257d48 -43:19.557 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3182a8f7-171e-0231-555b-0037e7257d48 -43:19.566 [INFO] TestAgent.server: New leader elected: payload=Node-3182a8f7-171e-0231-555b-0037e7257d48 -43:19.573 [INFO] TestAgent.server: federation state anti-entropy synced -43:19.679 [INFO] TestAgent: Requesting shutdown -43:19.679 [INFO] TestAgent.server: shutting down server -43:19.679 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:19.679 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:19.679 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:19.679 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:19.679 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:19.679 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:19.679 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:19.679 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:19.679 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:19.679 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:19.679 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:19.679 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:19.680Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:19.680Z [INFO] agent.router.manager: shutting down -43:19.680 [INFO] TestAgent: consul server down -43:19.680 [INFO] TestAgent: shutdown complete -43:19.680 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14079 network=tcp -43:19.680 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14079 network=udp -43:19.680 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14080 network=tcp -43:20.180 [INFO] TestAgent: Waiting for endpoints to shut down -43:20.180 [INFO] TestAgent: Endpoints down ---- PASS: TestSessionCreate_DefaultCheck (0.76s) -=== CONT TestIntentionsSpecificGet_invalidId -2022-09-15T21:43:19.454Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:19.471Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:19.474 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8e36963e-e133-1b2b-bf0f-078d7797bc5c Address:127.0.0.1:14077}]" -43:19.475 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8e36963e-e133-1b2b-bf0f-078d7797bc5c.dc1 127.0.0.1 -43:19.476 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8e36963e-e133-1b2b-bf0f-078d7797bc5c 127.0.0.1 -2022-09-15T21:43:19.476Z [INFO] agent.router: Initializing LAN area manager -43:19.476 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14065 network=udp -43:19.476 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14077 [Follower]" leader= -43:19.477 [INFO] TestAgent.server: Adding LAN server: server="Node-8e36963e-e133-1b2b-bf0f-078d7797bc5c (Addr: tcp/127.0.0.1:14077) (DC: dc1)" -43:19.477 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8e36963e-e133-1b2b-bf0f-078d7797bc5c.dc1 area=wan -43:19.477 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14065 network=tcp -43:19.478 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14066 network=tcp -43:19.478 [INFO] TestAgent: started state syncer -43:19.479 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14078 network=tcp -43:19.521 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:19.521 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14077 [Candidate]" term=2 -43:19.521 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:19.521 [DEBUG] TestAgent.server.raft: vote granted: from=8e36963e-e133-1b2b-bf0f-078d7797bc5c term=2 tally=1 -43:19.521 [INFO] TestAgent.server.raft: election won: tally=1 -43:19.521 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14077 [Leader]" -43:19.521 [INFO] TestAgent.server: cluster leadership acquired -43:19.522 [INFO] TestAgent.server: New leader elected: payload=Node-8e36963e-e133-1b2b-bf0f-078d7797bc5c -43:19.522 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14077 -43:19.524 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:19.530 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:19.530 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:19.530 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:19.530 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:19.530 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:19.532 [INFO] TestAgent.server: federation state anti-entropy synced -43:19.546 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8e36963e-e133-1b2b-bf0f-078d7797bc5c -43:19.546 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8e36963e-e133-1b2b-bf0f-078d7797bc5c -43:19.689 [INFO] TestAgent: Requesting shutdown -43:19.689 [INFO] TestAgent.server: shutting down server -43:19.689 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:19.689 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:19.689 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:19.689 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:19.689 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:19.689 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:19.689 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:19.689 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:19.689 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:19.689 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:19.689 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:19.690 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:19.690Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:19.690Z [INFO] agent.router.manager: shutting down -43:19.690 [INFO] TestAgent: consul server down -43:19.690 [INFO] TestAgent: shutdown complete -43:19.690 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14065 network=tcp -43:19.690 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14065 network=udp -43:19.690 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14066 network=tcp -43:20.194 [INFO] TestAgent: Waiting for endpoints to shut down -43:20.194 [INFO] TestAgent: Endpoints down ---- PASS: TestSessionGet (1.49s) - --- PASS: TestSessionGet/#00 (0.73s) - --- PASS: TestSessionGet/#01 (0.76s) -=== CONT TestACL_Legacy_Update -2022-09-15T21:43:19.498Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:19.516Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:19.519 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d7988f7d-10a2-992a-7750-b1bd4930e86e Address:127.0.0.1:13421}]" -43:19.519 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13421 [Follower]" leader= -43:19.520 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d7988f7d-10a2-992a-7750-b1bd4930e86e.dc1 127.0.0.1 -43:19.521 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d7988f7d-10a2-992a-7750-b1bd4930e86e 127.0.0.1 -2022-09-15T21:43:19.521Z [INFO] agent.router: Initializing LAN area manager -43:19.522 [INFO] TestAgent.server: Adding LAN server: server="Node-d7988f7d-10a2-992a-7750-b1bd4930e86e (Addr: tcp/127.0.0.1:13421) (DC: dc1)" -43:19.522 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d7988f7d-10a2-992a-7750-b1bd4930e86e.dc1 area=wan -43:19.523 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13486 network=udp -43:19.523 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13486 network=tcp -43:19.524 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13487 network=tcp -43:19.524 [INFO] TestAgent: started state syncer -43:19.524 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13422 network=tcp -43:19.589 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:19.589 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13421 [Candidate]" term=2 -43:19.589 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:19.589 [DEBUG] TestAgent.server.raft: vote granted: from=d7988f7d-10a2-992a-7750-b1bd4930e86e term=2 tally=1 -43:19.589 [INFO] TestAgent.server.raft: election won: tally=1 -43:19.589 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13421 [Leader]" -43:19.590 [INFO] TestAgent.server: cluster leadership acquired -43:19.590 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13421 -43:19.591 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:19.594 [INFO] TestAgent.server: New leader elected: payload=Node-d7988f7d-10a2-992a-7750-b1bd4930e86e -43:19.594 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:19.594 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:19.594 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:19.594 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:19.594 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:19.594 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d7988f7d-10a2-992a-7750-b1bd4930e86e -43:19.594 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d7988f7d-10a2-992a-7750-b1bd4930e86e -43:19.596 [INFO] TestAgent.server: federation state anti-entropy synced -43:19.674 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:19.675 [INFO] TestAgent: Synced node info -43:19.675 [DEBUG] TestAgent: Node info in sync -43:19.769 [INFO] TestAgent: Requesting shutdown -43:19.769 [INFO] TestAgent.server: shutting down server -43:19.769 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:19.769 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:19.769 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:19.769 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:19.769 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:19.770 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:19.770 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:19.770 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:19.770 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:19.770 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:19.770 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:19.770Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:19.770Z [INFO] agent.router.manager: shutting down -43:19.770 [INFO] TestAgent: consul server down -43:19.770 [INFO] TestAgent: shutdown complete -43:19.770 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13486 network=tcp -43:19.771 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13486 network=udp -43:19.771 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13487 network=tcp -43:20.271 [INFO] TestAgent: Waiting for endpoints to shut down -43:20.271 [INFO] TestAgent: Endpoints down ---- PASS: TestSessionCreate_NoCheck (0.79s) - --- PASS: TestSessionCreate_NoCheck/no_check_fields_should_yield_default_serfHealth (0.00s) - --- PASS: TestSessionCreate_NoCheck/overwrite_nodechecks_to_associate_with_no_checks (0.00s) - --- PASS: TestSessionCreate_NoCheck/overwrite_checks_to_associate_with_no_checks (0.00s) -=== CONT TestIntentionsSpecificGet_good -[WARN] freeport: 4 out of 25 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 4 out of 4 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:19.858Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:19.899Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:19.906 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:17b37617-488d-1d71-9344-4cc1bb08d28c Address:127.0.0.1:13456}]" -43:19.906 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13456 [Follower]" leader= -43:19.907 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-17b37617-488d-1d71-9344-4cc1bb08d28c.dc1 127.0.0.1 -43:19.908 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-17b37617-488d-1d71-9344-4cc1bb08d28c 127.0.0.1 -2022-09-15T21:43:19.908Z [INFO] agent.router: Initializing LAN area manager -43:19.908 [INFO] TestAgent.server: Adding LAN server: server="Node-17b37617-488d-1d71-9344-4cc1bb08d28c (Addr: tcp/127.0.0.1:13456) (DC: dc1)" -43:19.908 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-17b37617-488d-1d71-9344-4cc1bb08d28c.dc1 area=wan -43:19.908 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13423 network=udp -43:19.908 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13423 network=tcp -43:19.909 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13424 network=tcp -43:19.909 [INFO] TestAgent: started state syncer -43:19.910 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13457 network=tcp -43:19.982 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:19.982 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13456 [Candidate]" term=2 -43:19.982 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:19.982 [DEBUG] TestAgent.server.raft: vote granted: from=17b37617-488d-1d71-9344-4cc1bb08d28c term=2 tally=1 -43:19.982 [INFO] TestAgent.server.raft: election won: tally=1 -43:19.982 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13456 [Leader]" -43:19.982 [INFO] TestAgent.server: cluster leadership acquired -43:19.982 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13456 -43:19.983 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:19.986 [INFO] TestAgent.server: New leader elected: payload=Node-17b37617-488d-1d71-9344-4cc1bb08d28c -43:19.989 [INFO] TestAgent: Synced node info -43:19.989 [DEBUG] TestAgent: Node info in sync -43:19.992 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:19.992 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:19.992 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:19.992 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:19.992 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:19.992 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-17b37617-488d-1d71-9344-4cc1bb08d28c -43:19.992 [INFO] TestAgent.server: member joined, marking health alive: member=Node-17b37617-488d-1d71-9344-4cc1bb08d28c -43:19.993 [INFO] TestAgent.server: federation state anti-entropy synced -43:20.112 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query -43:20.114 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-17b37617-488d-1d71-9344-4cc1bb08d28c -43:20.114 [INFO] TestAgent: Requesting shutdown -43:20.114 [INFO] TestAgent.server: shutting down server -43:20.114 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:20.114 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:20.114 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:20.114 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:20.114 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:20.114 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:20.114 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:20.114 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:20.114 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:20.114 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:20.115 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:20.115Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:20.115Z [INFO] agent.router.manager: shutting down -43:20.115 [INFO] TestAgent: consul server down -43:20.115 [INFO] TestAgent: shutdown complete -43:20.115 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13423 network=tcp -43:20.115 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13423 network=udp -43:20.115 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13424 network=tcp -43:20.615 [INFO] TestAgent: Waiting for endpoints to shut down -43:20.615 [INFO] TestAgent: Endpoints down ---- PASS: TestOperator_Keyring_LocalOnly (0.78s) -=== CONT TestDNS_ServiceLookup_AnswerLimits/A_lookup_{0_0_0_0_0_0_0_0_0_0_0} -2022-09-15T21:43:19.703Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:19.703Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:19.722Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:19.722Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:19.725 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a66f8c8b-d8af-454c-df79-21cb034f805b Address:127.0.0.1:13428}]" -43:19.726 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13428 [Follower]" leader= -43:19.726 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a66f8c8b-d8af-454c-df79-21cb034f805b.dc1 127.0.0.1 -43:19.727 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a66f8c8b-d8af-454c-df79-21cb034f805b 127.0.0.1 -2022-09-15T21:43:19.728Z [INFO] agent.router: Initializing LAN area manager -43:19.728 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13458 network=udp -43:19.728 [INFO] TestAgent.server: Adding LAN server: server="Node-a66f8c8b-d8af-454c-df79-21cb034f805b (Addr: tcp/127.0.0.1:13428) (DC: dc1)" -43:19.728 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a66f8c8b-d8af-454c-df79-21cb034f805b.dc1 area=wan -43:19.728 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13458 network=tcp -43:19.729 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13459 network=tcp -43:19.729 [INFO] TestAgent: started state syncer -43:19.729 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13429 network=tcp -43:19.780 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -43:19.786 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:19.786 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13428 [Candidate]" term=2 -43:19.786 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:19.786 [DEBUG] TestAgent.server.raft: vote granted: from=a66f8c8b-d8af-454c-df79-21cb034f805b term=2 tally=1 -43:19.786 [INFO] TestAgent.server.raft: election won: tally=1 -43:19.786 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13428 [Leader]" -43:19.786 [INFO] TestAgent.server: cluster leadership acquired -43:19.787 [INFO] TestAgent.server: initializing acls -43:19.787 [INFO] TestAgent.server: Created ACL 'global-management' policy -43:19.787 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -43:19.788 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -43:19.788 [INFO] TestAgent.server: Created ACL anonymous token from configuration -43:19.788 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -43:19.788 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -43:19.789 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-a66f8c8b-d8af-454c-df79-21cb034f805b -43:19.789 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-a66f8c8b-d8af-454c-df79-21cb034f805b.dc1 -43:19.790 [INFO] TestAgent.server: New leader elected: payload=Node-a66f8c8b-d8af-454c-df79-21cb034f805b -43:19.790 [INFO] TestAgent.server: Updating LAN server: server="Node-a66f8c8b-d8af-454c-df79-21cb034f805b (Addr: tcp/127.0.0.1:13428) (DC: dc1)" -43:19.790 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-a66f8c8b-d8af-454c-df79-21cb034f805b.dc1 area=wan -43:19.791 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:19.793 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:19.793 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:19.793 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:19.793 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:19.793 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:19.794 [INFO] TestAgent.server: federation state anti-entropy synced -43:19.798 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a66f8c8b-d8af-454c-df79-21cb034f805b -43:19.798 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a66f8c8b-d8af-454c-df79-21cb034f805b -43:19.799 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a66f8c8b-d8af-454c-df79-21cb034f805b -43:20.062 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:20.062 [INFO] TestAgent: Synced node info -43:20.062 [DEBUG] TestAgent: Node info in sync -43:20.140 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-a66f8c8b-d8af-454c-df79-21cb034f805b -43:20.140 [INFO] TestAgent: Requesting shutdown -43:20.140 [INFO] TestAgent.server: shutting down server -43:20.140 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -43:20.140 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -43:20.140 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:20.140 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:20.140 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:20.140 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:20.140 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:20.141 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:20.141 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:20.141 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -43:20.141 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -43:20.141 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:20.141 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:20.141 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:20.141 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:20.141Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:20.141Z [INFO] agent.router.manager: shutting down -43:20.141 [INFO] TestAgent: consul server down -43:20.141 [INFO] TestAgent: shutdown complete -43:20.141 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13458 network=tcp -43:20.141 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13458 network=udp -43:20.141 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13459 network=tcp -43:20.642 [INFO] TestAgent: Waiting for endpoints to shut down -43:20.642 [INFO] TestAgent: Endpoints down ---- PASS: TestSnapshot_Options (5.09s) - --- PASS: TestSnapshot_Options/GET (0.86s) - --- PASS: TestSnapshot_Options/GET#01 (0.71s) - --- PASS: TestSnapshot_Options/GET#02 (1.01s) - --- PASS: TestSnapshot_Options/PUT (0.64s) - --- PASS: TestSnapshot_Options/PUT#01 (0.90s) - --- PASS: TestSnapshot_Options/PUT#02 (0.96s) -=== CONT TestDNS_ServiceLookup_AnswerLimits/A_lookup_{6_6_6_6_6_6_6_5_6_6_-5} -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:19.864Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:19.864Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:19.903Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:19.903Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:19.909 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:69db7769-57fc-b761-6561-8b7a82ab5c32 Address:127.0.0.1:13442}]" -43:19.910 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-69db7769-57fc-b761-6561-8b7a82ab5c32.dc1 127.0.0.1 -43:19.910 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13442 [Follower]" leader= -43:19.910 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-69db7769-57fc-b761-6561-8b7a82ab5c32 127.0.0.1 -2022-09-15T21:43:19.910Z [INFO] agent.router: Initializing LAN area manager -43:19.911 [INFO] TestAgent.server: Adding LAN server: server="Node-69db7769-57fc-b761-6561-8b7a82ab5c32 (Addr: tcp/127.0.0.1:13442) (DC: dc1)" -43:19.911 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-69db7769-57fc-b761-6561-8b7a82ab5c32.dc1 area=wan -43:19.911 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13402 network=udp -43:19.911 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13402 network=tcp -43:19.912 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13403 network=tcp -43:19.912 [INFO] TestAgent: started state syncer -43:19.912 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13443 network=tcp -43:19.962 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -43:19.979 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:19.979 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13442 [Candidate]" term=2 -43:19.980 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:19.980 [DEBUG] TestAgent.server.raft: vote granted: from=69db7769-57fc-b761-6561-8b7a82ab5c32 term=2 tally=1 -43:19.980 [INFO] TestAgent.server.raft: election won: tally=1 -43:19.980 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13442 [Leader]" -43:19.980 [INFO] TestAgent.server: cluster leadership acquired -43:19.980 [INFO] TestAgent.server: initializing acls -43:19.981 [INFO] TestAgent.server: Created ACL 'global-management' policy -43:19.981 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -43:19.981 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -43:19.982 [INFO] TestAgent.server: Created ACL anonymous token from configuration -43:19.982 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -43:19.982 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -43:19.982 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-69db7769-57fc-b761-6561-8b7a82ab5c32 -43:19.982 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-69db7769-57fc-b761-6561-8b7a82ab5c32.dc1 -43:19.983 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:19.986 [INFO] TestAgent.server: New leader elected: payload=Node-69db7769-57fc-b761-6561-8b7a82ab5c32 -43:19.986 [INFO] TestAgent.server: Updating LAN server: server="Node-69db7769-57fc-b761-6561-8b7a82ab5c32 (Addr: tcp/127.0.0.1:13442) (DC: dc1)" -43:19.986 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-69db7769-57fc-b761-6561-8b7a82ab5c32.dc1 area=wan -43:19.987 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:19.987 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:19.987 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:19.987 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:19.987 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:19.987 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-69db7769-57fc-b761-6561-8b7a82ab5c32 -43:19.987 [INFO] TestAgent.server: member joined, marking health alive: member=Node-69db7769-57fc-b761-6561-8b7a82ab5c32 -43:19.988 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-69db7769-57fc-b761-6561-8b7a82ab5c32 -43:19.988 [INFO] TestAgent.server: federation state anti-entropy synced -43:20.100 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:20.100 [INFO] TestAgent: Synced node info -43:20.100 [DEBUG] TestAgent: Node info in sync -43:20.289 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-69db7769-57fc-b761-6561-8b7a82ab5c32 -43:20.289 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-69db7769-57fc-b761-6561-8b7a82ab5c32 -43:20.290 [INFO] TestAgent: Requesting shutdown -43:20.290 [INFO] TestAgent.server: shutting down server -43:20.290 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -43:20.290 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -43:20.290 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:20.290 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:20.290 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:20.290 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:20.290 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:20.290 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -43:20.291 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:20.291 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:20.291 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -43:20.291 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:20.291 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:20.291 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:20.291 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:20.291Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:20.291Z [INFO] agent.router.manager: shutting down -43:20.291 [INFO] TestAgent: consul server down -43:20.291 [INFO] TestAgent: shutdown complete -43:20.291 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13402 network=tcp -43:20.291 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13402 network=udp -43:20.291 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13403 network=tcp -43:20.793 [INFO] TestAgent: Waiting for endpoints to shut down -43:20.793 [INFO] TestAgent: Endpoints down ---- PASS: TestACL_Legacy_UpdateUpsert (0.95s) +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._udp_(qType:=33)_compressed=true +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._udp_(qType:=33)_compressed=false +2021-08-13T15:34:55.721Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:55.764Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:55.786 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6526b460-412c-aef9-fb32-8b0ef5926924 Address:127.0.0.1:24618}]" +34:55.787 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24618 [Follower]" leader= +34:55.795 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6526b460-412c-aef9-fb32-8b0ef5926924.dc1 127.0.0.1 +34:55.796 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6526b460-412c-aef9-fb32-8b0ef5926924 127.0.0.1 +2021-08-13T15:34:55.796Z [INFO] agent.router: Initializing LAN area manager +34:55.797 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24351 network=udp +34:55.798 [INFO] TestAgent.server: Adding LAN server: server="Node-6526b460-412c-aef9-fb32-8b0ef5926924 (Addr: tcp/127.0.0.1:24618) (DC: dc1)" +34:55.798 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6526b460-412c-aef9-fb32-8b0ef5926924.dc1 area=wan +34:55.798 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24351 network=tcp +34:55.817 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24352 network=tcp +34:55.818 [INFO] TestAgent: started state syncer +34:55.818 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24619 network=tcp +34:55.834 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:55.834 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24618 [Candidate]" term=2 +34:55.835 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:55.835 [DEBUG] TestAgent.server.raft: vote granted: from=6526b460-412c-aef9-fb32-8b0ef5926924 term=2 tally=1 +34:55.835 [INFO] TestAgent.server.raft: election won: tally=1 +34:55.835 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24618 [Leader]" +34:55.835 [INFO] TestAgent.server: cluster leadership acquired +34:55.835 [INFO] TestAgent.server: New leader elected: payload=Node-6526b460-412c-aef9-fb32-8b0ef5926924 +34:55.835 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24618 +34:55.836 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:55.839 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:55.839 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:55.839 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:55.839 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:55.839 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:55.839 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6526b460-412c-aef9-fb32-8b0ef5926924 +34:55.839 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6526b460-412c-aef9-fb32-8b0ef5926924 +34:55.841 [INFO] TestAgent.server: federation state anti-entropy synced +34:56.148 [INFO] TestAgent: Requesting shutdown +34:56.148 [INFO] TestAgent.server: shutting down server +34:56.148 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:56.148 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:56.148 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:56.148 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:56.148 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:56.148 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:56.148 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:56.149 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:56.149 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:56.149 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:56.149 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:56.149 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:56.150Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:56.150Z [INFO] agent.router.manager: shutting down +34:56.150 [INFO] TestAgent: consul server down +34:56.150 [INFO] TestAgent: shutdown complete +34:56.150 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24351 network=tcp +34:56.150 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24351 network=udp +34:56.150 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24352 network=tcp +34:56.650 [INFO] TestAgent: Waiting for endpoints to shut down +34:56.650 [INFO] TestAgent: Endpoints down +2021-08-13T15:34:55.741Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:55.758Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:55.780 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f1277300-c4e3-eda1-29ae-8ef517ec4e8a Address:127.0.0.1:24541}]" +34:55.781 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f1277300-c4e3-eda1-29ae-8ef517ec4e8a.dc1 127.0.0.1 +34:55.783 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f1277300-c4e3-eda1-29ae-8ef517ec4e8a 127.0.0.1 +2021-08-13T15:34:55.783Z [INFO] agent.router: Initializing LAN area manager +34:55.787 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24541 [Follower]" leader= +34:55.791 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f1277300-c4e3-eda1-29ae-8ef517ec4e8a.dc1 area=wan +34:55.791 [INFO] TestAgent.server: Adding LAN server: server="Node-f1277300-c4e3-eda1-29ae-8ef517ec4e8a (Addr: tcp/127.0.0.1:24541) (DC: dc1)" +34:55.791 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24620 network=udp +34:55.791 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24620 network=tcp +34:55.793 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24621 network=tcp +34:55.793 [INFO] TestAgent: started state syncer +34:55.793 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24542 network=tcp +34:55.838 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:55.838 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24541 [Candidate]" term=2 +34:55.839 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:55.839 [DEBUG] TestAgent.server.raft: vote granted: from=f1277300-c4e3-eda1-29ae-8ef517ec4e8a term=2 tally=1 +34:55.839 [INFO] TestAgent.server.raft: election won: tally=1 +34:55.839 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24541 [Leader]" +34:55.839 [INFO] TestAgent.server: cluster leadership acquired +34:55.839 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24541 +34:55.840 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:55.841 [INFO] TestAgent.server: New leader elected: payload=Node-f1277300-c4e3-eda1-29ae-8ef517ec4e8a +34:55.855 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:55.855 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:55.855 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:55.855 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:55.855 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:55.855 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f1277300-c4e3-eda1-29ae-8ef517ec4e8a +34:55.855 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f1277300-c4e3-eda1-29ae-8ef517ec4e8a +34:55.857 [INFO] TestAgent.server: federation state anti-entropy synced +34:55.911 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:55.912 [INFO] TestAgent: Synced node info +34:55.912 [DEBUG] TestAgent: Node info in sync +34:56.148 [INFO] TestAgent: Requesting shutdown +34:56.148 [INFO] TestAgent.server: shutting down server +34:56.148 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:56.148 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:56.148 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:56.148 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:56.148 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:56.149 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:56.149 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:56.149 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:56.149 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:56.149 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:56.149 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:56.150Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:56.150Z [INFO] agent.router.manager: shutting down +34:56.150 [INFO] TestAgent: consul server down +34:56.150 [INFO] TestAgent: shutdown complete +34:56.150 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24620 network=tcp +34:56.150 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24620 network=udp +34:56.150 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24621 network=tcp +34:56.650 [INFO] TestAgent: Waiting for endpoints to shut down +34:56.650 [INFO] TestAgent: Endpoints down +--- PASS: TestUIGatewayServiceNodes_Terminating (0.93s) === CONT TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{30_30_8_8_6_8_8_5_8_-5_-5} -2022-09-15T21:43:19.169Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:19.189Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:19.192 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:74c1bebd-b36c-0924-3af7-525236af250f Address:127.0.0.1:13842}]" -43:19.193 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-74c1bebd-b36c-0924-3af7-525236af250f.dc1 127.0.0.1 -43:19.193 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-74c1bebd-b36c-0924-3af7-525236af250f 127.0.0.1 -43:19.194 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13842 [Follower]" leader= -2022-09-15T21:43:19.198Z [INFO] agent.router: Initializing LAN area manager -43:19.198 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13865 network=udp -43:19.198 [INFO] TestAgent.server: Adding LAN server: server="Node-74c1bebd-b36c-0924-3af7-525236af250f (Addr: tcp/127.0.0.1:13842) (DC: dc1)" -43:19.198 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-74c1bebd-b36c-0924-3af7-525236af250f.dc1 area=wan -43:19.199 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13865 network=tcp -43:19.199 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13866 network=tcp -43:19.200 [INFO] TestAgent: started state syncer -43:19.200 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13843 network=tcp -43:19.241 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:19.241 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13842 [Candidate]" term=2 -43:19.241 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:19.242 [DEBUG] TestAgent.server.raft: vote granted: from=74c1bebd-b36c-0924-3af7-525236af250f term=2 tally=1 -43:19.242 [INFO] TestAgent.server.raft: election won: tally=1 -43:19.242 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13842 [Leader]" -43:19.242 [INFO] TestAgent.server: cluster leadership acquired -43:19.242 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13842 -43:19.243 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:19.246 [INFO] TestAgent.server: New leader elected: payload=Node-74c1bebd-b36c-0924-3af7-525236af250f -43:19.255 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:19.255 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:19.255 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:19.255 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:19.255 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:19.255 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-74c1bebd-b36c-0924-3af7-525236af250f -43:19.255 [INFO] TestAgent.server: member joined, marking health alive: member=Node-74c1bebd-b36c-0924-3af7-525236af250f -43:19.258 [INFO] TestAgent.server: federation state anti-entropy synced -43:19.402 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:19.403 [INFO] TestAgent: Synced node info -43:19.403 [DEBUG] TestAgent: Node info in sync -43:20.142 [DEBUG] TestAgent.server: Session TTL expired: session=c036d928-b5df-863f-609d-5ae86701215b -43:20.391 [INFO] TestAgent: Requesting shutdown -43:20.391 [INFO] TestAgent.server: shutting down server -43:20.391 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:20.391 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:20.391 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:20.391 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:20.391 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:20.391 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:20.391 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:20.391 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:20.391 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:20.391 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:20.392 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:20.392Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:20.392Z [INFO] agent.router.manager: shutting down -43:20.392 [INFO] TestAgent: consul server down -43:20.392 [INFO] TestAgent: shutdown complete -43:20.392 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13865 network=tcp -43:20.392 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13865 network=udp -43:20.392 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13866 network=tcp -43:20.894 [INFO] TestAgent: Waiting for endpoints to shut down -43:20.894 [INFO] TestAgent: Endpoints down ---- PASS: TestSessionCustomTTL (1.75s) +=== RUN TestConnectCAConfig/updating_config_with_same_state +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=255)_compressed=true#01 +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! === CONT TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{30_30_8_8_6_8_8_5_8_-5_-5} -2022-09-15T21:43:19.967Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:19.989Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:19.994 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:def6755a-4aa8-6c3d-a0e4-6bc11865fcd5 Address:127.0.0.1:13463}]" -43:19.994 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13463 [Follower]" leader= -43:19.995 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-def6755a-4aa8-6c3d-a0e4-6bc11865fcd5.dc1 127.0.0.1 -43:19.996 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-def6755a-4aa8-6c3d-a0e4-6bc11865fcd5 127.0.0.1 -2022-09-15T21:43:19.996Z [INFO] agent.router: Initializing LAN area manager -43:19.997 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13409 network=udp -43:19.997 [INFO] TestAgent.server: Adding LAN server: server="Node-def6755a-4aa8-6c3d-a0e4-6bc11865fcd5 (Addr: tcp/127.0.0.1:13463) (DC: dc1)" -43:19.997 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-def6755a-4aa8-6c3d-a0e4-6bc11865fcd5.dc1 area=wan -43:19.997 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13409 network=tcp -43:19.998 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13410 network=tcp -43:19.998 [INFO] TestAgent: started state syncer -43:19.998 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13464 network=tcp -43:20.056 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:20.056 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13463 [Candidate]" term=2 -43:20.057 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:20.057 [DEBUG] TestAgent.server.raft: vote granted: from=def6755a-4aa8-6c3d-a0e4-6bc11865fcd5 term=2 tally=1 -43:20.057 [INFO] TestAgent.server.raft: election won: tally=1 -43:20.057 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13463 [Leader]" -43:20.057 [INFO] TestAgent.server: cluster leadership acquired -43:20.057 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13463 -43:20.058 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:20.060 [INFO] TestAgent.server: New leader elected: payload=Node-def6755a-4aa8-6c3d-a0e4-6bc11865fcd5 -43:20.063 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:20.063 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:20.063 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:20.063 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:20.063 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:20.064 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-def6755a-4aa8-6c3d-a0e4-6bc11865fcd5 -43:20.064 [INFO] TestAgent.server: member joined, marking health alive: member=Node-def6755a-4aa8-6c3d-a0e4-6bc11865fcd5 -43:20.065 [INFO] TestAgent.server: federation state anti-entropy synced -43:20.350 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:20.350 [INFO] TestAgent: Synced node info -43:20.350 [DEBUG] TestAgent: Node info in sync -43:20.401 [INFO] TestAgent: Requesting shutdown -43:20.401 [INFO] TestAgent.server: shutting down server -43:20.401 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:20.401 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:20.401 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:20.401 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:20.401 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:20.401 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:20.401 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:20.401 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:20.401 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:20.401 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:20.401 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:20.406Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:20.406Z [INFO] agent.router.manager: shutting down -43:20.406 [INFO] TestAgent: consul server down -43:20.406 [INFO] TestAgent: shutdown complete -43:20.406 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13409 network=tcp -43:20.406 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13409 network=udp -43:20.406 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13410 network=tcp -43:20.908 [INFO] TestAgent: Waiting for endpoints to shut down -43:20.908 [INFO] TestAgent: Endpoints down ---- PASS: TestSessionCreate_Delete (0.97s) +2021-08-13T15:34:55.898Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:55.916Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:55.919 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:da4c4465-6de4-d811-bcdd-e079b4a59b06 Address:127.0.0.1:25085}]" +34:55.920 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-da4c4465-6de4-d811-bcdd-e079b4a59b06.dc1 127.0.0.1 +34:55.921 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-da4c4465-6de4-d811-bcdd-e079b4a59b06 127.0.0.1 +2021-08-13T15:34:55.921Z [INFO] agent.router: Initializing LAN area manager +34:55.922 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24543 network=udp +34:55.922 [INFO] TestAgent.server: Adding LAN server: server="Node-da4c4465-6de4-d811-bcdd-e079b4a59b06 (Addr: tcp/127.0.0.1:25085) (DC: dc1)" +34:55.923 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-da4c4465-6de4-d811-bcdd-e079b4a59b06.dc1 area=wan +34:55.923 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25085 [Follower]" leader= +34:55.923 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24543 network=tcp +34:55.924 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24544 network=tcp +34:55.924 [INFO] TestAgent: started state syncer +34:55.924 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25072 network=tcp +34:55.993 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:55.993 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25085 [Candidate]" term=2 +34:55.993 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:55.993 [DEBUG] TestAgent.server.raft: vote granted: from=da4c4465-6de4-d811-bcdd-e079b4a59b06 term=2 tally=1 +34:55.993 [INFO] TestAgent.server.raft: election won: tally=1 +34:55.993 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25085 [Leader]" +34:55.994 [INFO] TestAgent.server: cluster leadership acquired +34:55.994 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25085 +34:55.996 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:55.999 [INFO] TestAgent.server: New leader elected: payload=Node-da4c4465-6de4-d811-bcdd-e079b4a59b06 +34:55.999 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:55.999 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:55.999 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:56.000 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:56.000 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:56.000 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-da4c4465-6de4-d811-bcdd-e079b4a59b06 +34:56.000 [INFO] TestAgent.server: member joined, marking health alive: member=Node-da4c4465-6de4-d811-bcdd-e079b4a59b06 +34:56.001 [INFO] TestAgent.server: federation state anti-entropy synced +34:56.095 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:56.136 [INFO] TestAgent: Synced node info +34:56.311 [INFO] TestAgent: Requesting shutdown +34:56.311 [INFO] TestAgent.server: shutting down server +34:56.311 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:56.311 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:56.311 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:56.311 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:56.311 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:56.311 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:56.311 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:56.311 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:56.311 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:56.311 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:56.312 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:56.312Z [INFO] agent.router.manager: shutting down +34:56.312 [INFO] TestAgent: consul server down +34:56.312 [INFO] TestAgent: shutdown complete +2021-08-13T15:34:56.312Z [INFO] agent.router.manager: shutting down +34:56.312 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24543 network=tcp +34:56.312 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24543 network=udp +34:56.312 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24544 network=tcp +34:56.812 [INFO] TestAgent: Waiting for endpoints to shut down +34:56.812 [INFO] TestAgent: Endpoints down +--- PASS: TestUiNodeInfo (0.94s) === CONT TestDNS_ServiceLookup_AnswerLimits/A_lookup_{30_30_8_8_6_8_8_5_8_-5_-5} -2022-09-15T21:43:20.237Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:20.237Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:20.281Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:43:20.281Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:20.284 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bacd9954-304a-7e1d-87d4-57cd799136ab Address:127.0.0.1:13435}]" -43:20.285 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13435 [Follower]" leader= -43:20.285 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bacd9954-304a-7e1d-87d4-57cd799136ab.dc1 127.0.0.1 -43:20.286 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bacd9954-304a-7e1d-87d4-57cd799136ab 127.0.0.1 -2022-09-15T21:43:20.286Z [INFO] agent.router: Initializing LAN area manager -43:20.286 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13444 network=udp -43:20.286 [INFO] TestAgent.server: Adding LAN server: server="Node-bacd9954-304a-7e1d-87d4-57cd799136ab (Addr: tcp/127.0.0.1:13435) (DC: dc1)" -43:20.287 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bacd9954-304a-7e1d-87d4-57cd799136ab.dc1 area=wan -43:20.287 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13444 network=tcp -43:20.288 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13445 network=tcp -43:20.288 [INFO] TestAgent: started state syncer -43:20.288 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13436 network=tcp -43:20.337 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -43:20.342 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:20.342 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13435 [Candidate]" term=2 -43:20.342 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:20.342 [DEBUG] TestAgent.server.raft: vote granted: from=bacd9954-304a-7e1d-87d4-57cd799136ab term=2 tally=1 -43:20.342 [INFO] TestAgent.server.raft: election won: tally=1 -43:20.342 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13435 [Leader]" -43:20.342 [INFO] TestAgent.server: cluster leadership acquired -43:20.342 [INFO] TestAgent.server: initializing acls -43:20.343 [INFO] TestAgent.server: Created ACL 'global-management' policy -43:20.343 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -43:20.344 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -43:20.344 [INFO] TestAgent.server: Created ACL anonymous token from configuration -43:20.344 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -43:20.344 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -43:20.344 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-bacd9954-304a-7e1d-87d4-57cd799136ab -43:20.345 [INFO] TestAgent.server: New leader elected: payload=Node-bacd9954-304a-7e1d-87d4-57cd799136ab -43:20.345 [INFO] TestAgent.server: Updating LAN server: server="Node-bacd9954-304a-7e1d-87d4-57cd799136ab (Addr: tcp/127.0.0.1:13435) (DC: dc1)" -43:20.345 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-bacd9954-304a-7e1d-87d4-57cd799136ab.dc1 -43:20.345 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-bacd9954-304a-7e1d-87d4-57cd799136ab.dc1 area=wan -43:20.346 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:20.348 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:20.348 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:20.348 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:20.348 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:20.348 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:20.348 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bacd9954-304a-7e1d-87d4-57cd799136ab -43:20.348 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bacd9954-304a-7e1d-87d4-57cd799136ab -43:20.349 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bacd9954-304a-7e1d-87d4-57cd799136ab -43:20.350 [INFO] TestAgent.server: federation state anti-entropy synced -43:20.372 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:20.373 [INFO] TestAgent: Synced node info -43:20.443 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-bacd9954-304a-7e1d-87d4-57cd799136ab -43:20.443 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-bacd9954-304a-7e1d-87d4-57cd799136ab -43:20.445 [INFO] TestAgent: Requesting shutdown -43:20.445 [INFO] TestAgent.server: shutting down server -43:20.445 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:20.445 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:20.445 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:20.445 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -43:20.445 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -43:20.445 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:20.445 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:20.445 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -43:20.445 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:20.445 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:20.445 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:20.445 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:20.445 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:20.445 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -43:20.445 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:20.445Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:20.445Z [INFO] agent.router.manager: shutting down -43:20.446 [INFO] TestAgent: consul server down -43:20.446 [INFO] TestAgent: shutdown complete -43:20.446 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13444 network=tcp -43:20.446 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13444 network=udp -43:20.446 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13445 network=tcp -43:20.947 [INFO] TestAgent: Waiting for endpoints to shut down -43:20.947 [INFO] TestAgent: Endpoints down ---- PASS: TestACL_Legacy_Update (0.75s) +2021-08-13T15:34:55.956Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:56.003Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:56.007 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ac6ed4a0-7e4b-f86e-c72a-af000b54ccb5 Address:127.0.0.1:24660}]" +34:56.007 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24660 [Follower]" leader= +34:56.008 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ac6ed4a0-7e4b-f86e-c72a-af000b54ccb5.dc1 127.0.0.1 +34:56.024 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ac6ed4a0-7e4b-f86e-c72a-af000b54ccb5 127.0.0.1 +2021-08-13T15:34:56.024Z [INFO] agent.router: Initializing LAN area manager +34:56.025 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25073 network=udp +34:56.025 [INFO] TestAgent.server: Adding LAN server: server="Node-ac6ed4a0-7e4b-f86e-c72a-af000b54ccb5 (Addr: tcp/127.0.0.1:24660) (DC: dc1)" +34:56.025 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ac6ed4a0-7e4b-f86e-c72a-af000b54ccb5.dc1 area=wan +34:56.025 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25073 network=tcp +34:56.035 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25074 network=tcp +34:56.035 [INFO] TestAgent: started state syncer +34:56.035 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24661 network=tcp +34:56.067 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:56.067 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24660 [Candidate]" term=2 +34:56.067 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:56.067 [DEBUG] TestAgent.server.raft: vote granted: from=ac6ed4a0-7e4b-f86e-c72a-af000b54ccb5 term=2 tally=1 +34:56.067 [INFO] TestAgent.server.raft: election won: tally=1 +34:56.067 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24660 [Leader]" +34:56.068 [INFO] TestAgent.server: cluster leadership acquired +34:56.068 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24660 +34:56.070 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:56.071 [INFO] TestAgent.server: New leader elected: payload=Node-ac6ed4a0-7e4b-f86e-c72a-af000b54ccb5 +34:56.075 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:56.075 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:56.075 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:56.075 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:56.075 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:56.076 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ac6ed4a0-7e4b-f86e-c72a-af000b54ccb5 +34:56.076 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ac6ed4a0-7e4b-f86e-c72a-af000b54ccb5 +34:56.077 [INFO] TestAgent.server: federation state anti-entropy synced +34:56.092 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:56.093 [INFO] TestAgent: Synced node info +34:56.422 [INFO] TestAgent: Requesting shutdown +34:56.422 [INFO] TestAgent.server: shutting down server +34:56.422 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:56.422 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:56.422 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:56.422 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:56.422 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:56.422 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:56.422 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:56.423 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:56.423 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:56.423 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:56.423 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:56.431Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:56.431Z [INFO] agent.router.manager: shutting down +34:56.431 [INFO] TestAgent: consul server down +34:56.431 [INFO] TestAgent: shutdown complete +34:56.431 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25073 network=tcp +34:56.431 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25073 network=udp +34:56.431 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25074 network=tcp +34:56.932 [INFO] TestAgent: Waiting for endpoints to shut down +34:56.932 [INFO] TestAgent: Endpoints down +--- PASS: TestUIGatewayServiceNodes_Ingress (1.02s) === CONT TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{20_20_8_8_6_8_8_5_8_-5_-5} -2022-09-15T21:43:20.292Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:20.309Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:20.313 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:acf23d21-6b5a-6dc3-e1f6-c5afca8c4a9b Address:127.0.0.1:13484}]" -43:20.313 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13484 [Follower]" leader= -43:20.314 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-acf23d21-6b5a-6dc3-e1f6-c5afca8c4a9b.dc1 127.0.0.1 -43:20.315 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-acf23d21-6b5a-6dc3-e1f6-c5afca8c4a9b 127.0.0.1 -2022-09-15T21:43:20.315Z [INFO] agent.router: Initializing LAN area manager -43:20.315 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13465 network=udp -43:20.315 [INFO] TestAgent.server: Adding LAN server: server="Node-acf23d21-6b5a-6dc3-e1f6-c5afca8c4a9b (Addr: tcp/127.0.0.1:13484) (DC: dc1)" -43:20.315 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-acf23d21-6b5a-6dc3-e1f6-c5afca8c4a9b.dc1 area=wan -43:20.316 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13465 network=tcp -43:20.317 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13466 network=tcp -43:20.317 [INFO] TestAgent: started state syncer -43:20.317 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13485 network=tcp -43:20.365 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:20.366 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13484 [Candidate]" term=2 -43:20.366 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:20.366 [DEBUG] TestAgent.server.raft: vote granted: from=acf23d21-6b5a-6dc3-e1f6-c5afca8c4a9b term=2 tally=1 -43:20.366 [INFO] TestAgent.server.raft: election won: tally=1 -43:20.366 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13484 [Leader]" -43:20.366 [INFO] TestAgent.server: cluster leadership acquired -43:20.366 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13484 -43:20.368 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:20.370 [INFO] TestAgent.server: New leader elected: payload=Node-acf23d21-6b5a-6dc3-e1f6-c5afca8c4a9b -43:20.371 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:20.371 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:20.371 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:20.371 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:20.371 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:20.371 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-acf23d21-6b5a-6dc3-e1f6-c5afca8c4a9b -43:20.371 [INFO] TestAgent.server: member joined, marking health alive: member=Node-acf23d21-6b5a-6dc3-e1f6-c5afca8c4a9b -43:20.374 [INFO] TestAgent.server: federation state anti-entropy synced -43:20.483 [INFO] TestAgent: Requesting shutdown -43:20.484 [INFO] TestAgent.server: shutting down server -43:20.484 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:20.484 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:20.484 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:20.484 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:20.484 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:20.484 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:20.484 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:20.484 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:20.484 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:20.484 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:20.484 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:20.484 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:20.484Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:20.484Z [INFO] agent.router.manager: shutting down -43:20.484 [INFO] TestAgent: consul server down -43:20.484 [INFO] TestAgent: shutdown complete -43:20.484 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13465 network=tcp -43:20.484 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13465 network=udp -43:20.484 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13466 network=tcp -43:20.985 [INFO] TestAgent: Waiting for endpoints to shut down -43:20.985 [INFO] TestAgent: Endpoints down ---- PASS: TestIntentionsSpecificGet_good (0.71s) +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:56.252Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:56.317Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:56.321 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8f2ddcdc-b4e2-21bc-7b10-dcbe084f1687 Address:127.0.0.1:24688}]" +34:56.321 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24688 [Follower]" leader= +34:56.322 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +34:56.324 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2021-08-13T15:34:56.324Z [INFO] agent.router: Initializing LAN area manager +34:56.324 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24683 network=udp +34:56.325 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +34:56.325 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:24688) (DC: dc1)" +34:56.325 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24683 network=tcp +34:56.326 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24684 network=tcp +34:56.326 [INFO] TestAgent: started state syncer +34:56.326 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24689 network=tcp +34:56.367 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:56.367 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24688 [Candidate]" term=2 +34:56.367 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:56.367 [DEBUG] TestAgent.server.raft: vote granted: from=8f2ddcdc-b4e2-21bc-7b10-dcbe084f1687 term=2 tally=1 +34:56.367 [INFO] TestAgent.server.raft: election won: tally=1 +34:56.367 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24688 [Leader]" +34:56.368 [INFO] TestAgent.server: cluster leadership acquired +34:56.368 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24688 +34:56.370 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:56.371 [INFO] TestAgent.server: New leader elected: payload=test-node +34:56.376 [INFO] TestAgent: Synced node info +34:56.376 [DEBUG] TestAgent: Node info in sync +34:56.377 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:56.377 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:56.377 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:56.377 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:56.377 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:56.378 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +34:56.378 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +34:56.383 [INFO] TestAgent.server: federation state anti-entropy synced +34:56.621 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=1.739324ms client=127.0.0.1:40616 client_network=udp +34:56.623 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=1.593463ms client=127.0.0.1:57018 client_network=udp +34:56.625 [DEBUG] TestAgent.dns: request served from client: name=cfbbd89f-6d95-8ee7-d337-8c6b17601f05.query.consul. type=A class=IN latency=1.763064ms client=127.0.0.1:60455 client_network=udp +34:56.625 [INFO] TestAgent: Requesting shutdown +34:56.625 [INFO] TestAgent.server: shutting down server +34:56.625 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:56.625 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:56.625 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:56.625 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:56.625 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:56.625 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:56.625 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:56.625 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:56.625 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:56.625 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:56.625 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:56.625Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:56.625Z [INFO] agent.router.manager: shutting down +34:56.626 [INFO] TestAgent: consul server down +34:56.626 [INFO] TestAgent: shutdown complete +34:56.626 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24683 network=tcp +34:56.626 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24683 network=udp +34:56.626 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24684 network=tcp +34:57.127 [INFO] TestAgent: Waiting for endpoints to shut down +34:57.127 [INFO] TestAgent: Endpoints down === CONT TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{20_20_8_8_6_8_8_5_8_-5_-5} -[WARN] freeport: 4 out of 39 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:20.219Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:20.261Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:20.269 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d8394cef-9e92-8746-1a71-defceb40135d Address:127.0.0.1:13393}]" -43:20.270 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13393 [Follower]" leader= -43:20.275 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d8394cef-9e92-8746-1a71-defceb40135d.dc1 127.0.0.1 -43:20.277 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d8394cef-9e92-8746-1a71-defceb40135d 127.0.0.1 -2022-09-15T21:43:20.277Z [INFO] agent.router: Initializing LAN area manager -43:20.277 [INFO] TestAgent.server: Adding LAN server: server="Node-d8394cef-9e92-8746-1a71-defceb40135d (Addr: tcp/127.0.0.1:13393) (DC: dc1)" -43:20.277 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d8394cef-9e92-8746-1a71-defceb40135d.dc1 area=wan -43:20.278 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13381 network=udp -43:20.285 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13381 network=tcp -43:20.286 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13382 network=tcp -43:20.286 [INFO] TestAgent: started state syncer -43:20.287 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13394 network=tcp -43:20.337 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:20.337 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13393 [Candidate]" term=2 -43:20.338 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:20.338 [DEBUG] TestAgent.server.raft: vote granted: from=d8394cef-9e92-8746-1a71-defceb40135d term=2 tally=1 -43:20.338 [INFO] TestAgent.server.raft: election won: tally=1 -43:20.338 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13393 [Leader]" -43:20.338 [INFO] TestAgent.server: cluster leadership acquired -43:20.338 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13393 -43:20.340 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:20.342 [INFO] TestAgent.server: New leader elected: payload=Node-d8394cef-9e92-8746-1a71-defceb40135d -43:20.351 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:20.351 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:20.351 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:20.351 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:20.351 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:20.351 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d8394cef-9e92-8746-1a71-defceb40135d -43:20.351 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d8394cef-9e92-8746-1a71-defceb40135d -43:20.353 [INFO] TestAgent.server: federation state anti-entropy synced -43:20.501 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:20.501 [INFO] TestAgent: Synced node info -43:20.637 [INFO] TestAgent: Requesting shutdown -43:20.638 [INFO] TestAgent.server: shutting down server -43:20.638 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:20.638 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:20.638 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:20.638 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:20.638 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:20.639 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:20.639 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:20.639 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:20.639 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:20.639 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:20.639 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:20.639Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:20.639Z [INFO] agent.router.manager: shutting down -43:20.640 [INFO] TestAgent: consul server down -43:20.640 [INFO] TestAgent: shutdown complete -43:20.640 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13381 network=tcp -43:20.640 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13381 network=udp -43:20.640 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13382 network=tcp -43:21.141 [INFO] TestAgent: Waiting for endpoints to shut down -43:21.141 [INFO] TestAgent: Endpoints down ---- PASS: TestIntentionsSpecificGet_invalidId (0.98s) +2021-08-13T15:34:56.242Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:56.312Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:56.317 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:50743e68-56f1-e5c9-e531-73bd4992117a Address:127.0.0.1:24681}]" +34:56.317 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24681 [Follower]" leader= +34:56.319 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +34:56.320 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2021-08-13T15:34:56.320Z [INFO] agent.router: Initializing LAN area manager +34:56.320 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:24681) (DC: dc1)" +34:56.320 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +34:56.320 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24606 network=tcp +34:56.320 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24606 network=udp +34:56.321 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24607 network=tcp +34:56.322 [INFO] TestAgent: started state syncer +34:56.322 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24682 network=tcp +34:56.371 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:56.371 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24681 [Candidate]" term=2 +34:56.371 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:56.371 [DEBUG] TestAgent.server.raft: vote granted: from=50743e68-56f1-e5c9-e531-73bd4992117a term=2 tally=1 +34:56.371 [INFO] TestAgent.server.raft: election won: tally=1 +34:56.371 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24681 [Leader]" +34:56.372 [INFO] TestAgent.server: cluster leadership acquired +34:56.372 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24681 +34:56.374 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:56.376 [INFO] TestAgent.server: New leader elected: payload=test-node +34:56.387 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:56.387 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:56.387 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:56.387 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:56.387 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:56.388 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +34:56.388 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +34:56.388 [INFO] TestAgent.server: federation state anti-entropy synced +34:56.595 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:56.596 [INFO] TestAgent: Synced node info +34:56.752 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=2.568246ms client=127.0.0.1:37273 client_network=udp +34:56.759 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=5.812913ms client=127.0.0.1:42191 client_network=udp +34:56.762 [DEBUG] TestAgent.dns: request served from client: name=f08c3b5e-f2de-a478-6d41-aa9bb10c50d1.query.consul. type=A class=IN latency=2.927967ms client=127.0.0.1:46351 client_network=udp +34:56.762 [INFO] TestAgent: Requesting shutdown +34:56.762 [INFO] TestAgent.server: shutting down server +34:56.762 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:56.762 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:56.762 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:56.762 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:56.762 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:56.763 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:56.763 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:56.763 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:56.763 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:56.763 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +34:56.763 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2021-08-13T15:34:56.763Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:56.763Z [INFO] agent.router.manager: shutting down +34:56.763 [INFO] TestAgent: consul server down +34:56.763 [INFO] TestAgent: shutdown complete +34:56.763 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24606 network=tcp +34:56.763 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24606 network=udp +34:56.763 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24607 network=tcp +34:57.269 [INFO] TestAgent: Waiting for endpoints to shut down +34:57.269 [INFO] TestAgent: Endpoints down === CONT TestDNS_ServiceLookup_AnswerLimits/A_lookup_{20_20_8_8_6_8_8_5_8_-5_-5} -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 4 out of 4 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 4 out of 4 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:20.816Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:20.868Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:20.872 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9b0a1371-2ace-9f94-469b-e3e255f5f1bf Address:127.0.0.1:13453}]" -43:20.873 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -43:20.873 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2022-09-15T21:43:20.873Z [INFO] agent.router: Initializing LAN area manager -43:20.874 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13453 [Follower]" leader= -43:20.874 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:13453) (DC: dc1)" -43:20.874 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -43:20.889 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13476 network=udp -43:20.889 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13476 network=tcp -43:20.890 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13498 network=tcp -43:20.890 [INFO] TestAgent: started state syncer -43:20.891 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13454 network=tcp -43:20.922 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:20.922 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13453 [Candidate]" term=2 -43:20.922 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:20.922 [DEBUG] TestAgent.server.raft: vote granted: from=9b0a1371-2ace-9f94-469b-e3e255f5f1bf term=2 tally=1 -43:20.922 [INFO] TestAgent.server.raft: election won: tally=1 -43:20.922 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13453 [Leader]" -43:20.922 [INFO] TestAgent.server: cluster leadership acquired -43:20.922 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13453 -43:20.924 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:20.926 [INFO] TestAgent.server: New leader elected: payload=test-node -43:20.929 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:20.929 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:20.929 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:20.929 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:20.929 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:20.929 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -43:20.929 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -43:20.930 [INFO] TestAgent.server: federation state anti-entropy synced -43:21.190 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:21.219 [INFO] TestAgent: Synced node info -43:21.219 [DEBUG] TestAgent: Node info in sync -43:21.260 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=2.2343ms client=127.0.0.1:50376 client_network=udp -43:21.275 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=14.36984ms client=127.0.0.1:56807 client_network=udp -43:21.287 [DEBUG] TestAgent.dns: request served from client: name=6bdcca46-7940-80a4-a79f-0571ce5cb596.query.consul. type=ANY class=IN latency=11.24224ms client=127.0.0.1:46142 client_network=udp -43:21.287 [INFO] TestAgent: Requesting shutdown -43:21.287 [INFO] TestAgent.server: shutting down server -43:21.287 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:21.287 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:21.287 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:21.287 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:21.287 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:21.287 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:21.287 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:21.287 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:21.287 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:21.287 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:21.287 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:21.288Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:21.288Z [INFO] agent.router.manager: shutting down -43:21.288 [INFO] TestAgent: consul server down -43:21.288 [INFO] TestAgent: shutdown complete -43:21.288 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13476 network=tcp -43:21.288 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13476 network=udp -43:21.288 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13498 network=tcp -43:21.788 [INFO] TestAgent: Waiting for endpoints to shut down -43:21.788 [INFO] TestAgent: Endpoints down +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=255)_compressed=false#01 +2021-08-13T15:34:56.669Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:56.686Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:56.695 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d57a6a7a-182a-675a-c3b7-59263a871141 Address:127.0.0.1:24695}]" +34:56.696 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d57a6a7a-182a-675a-c3b7-59263a871141.dc1 127.0.0.1 +34:56.698 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d57a6a7a-182a-675a-c3b7-59263a871141 127.0.0.1 +2021-08-13T15:34:56.698Z [INFO] agent.router: Initializing LAN area manager +34:56.699 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24732 network=udp +34:56.700 [INFO] TestAgent.server: Adding LAN server: server="Node-d57a6a7a-182a-675a-c3b7-59263a871141 (Addr: tcp/127.0.0.1:24695) (DC: dc1)" +34:56.701 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d57a6a7a-182a-675a-c3b7-59263a871141.dc1 area=wan +34:56.702 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24732 network=tcp +34:56.704 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24733 network=tcp +34:56.704 [INFO] TestAgent: started state syncer +34:56.704 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24696 network=tcp +34:56.704 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24695 [Follower]" leader= +34:56.763 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:56.763 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24695 [Candidate]" term=2 +34:56.763 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:56.763 [DEBUG] TestAgent.server.raft: vote granted: from=d57a6a7a-182a-675a-c3b7-59263a871141 term=2 tally=1 +34:56.763 [INFO] TestAgent.server.raft: election won: tally=1 +34:56.763 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24695 [Leader]" +34:56.764 [INFO] TestAgent.server: cluster leadership acquired +34:56.764 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24695 +34:56.764 [INFO] TestAgent.server: New leader elected: payload=Node-d57a6a7a-182a-675a-c3b7-59263a871141 +34:56.766 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:56.769 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:56.769 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:56.769 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:56.769 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:56.769 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:56.769 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d57a6a7a-182a-675a-c3b7-59263a871141 +34:56.769 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d57a6a7a-182a-675a-c3b7-59263a871141 +34:56.771 [INFO] TestAgent.server: federation state anti-entropy synced +34:56.929 [INFO] TestAgent.server.connect: CA provider config updated +34:56.929 [INFO] TestAgent: Requesting shutdown +34:56.930 [INFO] TestAgent.server: shutting down server +34:56.930 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:56.930 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:56.930 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:56.930 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:56.930 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:56.930 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:56.930 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:56.930 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:56.931 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:56.931 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:56.931 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:56.931 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:56.932Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:56.932Z [INFO] agent.router.manager: shutting down +34:56.932 [INFO] TestAgent: consul server down +34:56.932 [INFO] TestAgent: shutdown complete +34:56.932 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24732 network=tcp +34:56.932 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24732 network=udp +34:56.932 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24733 network=tcp +34:57.435 [INFO] TestAgent: Waiting for endpoints to shut down +34:57.435 [INFO] TestAgent: Endpoints down +--- PASS: TestConnectCAConfig (5.20s) + --- PASS: TestConnectCAConfig/basic (1.02s) + --- PASS: TestConnectCAConfig/basic_with_IntermediateCertTTL (0.82s) + --- PASS: TestConnectCAConfig/force_without_cross_sign_CamelCase (0.88s) + --- PASS: TestConnectCAConfig/force_without_cross_sign_snake_case (0.74s) + --- PASS: TestConnectCAConfig/setting_state_fails (0.96s) + --- PASS: TestConnectCAConfig/updating_config_with_same_state (0.78s) === CONT TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{9_9_8_8_6_8_8_5_8_8_-5} -2022-09-15T21:43:20.718Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:20.774Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:20.778 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:561e7494-b40c-fa4f-98b3-29058692a14f Address:127.0.0.1:13407}]" -43:20.779 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13407 [Follower]" leader= -43:20.779 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -43:20.780 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2022-09-15T21:43:20.780Z [INFO] agent.router: Initializing LAN area manager -43:20.780 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13437 network=udp -43:20.781 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:13407) (DC: dc1)" -43:20.781 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -43:20.781 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13437 network=tcp -43:20.782 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13438 network=tcp -43:20.782 [INFO] TestAgent: started state syncer -43:20.782 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13408 network=tcp -43:20.838 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:20.838 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13407 [Candidate]" term=2 -43:20.838 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:20.838 [DEBUG] TestAgent.server.raft: vote granted: from=561e7494-b40c-fa4f-98b3-29058692a14f term=2 tally=1 -43:20.838 [INFO] TestAgent.server.raft: election won: tally=1 -43:20.838 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13407 [Leader]" -43:20.842 [INFO] TestAgent.server: cluster leadership acquired -43:20.842 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13407 -43:20.843 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:20.846 [INFO] TestAgent.server: New leader elected: payload=test-node -43:20.854 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:20.854 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:20.854 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:20.854 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:20.854 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:20.854 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -43:20.854 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -43:20.855 [INFO] TestAgent.server: federation state anti-entropy synced -43:20.882 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:20.883 [INFO] TestAgent: Synced node info -43:21.282 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=1.83232ms client=127.0.0.1:35450 client_network=udp -43:21.284 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=1.69644ms client=127.0.0.1:35284 client_network=udp -43:21.292 [DEBUG] TestAgent.dns: request served from client: name=e351edeb-e6f6-7de0-8903-88c001f14ce4.query.consul. type=A class=IN latency=7.81586ms client=127.0.0.1:45279 client_network=udp -43:21.293 [INFO] TestAgent: Requesting shutdown -43:21.293 [INFO] TestAgent.server: shutting down server -43:21.293 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:21.293 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:21.293 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:21.293 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:21.293 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:21.293 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:21.293 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:21.293 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:21.293 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:21.293 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:21.293 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:21.293Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:21.293Z [INFO] agent.router.manager: shutting down -43:21.298 [INFO] TestAgent: consul server down -43:21.298 [INFO] TestAgent: shutdown complete -43:21.298 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13437 network=tcp -43:21.298 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13437 network=udp -43:21.298 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13438 network=tcp -43:21.800 [INFO] TestAgent: Waiting for endpoints to shut down -43:21.800 [INFO] TestAgent: Endpoints down +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:56.669Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:56.696Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:56.710 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8d59a01e-2b4d-d279-de6f-8de7064121fb Address:127.0.0.1:24730}]" +34:56.711 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24730 [Follower]" leader= +34:56.712 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +34:56.713 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2021-08-13T15:34:56.713Z [INFO] agent.router: Initializing LAN area manager +34:56.714 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:24730) (DC: dc1)" +34:56.714 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +34:56.714 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24690 network=tcp +34:56.715 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24690 network=udp +34:56.716 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24691 network=tcp +34:56.716 [INFO] TestAgent: started state syncer +34:56.717 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24731 network=tcp +34:56.780 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:56.780 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24730 [Candidate]" term=2 +34:56.780 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:56.780 [DEBUG] TestAgent.server.raft: vote granted: from=8d59a01e-2b4d-d279-de6f-8de7064121fb term=2 tally=1 +34:56.780 [INFO] TestAgent.server.raft: election won: tally=1 +34:56.780 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24730 [Leader]" +34:56.780 [INFO] TestAgent.server: cluster leadership acquired +34:56.781 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24730 +34:56.781 [INFO] TestAgent.server: New leader elected: payload=test-node +34:56.782 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:56.785 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:56.785 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:56.785 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:56.785 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:56.785 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:56.785 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +34:56.785 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +34:56.786 [INFO] TestAgent.server: federation state anti-entropy synced +34:57.068 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:57.069 [INFO] TestAgent: Synced node info +34:57.069 [DEBUG] TestAgent: Node info in sync +34:57.195 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=2.918527ms client=127.0.0.1:56473 client_network=udp +34:57.199 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=3.186088ms client=127.0.0.1:58560 client_network=udp +34:57.204 [DEBUG] TestAgent.dns: request served from client: name=c5070c3f-3cca-2f5e-02f0-ee0c44f62627.query.consul. type=ANY class=IN latency=3.887809ms client=127.0.0.1:53185 client_network=udp +34:57.211 [INFO] TestAgent: Requesting shutdown +34:57.211 [INFO] TestAgent.server: shutting down server +34:57.211 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:57.211 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:57.211 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:57.211 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:57.211 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:57.211 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:57.211 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:57.211 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:57.211 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:57.212 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:57.212 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:57.212Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:57.212Z [INFO] agent.router.manager: shutting down +34:57.212 [INFO] TestAgent: consul server down +34:57.212 [INFO] TestAgent: shutdown complete +34:57.212 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24690 network=tcp +34:57.212 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24690 network=udp +34:57.212 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24691 network=tcp +34:57.713 [INFO] TestAgent: Waiting for endpoints to shut down +34:57.713 [INFO] TestAgent: Endpoints down === CONT TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{9_9_8_8_6_8_8_5_8_8_-5} -2022-09-15T21:43:20.917Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:20.942Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:20.945 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:902d2c90-d7e4-6745-80d1-c8570e728fa6 Address:127.0.0.1:13474}]" -43:20.946 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13474 [Follower]" leader= -43:20.953 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -43:20.954 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2022-09-15T21:43:20.954Z [INFO] agent.router: Initializing LAN area manager -43:20.955 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13483 network=udp -43:20.955 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:13474) (DC: dc1)" -43:20.955 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -43:20.955 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13483 network=tcp -43:20.956 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13470 network=tcp -43:20.956 [INFO] TestAgent: started state syncer -43:20.956 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13475 network=tcp -43:20.994 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:20.994 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13474 [Candidate]" term=2 -43:20.994 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:20.994 [DEBUG] TestAgent.server.raft: vote granted: from=902d2c90-d7e4-6745-80d1-c8570e728fa6 term=2 tally=1 -43:20.994 [INFO] TestAgent.server.raft: election won: tally=1 -43:20.994 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13474 [Leader]" -43:20.994 [INFO] TestAgent.server: cluster leadership acquired -43:20.994 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13474 -43:20.995 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:20.998 [INFO] TestAgent.server: New leader elected: payload=test-node -43:20.999 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:20.999 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:20.999 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:20.999 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:20.999 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:20.999 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -43:20.999 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -43:21.000 [INFO] TestAgent.server: federation state anti-entropy synced -43:21.298 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:21.302 [INFO] TestAgent: Synced node info -43:21.315 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=1.83084ms client=127.0.0.1:43441 client_network=udp -43:21.317 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=1.4813ms client=127.0.0.1:49311 client_network=udp -43:21.319 [DEBUG] TestAgent.dns: request served from client: name=10eb36ea-256b-563f-de52-80b854a20544.query.consul. type=AAAA class=IN latency=1.94272ms client=127.0.0.1:54729 client_network=udp -43:21.320 [INFO] TestAgent: Requesting shutdown -43:21.320 [INFO] TestAgent.server: shutting down server -43:21.320 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:21.320 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:21.320 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:21.320 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:21.320 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:21.320 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:21.320 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:21.320 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:21.320 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:21.320 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:21.320 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:21.321Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:21.321Z [INFO] agent.router.manager: shutting down -43:21.321 [INFO] TestAgent: consul server down -43:21.321 [INFO] TestAgent: shutdown complete -43:21.321 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13483 network=tcp -43:21.321 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13483 network=udp -43:21.321 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13470 network=tcp -43:21.822 [INFO] TestAgent: Waiting for endpoints to shut down -43:21.822 [INFO] TestAgent: Endpoints down +2021-08-13T15:34:56.807Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:56.824Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:56.827 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a8293543-8dae-4ec2-3afc-919e08db0c22 Address:127.0.0.1:24709}]" +34:56.827 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24709 [Follower]" leader= +34:56.828 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +34:56.829 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2021-08-13T15:34:56.829Z [INFO] agent.router: Initializing LAN area manager +34:56.829 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:24709) (DC: dc1)" +34:56.829 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +34:56.830 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24697 network=udp +34:56.830 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24697 network=tcp +34:56.831 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24698 network=tcp +34:56.831 [INFO] TestAgent: started state syncer +34:56.831 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24710 network=tcp +34:56.866 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:56.866 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24709 [Candidate]" term=2 +34:56.866 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:56.867 [DEBUG] TestAgent.server.raft: vote granted: from=a8293543-8dae-4ec2-3afc-919e08db0c22 term=2 tally=1 +34:56.867 [INFO] TestAgent.server.raft: election won: tally=1 +34:56.867 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24709 [Leader]" +34:56.867 [INFO] TestAgent.server: cluster leadership acquired +34:56.867 [INFO] TestAgent.server: New leader elected: payload=test-node +34:56.867 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24709 +34:56.869 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:56.873 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:56.873 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:56.873 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:56.873 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:56.873 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:56.873 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +34:56.873 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +34:56.874 [INFO] TestAgent.server: federation state anti-entropy synced +34:57.152 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:57.153 [INFO] TestAgent: Synced node info +34:57.201 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:57.201 [DEBUG] TestAgent: Node info in sync +34:57.201 [DEBUG] TestAgent: Node info in sync +34:57.229 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=2.315006ms client=127.0.0.1:32978 client_network=udp +34:57.232 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=2.415365ms client=127.0.0.1:33562 client_network=udp +34:57.235 [DEBUG] TestAgent.dns: request served from client: name=0a87a96a-f7c5-3d15-f26e-14bcbf6cd869.query.consul. type=AAAA class=IN latency=2.172905ms client=127.0.0.1:35233 client_network=udp +34:57.235 [INFO] TestAgent: Requesting shutdown +34:57.235 [INFO] TestAgent.server: shutting down server +34:57.235 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:57.235 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:57.235 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:57.235 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:57.235 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:57.235 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:57.235 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:57.235 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:57.235 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:57.235 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:57.235 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:57.236Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:57.236Z [INFO] agent.router.manager: shutting down +34:57.236 [INFO] TestAgent: consul server down +34:57.236 [INFO] TestAgent: shutdown complete +34:57.236 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24697 network=tcp +34:57.236 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24697 network=udp +34:57.236 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24698 network=tcp +34:57.739 [INFO] TestAgent: Waiting for endpoints to shut down +34:57.739 [INFO] TestAgent: Endpoints down === CONT TestDNS_ServiceLookup_AnswerLimits/A_lookup_{9_9_8_8_6_8_8_5_8_8_-5} -2022-09-15T21:43:20.949Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:20.970Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:20.973 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:14c871f4-bbd7-51c0-6b19-3354b9827068 Address:127.0.0.1:13495}]" -43:20.974 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13495 [Follower]" leader= -43:20.977 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -43:20.979 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2022-09-15T21:43:20.979Z [INFO] agent.router: Initializing LAN area manager -43:20.979 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:13495) (DC: dc1)" -43:20.979 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -43:20.980 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13455 network=tcp -43:20.980 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13455 network=udp -43:20.981 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13477 network=tcp -43:20.981 [INFO] TestAgent: started state syncer -43:20.981 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13496 network=tcp -43:21.045 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:21.045 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13495 [Candidate]" term=2 -43:21.045 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:21.045 [DEBUG] TestAgent.server.raft: vote granted: from=14c871f4-bbd7-51c0-6b19-3354b9827068 term=2 tally=1 -43:21.045 [INFO] TestAgent.server.raft: election won: tally=1 -43:21.045 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13495 [Leader]" -43:21.045 [INFO] TestAgent.server: cluster leadership acquired -43:21.045 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13495 -43:21.046 [INFO] TestAgent.server: New leader elected: payload=test-node -43:21.047 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:21.049 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:21.049 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:21.049 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:21.049 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:21.049 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:21.050 [INFO] TestAgent.server: federation state anti-entropy synced -43:21.050 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -43:21.050 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -43:21.186 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:21.188 [INFO] TestAgent: Synced node info -43:21.352 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=1.70818ms client=127.0.0.1:60027 client_network=udp -43:21.355 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=1.55506ms client=127.0.0.1:59705 client_network=udp -43:21.359 [DEBUG] TestAgent.dns: request served from client: name=b704eced-0672-4111-1ad5-be89cf25fca8.query.consul. type=A class=IN latency=1.58852ms client=127.0.0.1:33171 client_network=udp -43:21.359 [INFO] TestAgent: Requesting shutdown -43:21.360 [INFO] TestAgent.server: shutting down server -43:21.360 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:21.360 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:21.360 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:21.360 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:21.360 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:21.360 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:21.360 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:21.360 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:21.360 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:21.360 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:21.360 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:21.360Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:21.360Z [INFO] agent.router.manager: shutting down -43:21.360 [INFO] TestAgent: consul server down -43:21.360 [INFO] TestAgent: shutdown complete -43:21.360 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13455 network=tcp -43:21.360 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13455 network=udp -43:21.360 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13477 network=tcp -43:21.862 [INFO] TestAgent: Waiting for endpoints to shut down -43:21.862 [INFO] TestAgent: Endpoints down +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=255)_compressed=true#01 +2021-08-13T15:34:57.001Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:57.105Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:57.108 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:32b32350-b5ed-4a36-a8b4-f7e76483ffcb Address:127.0.0.1:24716}]" +34:57.109 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +34:57.111 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2021-08-13T15:34:57.111Z [INFO] agent.router: Initializing LAN area manager +34:57.111 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24716 [Follower]" leader= +34:57.119 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24739 network=udp +34:57.120 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:24716) (DC: dc1)" +34:57.120 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +34:57.120 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24739 network=tcp +34:57.121 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24740 network=tcp +34:57.121 [INFO] TestAgent: started state syncer +34:57.121 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24717 network=tcp +34:57.171 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:57.171 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24716 [Candidate]" term=2 +34:57.171 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:57.171 [DEBUG] TestAgent.server.raft: vote granted: from=32b32350-b5ed-4a36-a8b4-f7e76483ffcb term=2 tally=1 +34:57.171 [INFO] TestAgent.server.raft: election won: tally=1 +34:57.171 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24716 [Leader]" +34:57.172 [INFO] TestAgent.server: cluster leadership acquired +34:57.172 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24716 +34:57.174 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:57.180 [INFO] TestAgent.server: New leader elected: payload=test-node +34:57.186 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:57.186 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:57.186 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:57.186 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:57.186 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:57.186 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +34:57.186 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +34:57.201 [INFO] TestAgent.server: federation state anti-entropy synced +34:57.320 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:57.321 [INFO] TestAgent: Synced node info +34:57.437 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=2.159965ms client=127.0.0.1:60181 client_network=udp +34:57.445 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=2.063424ms client=127.0.0.1:39576 client_network=udp +34:57.448 [DEBUG] TestAgent.dns: request served from client: name=6cf6a65b-7a2d-8f49-8d70-5a7cb34b7c5a.query.consul. type=ANY class=IN latency=2.113144ms client=127.0.0.1:53574 client_network=udp +34:57.451 [INFO] TestAgent: Requesting shutdown +34:57.451 [INFO] TestAgent.server: shutting down server +34:57.451 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:57.451 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:57.451 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:57.451 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:57.451 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:57.451 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:57.451 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:57.451 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:57.451 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:57.451 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:57.451 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:57.452Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:57.452Z [INFO] agent.router.manager: shutting down +34:57.452 [INFO] TestAgent: consul server down +34:57.452 [INFO] TestAgent: shutdown complete +34:57.452 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24739 network=tcp +34:57.452 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24739 network=udp +34:57.452 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24740 network=tcp +34:57.953 [INFO] TestAgent: Waiting for endpoints to shut down +34:57.953 [INFO] TestAgent: Endpoints down === CONT TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{8_8_8_8_6_8_8_5_8_8_-5} -2022-09-15T21:43:20.740Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:20.773Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:20.777 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ee9e84eb-3b27-bc1f-9136-55b7aaf8f3ea Address:127.0.0.1:13481}]" -43:20.778 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13481 [Follower]" leader= -43:20.778 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -43:20.779 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2022-09-15T21:43:20.779Z [INFO] agent.router: Initializing LAN area manager -43:20.780 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:13481) (DC: dc1)" -43:20.780 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -43:20.780 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13430 network=tcp -43:20.780 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13430 network=udp -43:20.781 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13431 network=tcp -43:20.781 [INFO] TestAgent: started state syncer -43:20.782 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13482 network=tcp -43:20.838 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:20.838 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13481 [Candidate]" term=2 -43:20.838 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:20.838 [DEBUG] TestAgent.server.raft: vote granted: from=ee9e84eb-3b27-bc1f-9136-55b7aaf8f3ea term=2 tally=1 -43:20.838 [INFO] TestAgent.server.raft: election won: tally=1 -43:20.838 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13481 [Leader]" -43:20.839 [INFO] TestAgent.server: cluster leadership acquired -43:20.839 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13481 -43:20.840 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:20.842 [INFO] TestAgent.server: New leader elected: payload=test-node -43:20.851 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:20.851 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:20.851 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:20.851 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:20.851 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:20.852 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -43:20.852 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -43:20.853 [INFO] TestAgent.server: federation state anti-entropy synced -43:21.158 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:21.175 [INFO] TestAgent: Synced node info -43:21.361 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=1.71432ms client=127.0.0.1:50385 client_network=udp -43:21.363 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=1.6612ms client=127.0.0.1:35102 client_network=udp -43:21.366 [INFO] TestAgent: Requesting shutdown -43:21.366 [INFO] TestAgent.server: shutting down server -43:21.366 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:21.366 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:21.366 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:21.366 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:21.366 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:21.366 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:21.366 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:21.366 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:21.366 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:21.366 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:21.366 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:21.366Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:21.366Z [INFO] agent.router.manager: shutting down -43:21.367 [INFO] TestAgent: consul server down -43:21.367 [INFO] TestAgent: shutdown complete -43:21.367 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13430 network=tcp -43:21.367 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13430 network=udp -43:21.367 [DEBUG] TestAgent.dns: request served from client: name=b973fa6d-0b48-6ea5-9723-f2be8c83a631.query.consul. type=A class=IN latency=3.54324ms client=127.0.0.1:44982 client_network=udp -43:21.367 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13431 network=tcp -43:21.868 [INFO] TestAgent: Waiting for endpoints to shut down -43:21.868 [INFO] TestAgent: Endpoints down +2021-08-13T15:34:56.831Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:56.849Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:56.852 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6a36396e-93cc-d7cc-d565-675a1df84215 Address:127.0.0.1:24737}]" +34:56.852 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24737 [Follower]" leader= +34:56.853 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +34:56.854 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2021-08-13T15:34:56.854Z [INFO] agent.router: Initializing LAN area manager +34:56.854 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:24737) (DC: dc1)" +34:56.854 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +34:56.854 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24711 network=udp +34:56.854 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24711 network=tcp +34:56.855 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24712 network=tcp +34:56.855 [INFO] TestAgent: started state syncer +34:56.855 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24738 network=tcp +34:56.923 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:56.923 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24737 [Candidate]" term=2 +34:56.923 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:56.923 [DEBUG] TestAgent.server.raft: vote granted: from=6a36396e-93cc-d7cc-d565-675a1df84215 term=2 tally=1 +34:56.923 [INFO] TestAgent.server.raft: election won: tally=1 +34:56.923 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24737 [Leader]" +34:56.924 [INFO] TestAgent.server: cluster leadership acquired +34:56.924 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24737 +34:56.926 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:56.932 [INFO] TestAgent.server: New leader elected: payload=test-node +34:56.950 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:56.950 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:56.950 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:56.950 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:56.950 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:56.951 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +34:56.951 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +34:56.952 [INFO] TestAgent.server: federation state anti-entropy synced +34:57.068 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:57.069 [INFO] TestAgent: Synced node info +34:57.466 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=1.815424ms client=127.0.0.1:60987 client_network=udp +34:57.468 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=1.633404ms client=127.0.0.1:57787 client_network=udp +34:57.472 [DEBUG] TestAgent.dns: request served from client: name=5051f2a1-87ef-95f2-c947-35b860914e93.query.consul. type=A class=IN latency=1.639924ms client=127.0.0.1:54414 client_network=udp +34:57.472 [INFO] TestAgent: Requesting shutdown +34:57.472 [INFO] TestAgent.server: shutting down server +34:57.472 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:57.472 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:57.472 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:57.472 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:57.472 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:57.472 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:57.472 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:57.472 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:57.472 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:57.472 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:57.472 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:57.473Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:57.473Z [INFO] agent.router.manager: shutting down +34:57.473 [INFO] TestAgent: consul server down +34:57.473 [INFO] TestAgent: shutdown complete +34:57.473 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24711 network=tcp +34:57.473 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24711 network=udp +34:57.473 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24712 network=tcp +34:57.974 [INFO] TestAgent: Waiting for endpoints to shut down +34:57.974 [INFO] TestAgent: Endpoints down === CONT TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{8_8_8_8_6_8_8_5_8_8_-5} -2022-09-15T21:43:20.970Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:21.001Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:21.016 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:aebb4c68-d236-46d3-91c5-6ce36395eae6 Address:127.0.0.1:13502}]" -43:21.017 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -43:21.018 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13502 [Follower]" leader= -43:21.026 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2022-09-15T21:43:21.026Z [INFO] agent.router: Initializing LAN area manager -43:21.026 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13497 network=udp -43:21.027 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:13502) (DC: dc1)" -43:21.027 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -43:21.027 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13497 network=tcp -43:21.028 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13505 network=tcp -43:21.038 [INFO] TestAgent: started state syncer -43:21.038 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13503 network=tcp -43:21.099 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:21.099 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13502 [Candidate]" term=2 -43:21.100 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:21.101 [DEBUG] TestAgent.server.raft: vote granted: from=aebb4c68-d236-46d3-91c5-6ce36395eae6 term=2 tally=1 -43:21.101 [INFO] TestAgent.server.raft: election won: tally=1 -43:21.101 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13502 [Leader]" -43:21.101 [INFO] TestAgent.server: cluster leadership acquired -43:21.106 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13502 -43:21.106 [INFO] TestAgent.server: New leader elected: payload=test-node -43:21.110 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:21.138 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:21.138 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:21.138 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:21.138 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:21.138 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:21.138 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -43:21.138 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -43:21.140 [INFO] TestAgent.server: federation state anti-entropy synced -43:21.354 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:21.356 [INFO] TestAgent: Synced node info -43:21.356 [DEBUG] TestAgent: Node info in sync -43:21.376 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=2.11268ms client=127.0.0.1:41966 client_network=udp -43:21.378 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=2.1536ms client=127.0.0.1:33395 client_network=udp -43:21.388 [DEBUG] TestAgent.dns: request served from client: name=af877524-02c4-2d66-5e94-cb7cf7140910.query.consul. type=ANY class=IN latency=2.14276ms client=127.0.0.1:35924 client_network=udp -43:21.388 [INFO] TestAgent: Requesting shutdown -43:21.388 [INFO] TestAgent.server: shutting down server -43:21.388 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:21.388 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:21.388 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:21.388 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:21.388 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:21.388 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:21.389 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:21.389 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:21.389 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:21.389 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:21.389 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:21.389Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:21.389Z [INFO] agent.router.manager: shutting down -43:21.389 [INFO] TestAgent: consul server down -43:21.389 [INFO] TestAgent: shutdown complete -43:21.389 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13497 network=tcp -43:21.389 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13497 network=udp -43:21.389 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13505 network=tcp -43:21.890 [INFO] TestAgent: Waiting for endpoints to shut down -43:21.890 [INFO] TestAgent: Endpoints down +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=255)_compressed=false#01 +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._tcp_(qType:=255)_compressed=true#01 +2021-08-13T15:34:57.465Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:57.483Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:57.486 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c04d0d78-f774-ba42-f31a-867c7e33a8a4 Address:127.0.0.1:24723}]" +34:57.486 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24723 [Follower]" leader= +34:57.487 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +34:57.488 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2021-08-13T15:34:57.488Z [INFO] agent.router: Initializing LAN area manager +34:57.489 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:24723) (DC: dc1)" +34:57.489 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +34:57.489 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24704 network=udp +34:57.489 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24704 network=tcp +34:57.490 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24705 network=tcp +34:57.490 [INFO] TestAgent: started state syncer +34:57.490 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24724 network=tcp +34:57.528 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:57.528 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24723 [Candidate]" term=2 +34:57.529 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:57.529 [DEBUG] TestAgent.server.raft: vote granted: from=c04d0d78-f774-ba42-f31a-867c7e33a8a4 term=2 tally=1 +34:57.529 [INFO] TestAgent.server.raft: election won: tally=1 +34:57.529 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24723 [Leader]" +34:57.529 [INFO] TestAgent.server: cluster leadership acquired +34:57.529 [INFO] TestAgent.server: New leader elected: payload=test-node +34:57.529 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24723 +34:57.530 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:57.533 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:57.533 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:57.533 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:57.533 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:57.533 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:57.533 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +34:57.533 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +34:57.534 [INFO] TestAgent.server: federation state anti-entropy synced +34:57.716 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=2.208685ms client=127.0.0.1:52302 client_network=udp +34:57.721 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=1.965965ms client=127.0.0.1:58733 client_network=udp +34:57.752 [DEBUG] TestAgent.dns: request served from client: name=bda8944a-bdc9-26f3-37ec-c336d3f822e8.query.consul. type=ANY class=IN latency=27.438264ms client=127.0.0.1:50249 client_network=udp +34:57.759 [INFO] TestAgent: Requesting shutdown +34:57.759 [INFO] TestAgent.server: shutting down server +34:57.759 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:57.759 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:57.759 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:57.759 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:57.759 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:57.759 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:57.759 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:57.759 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:57.760 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:57.760 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:57.760 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:57.760 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:57.760Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:57.760Z [INFO] agent.router.manager: shutting down +34:57.760 [INFO] TestAgent: consul server down +34:57.760 [INFO] TestAgent: shutdown complete +34:57.760 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24704 network=tcp +34:57.760 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24704 network=udp +34:57.760 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24705 network=tcp +34:58.260 [INFO] TestAgent: Waiting for endpoints to shut down +34:58.260 [INFO] TestAgent: Endpoints down === CONT TestDNS_ServiceLookup_AnswerLimits/A_lookup_{8_8_8_8_6_8_8_5_8_8_-5} -2022-09-15T21:43:21.283Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:21.307Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:21.310 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3fbbc378-e13f-4646-e0a0-e70f0face30b Address:127.0.0.1:13509}]" -43:21.311 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -43:21.311 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13509 [Follower]" leader= -43:21.312 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2022-09-15T21:43:21.312Z [INFO] agent.router: Initializing LAN area manager -43:21.312 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13504 network=udp -43:21.312 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:13509) (DC: dc1)" -43:21.312 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -43:21.313 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13504 network=tcp -43:21.313 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13491 network=tcp -43:21.313 [INFO] TestAgent: started state syncer -43:21.314 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13510 network=tcp -43:21.361 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:21.361 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13509 [Candidate]" term=2 -43:21.362 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:21.362 [DEBUG] TestAgent.server.raft: vote granted: from=3fbbc378-e13f-4646-e0a0-e70f0face30b term=2 tally=1 -43:21.362 [INFO] TestAgent.server.raft: election won: tally=1 -43:21.362 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13509 [Leader]" -43:21.362 [INFO] TestAgent.server: cluster leadership acquired -43:21.362 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13509 -43:21.363 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:21.366 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:21.366 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:21.366 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:21.366 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:21.366 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:21.366 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -43:21.366 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -43:21.367 [INFO] TestAgent.server: New leader elected: payload=test-node -43:21.367 [INFO] TestAgent.server: federation state anti-entropy synced -43:21.494 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:21.497 [INFO] TestAgent: Synced node info -43:21.501 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=2.39452ms client=127.0.0.1:56593 client_network=udp -43:21.505 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=2.65186ms client=127.0.0.1:47352 client_network=udp -43:21.508 [DEBUG] TestAgent.dns: request served from client: name=9d10439d-4229-4baf-2769-ae577a6929f5.query.consul. type=A class=IN latency=1.64236ms client=127.0.0.1:43208 client_network=udp -43:21.509 [INFO] TestAgent: Requesting shutdown -43:21.509 [INFO] TestAgent.server: shutting down server -43:21.509 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:21.509 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:21.509 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:21.509 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:21.509 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:21.509 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:21.509 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:21.509 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:21.509 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:21.509 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:21.509 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:21.509Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:21.509Z [INFO] agent.router.manager: shutting down -43:21.509 [INFO] TestAgent: consul server down -43:21.509 [INFO] TestAgent: shutdown complete -43:21.509 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13504 network=tcp -43:21.510 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13504 network=udp -43:21.510 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13491 network=tcp -43:22.010 [INFO] TestAgent: Waiting for endpoints to shut down -43:22.010 [INFO] TestAgent: Endpoints down +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:57.224Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:57.248Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:57.253 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d55040b9-0747-4753-fe9a-d628d1b447ce Address:127.0.0.1:24744}]" +34:57.255 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +34:57.256 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24744 [Follower]" leader= +34:57.256 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2021-08-13T15:34:57.256Z [INFO] agent.router: Initializing LAN area manager +34:57.257 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24718 network=udp +34:57.258 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:24744) (DC: dc1)" +34:57.258 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24718 network=tcp +34:57.258 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +34:57.260 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24719 network=tcp +34:57.260 [INFO] TestAgent: started state syncer +34:57.260 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24745 network=tcp +34:57.309 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:57.309 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24744 [Candidate]" term=2 +34:57.309 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:57.309 [DEBUG] TestAgent.server.raft: vote granted: from=d55040b9-0747-4753-fe9a-d628d1b447ce term=2 tally=1 +34:57.309 [INFO] TestAgent.server.raft: election won: tally=1 +34:57.309 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24744 [Leader]" +34:57.309 [INFO] TestAgent.server: cluster leadership acquired +34:57.310 [INFO] TestAgent.server: New leader elected: payload=test-node +34:57.310 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24744 +34:57.313 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:57.316 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:57.316 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:57.316 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:57.316 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:57.316 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:57.316 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +34:57.316 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +34:57.318 [INFO] TestAgent.server: federation state anti-entropy synced +34:57.399 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:57.400 [INFO] TestAgent: Synced node info +34:57.862 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=2.877667ms client=127.0.0.1:34645 client_network=udp +34:57.866 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=2.506546ms client=127.0.0.1:47131 client_network=udp +34:57.873 [DEBUG] TestAgent.dns: request served from client: name=b46c8bf3-cfc3-9431-38f1-2e331b5e24d0.query.consul. type=AAAA class=IN latency=6.813516ms client=127.0.0.1:42184 client_network=udp +34:57.873 [INFO] TestAgent: Requesting shutdown +34:57.873 [INFO] TestAgent.server: shutting down server +34:57.873 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:57.873 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:57.873 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:57.873 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:57.873 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:57.874 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:57.874 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:57.874 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:57.874 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:57.874 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:57.874 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:57.874Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:57.874Z [INFO] agent.router.manager: shutting down +34:57.875 [INFO] TestAgent: consul server down +34:57.875 [INFO] TestAgent: shutdown complete +34:57.875 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24718 network=tcp +34:57.875 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24718 network=udp +34:57.875 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24719 network=tcp +34:58.375 [INFO] TestAgent: Waiting for endpoints to shut down +34:58.375 [INFO] TestAgent: Endpoints down === CONT TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{7_7_7_7_6_7_7_5_7_7_-5} -[WARN] freeport: 4 out of 53 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:21.043Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:21.167Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:21.223 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3044bf46-4db0-d366-b872-d40762319d0c Address:127.0.0.1:13530}]" -43:21.224 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -43:21.225 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2022-09-15T21:43:21.225Z [INFO] agent.router: Initializing LAN area manager -43:21.226 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13530 [Follower]" leader= -43:21.226 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:13530) (DC: dc1)" -43:21.226 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -43:21.230 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13539 network=udp -43:21.230 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13539 network=tcp -43:21.231 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13526 network=tcp -43:21.232 [INFO] TestAgent: started state syncer -43:21.232 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13531 network=tcp -43:21.282 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:21.282 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13530 [Candidate]" term=2 -43:21.282 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:21.282 [DEBUG] TestAgent.server.raft: vote granted: from=3044bf46-4db0-d366-b872-d40762319d0c term=2 tally=1 -43:21.282 [INFO] TestAgent.server.raft: election won: tally=1 -43:21.282 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13530 [Leader]" -43:21.285 [INFO] TestAgent.server: cluster leadership acquired -43:21.290 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13530 -43:21.291 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:21.292 [INFO] TestAgent.server: New leader elected: payload=test-node -43:21.300 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:21.300 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:21.300 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:21.300 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:21.300 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:21.300 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -43:21.300 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -43:21.302 [INFO] TestAgent.server: federation state anti-entropy synced -43:21.303 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:21.303 [INFO] TestAgent: Synced node info -43:21.551 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=2.22158ms client=127.0.0.1:48496 client_network=udp -43:21.553 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=1.77842ms client=127.0.0.1:35493 client_network=udp -43:21.555 [DEBUG] TestAgent.dns: request served from client: name=46b8c3a4-e8ab-b997-bad1-baad9b72d4e1.query.consul. type=AAAA class=IN latency=1.8063ms client=127.0.0.1:54816 client_network=udp -43:21.555 [INFO] TestAgent: Requesting shutdown -43:21.555 [INFO] TestAgent.server: shutting down server -43:21.555 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:21.555 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:21.555 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:21.555 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:21.555 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:21.555 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:21.556 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:21.556 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:21.556 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -43:21.556 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:21.556 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2022-09-15T21:43:21.556Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:21.556Z [INFO] agent.router.manager: shutting down -43:21.556 [INFO] TestAgent: consul server down -43:21.556 [INFO] TestAgent: shutdown complete -43:21.556 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13539 network=tcp -43:21.556 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13539 network=udp -43:21.556 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13526 network=tcp -43:22.058 [INFO] TestAgent: Waiting for endpoints to shut down -43:22.058 [INFO] TestAgent: Endpoints down +2021-08-13T15:34:57.305Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:57.329Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:57.333 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cbc5d632-16ea-467e-54fc-b62d6809dad8 Address:127.0.0.1:24702}]" +34:57.334 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +34:57.339 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24702 [Follower]" leader= +34:57.340 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2021-08-13T15:34:57.341Z [INFO] agent.router: Initializing LAN area manager +34:57.341 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24746 network=udp +34:57.342 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +34:57.342 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:24702) (DC: dc1)" +34:57.342 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24746 network=tcp +34:57.344 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24747 network=tcp +34:57.344 [INFO] TestAgent: started state syncer +34:57.344 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24703 network=tcp +34:57.407 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:57.407 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24702 [Candidate]" term=2 +34:57.407 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:57.407 [DEBUG] TestAgent.server.raft: vote granted: from=cbc5d632-16ea-467e-54fc-b62d6809dad8 term=2 tally=1 +34:57.408 [INFO] TestAgent.server.raft: election won: tally=1 +34:57.408 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24702 [Leader]" +34:57.408 [INFO] TestAgent.server: cluster leadership acquired +34:57.409 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24702 +34:57.409 [INFO] TestAgent.server: New leader elected: payload=test-node +34:57.411 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:57.414 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:57.414 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:57.414 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:57.414 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:57.414 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:57.414 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +34:57.414 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +34:57.416 [INFO] TestAgent.server: federation state anti-entropy synced +34:57.755 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:57.756 [INFO] TestAgent: Synced node info +34:57.924 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=1.779724ms client=127.0.0.1:34907 client_network=udp +34:57.926 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=1.517063ms client=127.0.0.1:52522 client_network=udp +34:57.928 [DEBUG] TestAgent.dns: request served from client: name=7c78ec96-578c-07a8-b7fe-bf67ccad2fb3.query.consul. type=A class=IN latency=1.657164ms client=127.0.0.1:37516 client_network=udp +34:57.928 [INFO] TestAgent: Requesting shutdown +34:57.928 [INFO] TestAgent.server: shutting down server +34:57.928 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:57.928 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:57.928 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:57.928 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:57.928 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:57.928 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:57.929 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:57.929 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:57.929 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:57.929 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:57.929 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:57.929Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:57.929Z [INFO] agent.router.manager: shutting down +34:57.929 [INFO] TestAgent: consul server down +34:57.929 [INFO] TestAgent: shutdown complete +34:57.929 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24746 network=tcp +34:57.929 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24746 network=udp +34:57.929 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24747 network=tcp +34:58.430 [INFO] TestAgent: Waiting for endpoints to shut down +34:58.431 [INFO] TestAgent: Endpoints down === CONT TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{7_7_7_7_6_7_7_5_7_7_-5} -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 4 out of 4 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:21.863Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:21.895Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:21.898 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c05b26c3-a10c-aff0-477d-449db21011d0 Address:127.0.0.1:13523}]" -43:21.899 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13523 [Follower]" leader= -43:21.899 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -43:21.900 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2022-09-15T21:43:21.900Z [INFO] agent.router: Initializing LAN area manager -43:21.901 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13553 network=udp -43:21.901 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:13523) (DC: dc1)" -43:21.901 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -43:21.901 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13553 network=tcp -43:21.903 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13556 network=tcp -43:21.903 [INFO] TestAgent: started state syncer -43:21.904 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13524 network=tcp -43:21.951 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:21.952 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13523 [Candidate]" term=2 -43:21.952 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:21.952 [DEBUG] TestAgent.server.raft: vote granted: from=c05b26c3-a10c-aff0-477d-449db21011d0 term=2 tally=1 -43:21.952 [INFO] TestAgent.server.raft: election won: tally=1 -43:21.952 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13523 [Leader]" -43:21.952 [INFO] TestAgent.server: cluster leadership acquired -43:21.952 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13523 -43:21.953 [INFO] TestAgent.server: New leader elected: payload=test-node -43:21.954 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:21.957 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:21.957 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:21.957 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:21.957 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:21.957 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:21.957 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -43:21.957 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -43:21.958 [INFO] TestAgent.server: federation state anti-entropy synced -43:22.161 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:22.162 [INFO] TestAgent: Synced node info -43:22.162 [DEBUG] TestAgent: Node info in sync -43:22.188 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=1.8807ms client=127.0.0.1:55310 client_network=udp -43:22.190 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=1.58606ms client=127.0.0.1:37753 client_network=udp -43:22.195 [DEBUG] TestAgent.dns: request served from client: name=6f9cf975-9836-fc2c-076e-56c88fe45273.query.consul. type=A class=IN latency=1.68036ms client=127.0.0.1:59112 client_network=udp -43:22.195 [INFO] TestAgent: Requesting shutdown -43:22.196 [INFO] TestAgent.server: shutting down server -43:22.196 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:22.196 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:22.196 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:22.196 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:22.196 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:22.196 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:22.196 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:22.196 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -43:22.196 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:22.196 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:22.196 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2022-09-15T21:43:22.196Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:22.196Z [INFO] agent.router.manager: shutting down -43:22.196 [INFO] TestAgent: consul server down -43:22.196 [INFO] TestAgent: shutdown complete -43:22.196 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13553 network=tcp -43:22.196 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13553 network=udp -43:22.196 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13556 network=tcp -43:22.697 [INFO] TestAgent: Waiting for endpoints to shut down -43:22.697 [INFO] TestAgent: Endpoints down +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:57.771Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:57.821Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:57.837 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4957209a-d741-8b6e-7485-2b85543e6fef Address:127.0.0.1:24751}]" +34:57.839 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24751 [Follower]" leader= +34:57.848 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +34:57.849 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2021-08-13T15:34:57.849Z [INFO] agent.router: Initializing LAN area manager +34:57.850 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24725 network=udp +34:57.850 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +34:57.850 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:24751) (DC: dc1)" +34:57.851 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24725 network=tcp +34:57.852 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24726 network=tcp +34:57.852 [INFO] TestAgent: started state syncer +34:57.853 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24752 network=tcp +34:57.904 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:57.904 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24751 [Candidate]" term=2 +34:57.904 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:57.904 [DEBUG] TestAgent.server.raft: vote granted: from=4957209a-d741-8b6e-7485-2b85543e6fef term=2 tally=1 +34:57.904 [INFO] TestAgent.server.raft: election won: tally=1 +34:57.904 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24751 [Leader]" +34:57.904 [INFO] TestAgent.server: cluster leadership acquired +34:57.905 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24751 +34:57.906 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:57.907 [INFO] TestAgent.server: New leader elected: payload=test-node +34:57.909 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:57.909 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:57.909 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:57.909 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:57.909 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:57.909 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +34:57.909 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +34:57.910 [INFO] TestAgent.server: federation state anti-entropy synced +34:57.912 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:57.912 [INFO] TestAgent: Synced node info +34:57.912 [DEBUG] TestAgent: Node info in sync +34:58.125 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=1.673324ms client=127.0.0.1:60806 client_network=udp +34:58.126 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=1.614884ms client=127.0.0.1:59467 client_network=udp +34:58.129 [DEBUG] TestAgent.dns: request served from client: name=ba72aea1-2d3d-30be-f925-4e1e197fc5cc.query.consul. type=AAAA class=IN latency=1.530984ms client=127.0.0.1:40106 client_network=udp +34:58.129 [INFO] TestAgent: Requesting shutdown +34:58.129 [INFO] TestAgent.server: shutting down server +34:58.129 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:58.129 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:58.129 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:58.129 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:58.129 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:58.129 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:58.129 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:58.129 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:58.129 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:58.129 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:58.129 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:58.129Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:58.129Z [INFO] agent.router.manager: shutting down +34:58.129 [INFO] TestAgent: consul server down +34:58.130 [INFO] TestAgent: shutdown complete +34:58.130 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24725 network=tcp +34:58.130 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24725 network=udp +34:58.130 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24726 network=tcp +34:58.631 [INFO] TestAgent: Waiting for endpoints to shut down +34:58.631 [INFO] TestAgent: Endpoints down === CONT TestDNS_ServiceLookup_AnswerLimits/A_lookup_{7_7_7_7_6_7_7_5_7_7_-5} -2022-09-15T21:43:21.810Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:21.843Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:21.848 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:19a2ea44-eab1-e46e-de01-ad33cb6ceac0 Address:127.0.0.1:13537}]" -43:21.849 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -43:21.850 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13537 [Follower]" leader= -43:21.850 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2022-09-15T21:43:21.850Z [INFO] agent.router: Initializing LAN area manager -43:21.850 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:13537) (DC: dc1)" -43:21.850 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -43:21.850 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13511 network=udp -43:21.850 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13511 network=tcp -43:21.851 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13512 network=tcp -43:21.851 [INFO] TestAgent: started state syncer -43:21.851 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13538 network=tcp -43:21.902 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:21.902 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13537 [Candidate]" term=2 -43:21.902 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:21.902 [DEBUG] TestAgent.server.raft: vote granted: from=19a2ea44-eab1-e46e-de01-ad33cb6ceac0 term=2 tally=1 -43:21.902 [INFO] TestAgent.server.raft: election won: tally=1 -43:21.902 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13537 [Leader]" -43:21.902 [INFO] TestAgent.server: cluster leadership acquired -43:21.902 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13537 -43:21.904 [INFO] TestAgent.server: New leader elected: payload=test-node -43:21.905 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:21.908 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:21.908 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:21.908 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:21.908 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:21.908 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:21.908 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -43:21.908 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -43:21.909 [INFO] TestAgent.server: federation state anti-entropy synced -43:22.016 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:22.017 [INFO] TestAgent: Synced node info -43:22.204 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=2.19714ms client=127.0.0.1:53280 client_network=udp -43:22.207 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=2.0025ms client=127.0.0.1:60084 client_network=udp -43:22.212 [DEBUG] TestAgent.dns: request served from client: name=af679ad9-b5e3-569a-0903-8ff109d86229.query.consul. type=ANY class=IN latency=2.13516ms client=127.0.0.1:36693 client_network=udp -43:22.216 [INFO] TestAgent: Requesting shutdown -43:22.216 [INFO] TestAgent.server: shutting down server -43:22.216 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:22.216 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:22.216 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:22.216 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:22.216 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:22.216 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:22.216 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:22.216 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:22.216 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:22.216 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:22.216 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:22.216Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:22.216Z [INFO] agent.router.manager: shutting down -43:22.217 [INFO] TestAgent: consul server down -43:22.217 [INFO] TestAgent: shutdown complete -43:22.217 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13511 network=tcp -43:22.217 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13511 network=udp -43:22.217 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13512 network=tcp -43:22.717 [INFO] TestAgent: Waiting for endpoints to shut down -43:22.717 [INFO] TestAgent: Endpoints down +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._tcp_(qType:=255)_compressed=false#01 +2021-08-13T15:34:57.794Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:57.827Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:57.831 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:280131fa-29a4-cda9-670e-88bdbdcd1dc0 Address:127.0.0.1:24674}]" +34:57.831 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24674 [Follower]" leader= +34:57.840 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +34:57.841 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2021-08-13T15:34:57.841Z [INFO] agent.router: Initializing LAN area manager +34:57.842 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:24674) (DC: dc1)" +34:57.842 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +34:57.843 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24753 network=tcp +34:57.843 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24753 network=udp +34:57.844 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24754 network=tcp +34:57.844 [INFO] TestAgent: started state syncer +34:57.845 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24675 network=tcp +34:57.901 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:57.901 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24674 [Candidate]" term=2 +34:57.902 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:57.902 [DEBUG] TestAgent.server.raft: vote granted: from=280131fa-29a4-cda9-670e-88bdbdcd1dc0 term=2 tally=1 +34:57.902 [INFO] TestAgent.server.raft: election won: tally=1 +34:57.902 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24674 [Leader]" +34:57.902 [INFO] TestAgent.server: cluster leadership acquired +34:57.902 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24674 +34:57.904 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:57.907 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:57.907 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:57.907 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:57.907 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:57.907 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:57.907 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +34:57.907 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +34:57.909 [INFO] TestAgent.server: federation state anti-entropy synced +34:57.910 [INFO] TestAgent.server: New leader elected: payload=test-node +34:57.981 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:57.982 [INFO] TestAgent: Synced node info +34:58.347 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=2.066045ms client=127.0.0.1:55677 client_network=udp +34:58.349 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=2.008345ms client=127.0.0.1:51140 client_network=udp +34:58.353 [DEBUG] TestAgent.dns: request served from client: name=543e571c-df28-886e-4adc-cffde784b703.query.consul. type=A class=IN latency=1.805684ms client=127.0.0.1:48403 client_network=udp +34:58.353 [INFO] TestAgent: Requesting shutdown +34:58.353 [INFO] TestAgent.server: shutting down server +34:58.353 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:58.353 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:58.353 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:58.353 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:58.353 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:58.353 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:58.353 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:58.353 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:58.353 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:58.353 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:58.353 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:58.354Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:58.354Z [INFO] agent.router.manager: shutting down +34:58.354 [INFO] TestAgent: consul server down +34:58.354 [INFO] TestAgent: shutdown complete +34:58.354 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24753 network=tcp +34:58.354 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24753 network=udp +34:58.354 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24754 network=tcp +34:58.854 [INFO] TestAgent: Waiting for endpoints to shut down +34:58.855 [INFO] TestAgent: Endpoints down === CONT TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{6_6_6_6_6_6_6_5_6_6_-5} -2022-09-15T21:43:21.821Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:21.869Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:21.872 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:90a0446c-f602-3a9a-afb6-a5e975ed3de3 Address:127.0.0.1:13551}]" -43:21.873 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13551 [Follower]" leader= -43:21.875 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -43:21.876 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2022-09-15T21:43:21.876Z [INFO] agent.router: Initializing LAN area manager -43:21.877 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13532 network=udp -43:21.877 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:13551) (DC: dc1)" -43:21.877 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -43:21.877 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13532 network=tcp -43:21.878 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13540 network=tcp -43:21.878 [INFO] TestAgent: started state syncer -43:21.878 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13552 network=tcp -43:21.930 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:21.930 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13551 [Candidate]" term=2 -43:21.931 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:21.931 [DEBUG] TestAgent.server.raft: vote granted: from=90a0446c-f602-3a9a-afb6-a5e975ed3de3 term=2 tally=1 -43:21.931 [INFO] TestAgent.server.raft: election won: tally=1 -43:21.931 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13551 [Leader]" -43:21.931 [INFO] TestAgent.server: cluster leadership acquired -43:21.931 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13551 -43:21.932 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:21.934 [INFO] TestAgent.server: New leader elected: payload=test-node -43:21.935 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:21.935 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:21.935 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:21.935 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:21.935 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:21.935 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -43:21.935 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -43:21.936 [INFO] TestAgent.server: federation state anti-entropy synced -43:22.212 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:22.213 [INFO] TestAgent: Synced node info -43:22.216 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=1.81876ms client=127.0.0.1:40140 client_network=udp -43:22.219 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=1.83406ms client=127.0.0.1:57732 client_network=udp -43:22.221 [DEBUG] TestAgent.dns: request served from client: name=1dc65548-eb43-741c-b66e-da6a2dee6339.query.consul. type=AAAA class=IN latency=1.67068ms client=127.0.0.1:54177 client_network=udp -43:22.221 [INFO] TestAgent: Requesting shutdown -43:22.222 [INFO] TestAgent.server: shutting down server -43:22.222 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:22.222 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:22.222 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:22.222 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:22.222 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:22.222 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:22.222 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:22.222 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:22.222 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:22.222 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:22.222 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:22.223Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:22.223Z [INFO] agent.router.manager: shutting down -43:22.223 [INFO] TestAgent: consul server down -43:22.223 [INFO] TestAgent: shutdown complete -43:22.223 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13532 network=tcp -43:22.223 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13532 network=udp -43:22.223 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13540 network=tcp -43:22.730 [INFO] TestAgent: Waiting for endpoints to shut down -43:22.730 [INFO] TestAgent: Endpoints down +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:57.991Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:58.009Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:58.013 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2843a64a-4a73-94a0-3a71-f085e3882f55 Address:127.0.0.1:24758}]" +34:58.013 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24758 [Follower]" leader= +34:58.014 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +34:58.015 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2021-08-13T15:34:58.015Z [INFO] agent.router: Initializing LAN area manager +34:58.015 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24669 network=udp +34:58.015 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +34:58.015 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:24758) (DC: dc1)" +34:58.016 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24669 network=tcp +34:58.017 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24670 network=tcp +34:58.017 [INFO] TestAgent: started state syncer +34:58.017 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24759 network=tcp +34:58.057 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:58.057 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24758 [Candidate]" term=2 +34:58.058 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:58.058 [DEBUG] TestAgent.server.raft: vote granted: from=2843a64a-4a73-94a0-3a71-f085e3882f55 term=2 tally=1 +34:58.058 [INFO] TestAgent.server.raft: election won: tally=1 +34:58.058 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24758 [Leader]" +34:58.058 [INFO] TestAgent.server: cluster leadership acquired +34:58.058 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24758 +34:58.058 [INFO] TestAgent.server: New leader elected: payload=test-node +34:58.059 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:58.062 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:58.062 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:58.062 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:58.062 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:58.062 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:58.062 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +34:58.062 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +34:58.063 [INFO] TestAgent.server: federation state anti-entropy synced +34:58.455 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:58.456 [INFO] TestAgent: Synced node info +34:58.456 [DEBUG] TestAgent: Node info in sync +34:58.550 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=2.511966ms client=127.0.0.1:36279 client_network=udp +34:58.554 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=3.684088ms client=127.0.0.1:33231 client_network=udp +34:58.570 [DEBUG] TestAgent.dns: request served from client: name=bad2fe34-4216-37e6-a0d1-fa0a2728eab8.query.consul. type=AAAA class=IN latency=9.385621ms client=127.0.0.1:52284 client_network=udp +34:58.570 [INFO] TestAgent: Requesting shutdown +34:58.570 [INFO] TestAgent.server: shutting down server +34:58.570 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:58.570 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:58.570 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:58.570 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:58.570 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:58.570 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:58.570 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:58.570 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:58.570 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:58.570 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:58.570 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:58.570Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:58.570Z [INFO] agent.router.manager: shutting down +34:58.571 [INFO] TestAgent: consul server down +34:58.571 [INFO] TestAgent: shutdown complete +34:58.571 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24669 network=tcp +34:58.571 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24669 network=udp +34:58.571 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24670 network=tcp +34:59.071 [INFO] TestAgent: Waiting for endpoints to shut down +34:59.071 [INFO] TestAgent: Endpoints down === CONT TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{6_6_6_6_6_6_6_5_6_6_-5} -2022-09-15T21:43:21.901Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:21.918Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:21.921 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0c022755-9ea7-8c69-7006-31f1b0b366c6 Address:127.0.0.1:13516}]" -43:21.922 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13516 [Follower]" leader= -43:21.922 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -43:21.923 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2022-09-15T21:43:21.923Z [INFO] agent.router: Initializing LAN area manager -43:21.925 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13573 network=udp -43:21.925 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:13516) (DC: dc1)" -43:21.925 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -43:21.925 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13573 network=tcp -43:21.926 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13574 network=tcp -43:21.926 [INFO] TestAgent: started state syncer -43:21.926 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13517 network=tcp -43:21.981 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:21.981 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13516 [Candidate]" term=2 -43:21.981 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:21.981 [DEBUG] TestAgent.server.raft: vote granted: from=0c022755-9ea7-8c69-7006-31f1b0b366c6 term=2 tally=1 -43:21.981 [INFO] TestAgent.server.raft: election won: tally=1 -43:21.981 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13516 [Leader]" -43:21.981 [INFO] TestAgent.server: cluster leadership acquired -43:21.981 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13516 -43:21.982 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:21.985 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:21.985 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:21.985 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:21.985 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:21.985 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:21.985 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -43:21.985 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -43:21.986 [INFO] TestAgent.server: New leader elected: payload=test-node -43:21.986 [INFO] TestAgent.server: federation state anti-entropy synced -43:22.069 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:22.083 [INFO] TestAgent: Synced node info -43:22.233 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=1.98092ms client=127.0.0.1:39017 client_network=udp -43:22.241 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=1.78946ms client=127.0.0.1:50896 client_network=udp -43:22.243 [DEBUG] TestAgent.dns: request served from client: name=00c8f48a-6799-0b9f-3e32-3d2b4b37a1c7.query.consul. type=AAAA class=IN latency=1.86162ms client=127.0.0.1:45315 client_network=udp -43:22.243 [INFO] TestAgent: Requesting shutdown -43:22.244 [INFO] TestAgent.server: shutting down server -43:22.244 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:22.244 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:22.244 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:22.244 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:22.244 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:22.244 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:22.244 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:22.244 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:22.244 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:22.244 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:22.244 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:22.244Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:22.244Z [INFO] agent.router.manager: shutting down -43:22.244 [INFO] TestAgent: consul server down -43:22.244 [INFO] TestAgent: shutdown complete -43:22.244 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13573 network=tcp -43:22.244 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13573 network=udp -43:22.244 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13574 network=tcp -43:22.746 [INFO] TestAgent: Waiting for endpoints to shut down -43:22.746 [INFO] TestAgent: Endpoints down +2021-08-13T15:34:57.970Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:57.987Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:57.990 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:60b35003-0ab7-e7b0-12d1-94af4767f355 Address:127.0.0.1:24667}]" +34:57.990 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24667 [Follower]" leader= +34:57.991 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +34:57.992 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2021-08-13T15:34:57.992Z [INFO] agent.router: Initializing LAN area manager +34:57.992 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:24667) (DC: dc1)" +34:57.992 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +34:57.992 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24676 network=udp +34:57.993 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24676 network=tcp +34:57.993 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24677 network=tcp +34:57.994 [INFO] TestAgent: started state syncer +34:57.994 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24668 network=tcp +34:58.040 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:58.040 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24667 [Candidate]" term=2 +34:58.040 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:58.040 [DEBUG] TestAgent.server.raft: vote granted: from=60b35003-0ab7-e7b0-12d1-94af4767f355 term=2 tally=1 +34:58.040 [INFO] TestAgent.server.raft: election won: tally=1 +34:58.040 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24667 [Leader]" +34:58.040 [INFO] TestAgent.server: cluster leadership acquired +34:58.040 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24667 +34:58.041 [INFO] TestAgent.server: New leader elected: payload=test-node +34:58.042 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:58.044 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:58.044 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:58.044 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:58.044 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:58.044 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:58.044 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +34:58.044 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +34:58.046 [INFO] TestAgent.server: federation state anti-entropy synced +34:58.307 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:58.308 [INFO] TestAgent: Synced node info +34:58.586 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=4.15959ms client=127.0.0.1:57977 client_network=udp +34:58.590 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=2.833546ms client=127.0.0.1:59109 client_network=udp +34:58.594 [DEBUG] TestAgent.dns: request served from client: name=48fb7969-9e72-19cf-c5b4-754cc7e5965e.query.consul. type=ANY class=IN latency=2.997407ms client=127.0.0.1:52871 client_network=udp +34:58.595 [INFO] TestAgent: Requesting shutdown +34:58.595 [INFO] TestAgent.server: shutting down server +34:58.595 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:58.595 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:58.595 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:58.595 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:58.595 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:58.596 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:58.596 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:58.596 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:58.596 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:58.596 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:58.596 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:58.597Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:58.597Z [INFO] agent.router.manager: shutting down +34:58.598 [INFO] TestAgent: consul server down +34:58.598 [INFO] TestAgent: shutdown complete +34:58.598 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24676 network=tcp +34:58.598 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24676 network=udp +34:58.598 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24677 network=tcp +34:59.103 [INFO] TestAgent: Waiting for endpoints to shut down +34:59.103 [INFO] TestAgent: Endpoints down === CONT TestDNS_ServiceLookup_AnswerLimits/A_lookup_{3_3_3_3_3_3_3_3_3_3_3} -[WARN] freeport: 4 out of 32 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:21.923Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:21.940Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:21.944 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cd2882a8-fec8-d372-ab07-706ff2e3e83a Address:127.0.0.1:13578}]" -43:21.944 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13578 [Follower]" leader= -43:21.945 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -43:21.946 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2022-09-15T21:43:21.946Z [INFO] agent.router: Initializing LAN area manager -43:21.946 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:13578) (DC: dc1)" -43:21.946 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -43:21.947 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13518 network=udp -43:21.947 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13518 network=tcp -43:21.948 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13519 network=tcp -43:21.948 [INFO] TestAgent: started state syncer -43:21.948 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13579 network=tcp -43:21.994 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:21.994 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13578 [Candidate]" term=2 -43:21.994 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:21.995 [DEBUG] TestAgent.server.raft: vote granted: from=cd2882a8-fec8-d372-ab07-706ff2e3e83a term=2 tally=1 -43:21.995 [INFO] TestAgent.server.raft: election won: tally=1 -43:21.995 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13578 [Leader]" -43:21.995 [INFO] TestAgent.server: cluster leadership acquired -43:21.995 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13578 -43:21.996 [INFO] TestAgent.server: New leader elected: payload=test-node -43:21.996 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:21.999 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:21.999 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:21.999 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:21.999 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:21.999 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:21.999 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -43:21.999 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -43:22.000 [INFO] TestAgent.server: federation state anti-entropy synced -43:22.050 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:22.051 [INFO] TestAgent: Synced node info -43:22.352 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=1.82154ms client=127.0.0.1:55526 client_network=udp -43:22.356 [DEBUG] TestAgent.dns: request served from client: name=5208cf95-83d6-1eb4-4d36-2d900e834d66.query.consul. type=A class=IN latency=1.75382ms client=127.0.0.1:55914 client_network=udp -43:22.356 [INFO] TestAgent: Requesting shutdown -43:22.356 [INFO] TestAgent.server: shutting down server -43:22.356 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:22.356 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:22.356 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:22.356 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:22.356 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:22.356 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:22.356 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:22.356 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:22.356 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:22.356 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:22.356 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:22.357Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:22.357Z [INFO] agent.router.manager: shutting down -43:22.357 [INFO] TestAgent: consul server down -43:22.357 [INFO] TestAgent: shutdown complete -43:22.357 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13518 network=tcp -43:22.357 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13518 network=udp -43:22.358 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=1.57518ms client=127.0.0.1:41794 client_network=udp -43:22.358 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13519 network=tcp -43:22.862 [INFO] TestAgent: Waiting for endpoints to shut down -43:22.862 [INFO] TestAgent: Endpoints down +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:58.500Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:58.544Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:58.548 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f95ab8a0-2632-714c-f7b0-bf9ada726b14 Address:127.0.0.1:24792}]" +34:58.549 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +34:58.550 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2021-08-13T15:34:58.551Z [INFO] agent.router: Initializing LAN area manager +34:58.551 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24801 network=udp +34:58.552 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24792 [Follower]" leader= +34:58.555 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24801 network=tcp +34:58.557 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:24792) (DC: dc1)" +34:58.558 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24802 network=tcp +34:58.558 [INFO] TestAgent: started state syncer +34:58.559 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24793 network=tcp +34:58.560 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +34:58.623 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:58.623 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24792 [Candidate]" term=2 +34:58.623 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:58.623 [DEBUG] TestAgent.server.raft: vote granted: from=f95ab8a0-2632-714c-f7b0-bf9ada726b14 term=2 tally=1 +34:58.623 [INFO] TestAgent.server.raft: election won: tally=1 +34:58.623 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24792 [Leader]" +34:58.624 [INFO] TestAgent.server: cluster leadership acquired +34:58.625 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24792 +34:58.625 [INFO] TestAgent.server: New leader elected: payload=test-node +34:58.629 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:58.637 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:58.637 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:58.637 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:58.637 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:58.637 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:58.637 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +34:58.637 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +34:58.639 [INFO] TestAgent.server: federation state anti-entropy synced +34:58.850 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=1.726144ms client=127.0.0.1:34431 client_network=udp +34:58.852 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=1.558923ms client=127.0.0.1:35499 client_network=udp +34:58.861 [DEBUG] TestAgent.dns: request served from client: name=8fc76f79-3e68-9d84-1093-94fc971b7c23.query.consul. type=AAAA class=IN latency=8.288379ms client=127.0.0.1:39782 client_network=udp +34:58.861 [INFO] TestAgent: Requesting shutdown +34:58.861 [INFO] TestAgent.server: shutting down server +34:58.861 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:58.861 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:58.861 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:58.861 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:58.861 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:58.862 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:58.862 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:58.862 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:58.862 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:58.862 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:58.862 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:58.862 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:58.862Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:58.862Z [INFO] agent.router.manager: shutting down +34:58.862 [INFO] TestAgent: consul server down +34:58.862 [INFO] TestAgent: shutdown complete +34:58.862 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24801 network=tcp +34:58.862 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24801 network=udp +34:58.862 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24802 network=tcp +34:59.363 [INFO] TestAgent: Waiting for endpoints to shut down +34:59.363 [INFO] TestAgent: Endpoints down === CONT TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{5_5_5_5_5_5_5_5_5_5_5} -2022-09-15T21:43:21.898Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:21.915Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:21.919 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d3b280e2-fcf3-cbee-e3b3-c9841e49b1d0 Address:127.0.0.1:13571}]" -43:21.920 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -43:21.920 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2022-09-15T21:43:21.921Z [INFO] agent.router: Initializing LAN area manager -43:21.921 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13525 network=udp -43:21.921 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13571 [Follower]" leader= -43:21.921 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:13571) (DC: dc1)" -43:21.922 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13525 network=tcp -43:21.923 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13533 network=tcp -43:21.923 [INFO] TestAgent: started state syncer -43:21.923 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13572 network=tcp -43:21.930 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -43:21.980 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:21.980 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13571 [Candidate]" term=2 -43:21.980 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:21.980 [DEBUG] TestAgent.server.raft: vote granted: from=d3b280e2-fcf3-cbee-e3b3-c9841e49b1d0 term=2 tally=1 -43:21.980 [INFO] TestAgent.server.raft: election won: tally=1 -43:21.980 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13571 [Leader]" -43:21.981 [INFO] TestAgent.server: cluster leadership acquired -43:21.981 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13571 -43:21.982 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:21.985 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:21.985 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:21.985 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:21.985 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:21.985 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:21.985 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -43:21.985 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -43:21.986 [INFO] TestAgent.server: New leader elected: payload=test-node -43:21.986 [INFO] TestAgent.server: federation state anti-entropy synced -43:22.094 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:22.095 [INFO] TestAgent: Synced node info -43:22.369 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=2.2146ms client=127.0.0.1:41856 client_network=udp -43:22.371 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=2.17136ms client=127.0.0.1:52314 client_network=udp -43:22.383 [DEBUG] TestAgent.dns: request served from client: name=268e33e8-8c00-fd1f-29ce-b0583fdfddd9.query.consul. type=ANY class=IN latency=10.97212ms client=127.0.0.1:51544 client_network=udp -43:22.383 [INFO] TestAgent: Requesting shutdown -43:22.384 [INFO] TestAgent.server: shutting down server -43:22.384 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:22.384 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:22.384 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:22.384 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:22.384 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:22.384 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:22.384 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:22.384 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:22.384 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:22.384 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:22.385 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:22.385Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:22.385Z [INFO] agent.router.manager: shutting down -43:22.385 [INFO] TestAgent: consul server down -43:22.385 [INFO] TestAgent: shutdown complete -43:22.385 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13525 network=tcp -43:22.385 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13525 network=udp -43:22.385 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13533 network=tcp -43:22.887 [INFO] TestAgent: Waiting for endpoints to shut down -43:22.887 [INFO] TestAgent: Endpoints down +2021-08-13T15:34:58.291Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:58.355Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:58.359 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:65bf85bb-df2b-a208-79f1-3b113e5950fe Address:127.0.0.1:24813}]" +34:58.359 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24813 [Follower]" leader= +34:58.372 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +34:58.374 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2021-08-13T15:34:58.374Z [INFO] agent.router: Initializing LAN area manager +34:58.374 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24760 network=udp +34:58.381 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:24813) (DC: dc1)" +34:58.382 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +34:58.383 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24760 network=tcp +34:58.389 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24761 network=tcp +34:58.389 [INFO] TestAgent: started state syncer +34:58.390 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24814 network=tcp +34:58.431 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:58.446 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24813 [Candidate]" term=2 +34:58.447 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:58.447 [DEBUG] TestAgent.server.raft: vote granted: from=65bf85bb-df2b-a208-79f1-3b113e5950fe term=2 tally=1 +34:58.447 [INFO] TestAgent.server.raft: election won: tally=1 +34:58.447 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24813 [Leader]" +34:58.447 [INFO] TestAgent.server: cluster leadership acquired +34:58.447 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24813 +34:58.449 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:58.452 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:58.452 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:58.452 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:58.452 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:58.452 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:58.452 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +34:58.452 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +34:58.452 [INFO] TestAgent.server: New leader elected: payload=test-node +34:58.453 [INFO] TestAgent.server: federation state anti-entropy synced +34:58.776 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:58.777 [INFO] TestAgent: Synced node info +34:58.777 [DEBUG] TestAgent: Node info in sync +34:58.891 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=10.552884ms client=127.0.0.1:50098 client_network=udp +34:58.894 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=2.277665ms client=127.0.0.1:55360 client_network=udp +34:58.897 [DEBUG] TestAgent.dns: request served from client: name=926c8d6a-916e-7f0f-e0b1-7531830c3389.query.consul. type=A class=IN latency=2.332905ms client=127.0.0.1:44906 client_network=udp +34:58.897 [INFO] TestAgent: Requesting shutdown +34:58.897 [INFO] TestAgent.server: shutting down server +34:58.897 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:58.897 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:58.897 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:58.897 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:58.897 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:58.903 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:58.903 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:58.903 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:58.903 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:58.903 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:58.903 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:58.904Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:58.904Z [INFO] agent.router.manager: shutting down +34:58.904 [INFO] TestAgent: consul server down +34:58.904 [INFO] TestAgent: shutdown complete +34:58.904 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24760 network=tcp +34:58.904 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24760 network=udp +34:58.904 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24761 network=tcp +34:59.407 [INFO] TestAgent: Waiting for endpoints to shut down +34:59.407 [INFO] TestAgent: Endpoints down +=== CONT TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{1_1_1_1_1_1_1_1_1_1_1} +2021-08-13T15:34:58.455Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:58.474Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:58.489 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c2d33987-74e1-dd0b-49d2-7b95f43f2781 Address:127.0.0.1:24799}]" +34:58.490 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +34:58.491 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24799 [Follower]" leader= +34:58.496 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2021-08-13T15:34:58.496Z [INFO] agent.router: Initializing LAN area manager +34:58.497 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24815 network=udp +34:58.500 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:24799) (DC: dc1)" +34:58.500 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +34:58.501 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24815 network=tcp +34:58.502 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24816 network=tcp +34:58.503 [INFO] TestAgent: started state syncer +34:58.503 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24800 network=tcp +34:58.535 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:58.535 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24799 [Candidate]" term=2 +34:58.535 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:58.535 [DEBUG] TestAgent.server.raft: vote granted: from=c2d33987-74e1-dd0b-49d2-7b95f43f2781 term=2 tally=1 +34:58.535 [INFO] TestAgent.server.raft: election won: tally=1 +34:58.535 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24799 [Leader]" +34:58.536 [INFO] TestAgent.server: cluster leadership acquired +34:58.536 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24799 +34:58.538 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:58.538 [INFO] TestAgent.server: New leader elected: payload=test-node +34:58.545 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:58.545 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:58.545 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:58.545 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:58.545 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:58.545 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +34:58.545 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +34:58.547 [INFO] TestAgent.server: federation state anti-entropy synced +34:58.626 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:58.627 [INFO] TestAgent: Synced node info +34:58.627 [DEBUG] TestAgent: Node info in sync +34:58.892 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=10.848405ms client=127.0.0.1:53876 client_network=udp +34:58.895 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=2.613786ms client=127.0.0.1:45835 client_network=udp +34:58.908 [DEBUG] TestAgent.dns: request served from client: name=c4dd2019-cea9-de8f-1556-50d5ba578b5a.query.consul. type=ANY class=IN latency=10.900885ms client=127.0.0.1:36181 client_network=udp +34:58.908 [INFO] TestAgent: Requesting shutdown +34:58.908 [INFO] TestAgent.server: shutting down server +34:58.908 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:58.908 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:58.908 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:58.908 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:58.908 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:58.908 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:58.908 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:58.909 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:58.909 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:58.909 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:58.909 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:58.909Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:58.909Z [INFO] agent.router.manager: shutting down +34:58.909 [INFO] TestAgent: consul server down +34:58.909 [INFO] TestAgent: shutdown complete +34:58.909 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24815 network=tcp +34:58.909 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24815 network=udp +34:58.909 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24816 network=tcp +34:59.410 [INFO] TestAgent: Waiting for endpoints to shut down +34:59.410 [INFO] TestAgent: Endpoints down +=== CONT TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{2_2_2_2_2_2_2_2_2_2_2} +2021-08-13T15:34:58.658Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:58.688Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:58.693 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5909fbaf-0817-e815-3788-0cc3f080a78f Address:127.0.0.1:24779}]" +34:58.694 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +34:58.696 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24779 [Follower]" leader= +34:58.696 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2021-08-13T15:34:58.696Z [INFO] agent.router: Initializing LAN area manager +34:58.696 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24794 network=udp +34:58.698 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +34:58.700 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24794 network=tcp +34:58.700 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:24779) (DC: dc1)" +34:58.702 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24795 network=tcp +34:58.702 [INFO] TestAgent: started state syncer +34:58.702 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24780 network=tcp +34:58.764 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:58.764 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24779 [Candidate]" term=2 +34:58.764 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:58.764 [DEBUG] TestAgent.server.raft: vote granted: from=5909fbaf-0817-e815-3788-0cc3f080a78f term=2 tally=1 +34:58.764 [INFO] TestAgent.server.raft: election won: tally=1 +34:58.764 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24779 [Leader]" +34:58.764 [INFO] TestAgent.server: cluster leadership acquired +34:58.765 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24779 +34:58.767 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:58.767 [INFO] TestAgent.server: New leader elected: payload=test-node +34:58.771 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:58.779 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:58.779 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:58.779 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:58.779 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:58.779 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +34:58.779 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +34:58.783 [INFO] TestAgent.server: federation state anti-entropy synced +34:58.988 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=1.719484ms client=127.0.0.1:46573 client_network=udp +34:58.990 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=1.535644ms client=127.0.0.1:53254 client_network=udp +34:58.991 [DEBUG] TestAgent.dns: request served from client: name=16808134-4ac5-dd3b-08ed-6108c726d14c.query.consul. type=A class=IN latency=1.648284ms client=127.0.0.1:42006 client_network=udp +34:58.992 [INFO] TestAgent: Requesting shutdown +34:58.992 [INFO] TestAgent.server: shutting down server +34:58.992 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:58.992 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:58.992 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:58.992 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:58.992 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:58.992 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:58.992 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:58.992 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:58.992 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:58.992 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:58.992 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:58.992 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:58.992Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:58.992Z [INFO] agent.router.manager: shutting down +34:58.992 [INFO] TestAgent: consul server down +34:58.992 [INFO] TestAgent: shutdown complete +34:58.992 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24794 network=tcp +34:58.992 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24794 network=udp +34:58.992 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24795 network=tcp +34:59.496 [INFO] TestAgent: Waiting for endpoints to shut down +34:59.496 [INFO] TestAgent: Endpoints down === CONT TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{5_5_5_5_5_5_5_5_5_5_5} -2022-09-15T21:43:22.123Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:22.161Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:22.168 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:50861e06-8a7c-785f-d29b-19049a8352be Address:127.0.0.1:13414}]" -43:22.169 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -43:22.170 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13414 [Follower]" leader= -43:22.170 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2022-09-15T21:43:22.170Z [INFO] agent.router: Initializing LAN area manager -43:22.171 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13416 network=udp -43:22.171 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:13414) (DC: dc1)" -43:22.171 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -43:22.171 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13416 network=tcp -43:22.172 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13417 network=tcp -43:22.172 [INFO] TestAgent: started state syncer -43:22.173 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13415 network=tcp -43:22.207 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:22.207 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13414 [Candidate]" term=2 -43:22.207 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:22.207 [DEBUG] TestAgent.server.raft: vote granted: from=50861e06-8a7c-785f-d29b-19049a8352be term=2 tally=1 -43:22.207 [INFO] TestAgent.server.raft: election won: tally=1 -43:22.207 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13414 [Leader]" -43:22.208 [INFO] TestAgent.server: cluster leadership acquired -43:22.208 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13414 -43:22.209 [INFO] TestAgent.server: New leader elected: payload=test-node -43:22.210 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:22.213 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:22.213 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:22.213 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:22.213 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:22.213 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:22.213 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -43:22.213 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -43:22.222 [INFO] TestAgent.server: federation state anti-entropy synced -43:22.270 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:22.271 [INFO] TestAgent: Synced node info -43:22.485 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=1.71854ms client=127.0.0.1:40986 client_network=udp -43:22.487 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=1.50532ms client=127.0.0.1:41713 client_network=udp -43:22.489 [DEBUG] TestAgent.dns: request served from client: name=80d96ad1-314a-3443-4da5-3dfd563b5fa9.query.consul. type=AAAA class=IN latency=1.58366ms client=127.0.0.1:52911 client_network=udp -43:22.489 [INFO] TestAgent: Requesting shutdown -43:22.489 [INFO] TestAgent.server: shutting down server -43:22.489 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:22.489 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:22.489 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:22.489 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:22.489 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:22.489 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:22.489 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:22.489 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:22.489 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:22.489 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:22.489 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:22.489Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:22.489Z [INFO] agent.router.manager: shutting down -43:22.490 [INFO] TestAgent: consul server down -43:22.490 [INFO] TestAgent: shutdown complete -43:22.490 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13416 network=tcp -43:22.490 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13416 network=udp -43:22.490 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13417 network=tcp -43:22.991 [INFO] TestAgent: Waiting for endpoints to shut down -43:22.991 [INFO] TestAgent: Endpoints down +[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._udp_(qType:=255)_compressed=true#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._udp_(qType:=255)_compressed=false#01 +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:58.917Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:58.947Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:58.950 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1a963eb2-2525-65ba-9ab3-3aef7f464e39 Address:127.0.0.1:24765}]" +34:58.950 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24765 [Follower]" leader= +34:58.951 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +34:58.951 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2021-08-13T15:34:58.951Z [INFO] agent.router: Initializing LAN area manager +34:58.952 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24781 network=udp +34:58.952 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +34:58.952 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:24765) (DC: dc1)" +34:58.952 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24781 network=tcp +34:58.953 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24782 network=tcp +34:58.953 [INFO] TestAgent: started state syncer +34:58.953 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24766 network=tcp +34:58.999 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:58.999 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24765 [Candidate]" term=2 +34:58.999 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:58.999 [DEBUG] TestAgent.server.raft: vote granted: from=1a963eb2-2525-65ba-9ab3-3aef7f464e39 term=2 tally=1 +34:58.999 [INFO] TestAgent.server.raft: election won: tally=1 +34:58.999 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24765 [Leader]" +34:58.999 [INFO] TestAgent.server: cluster leadership acquired +34:59.000 [INFO] TestAgent.server: New leader elected: payload=test-node +34:59.000 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24765 +34:59.001 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:59.012 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:59.012 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:59.012 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:59.012 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:59.012 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:59.012 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +34:59.012 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +34:59.013 [INFO] TestAgent.server: federation state anti-entropy synced +34:59.275 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=2.782426ms client=127.0.0.1:43914 client_network=udp +34:59.278 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=2.673827ms client=127.0.0.1:47794 client_network=udp +34:59.282 [DEBUG] TestAgent.dns: request served from client: name=f7ae8bd7-0f9f-04a4-bd44-439f366f8dc6.query.consul. type=ANY class=IN latency=3.717949ms client=127.0.0.1:32821 client_network=udp +34:59.282 [INFO] TestAgent: Requesting shutdown +34:59.283 [INFO] TestAgent.server: shutting down server +34:59.283 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:59.283 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:59.283 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:59.283 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:59.283 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:59.286 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:59.286 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:59.286 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:59.287 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:59.287 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:59.287 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:59.303 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:59.304Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:59.304Z [INFO] agent.router.manager: shutting down +34:59.304 [INFO] TestAgent: consul server down +34:59.304 [INFO] TestAgent: shutdown complete +34:59.304 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24781 network=tcp +34:59.304 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24781 network=udp +34:59.304 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24782 network=tcp +34:59.807 [INFO] TestAgent: Waiting for endpoints to shut down +34:59.807 [INFO] TestAgent: Endpoints down +=== CONT TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{2_2_2_2_2_2_2_2_2_2_2} +2021-08-13T15:34:59.099Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:59.137Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:59.141 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cb8827a9-74ed-6425-cbfb-1cbbdd951dd7 Address:127.0.0.1:24772}]" +34:59.142 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +34:59.144 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24772 [Follower]" leader= +34:59.151 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2021-08-13T15:34:59.151Z [INFO] agent.router: Initializing LAN area manager +34:59.156 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +34:59.156 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24767 network=tcp +34:59.157 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:24772) (DC: dc1)" +34:59.157 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24767 network=udp +34:59.159 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24768 network=tcp +34:59.159 [INFO] TestAgent: started state syncer +34:59.159 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24773 network=tcp +34:59.183 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:59.184 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24772 [Candidate]" term=2 +34:59.184 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:59.184 [DEBUG] TestAgent.server.raft: vote granted: from=cb8827a9-74ed-6425-cbfb-1cbbdd951dd7 term=2 tally=1 +34:59.184 [INFO] TestAgent.server.raft: election won: tally=1 +34:59.184 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24772 [Leader]" +34:59.184 [INFO] TestAgent.server: cluster leadership acquired +34:59.184 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24772 +34:59.186 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:59.187 [INFO] TestAgent.server: New leader elected: payload=test-node +34:59.189 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:59.189 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:59.189 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:59.189 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:59.189 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:59.189 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +34:59.189 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +34:59.191 [INFO] TestAgent.server: federation state anti-entropy synced +34:59.341 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=2.006045ms client=127.0.0.1:45480 client_network=udp +34:59.344 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=2.035005ms client=127.0.0.1:40271 client_network=udp +34:59.346 [DEBUG] TestAgent.dns: request served from client: name=88f54965-2f0d-3625-5d01-9158e2f7061c.query.consul. type=AAAA class=IN latency=2.002504ms client=127.0.0.1:34882 client_network=udp +34:59.347 [INFO] TestAgent: Requesting shutdown +34:59.347 [INFO] TestAgent.server: shutting down server +34:59.347 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:59.347 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:59.347 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:59.347 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:59.347 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:59.347 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:59.347 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:59.347 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:59.347 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:59.347 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:59.347 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:59.347 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:59.347Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:59.347Z [INFO] agent.router.manager: shutting down +34:59.348 [INFO] TestAgent: consul server down +34:59.348 [INFO] TestAgent: shutdown complete +34:59.348 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24767 network=tcp +34:59.348 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24767 network=udp +34:59.348 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24768 network=tcp +34:59.848 [INFO] TestAgent: Waiting for endpoints to shut down +34:59.848 [INFO] TestAgent: Endpoints down === CONT TestDNS_ServiceLookup_AnswerLimits/A_lookup_{5_5_5_5_5_5_5_5_5_5_5} -[WARN] freeport: 4 out of 25 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:22.090Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:22.132Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:22.142 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fb434873-f756-198a-98d6-21a8d1f25d2f Address:127.0.0.1:13564}]" -43:22.142 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13564 [Follower]" leader= -43:22.143 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -43:22.143 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2022-09-15T21:43:22.143Z [INFO] agent.router: Initializing LAN area manager -43:22.144 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13580 network=udp -43:22.144 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:13564) (DC: dc1)" -43:22.144 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -43:22.145 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13580 network=tcp -43:22.145 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13581 network=tcp -43:22.154 [INFO] TestAgent: started state syncer -43:22.154 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13565 network=tcp -43:22.182 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:22.182 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13564 [Candidate]" term=2 -43:22.183 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:22.183 [DEBUG] TestAgent.server.raft: vote granted: from=fb434873-f756-198a-98d6-21a8d1f25d2f term=2 tally=1 -43:22.183 [INFO] TestAgent.server.raft: election won: tally=1 -43:22.183 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13564 [Leader]" -43:22.183 [INFO] TestAgent.server: cluster leadership acquired -43:22.183 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13564 -43:22.184 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:22.186 [INFO] TestAgent.server: New leader elected: payload=test-node -43:22.191 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:22.191 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:22.191 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:22.191 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:22.191 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:22.191 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -43:22.191 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -43:22.192 [INFO] TestAgent.server: federation state anti-entropy synced -43:22.532 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=2.16102ms client=127.0.0.1:39443 client_network=udp -43:22.542 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=2.81906ms client=127.0.0.1:52671 client_network=udp -43:22.545 [DEBUG] TestAgent.dns: request served from client: name=2e05d373-6005-ecf4-a7c9-367ffcb7805e.query.consul. type=ANY class=IN latency=2.23064ms client=127.0.0.1:32768 client_network=udp -43:22.545 [INFO] TestAgent: Requesting shutdown -43:22.545 [INFO] TestAgent.server: shutting down server -43:22.545 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:22.545 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:22.545 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:22.545 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:22.545 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:22.545 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:22.545 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:22.545 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:22.545 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:22.545 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:22.545 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:22.545 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:22.546Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:22.546Z [INFO] agent.router.manager: shutting down -43:22.546 [INFO] TestAgent: consul server down -43:22.546 [INFO] TestAgent: shutdown complete -43:22.546 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13580 network=tcp -43:22.546 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13580 network=udp -43:22.546 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13581 network=tcp -43:23.050 [INFO] TestAgent: Waiting for endpoints to shut down -43:23.050 [INFO] TestAgent: Endpoints down +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=1)_compressed=true#01 +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:59.162Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:59.203Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:59.215 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ab948497-4ef9-fef6-e4e0-0a17b20dda14 Address:127.0.0.1:24841}]" +34:59.216 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +34:59.217 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24841 [Follower]" leader= +34:59.218 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2021-08-13T15:34:59.218Z [INFO] agent.router: Initializing LAN area manager +34:59.218 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24774 network=udp +34:59.219 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:24841) (DC: dc1)" +34:59.219 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +34:59.220 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24774 network=tcp +34:59.222 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24775 network=tcp +34:59.222 [INFO] TestAgent: started state syncer +34:59.222 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24842 network=tcp +34:59.276 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:59.276 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24841 [Candidate]" term=2 +34:59.276 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:59.276 [DEBUG] TestAgent.server.raft: vote granted: from=ab948497-4ef9-fef6-e4e0-0a17b20dda14 term=2 tally=1 +34:59.276 [INFO] TestAgent.server.raft: election won: tally=1 +34:59.276 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24841 [Leader]" +34:59.276 [INFO] TestAgent.server: cluster leadership acquired +34:59.277 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24841 +34:59.278 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:59.282 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:59.282 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:59.282 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:59.282 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:59.282 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:59.282 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +34:59.282 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +34:59.284 [INFO] TestAgent.server: New leader elected: payload=test-node +34:59.285 [INFO] TestAgent.server: federation state anti-entropy synced +34:59.387 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:59.388 [INFO] TestAgent: Synced node info +34:59.388 [DEBUG] TestAgent: Node info in sync +34:59.693 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=1.746424ms client=127.0.0.1:33783 client_network=udp +34:59.695 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=1.485343ms client=127.0.0.1:57338 client_network=udp +34:59.697 [DEBUG] TestAgent.dns: request served from client: name=91f3c7d8-f8f4-e052-1085-7e08f12803f9.query.consul. type=A class=IN latency=1.548624ms client=127.0.0.1:54440 client_network=udp +34:59.697 [INFO] TestAgent: Requesting shutdown +34:59.697 [INFO] TestAgent.server: shutting down server +34:59.697 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:59.697 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:59.697 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:59.697 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:59.697 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:59.697 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:59.697 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:59.697 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:59.697 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:59.697 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:59.698 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:59.698Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:59.698Z [INFO] agent.router.manager: shutting down +34:59.698 [INFO] TestAgent: consul server down +34:59.698 [INFO] TestAgent: shutdown complete +34:59.698 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24774 network=tcp +34:59.698 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24774 network=udp +34:59.698 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24775 network=tcp +35:00.199 [INFO] TestAgent: Waiting for endpoints to shut down +35:00.199 [INFO] TestAgent: Endpoints down +=== CONT TestDNS_ServiceLookup_AnswerLimits/A_lookup_{2_2_2_2_2_2_2_2_2_2_2} +2021-08-13T15:34:59.528Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:59.548Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:59.551 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ac89f41f-ab08-1a53-c1b8-3954e9ce7c98 Address:127.0.0.1:24862}]" +34:59.551 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24862 [Follower]" leader= +34:59.552 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +34:59.552 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2021-08-13T15:34:59.552Z [INFO] agent.router: Initializing LAN area manager +34:59.553 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:24862) (DC: dc1)" +34:59.553 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +34:59.553 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24829 network=udp +34:59.553 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24829 network=tcp +34:59.554 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24830 network=tcp +34:59.554 [INFO] TestAgent: started state syncer +34:59.554 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24863 network=tcp +34:59.604 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:59.604 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24862 [Candidate]" term=2 +34:59.604 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:59.604 [DEBUG] TestAgent.server.raft: vote granted: from=ac89f41f-ab08-1a53-c1b8-3954e9ce7c98 term=2 tally=1 +34:59.604 [INFO] TestAgent.server.raft: election won: tally=1 +34:59.604 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24862 [Leader]" +34:59.605 [INFO] TestAgent.server: cluster leadership acquired +34:59.605 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24862 +34:59.607 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:59.608 [INFO] TestAgent.server: New leader elected: payload=test-node +34:59.610 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:59.610 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:59.610 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:59.610 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:59.610 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:59.610 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +34:59.610 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +34:59.620 [INFO] TestAgent.server: federation state anti-entropy synced +34:59.703 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=1.807204ms client=127.0.0.1:59211 client_network=udp +34:59.705 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=1.532844ms client=127.0.0.1:53887 client_network=udp +34:59.707 [DEBUG] TestAgent.dns: request served from client: name=ba9fa0f9-a54a-30ef-b6bd-9dc87db686c0.query.consul. type=AAAA class=IN latency=1.926265ms client=127.0.0.1:50244 client_network=udp +34:59.708 [INFO] TestAgent: Requesting shutdown +34:59.708 [INFO] TestAgent.server: shutting down server +34:59.708 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:59.708 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:59.708 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:59.708 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:59.708 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:59.708 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:59.708 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:59.708 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:59.708 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:59.708 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:59.708 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:59.708 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:59.708Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:59.708Z [INFO] agent.router.manager: shutting down +34:59.708 [INFO] TestAgent: consul server down +34:59.708 [INFO] TestAgent: shutdown complete +34:59.708 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24829 network=tcp +34:59.708 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24829 network=udp +34:59.709 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24830 network=tcp +35:00.219 [INFO] TestAgent: Waiting for endpoints to shut down +35:00.227 [INFO] TestAgent: Endpoints down === CONT TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{4_4_4_4_4_4_4_4_4_4_4} -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 4 out of 4 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:22.716Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:22.734Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:22.737 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6fe52e46-8e0f-97bc-4840-e2ad90ee22b6 Address:127.0.0.1:13386}]" -43:22.738 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13386 [Follower]" leader= -43:22.738 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -43:22.739 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2022-09-15T21:43:22.739Z [INFO] agent.router: Initializing LAN area manager -43:22.740 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13388 network=udp -43:22.741 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:13386) (DC: dc1)" -43:22.741 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13388 network=tcp -43:22.743 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13389 network=tcp -43:22.743 [INFO] TestAgent: started state syncer -43:22.743 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13387 network=tcp -43:22.755 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -43:22.803 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:22.803 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13386 [Candidate]" term=2 -43:22.803 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:22.803 [DEBUG] TestAgent.server.raft: vote granted: from=6fe52e46-8e0f-97bc-4840-e2ad90ee22b6 term=2 tally=1 -43:22.803 [INFO] TestAgent.server.raft: election won: tally=1 -43:22.803 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13386 [Leader]" -43:22.803 [INFO] TestAgent.server: cluster leadership acquired -43:22.804 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13386 -43:22.805 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:22.806 [INFO] TestAgent.server: New leader elected: payload=test-node -43:22.808 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:22.808 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:22.808 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:22.808 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:22.808 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:22.808 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -43:22.808 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -43:22.809 [INFO] TestAgent.server: federation state anti-entropy synced -43:23.041 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:23.043 [INFO] TestAgent: Synced node info -43:23.043 [DEBUG] TestAgent: Node info in sync -43:23.077 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=1.78424ms client=127.0.0.1:41156 client_network=udp -43:23.079 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=1.62204ms client=127.0.0.1:59546 client_network=udp -43:23.085 [DEBUG] TestAgent.dns: request served from client: name=456e9170-c20e-73a2-ca65-e21a7c9f6364.query.consul. type=A class=IN latency=1.68394ms client=127.0.0.1:54996 client_network=udp -43:23.085 [INFO] TestAgent: Requesting shutdown -43:23.085 [INFO] TestAgent.server: shutting down server -43:23.085 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:23.085 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:23.085 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:23.085 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:23.085 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:23.085 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:23.085 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:23.085 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:23.085 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:23.085 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:23.085 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:23.085Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:23.085Z [INFO] agent.router.manager: shutting down -43:23.086 [INFO] TestAgent: consul server down -43:23.086 [INFO] TestAgent: shutdown complete -43:23.086 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13388 network=tcp -43:23.086 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13388 network=udp -43:23.086 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13389 network=tcp -43:23.587 [INFO] TestAgent: Waiting for endpoints to shut down -43:23.587 [INFO] TestAgent: Endpoints down +2021-08-13T15:34:59.479Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:59.525Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:59.528 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:22f35900-d818-57a5-0450-1ecb9bde32ee Address:127.0.0.1:24820}]" +34:59.528 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24820 [Follower]" leader= +34:59.529 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +34:59.530 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2021-08-13T15:34:59.530Z [INFO] agent.router: Initializing LAN area manager +34:59.531 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24808 network=udp +34:59.531 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +34:59.531 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:24820) (DC: dc1)" +34:59.531 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24808 network=tcp +34:59.532 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24809 network=tcp +34:59.532 [INFO] TestAgent: started state syncer +34:59.533 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24821 network=tcp +34:59.587 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:59.587 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24820 [Candidate]" term=2 +34:59.587 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:59.587 [DEBUG] TestAgent.server.raft: vote granted: from=22f35900-d818-57a5-0450-1ecb9bde32ee term=2 tally=1 +34:59.587 [INFO] TestAgent.server.raft: election won: tally=1 +34:59.587 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24820 [Leader]" +34:59.588 [INFO] TestAgent.server: cluster leadership acquired +34:59.588 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24820 +34:59.590 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:59.591 [INFO] TestAgent.server: New leader elected: payload=test-node +34:59.594 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:59.594 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:59.594 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:59.594 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:59.594 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:59.594 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +34:59.594 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +34:59.604 [INFO] TestAgent.server: federation state anti-entropy synced +34:59.743 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:59.744 [INFO] TestAgent: Synced node info +34:59.747 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=1.936884ms client=127.0.0.1:54221 client_network=udp +34:59.749 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=1.838304ms client=127.0.0.1:44632 client_network=udp +34:59.751 [DEBUG] TestAgent.dns: request served from client: name=f86ff634-fba9-0df1-3882-9dea81e09923.query.consul. type=ANY class=IN latency=2.143785ms client=127.0.0.1:53130 client_network=udp +34:59.755 [INFO] TestAgent: Requesting shutdown +34:59.755 [INFO] TestAgent.server: shutting down server +34:59.755 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:59.755 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:59.755 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:59.755 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:59.755 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:59.755 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:59.755 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:59.755 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:59.755 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:59.755 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:59.755 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:59.756Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:59.756Z [INFO] agent.router.manager: shutting down +34:59.756 [INFO] TestAgent: consul server down +34:59.756 [INFO] TestAgent: shutdown complete +34:59.756 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24808 network=tcp +34:59.756 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24808 network=udp +34:59.756 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24809 network=tcp +35:00.257 [INFO] TestAgent: Waiting for endpoints to shut down +35:00.257 [INFO] TestAgent: Endpoints down === CONT TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{4_4_4_4_4_4_4_4_4_4_4} -2022-09-15T21:43:22.772Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:22.804Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:22.811 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9b71407f-7628-00fd-ab99-359ee25c56d6 Address:127.0.0.1:13627}]" -43:22.812 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -43:22.813 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2022-09-15T21:43:22.813Z [INFO] agent.router: Initializing LAN area manager -43:22.814 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13627 [Follower]" leader= -43:22.814 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -43:22.814 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:13627) (DC: dc1)" -43:22.814 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13601 network=udp -43:22.814 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13601 network=tcp -43:22.815 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13602 network=tcp -43:22.815 [INFO] TestAgent: started state syncer -43:22.815 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13628 network=tcp -43:22.863 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:22.863 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13627 [Candidate]" term=2 -43:22.863 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:22.863 [DEBUG] TestAgent.server.raft: vote granted: from=9b71407f-7628-00fd-ab99-359ee25c56d6 term=2 tally=1 -43:22.863 [INFO] TestAgent.server.raft: election won: tally=1 -43:22.863 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13627 [Leader]" -43:22.863 [INFO] TestAgent.server: cluster leadership acquired -43:22.863 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13627 -43:22.865 [INFO] TestAgent.server: New leader elected: payload=test-node -43:22.865 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:22.867 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:22.867 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:22.867 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:22.867 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:22.867 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:22.867 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -43:22.867 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -43:22.869 [INFO] TestAgent.server: federation state anti-entropy synced -43:22.971 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:22.994 [INFO] TestAgent: Synced node info -43:23.105 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=1.8379ms client=127.0.0.1:46723 client_network=udp -43:23.107 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=2.3584ms client=127.0.0.1:34412 client_network=udp -43:23.111 [DEBUG] TestAgent.dns: request served from client: name=ef9240fd-b282-2729-5cb2-9b5d76565af5.query.consul. type=AAAA class=IN latency=1.74562ms client=127.0.0.1:40091 client_network=udp -43:23.111 [INFO] TestAgent: Requesting shutdown -43:23.111 [INFO] TestAgent.server: shutting down server -43:23.111 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:23.111 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:23.111 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:23.111 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:23.111 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:23.111 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:23.111 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:23.111 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:23.111 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:23.111 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:23.111 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:23.111Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:23.111Z [INFO] agent.router.manager: shutting down -43:23.112 [INFO] TestAgent: consul server down -43:23.112 [INFO] TestAgent: shutdown complete -43:23.112 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13601 network=tcp -43:23.112 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13601 network=udp -43:23.112 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13602 network=tcp -43:23.612 [INFO] TestAgent: Waiting for endpoints to shut down -43:23.612 [INFO] TestAgent: Endpoints down -=== CONT TestDNS_ServiceLookup_AnswerLimits/A_lookup_{4_4_4_4_4_4_4_4_4_4_4} -2022-09-15T21:43:22.883Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:22.917Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:22.920 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5abef54f-c316-47ac-dbf0-3af55f0de428 Address:127.0.0.1:13544}]" -43:22.921 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -43:22.922 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13544 [Follower]" leader= -43:22.932 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2022-09-15T21:43:22.932Z [INFO] agent.router: Initializing LAN area manager -43:22.935 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13629 network=tcp -43:22.937 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -43:22.939 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13629 network=udp -43:22.942 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:13544) (DC: dc1)" -43:22.961 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13630 network=tcp -43:22.961 [INFO] TestAgent: started state syncer -43:22.970 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13545 network=tcp -43:22.971 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:22.971 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13544 [Candidate]" term=2 -43:22.972 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:22.972 [DEBUG] TestAgent.server.raft: vote granted: from=5abef54f-c316-47ac-dbf0-3af55f0de428 term=2 tally=1 -43:22.972 [INFO] TestAgent.server.raft: election won: tally=1 -43:22.972 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13544 [Leader]" -43:22.973 [INFO] TestAgent.server: cluster leadership acquired -43:22.973 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13544 -43:22.973 [INFO] TestAgent.server: New leader elected: payload=test-node -43:22.976 [INFO] TestAgent: Synced node info -43:22.984 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:23.014 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:23.014 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:23.014 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:23.014 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:23.014 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:23.014 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -43:23.014 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -43:23.016 [INFO] TestAgent.server: federation state anti-entropy synced -43:23.140 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=2.5407ms client=127.0.0.1:60283 client_network=udp -43:23.142 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=1.9332ms client=127.0.0.1:57356 client_network=udp -43:23.145 [DEBUG] TestAgent.dns: request served from client: name=15f4da87-ab98-ae60-20e9-ceee0c311243.query.consul. type=ANY class=IN latency=2.0187ms client=127.0.0.1:55708 client_network=udp -43:23.150 [INFO] TestAgent: Requesting shutdown -43:23.150 [INFO] TestAgent.server: shutting down server -43:23.150 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:23.150 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:23.150 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:23.150 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:23.150 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:23.150 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:23.150 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:23.150 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:23.150 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:23.150 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:23.150 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:23.150Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:23.151Z [INFO] agent.router.manager: shutting down -43:23.151 [INFO] TestAgent: consul server down -43:23.151 [INFO] TestAgent: shutdown complete -43:23.151 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13629 network=tcp -43:23.151 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13629 network=udp -43:23.151 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13630 network=tcp -43:23.652 [INFO] TestAgent: Waiting for endpoints to shut down -43:23.652 [INFO] TestAgent: Endpoints down +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:59.446Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:59.473Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:59.477 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3aedf773-2dfc-02ef-39b8-ac1555e03f84 Address:127.0.0.1:24827}]" +34:59.478 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +34:59.479 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2021-08-13T15:34:59.479Z [INFO] agent.router: Initializing LAN area manager +34:59.480 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24822 network=udp +34:59.480 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24827 [Follower]" leader= +34:59.481 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:24827) (DC: dc1)" +34:59.481 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +34:59.482 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24822 network=tcp +34:59.483 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24823 network=tcp +34:59.484 [INFO] TestAgent: started state syncer +34:59.485 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24828 network=tcp +34:59.540 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:59.540 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24827 [Candidate]" term=2 +34:59.540 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:59.540 [DEBUG] TestAgent.server.raft: vote granted: from=3aedf773-2dfc-02ef-39b8-ac1555e03f84 term=2 tally=1 +34:59.540 [INFO] TestAgent.server.raft: election won: tally=1 +34:59.540 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24827 [Leader]" +34:59.541 [INFO] TestAgent.server: cluster leadership acquired +34:59.541 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24827 +34:59.541 [INFO] TestAgent.server: New leader elected: payload=test-node +34:59.542 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:59.545 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:59.545 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:59.545 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:59.545 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:59.545 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:59.545 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +34:59.545 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +34:59.546 [INFO] TestAgent.server: federation state anti-entropy synced +34:59.591 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:59.592 [INFO] TestAgent: Synced node info +34:59.592 [DEBUG] TestAgent: Node info in sync +34:59.785 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=2.758567ms client=127.0.0.1:50361 client_network=udp +34:59.793 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=6.766136ms client=127.0.0.1:36679 client_network=udp +34:59.802 [DEBUG] TestAgent.dns: request served from client: name=14afd3fa-5c09-e31c-54f3-b596a1095dce.query.consul. type=ANY class=IN latency=9.032101ms client=127.0.0.1:34152 client_network=udp +34:59.802 [INFO] TestAgent: Requesting shutdown +34:59.802 [INFO] TestAgent.server: shutting down server +34:59.802 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:59.802 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:59.802 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:59.802 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:59.802 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:59.803 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:59.803 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:59.803 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:59.803 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:59.803 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:59.803 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:59.803Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:59.803Z [INFO] agent.router.manager: shutting down +34:59.803 [INFO] TestAgent: consul server down +34:59.803 [INFO] TestAgent: shutdown complete +34:59.803 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24822 network=tcp +34:59.803 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24822 network=udp +34:59.804 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24823 network=tcp +35:00.311 [INFO] TestAgent: Waiting for endpoints to shut down +35:00.311 [INFO] TestAgent: Endpoints down === CONT TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{3_3_3_3_3_3_3_3_3_3_3} -2022-09-15T21:43:22.780Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:22.819Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:22.824 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c29b8c20-bc47-0876-9b36-13f84af9e2b1 Address:127.0.0.1:13599}]" -43:22.824 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13599 [Follower]" leader= -43:22.825 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -43:22.825 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2022-09-15T21:43:22.825Z [INFO] agent.router: Initializing LAN area manager -43:22.830 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13615 network=udp -43:22.830 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:13599) (DC: dc1)" -43:22.830 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -43:22.831 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13615 network=tcp -43:22.832 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13616 network=tcp -43:22.832 [INFO] TestAgent: started state syncer -43:22.832 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13600 network=tcp -43:22.890 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:22.890 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13599 [Candidate]" term=2 -43:22.890 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:22.890 [DEBUG] TestAgent.server.raft: vote granted: from=c29b8c20-bc47-0876-9b36-13f84af9e2b1 term=2 tally=1 -43:22.890 [INFO] TestAgent.server.raft: election won: tally=1 -43:22.890 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13599 [Leader]" -43:22.890 [INFO] TestAgent.server: cluster leadership acquired -43:22.891 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13599 -43:22.892 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:22.894 [INFO] TestAgent.server: New leader elected: payload=test-node -43:22.901 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:22.901 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:22.901 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:22.901 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:22.901 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:22.901 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -43:22.901 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -43:22.903 [INFO] TestAgent.server: federation state anti-entropy synced -43:23.014 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:23.014 [INFO] TestAgent: Synced node info -43:23.182 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=1.7717ms client=127.0.0.1:54281 client_network=udp -43:23.189 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=5.9909ms client=127.0.0.1:36087 client_network=udp -43:23.217 [DEBUG] TestAgent.dns: request served from client: name=e4dfe525-6a74-28fe-08d9-ab71694b190e.query.consul. type=A class=IN latency=13.3847ms client=127.0.0.1:57843 client_network=udp -43:23.217 [INFO] TestAgent: Requesting shutdown -43:23.217 [INFO] TestAgent.server: shutting down server -43:23.218 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:23.218 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:23.218 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:23.218 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:23.218 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:23.218 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:23.218 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:23.218 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -43:23.218 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:23.218 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:23.218 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2022-09-15T21:43:23.219Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:23.219Z [INFO] agent.router.manager: shutting down -43:23.220 [INFO] TestAgent: consul server down -43:23.220 [INFO] TestAgent: shutdown complete -43:23.220 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13615 network=tcp -43:23.220 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13615 network=udp -43:23.220 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13616 network=tcp -43:23.721 [INFO] TestAgent: Waiting for endpoints to shut down -43:23.721 [INFO] TestAgent: Endpoints down +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=1)_compressed=false#01 +2021-08-13T15:34:59.405Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:59.442Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:59.447 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:64fc3d43-ae7e-e6eb-1275-f700cfa3a659 Address:127.0.0.1:24806}]" +34:59.451 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24806 [Follower]" leader= +34:59.456 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +34:59.457 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2021-08-13T15:34:59.457Z [INFO] agent.router: Initializing LAN area manager +34:59.458 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24843 network=udp +34:59.459 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:24806) (DC: dc1)" +34:59.459 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +34:59.463 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24843 network=tcp +34:59.465 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24844 network=tcp +34:59.465 [INFO] TestAgent: started state syncer +34:59.465 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24807 network=tcp +34:59.514 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:59.514 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24806 [Candidate]" term=2 +34:59.514 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:59.514 [DEBUG] TestAgent.server.raft: vote granted: from=64fc3d43-ae7e-e6eb-1275-f700cfa3a659 term=2 tally=1 +34:59.514 [INFO] TestAgent.server.raft: election won: tally=1 +34:59.514 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24806 [Leader]" +34:59.515 [INFO] TestAgent.server: cluster leadership acquired +34:59.520 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24806 +34:59.521 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:59.522 [INFO] TestAgent.server: New leader elected: payload=test-node +34:59.526 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:59.526 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:59.526 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:59.526 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:59.526 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:59.526 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +34:59.526 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +34:59.528 [INFO] TestAgent.server: federation state anti-entropy synced +34:59.575 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:59.576 [INFO] TestAgent: Synced node info +34:59.576 [DEBUG] TestAgent: Node info in sync +34:59.950 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=2.003685ms client=127.0.0.1:47899 client_network=udp +34:59.952 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=1.865824ms client=127.0.0.1:34394 client_network=udp +34:59.955 [DEBUG] TestAgent.dns: request served from client: name=c5085f11-bb0e-65c0-40ec-dbe4471bbdb0.query.consul. type=ANY class=IN latency=1.992204ms client=127.0.0.1:47298 client_network=udp +34:59.955 [INFO] TestAgent: Requesting shutdown +34:59.955 [INFO] TestAgent.server: shutting down server +34:59.955 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:59.955 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:59.955 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:59.955 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:59.955 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:59.955 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:59.955 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:59.955 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:59.955 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:59.955 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:59.955 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:59.955Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:59.955Z [INFO] agent.router.manager: shutting down +34:59.955 [INFO] TestAgent: consul server down +34:59.955 [INFO] TestAgent: shutdown complete +34:59.955 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24843 network=tcp +34:59.956 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24843 network=udp +34:59.956 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24844 network=tcp +35:00.456 [INFO] TestAgent: Waiting for endpoints to shut down +35:00.456 [INFO] TestAgent: Endpoints down === CONT TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{3_3_3_3_3_3_3_3_3_3_3} -2022-09-15T21:43:22.753Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:22.792Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:22.800 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ab21f9da-bb08-de69-8786-1739166a0be4 Address:127.0.0.1:13613}]" -43:22.801 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -43:22.802 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13613 [Follower]" leader= -43:22.806 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2022-09-15T21:43:22.806Z [INFO] agent.router: Initializing LAN area manager -43:22.807 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13566 network=udp -43:22.807 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:13613) (DC: dc1)" -43:22.807 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -43:22.807 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13566 network=tcp -43:22.808 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13567 network=tcp -43:22.808 [INFO] TestAgent: started state syncer -43:22.808 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13614 network=tcp -43:22.865 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:22.865 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13613 [Candidate]" term=2 -43:22.865 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:22.865 [DEBUG] TestAgent.server.raft: vote granted: from=ab21f9da-bb08-de69-8786-1739166a0be4 term=2 tally=1 -43:22.865 [INFO] TestAgent.server.raft: election won: tally=1 -43:22.866 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13613 [Leader]" -43:22.866 [INFO] TestAgent.server: cluster leadership acquired -43:22.866 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13613 -43:22.867 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:22.869 [INFO] TestAgent.server: New leader elected: payload=test-node -43:22.874 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:22.874 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:22.874 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:22.874 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:22.874 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:22.874 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -43:22.874 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -43:22.875 [INFO] TestAgent.server: federation state anti-entropy synced -43:22.998 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:23.004 [INFO] TestAgent: Synced node info -43:23.004 [DEBUG] TestAgent: Node info in sync -43:23.206 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=6.8346ms client=127.0.0.1:43534 client_network=udp -43:23.211 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=4.61506ms client=127.0.0.1:33235 client_network=udp -43:23.217 [DEBUG] TestAgent.dns: request served from client: name=a7b0c133-74dc-0ef3-9dba-53921025ca2d.query.consul. type=ANY class=IN latency=5.35716ms client=127.0.0.1:46778 client_network=udp -43:23.226 [INFO] TestAgent: Requesting shutdown -43:23.226 [INFO] TestAgent.server: shutting down server -43:23.226 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:23.226 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:23.226 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:23.226 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:23.226 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:23.226 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:23.226 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:23.226 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:23.226 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:23.226 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:23.226 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:23.227Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:23.227Z [INFO] agent.router.manager: shutting down -43:23.227 [INFO] TestAgent: consul server down -43:23.227 [INFO] TestAgent: shutdown complete -43:23.227 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13566 network=tcp -43:23.227 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13566 network=udp -43:23.227 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13567 network=tcp -43:23.727 [INFO] TestAgent: Waiting for endpoints to shut down -43:23.727 [INFO] TestAgent: Endpoints down -=== CONT TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{1_1_1_1_1_1_1_1_1_1_1} -[WARN] freeport: 4 out of 39 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:22.918Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:22.990Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:23.005 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a4939437-8b6a-e2bf-59bd-6ac402ebfc25 Address:127.0.0.1:13379}]" -43:23.006 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13379 [Follower]" leader= -43:23.006 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -43:23.018 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2022-09-15T21:43:23.018Z [INFO] agent.router: Initializing LAN area manager -43:23.018 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13395 network=udp -43:23.018 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:13379) (DC: dc1)" -43:23.019 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -43:23.019 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13395 network=tcp -43:23.020 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13396 network=tcp -43:23.020 [INFO] TestAgent: started state syncer -43:23.020 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13380 network=tcp -43:23.052 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:23.052 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13379 [Candidate]" term=2 -43:23.052 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:23.052 [DEBUG] TestAgent.server.raft: vote granted: from=a4939437-8b6a-e2bf-59bd-6ac402ebfc25 term=2 tally=1 -43:23.053 [INFO] TestAgent.server.raft: election won: tally=1 -43:23.053 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13379 [Leader]" -43:23.053 [INFO] TestAgent.server: cluster leadership acquired -43:23.053 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13379 -43:23.054 [INFO] TestAgent.server: New leader elected: payload=test-node -43:23.054 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:23.056 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:23.056 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:23.056 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:23.056 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:23.056 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:23.057 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -43:23.057 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -43:23.060 [INFO] TestAgent.server: federation state anti-entropy synced -43:23.178 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:23.178 [INFO] TestAgent: Synced node info -43:23.215 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:23.215 [DEBUG] TestAgent: Node info in sync -43:23.215 [DEBUG] TestAgent: Node info in sync -43:23.470 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=1.9966ms client=127.0.0.1:42073 client_network=udp -43:23.476 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=1.79008ms client=127.0.0.1:33461 client_network=udp -43:23.480 [DEBUG] TestAgent.dns: request served from client: name=52b8a8db-6e3a-e4e1-33d4-c5daff0a6253.query.consul. type=AAAA class=IN latency=1.87456ms client=127.0.0.1:48350 client_network=udp -43:23.480 [INFO] TestAgent: Requesting shutdown -43:23.480 [INFO] TestAgent.server: shutting down server -43:23.480 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:23.480 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:23.480 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:23.480 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:23.480 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:23.480 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:23.480 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:23.480 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:23.480 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:23.480 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:23.481 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:23.481Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:23.481Z [INFO] agent.router.manager: shutting down -43:23.481 [INFO] TestAgent: consul server down -43:23.481 [INFO] TestAgent: shutdown complete -43:23.481 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13395 network=tcp -43:23.481 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13395 network=udp -43:23.481 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13396 network=tcp -43:23.982 [INFO] TestAgent: Waiting for endpoints to shut down -43:23.982 [INFO] TestAgent: Endpoints down -=== CONT TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{2_2_2_2_2_2_2_2_2_2_2} -2022-09-15T21:43:23.050Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:23.073Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:23.076 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7a0b8efa-96df-3a92-6af3-8f5fc366d9d3 Address:127.0.0.1:13344}]" -43:23.076 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -43:23.077 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2022-09-15T21:43:23.077Z [INFO] agent.router: Initializing LAN area manager -43:23.078 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13344 [Follower]" leader= -43:23.078 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:13344) (DC: dc1)" -43:23.078 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -43:23.080 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13374 network=udp -43:23.080 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13374 network=tcp -43:23.081 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13375 network=tcp -43:23.081 [INFO] TestAgent: started state syncer -43:23.081 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13345 network=tcp -43:23.114 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:23.114 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13344 [Candidate]" term=2 -43:23.114 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:23.114 [DEBUG] TestAgent.server.raft: vote granted: from=7a0b8efa-96df-3a92-6af3-8f5fc366d9d3 term=2 tally=1 -43:23.114 [INFO] TestAgent.server.raft: election won: tally=1 -43:23.114 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13344 [Leader]" -43:23.114 [INFO] TestAgent.server: cluster leadership acquired -43:23.114 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13344 -43:23.115 [INFO] TestAgent.server: New leader elected: payload=test-node -43:23.116 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:23.118 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:23.118 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:23.118 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:23.118 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:23.118 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:23.118 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -43:23.118 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -43:23.119 [INFO] TestAgent.server: federation state anti-entropy synced -43:23.290 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:23.291 [INFO] TestAgent: Synced node info -43:23.484 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=1.82588ms client=127.0.0.1:41951 client_network=udp -43:23.486 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=1.596ms client=127.0.0.1:38148 client_network=udp -43:23.492 [DEBUG] TestAgent.dns: request served from client: name=535f1fc2-8aec-a4f7-bd40-80620d646e44.query.consul. type=A class=IN latency=1.90422ms client=127.0.0.1:34722 client_network=udp -43:23.496 [INFO] TestAgent: Requesting shutdown -43:23.496 [INFO] TestAgent.server: shutting down server -43:23.496 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:23.496 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:23.496 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:23.496 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:23.496 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:23.496 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:23.496 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:23.496 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:23.496 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:23.496 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:23.497 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:23.497Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:23.497Z [INFO] agent.router.manager: shutting down -43:23.497 [INFO] TestAgent: consul server down -43:23.497 [INFO] TestAgent: shutdown complete -43:23.497 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13374 network=tcp -43:23.497 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13374 network=udp -43:23.497 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13375 network=tcp -43:23.997 [INFO] TestAgent: Waiting for endpoints to shut down -43:23.997 [INFO] TestAgent: Endpoints down -=== CONT TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{2_2_2_2_2_2_2_2_2_2_2} -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:23.099Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:23.190Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:23.200 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:89a2f723-c5f1-60fb-269a-45f3efd2d468 Address:127.0.0.1:13641}]" -43:23.203 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -43:23.206 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13641 [Follower]" leader= -43:23.211 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2022-09-15T21:43:23.211Z [INFO] agent.router: Initializing LAN area manager -43:23.212 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13546 network=udp -43:23.214 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -43:23.214 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:13641) (DC: dc1)" -43:23.217 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13546 network=tcp -43:23.219 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13547 network=tcp -43:23.219 [INFO] TestAgent: started state syncer -43:23.220 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13642 network=tcp -43:23.254 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:23.254 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13641 [Candidate]" term=2 -43:23.255 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:23.255 [DEBUG] TestAgent.server.raft: vote granted: from=89a2f723-c5f1-60fb-269a-45f3efd2d468 term=2 tally=1 -43:23.255 [INFO] TestAgent.server.raft: election won: tally=1 -43:23.255 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13641 [Leader]" -43:23.255 [INFO] TestAgent.server: cluster leadership acquired -43:23.255 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13641 -43:23.256 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:23.257 [INFO] TestAgent.server: New leader elected: payload=test-node -43:23.260 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:23.260 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:23.260 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:23.260 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:23.260 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:23.260 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -43:23.260 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -43:23.261 [INFO] TestAgent.server: federation state anti-entropy synced -43:23.547 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:23.553 [INFO] TestAgent: Synced node info -43:23.560 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=2.18342ms client=127.0.0.1:48389 client_network=udp -43:23.563 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=2.0738ms client=127.0.0.1:55493 client_network=udp -43:23.565 [DEBUG] TestAgent.dns: request served from client: name=f28df1cd-1a59-f2c2-a83c-fc14224c10f3.query.consul. type=ANY class=IN latency=2.12606ms client=127.0.0.1:34791 client_network=udp -43:23.565 [INFO] TestAgent: Requesting shutdown -43:23.565 [INFO] TestAgent.server: shutting down server -43:23.565 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:23.565 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:23.565 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:23.565 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:23.565 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:23.566 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:23.566 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:23.566 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:23.566 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:23.566 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:23.566 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:23.566Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:23.566Z [INFO] agent.router.manager: shutting down -43:23.566 [INFO] TestAgent: consul server down -43:23.566 [INFO] TestAgent: shutdown complete -43:23.566 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13546 network=tcp -43:23.566 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13546 network=udp -43:23.566 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13547 network=tcp -43:24.067 [INFO] TestAgent: Waiting for endpoints to shut down -43:24.067 [INFO] TestAgent: Endpoints down -=== CONT TestDNS_ServiceLookup_AnswerLimits/A_lookup_{2_2_2_2_2_2_2_2_2_2_2} -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:23.607Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:23.627Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:23.634 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:47f6ebd5-2fba-12fc-d715-b99a1c9afba5 Address:127.0.0.1:13585}]" -43:23.635 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -43:23.636 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13585 [Follower]" leader= -43:23.636 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2022-09-15T21:43:23.636Z [INFO] agent.router: Initializing LAN area manager -43:23.636 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:13585) (DC: dc1)" -43:23.636 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -43:23.636 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13643 network=udp -43:23.636 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13643 network=tcp -43:23.637 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13644 network=tcp -43:23.638 [INFO] TestAgent: started state syncer -43:23.638 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13586 network=tcp -43:23.693 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:23.693 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13585 [Candidate]" term=2 -43:23.693 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:23.693 [DEBUG] TestAgent.server.raft: vote granted: from=47f6ebd5-2fba-12fc-d715-b99a1c9afba5 term=2 tally=1 -43:23.693 [INFO] TestAgent.server.raft: election won: tally=1 -43:23.693 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13585 [Leader]" -43:23.693 [INFO] TestAgent.server: cluster leadership acquired -43:23.693 [INFO] TestAgent.server: New leader elected: payload=test-node -43:23.694 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13585 -43:23.697 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:23.702 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:23.702 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:23.702 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:23.702 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:23.702 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:23.702 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -43:23.702 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -43:23.703 [INFO] TestAgent.server: federation state anti-entropy synced -43:23.893 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=2.09846ms client=127.0.0.1:52538 client_network=udp -43:23.895 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=1.72226ms client=127.0.0.1:45461 client_network=udp -43:23.900 [DEBUG] TestAgent.dns: request served from client: name=64627ff3-41c4-cd46-fdad-a46a976d6062.query.consul. type=AAAA class=IN latency=2.0491ms client=127.0.0.1:44059 client_network=udp -43:23.900 [INFO] TestAgent: Requesting shutdown -43:23.901 [INFO] TestAgent.server: shutting down server -43:23.901 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:23.901 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:23.901 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:23.901 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:23.901 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:23.901 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:23.901 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:23.901 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:23.901 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:23.901 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:23.901 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:23.901 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:23.901Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:23.901Z [INFO] agent.router.manager: shutting down -43:23.901 [INFO] TestAgent: consul server down -43:23.901 [INFO] TestAgent: shutdown complete -43:23.901 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13643 network=tcp -43:23.902 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13643 network=udp -43:23.902 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13644 network=tcp -43:24.403 [INFO] TestAgent: Waiting for endpoints to shut down -43:24.403 [INFO] TestAgent: Endpoints down +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:59.848Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:59.872Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:59.875 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a6c9bfa6-2848-e7c3-a31f-3f199eabbd34 Address:127.0.0.1:24869}]" +34:59.875 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24869 [Follower]" leader= +34:59.876 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +34:59.877 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2021-08-13T15:34:59.877Z [INFO] agent.router: Initializing LAN area manager +34:59.877 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24864 network=udp +34:59.878 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:24869) (DC: dc1)" +34:59.878 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +34:59.878 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24864 network=tcp +34:59.879 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24865 network=tcp +34:59.879 [INFO] TestAgent: started state syncer +34:59.879 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24870 network=tcp +34:59.925 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:59.925 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24869 [Candidate]" term=2 +34:59.926 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:59.926 [DEBUG] TestAgent.server.raft: vote granted: from=a6c9bfa6-2848-e7c3-a31f-3f199eabbd34 term=2 tally=1 +34:59.926 [INFO] TestAgent.server.raft: election won: tally=1 +34:59.926 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24869 [Leader]" +34:59.926 [INFO] TestAgent.server: cluster leadership acquired +34:59.926 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24869 +34:59.927 [INFO] TestAgent.server: New leader elected: payload=test-node +34:59.927 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:59.930 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:59.930 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:59.930 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:59.930 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:59.930 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:59.930 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +34:59.930 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +34:59.931 [INFO] TestAgent.server: federation state anti-entropy synced +35:00.221 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=2.476306ms client=127.0.0.1:34025 client_network=udp +35:00.224 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=2.421366ms client=127.0.0.1:39949 client_network=udp +35:00.227 [DEBUG] TestAgent.dns: request served from client: name=a328a052-35f2-475d-3251-ecc395922664.query.consul. type=AAAA class=IN latency=2.306885ms client=127.0.0.1:56950 client_network=udp +35:00.227 [INFO] TestAgent: Requesting shutdown +35:00.227 [INFO] TestAgent.server: shutting down server +35:00.227 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +35:00.227 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +35:00.227 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +35:00.227 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +35:00.227 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +35:00.227 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +35:00.227 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +35:00.227 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +35:00.227 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +35:00.227 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +35:00.227 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +35:00.227 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:35:00.228Z [INFO] agent.router.manager: shutting down +2021-08-13T15:35:00.228Z [INFO] agent.router.manager: shutting down +35:00.228 [INFO] TestAgent: consul server down +35:00.228 [INFO] TestAgent: shutdown complete +35:00.228 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24864 network=tcp +35:00.228 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24864 network=udp +35:00.228 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24865 network=tcp +35:00.731 [INFO] TestAgent: Waiting for endpoints to shut down +35:00.731 [INFO] TestAgent: Endpoints down +=== CONT TestDNS_ServiceLookup_AnswerLimits/A_lookup_{4_4_4_4_4_4_4_4_4_4_4} +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=1)_compressed=true#01 +2021-08-13T15:34:59.866Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:59.885Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:59.888 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:22481389-9e78-b5d7-b20b-ad533afb20b5 Address:127.0.0.1:24897}]" +34:59.888 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24897 [Follower]" leader= +34:59.889 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +34:59.890 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2021-08-13T15:34:59.890Z [INFO] agent.router: Initializing LAN area manager +34:59.890 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24871 network=udp +34:59.890 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:24897) (DC: dc1)" +34:59.890 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +34:59.891 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24871 network=tcp +34:59.891 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24872 network=tcp +34:59.892 [INFO] TestAgent: started state syncer +34:59.892 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24898 network=tcp +34:59.931 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:59.931 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24897 [Candidate]" term=2 +34:59.931 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:59.931 [DEBUG] TestAgent.server.raft: vote granted: from=22481389-9e78-b5d7-b20b-ad533afb20b5 term=2 tally=1 +34:59.931 [INFO] TestAgent.server.raft: election won: tally=1 +34:59.931 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24897 [Leader]" +34:59.931 [INFO] TestAgent.server: cluster leadership acquired +34:59.932 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24897 +34:59.932 [INFO] TestAgent.server: New leader elected: payload=test-node +34:59.933 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:59.935 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:59.935 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:59.935 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:59.935 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:59.935 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:59.935 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +34:59.935 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +34:59.936 [INFO] TestAgent.server: federation state anti-entropy synced +35:00.067 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +35:00.068 [INFO] TestAgent: Synced node info +35:00.367 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=2.414945ms client=127.0.0.1:56509 client_network=udp +35:00.374 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=2.428566ms client=127.0.0.1:45116 client_network=udp +35:00.384 [DEBUG] TestAgent.dns: request served from client: name=80560b65-00b4-fb1d-7555-377b29611084.query.consul. type=A class=IN latency=8.9197ms client=127.0.0.1:58803 client_network=udp +35:00.384 [INFO] TestAgent: Requesting shutdown +35:00.384 [INFO] TestAgent.server: shutting down server +35:00.384 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +35:00.384 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +35:00.384 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +35:00.384 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +35:00.384 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +35:00.384 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +35:00.384 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +35:00.384 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +35:00.384 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +35:00.384 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +35:00.385 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:35:00.385Z [INFO] agent.router.manager: shutting down +2021-08-13T15:35:00.385Z [INFO] agent.router.manager: shutting down +35:00.385 [INFO] TestAgent: consul server down +35:00.385 [INFO] TestAgent: shutdown complete +35:00.385 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24871 network=tcp +35:00.385 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24871 network=udp +35:00.385 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24872 network=tcp +35:00.885 [INFO] TestAgent: Waiting for endpoints to shut down +35:00.885 [INFO] TestAgent: Endpoints down === CONT TestDNS_ServiceLookup_AnswerLimits/A_lookup_{1_1_1_1_1_1_1_1_1_1_1} -2022-09-15T21:43:23.675Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:23.694Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:23.697 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cf71cf18-7eaa-f23f-e0d1-41b0cb44284e Address:127.0.0.1:13662}]" -43:23.698 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -43:23.699 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2022-09-15T21:43:23.699Z [INFO] agent.router: Initializing LAN area manager -43:23.700 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13608 network=udp -43:23.700 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13662 [Follower]" leader= -43:23.700 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:13662) (DC: dc1)" -43:23.700 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -43:23.700 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13608 network=tcp -43:23.701 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13609 network=tcp -43:23.701 [INFO] TestAgent: started state syncer -43:23.702 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13663 network=tcp -43:23.750 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:23.750 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13662 [Candidate]" term=2 -43:23.750 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:23.750 [DEBUG] TestAgent.server.raft: vote granted: from=cf71cf18-7eaa-f23f-e0d1-41b0cb44284e term=2 tally=1 -43:23.750 [INFO] TestAgent.server.raft: election won: tally=1 -43:23.750 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13662 [Leader]" -43:23.751 [INFO] TestAgent.server: cluster leadership acquired -43:23.751 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13662 -43:23.753 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:23.754 [INFO] TestAgent.server: New leader elected: payload=test-node -43:23.768 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:23.768 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:23.768 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:23.768 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:23.768 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:23.769 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -43:23.769 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -43:23.779 [INFO] TestAgent.server: federation state anti-entropy synced -43:23.984 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=2.00464ms client=127.0.0.1:52178 client_network=udp -43:23.987 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=1.99626ms client=127.0.0.1:49501 client_network=udp -43:23.989 [DEBUG] TestAgent.dns: request served from client: name=d499a6e0-ec79-b69f-6561-3949c5c368d7.query.consul. type=ANY class=IN latency=1.98736ms client=127.0.0.1:53758 client_network=udp -43:23.989 [INFO] TestAgent: Requesting shutdown -43:23.989 [INFO] TestAgent.server: shutting down server -43:23.989 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:23.989 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:23.989 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:23.989 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:23.989 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:23.989 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:23.989 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:23.989 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:23.989 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:23.989 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:23.990 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:23.990 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:23.990Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:23.990Z [INFO] agent.router.manager: shutting down -43:23.990 [INFO] TestAgent: consul server down -43:23.990 [INFO] TestAgent: shutdown complete -43:23.990 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13608 network=tcp -43:23.990 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13608 network=udp -43:23.990 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13609 network=tcp -43:24.491 [INFO] TestAgent: Waiting for endpoints to shut down -43:24.491 [INFO] TestAgent: Endpoints down +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=1)_compressed=false#01 +2021-08-13T15:35:00.280Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:35:00.309Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +35:00.313 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2f65fcef-80ad-94f6-fe3e-382635f0b5bc Address:127.0.0.1:24883}]" +35:00.314 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +35:00.315 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2021-08-13T15:35:00.315Z [INFO] agent.router: Initializing LAN area manager +35:00.316 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24850 network=udp +35:00.317 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24883 [Follower]" leader= +35:00.320 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:24883) (DC: dc1)" +35:00.320 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +35:00.322 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24850 network=tcp +35:00.324 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24851 network=tcp +35:00.324 [INFO] TestAgent: started state syncer +35:00.324 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24884 network=tcp +35:00.358 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +35:00.358 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24883 [Candidate]" term=2 +35:00.358 [DEBUG] TestAgent.server.raft: votes: needed=1 +35:00.358 [DEBUG] TestAgent.server.raft: vote granted: from=2f65fcef-80ad-94f6-fe3e-382635f0b5bc term=2 tally=1 +35:00.358 [INFO] TestAgent.server.raft: election won: tally=1 +35:00.358 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24883 [Leader]" +35:00.358 [INFO] TestAgent.server: cluster leadership acquired +35:00.364 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24883 +35:00.365 [INFO] TestAgent.server: New leader elected: payload=test-node +35:00.366 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +35:00.374 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +35:00.374 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +35:00.375 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +35:00.375 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +35:00.375 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +35:00.375 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +35:00.375 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +35:00.376 [INFO] TestAgent.server: federation state anti-entropy synced +35:00.478 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=1.976084ms client=127.0.0.1:40818 client_network=udp +35:00.480 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=1.797024ms client=127.0.0.1:46168 client_network=udp +35:00.482 [DEBUG] TestAgent.dns: request served from client: name=ccfbe809-fc03-3945-d9f3-c0a062726fd2.query.consul. type=ANY class=IN latency=1.877624ms client=127.0.0.1:50147 client_network=udp +35:00.482 [INFO] TestAgent: Requesting shutdown +35:00.482 [INFO] TestAgent.server: shutting down server +35:00.482 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +35:00.482 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +35:00.482 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +35:00.482 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +35:00.482 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +35:00.483 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +35:00.483 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +35:00.483 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +35:00.483 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +35:00.483 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +35:00.483 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +35:00.483 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:35:00.483Z [INFO] agent.router.manager: shutting down +2021-08-13T15:35:00.483Z [INFO] agent.router.manager: shutting down +35:00.483 [INFO] TestAgent: consul server down +35:00.483 [INFO] TestAgent: shutdown complete +35:00.483 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24850 network=tcp +35:00.483 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24850 network=udp +35:00.483 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24851 network=tcp +35:00.983 [INFO] TestAgent: Waiting for endpoints to shut down +35:00.983 [INFO] TestAgent: Endpoints down === CONT TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{1_1_1_1_1_1_1_1_1_1_1} -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:23.790Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:23.841Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:23.848 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9465f9f2-e3f1-2d9c-f38e-ee6bbe1ed370 Address:127.0.0.1:13648}]" -43:23.849 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13648 [Follower]" leader= -43:23.849 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -43:23.855 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2022-09-15T21:43:23.855Z [INFO] agent.router: Initializing LAN area manager -43:23.855 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13664 network=udp -43:23.855 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -43:23.856 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13664 network=tcp -43:23.857 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13665 network=tcp -43:23.857 [INFO] TestAgent: started state syncer -43:23.857 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13649 network=tcp -43:23.855 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:13648) (DC: dc1)" -43:23.914 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:23.914 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13648 [Candidate]" term=2 -43:23.914 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:23.914 [DEBUG] TestAgent.server.raft: vote granted: from=9465f9f2-e3f1-2d9c-f38e-ee6bbe1ed370 term=2 tally=1 -43:23.914 [INFO] TestAgent.server.raft: election won: tally=1 -43:23.914 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13648 [Leader]" -43:23.914 [INFO] TestAgent.server: cluster leadership acquired -43:23.915 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13648 -43:23.916 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:23.918 [INFO] TestAgent.server: New leader elected: payload=test-node -43:23.922 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:23.922 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:23.922 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:23.922 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:23.922 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:23.922 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -43:23.922 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -43:23.924 [INFO] TestAgent.server: federation state anti-entropy synced -43:24.195 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:24.197 [INFO] TestAgent: Synced node info -43:24.230 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=32.26634ms client=127.0.0.1:38029 client_network=udp -43:24.233 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=1.92108ms client=127.0.0.1:57751 client_network=udp -43:24.235 [DEBUG] TestAgent.dns: request served from client: name=4e4aa329-e28a-9c03-bd23-6323615da6ed.query.consul. type=ANY class=IN latency=2.26322ms client=127.0.0.1:52823 client_network=udp -43:24.238 [INFO] TestAgent: Requesting shutdown -43:24.238 [INFO] TestAgent.server: shutting down server -43:24.238 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:24.238 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:24.238 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:24.238 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:24.238 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:24.238 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:24.238 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:24.238 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:24.238 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:24.238 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:24.238 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:24.238Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:24.238Z [INFO] agent.router.manager: shutting down -43:24.238 [INFO] TestAgent: consul server down -43:24.238 [INFO] TestAgent: shutdown complete -43:24.238 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13664 network=tcp -43:24.239 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13664 network=udp -43:24.239 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13665 network=tcp -43:24.742 [INFO] TestAgent: Waiting for endpoints to shut down -43:24.742 [INFO] TestAgent: Endpoints down +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._tcp_(qType:=1)_compressed=true#01 +2021-08-13T15:35:00.300Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:35:00.366Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +35:00.370 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:913dcc30-80b6-1fd5-245b-876de316a84b Address:127.0.0.1:24834}]" +35:00.371 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24834 [Follower]" leader= +35:00.377 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +35:00.383 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2021-08-13T15:35:00.383Z [INFO] agent.router: Initializing LAN area manager +35:00.384 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24885 network=udp +35:00.384 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:24834) (DC: dc1)" +35:00.384 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +35:00.385 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24885 network=tcp +35:00.386 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24886 network=tcp +35:00.387 [INFO] TestAgent: started state syncer +35:00.387 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24835 network=tcp +35:00.441 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +35:00.441 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24834 [Candidate]" term=2 +35:00.441 [DEBUG] TestAgent.server.raft: votes: needed=1 +35:00.441 [DEBUG] TestAgent.server.raft: vote granted: from=913dcc30-80b6-1fd5-245b-876de316a84b term=2 tally=1 +35:00.441 [INFO] TestAgent.server.raft: election won: tally=1 +35:00.441 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24834 [Leader]" +35:00.441 [INFO] TestAgent.server: cluster leadership acquired +35:00.442 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24834 +35:00.443 [INFO] TestAgent.server: New leader elected: payload=test-node +35:00.443 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +35:00.446 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +35:00.446 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +35:00.446 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +35:00.446 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +35:00.446 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +35:00.446 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +35:00.446 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +35:00.447 [INFO] TestAgent.server: federation state anti-entropy synced +35:00.511 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +35:00.512 [INFO] TestAgent: Synced node info +35:00.512 [DEBUG] TestAgent: Node info in sync +35:00.700 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=2.204745ms client=127.0.0.1:48356 client_network=udp +35:00.703 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=2.073705ms client=127.0.0.1:54605 client_network=udp +35:00.706 [DEBUG] TestAgent.dns: request served from client: name=aa26ac84-1a50-77fc-a82a-67ab235b0617.query.consul. type=AAAA class=IN latency=2.365186ms client=127.0.0.1:55067 client_network=udp +35:00.706 [INFO] TestAgent: Requesting shutdown +35:00.706 [INFO] TestAgent.server: shutting down server +35:00.706 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +35:00.706 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +35:00.706 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +35:00.706 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +35:00.706 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +35:00.706 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +35:00.706 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +35:00.706 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +35:00.706 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +35:00.706 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +35:00.706 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:35:00.707Z [INFO] agent.router.manager: shutting down +2021-08-13T15:35:00.707Z [INFO] agent.router.manager: shutting down +35:00.707 [INFO] TestAgent: consul server down +35:00.707 [INFO] TestAgent: shutdown complete +35:00.707 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24885 network=tcp +35:00.707 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24885 network=udp +35:00.707 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24886 network=tcp +35:01.211 [INFO] TestAgent: Waiting for endpoints to shut down +35:01.211 [INFO] TestAgent: Endpoints down === CONT TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{0_0_0_0_0_0_0_0_0_0_0} -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:23.645Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:23.664Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:23.667 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1b93787a-9f02-f96c-0e99-d8eafc904032 Address:127.0.0.1:13606}]" -43:23.667 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13606 [Follower]" leader= -43:23.668 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -43:23.669 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2022-09-15T21:43:23.669Z [INFO] agent.router: Initializing LAN area manager -43:23.669 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13587 network=udp -43:23.670 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -43:23.670 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13587 network=tcp -43:23.671 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13588 network=tcp -43:23.671 [INFO] TestAgent: started state syncer -43:23.672 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13607 network=tcp -43:23.672 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:13606) (DC: dc1)" -43:23.709 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:23.709 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13606 [Candidate]" term=2 -43:23.709 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:23.709 [DEBUG] TestAgent.server.raft: vote granted: from=1b93787a-9f02-f96c-0e99-d8eafc904032 term=2 tally=1 -43:23.709 [INFO] TestAgent.server.raft: election won: tally=1 -43:23.709 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13606 [Leader]" -43:23.709 [INFO] TestAgent.server: cluster leadership acquired -43:23.710 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13606 -43:23.711 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:23.716 [INFO] TestAgent.server: New leader elected: payload=test-node -43:23.716 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:23.716 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:23.716 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:23.716 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:23.716 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:23.716 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -43:23.716 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -43:23.718 [INFO] TestAgent.server: federation state anti-entropy synced -43:23.774 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:23.775 [INFO] TestAgent: Synced node info -43:23.775 [DEBUG] TestAgent: Node info in sync -43:24.272 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=1.8493ms client=127.0.0.1:57918 client_network=udp -43:24.274 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=1.7028ms client=127.0.0.1:58717 client_network=udp -43:24.276 [DEBUG] TestAgent.dns: request served from client: name=9f12bbae-849a-5257-15a8-c9b77e5e9059.query.consul. type=A class=IN latency=1.6474ms client=127.0.0.1:33422 client_network=udp -43:24.278 [INFO] TestAgent: Requesting shutdown -43:24.278 [INFO] TestAgent.server: shutting down server -43:24.278 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:24.278 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:24.278 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:24.278 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:24.278 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:24.278 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:24.278 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:24.278 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:24.278 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:24.278 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:24.278 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:24.278Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:24.279Z [INFO] agent.router.manager: shutting down -43:24.279 [INFO] TestAgent: consul server down -43:24.279 [INFO] TestAgent: shutdown complete -43:24.279 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13587 network=tcp -43:24.279 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13587 network=udp -43:24.279 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13588 network=tcp -43:24.780 [INFO] TestAgent: Waiting for endpoints to shut down -43:24.780 [INFO] TestAgent: Endpoints down +2021-08-13T15:35:00.355Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:35:00.381Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +35:00.389 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ab8c4f44-be38-91e9-050a-ea15fff941df Address:127.0.0.1:24911}]" +35:00.391 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +35:00.391 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24911 [Follower]" leader= +35:00.393 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2021-08-13T15:35:00.393Z [INFO] agent.router: Initializing LAN area manager +35:00.393 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24836 network=udp +35:00.394 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:24911) (DC: dc1)" +35:00.395 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +35:00.395 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24836 network=tcp +35:00.397 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24837 network=tcp +35:00.397 [INFO] TestAgent: started state syncer +35:00.397 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24912 network=tcp +35:00.447 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +35:00.447 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24911 [Candidate]" term=2 +35:00.447 [DEBUG] TestAgent.server.raft: votes: needed=1 +35:00.447 [DEBUG] TestAgent.server.raft: vote granted: from=ab8c4f44-be38-91e9-050a-ea15fff941df term=2 tally=1 +35:00.447 [INFO] TestAgent.server.raft: election won: tally=1 +35:00.447 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24911 [Leader]" +35:00.447 [INFO] TestAgent.server: cluster leadership acquired +35:00.448 [INFO] TestAgent.server: New leader elected: payload=test-node +35:00.448 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24911 +35:00.449 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +35:00.451 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +35:00.451 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +35:00.451 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +35:00.451 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +35:00.451 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +35:00.451 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +35:00.451 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +35:00.452 [INFO] TestAgent.server: federation state anti-entropy synced +35:00.560 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +35:00.561 [INFO] TestAgent: Synced node info +35:00.561 [DEBUG] TestAgent: Node info in sync +35:00.711 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=1.940325ms client=127.0.0.1:39964 client_network=udp +35:00.718 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=6.786156ms client=127.0.0.1:48016 client_network=udp +35:00.726 [DEBUG] TestAgent.dns: request served from client: name=dc24d4c3-99a4-d186-f1db-0a8d9bfaa05a.query.consul. type=ANY class=IN latency=7.120396ms client=127.0.0.1:41360 client_network=udp +35:00.726 [INFO] TestAgent: Requesting shutdown +35:00.726 [INFO] TestAgent.server: shutting down server +35:00.726 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +35:00.726 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +35:00.726 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +35:00.726 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +35:00.726 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +35:00.726 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +35:00.726 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +35:00.727 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +35:00.727 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +35:00.727 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +35:00.727 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:35:00.727Z [INFO] agent.router.manager: shutting down +2021-08-13T15:35:00.727Z [INFO] agent.router.manager: shutting down +35:00.727 [INFO] TestAgent: consul server down +35:00.727 [INFO] TestAgent: shutdown complete +35:00.727 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24836 network=tcp +35:00.727 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24836 network=udp +35:00.727 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24837 network=tcp +35:01.231 [INFO] TestAgent: Waiting for endpoints to shut down +35:01.231 [INFO] TestAgent: Endpoints down === CONT TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{0_0_0_0_0_0_0_0_0_0_0} -2022-09-15T21:43:24.016Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:24.036Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:24.039 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:29e80213-33e1-5bfd-0f8a-3b96f1aa91f2 Address:127.0.0.1:13697}]" -43:24.040 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13697 [Follower]" leader= -43:24.041 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -43:24.042 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2022-09-15T21:43:24.042Z [INFO] agent.router: Initializing LAN area manager -43:24.043 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13656 network=udp -43:24.043 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:13697) (DC: dc1)" -43:24.043 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -43:24.043 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13656 network=tcp -43:24.044 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13672 network=tcp -43:24.044 [INFO] TestAgent: started state syncer -43:24.044 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13698 network=tcp -43:24.098 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:24.098 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13697 [Candidate]" term=2 -43:24.098 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:24.098 [DEBUG] TestAgent.server.raft: vote granted: from=29e80213-33e1-5bfd-0f8a-3b96f1aa91f2 term=2 tally=1 -43:24.098 [INFO] TestAgent.server.raft: election won: tally=1 -43:24.098 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13697 [Leader]" -43:24.098 [INFO] TestAgent.server: cluster leadership acquired -43:24.098 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13697 -43:24.099 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:24.100 [INFO] TestAgent.server: New leader elected: payload=test-node -43:24.102 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:24.102 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:24.102 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:24.102 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:24.102 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:24.102 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -43:24.102 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -43:24.103 [INFO] TestAgent.server: federation state anti-entropy synced -43:24.276 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:24.276 [INFO] TestAgent: Synced node info -43:24.276 [DEBUG] TestAgent: Node info in sync -43:24.304 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=1.94592ms client=127.0.0.1:50429 client_network=udp -43:24.308 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=1.79956ms client=127.0.0.1:33460 client_network=udp -43:24.310 [INFO] TestAgent: Requesting shutdown -43:24.310 [INFO] TestAgent.server: shutting down server -43:24.310 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:24.310 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:24.310 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:24.310 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:24.310 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:24.311 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:24.311 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:24.311 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:24.311 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:24.311 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:24.311 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:24.311Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:24.311Z [INFO] agent.router.manager: shutting down -43:24.311 [INFO] TestAgent: consul server down -43:24.311 [INFO] TestAgent: shutdown complete -43:24.311 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13656 network=tcp -43:24.311 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13656 network=udp -43:24.312 [DEBUG] TestAgent.dns: request served from client: name=d656a9eb-ecf0-77d3-95a8-944a53175252.query.consul. type=AAAA class=IN latency=3.4612ms client=127.0.0.1:45362 client_network=udp -43:24.312 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13672 network=tcp -43:24.813 [INFO] TestAgent: Waiting for endpoints to shut down -43:24.813 [INFO] TestAgent: Endpoints down +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:35:00.260Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:35:00.306Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +35:00.310 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d56f0b61-8008-9224-abcb-855a7365fdda Address:127.0.0.1:24848}]" +35:00.320 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +35:00.321 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2021-08-13T15:35:00.321Z [INFO] agent.router: Initializing LAN area manager +35:00.322 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24899 network=udp +35:00.322 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24848 [Follower]" leader= +35:00.325 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +35:00.325 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:24848) (DC: dc1)" +35:00.326 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24899 network=tcp +35:00.336 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24900 network=tcp +35:00.336 [INFO] TestAgent: started state syncer +35:00.336 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24849 network=tcp +35:00.364 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +35:00.364 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24848 [Candidate]" term=2 +35:00.364 [DEBUG] TestAgent.server.raft: votes: needed=1 +35:00.364 [DEBUG] TestAgent.server.raft: vote granted: from=d56f0b61-8008-9224-abcb-855a7365fdda term=2 tally=1 +35:00.364 [INFO] TestAgent.server.raft: election won: tally=1 +35:00.364 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24848 [Leader]" +35:00.365 [INFO] TestAgent.server: cluster leadership acquired +35:00.365 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24848 +35:00.366 [INFO] TestAgent.server: New leader elected: payload=test-node +35:00.367 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +35:00.370 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +35:00.370 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +35:00.370 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +35:00.370 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +35:00.370 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +35:00.371 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +35:00.371 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +35:00.382 [INFO] TestAgent.server: federation state anti-entropy synced +35:00.488 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +35:00.488 [INFO] TestAgent: Synced node info +35:00.870 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=3.344188ms client=127.0.0.1:51537 client_network=udp +35:00.875 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=2.619786ms client=127.0.0.1:35969 client_network=udp +35:00.878 [DEBUG] TestAgent.dns: request served from client: name=84685515-e556-0f3a-ecab-5f7610ec674a.query.consul. type=A class=IN latency=2.251825ms client=127.0.0.1:46521 client_network=udp +35:00.878 [INFO] TestAgent: Requesting shutdown +35:00.878 [INFO] TestAgent.server: shutting down server +35:00.878 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +35:00.878 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +35:00.878 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +35:00.883 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +35:00.883 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +35:00.883 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +35:00.883 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +35:00.883 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:35:00.884Z [INFO] agent.router.manager: shutting down +2021-08-13T15:35:00.884Z [INFO] agent.router.manager: shutting down +35:00.884 [INFO] TestAgent: consul server down +35:00.884 [INFO] TestAgent: shutdown complete +35:00.884 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24899 network=tcp +35:00.884 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24899 network=udp +35:00.884 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24900 network=tcp +35:00.884 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +35:00.884 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +35:00.884 [DEBUG] TestAgent.server: shutting down leader loop +35:00.884 [INFO] TestAgent.server: cluster leadership lost +35:00.884 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +35:01.387 [INFO] TestAgent: Waiting for endpoints to shut down +35:01.387 [INFO] TestAgent: Endpoints down === CONT TestUiServices/No_Filter -2022-09-15T21:43:14.301Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:14.333Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:14.336 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1a7f2da4-95ca-99e0-b980-1980fb97c961 Address:127.0.0.1:13134}]" -43:14.336 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13134 [Follower]" leader= -43:14.338 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1a7f2da4-95ca-99e0-b980-1980fb97c961.dc1 127.0.0.1 -43:14.339 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1a7f2da4-95ca-99e0-b980-1980fb97c961 127.0.0.1 -2022-09-15T21:43:14.339Z [INFO] agent.router: Initializing LAN area manager -43:14.340 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13094 network=udp -43:14.340 [INFO] TestAgent.server: Adding LAN server: server="Node-1a7f2da4-95ca-99e0-b980-1980fb97c961 (Addr: tcp/127.0.0.1:13134) (DC: dc1)" -43:14.340 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1a7f2da4-95ca-99e0-b980-1980fb97c961.dc1 area=wan -43:14.340 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13094 network=tcp -43:14.341 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13095 network=tcp -43:14.341 [INFO] TestAgent: started state syncer -43:14.342 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13135 network=tcp -43:14.383 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:14.383 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13134 [Candidate]" term=2 -43:14.384 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:14.384 [DEBUG] TestAgent.server.raft: vote granted: from=1a7f2da4-95ca-99e0-b980-1980fb97c961 term=2 tally=1 -43:14.384 [INFO] TestAgent.server.raft: election won: tally=1 -43:14.384 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13134 [Leader]" -43:14.384 [INFO] TestAgent.server: cluster leadership acquired -43:14.384 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13134 -43:14.386 [INFO] TestAgent.server: New leader elected: payload=Node-1a7f2da4-95ca-99e0-b980-1980fb97c961 -43:14.386 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:14.389 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:14.389 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:14.389 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:14.389 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:14.389 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:14.389 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1a7f2da4-95ca-99e0-b980-1980fb97c961 -43:14.389 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1a7f2da4-95ca-99e0-b980-1980fb97c961 -43:14.390 [INFO] TestAgent.server: federation state anti-entropy synced -43:14.399 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:14.399 [INFO] TestAgent: Synced node info -43:14.405 [INFO] TestAgent: Requesting shutdown -43:14.405 [INFO] TestAgent.server: shutting down server -43:14.405 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:14.405 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:14.405 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:14.405 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:14.405 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:14.405 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:14.405 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:14.405 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:14.405 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:14.405 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:14.405 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:14.405Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:14.405Z [INFO] agent.router.manager: shutting down -43:14.414 [INFO] TestAgent: consul server down -43:14.414 [INFO] TestAgent: shutdown complete -43:14.414 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13094 network=tcp -43:14.414 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13094 network=udp -43:14.414 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13095 network=tcp -43:14.918 [INFO] TestAgent: Waiting for endpoints to shut down -43:14.918 [INFO] TestAgent: Endpoints down ---- PASS: TestUiServices (0.65s) +2021-08-13T15:34:55.963Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:56.004Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:56.008 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8c4cd1a5-33c6-f07b-d737-15b8368c21f0 Address:127.0.0.1:24604}]" +34:56.008 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24604 [Follower]" leader= +34:56.010 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8c4cd1a5-33c6-f07b-d737-15b8368c21f0.dc1 127.0.0.1 +34:56.011 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8c4cd1a5-33c6-f07b-d737-15b8368c21f0 127.0.0.1 +2021-08-13T15:34:56.011Z [INFO] agent.router: Initializing LAN area manager +34:56.017 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8c4cd1a5-33c6-f07b-d737-15b8368c21f0.dc1 area=wan +34:56.018 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24662 network=tcp +34:56.018 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24662 network=udp +34:56.020 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24663 network=tcp +34:56.020 [INFO] TestAgent: started state syncer +34:56.021 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24605 network=tcp +34:56.021 [INFO] TestAgent.server: Adding LAN server: server="Node-8c4cd1a5-33c6-f07b-d737-15b8368c21f0 (Addr: tcp/127.0.0.1:24604) (DC: dc1)" +34:56.063 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:56.063 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24604 [Candidate]" term=2 +34:56.063 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:56.063 [DEBUG] TestAgent.server.raft: vote granted: from=8c4cd1a5-33c6-f07b-d737-15b8368c21f0 term=2 tally=1 +34:56.063 [INFO] TestAgent.server.raft: election won: tally=1 +34:56.063 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24604 [Leader]" +34:56.063 [INFO] TestAgent.server: cluster leadership acquired +34:56.063 [INFO] TestAgent.server: New leader elected: payload=Node-8c4cd1a5-33c6-f07b-d737-15b8368c21f0 +34:56.064 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24604 +34:56.065 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:56.073 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:56.073 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:56.073 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:56.073 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:56.073 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:56.074 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8c4cd1a5-33c6-f07b-d737-15b8368c21f0 +34:56.074 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8c4cd1a5-33c6-f07b-d737-15b8368c21f0 +34:56.075 [INFO] TestAgent.server: federation state anti-entropy synced +34:56.235 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:56.236 [INFO] TestAgent: Synced node info +34:56.236 [DEBUG] TestAgent: Node info in sync +34:56.288 [INFO] TestAgent: Requesting shutdown +34:56.288 [INFO] TestAgent.server: shutting down server +34:56.288 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:56.288 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:56.288 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:56.288 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:56.288 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:56.288 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:56.289 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:56.289 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:56.289 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:56.289 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:56.289 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:56.289Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:56.289Z [INFO] agent.router.manager: shutting down +34:56.289 [INFO] TestAgent: consul server down +34:56.289 [INFO] TestAgent: shutdown complete +34:56.289 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24662 network=tcp +34:56.289 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24662 network=udp +34:56.289 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24663 network=tcp +34:56.790 [INFO] TestAgent: Waiting for endpoints to shut down +34:56.790 [INFO] TestAgent: Endpoints down +--- PASS: TestUiServices (0.87s) --- PASS: TestUiServices/Filtered (0.00s) --- PASS: TestUiServices/No_Filter (0.00s) -2022-09-15T21:43:23.769Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:23.841Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:23.844 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:dbbcce0d-a176-d3d9-95b8-89f352ac5dc6 Address:127.0.0.1:13683}]" -43:23.845 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13683 [Follower]" leader= -43:23.845 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -43:23.846 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2022-09-15T21:43:23.846Z [INFO] agent.router: Initializing LAN area manager -43:23.847 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13650 network=udp -43:23.847 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -43:23.847 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:13683) (DC: dc1)" -43:23.847 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13650 network=tcp -43:23.848 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13651 network=tcp -43:23.848 [INFO] TestAgent: started state syncer -43:23.848 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13684 network=tcp -43:23.910 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:23.910 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13683 [Candidate]" term=2 -43:23.910 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:23.910 [DEBUG] TestAgent.server.raft: vote granted: from=dbbcce0d-a176-d3d9-95b8-89f352ac5dc6 term=2 tally=1 -43:23.910 [INFO] TestAgent.server.raft: election won: tally=1 -43:23.910 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13683 [Leader]" -43:23.910 [INFO] TestAgent.server: cluster leadership acquired -43:23.911 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13683 -43:23.912 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:23.914 [INFO] TestAgent.server: New leader elected: payload=test-node -43:23.919 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:23.919 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:23.919 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:23.919 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:23.919 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:23.919 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -43:23.919 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -43:23.921 [INFO] TestAgent.server: federation state anti-entropy synced -43:24.134 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:24.171 [INFO] TestAgent: Synced node info -43:24.171 [DEBUG] TestAgent: Node info in sync -43:24.319 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=2.69492ms client=127.0.0.1:39622 client_network=udp -43:24.321 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=1.6194ms client=127.0.0.1:53061 client_network=udp -43:24.323 [DEBUG] TestAgent.dns: request served from client: name=3d1f5ae8-c21f-764b-af98-18b76843dc29.query.consul. type=AAAA class=IN latency=1.61932ms client=127.0.0.1:36509 client_network=udp -43:24.323 [INFO] TestAgent: Requesting shutdown -43:24.323 [INFO] TestAgent.server: shutting down server -43:24.323 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:24.323 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:24.323 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:24.324 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:24.324 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:24.324 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:24.324 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:24.324 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:24.324 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:24.324 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:24.324 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:24.324Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:24.324Z [INFO] agent.router.manager: shutting down -43:24.324 [INFO] TestAgent: consul server down -43:24.324 [INFO] TestAgent: shutdown complete -43:24.324 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13650 network=tcp -43:24.324 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13650 network=udp -43:24.324 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13651 network=tcp -43:24.825 [INFO] TestAgent: Waiting for endpoints to shut down -43:24.825 [INFO] TestAgent: Endpoints down -2022-09-15T21:43:24.004Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:24.026Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:24.029 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:465ad114-aa67-69f6-1472-6cf61ab0cde5 Address:127.0.0.1:13696}]" -43:24.030 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13696 [Follower]" leader= -43:24.031 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -43:24.032 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2022-09-15T21:43:24.032Z [INFO] agent.router: Initializing LAN area manager -43:24.032 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:13696) (DC: dc1)" -43:24.032 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -43:24.032 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13706 network=udp -43:24.032 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13706 network=tcp -43:24.033 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13685 network=tcp -43:24.033 [INFO] TestAgent: started state syncer -43:24.033 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13655 network=tcp -43:24.078 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:24.078 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13696 [Candidate]" term=2 -43:24.078 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:24.078 [DEBUG] TestAgent.server.raft: vote granted: from=465ad114-aa67-69f6-1472-6cf61ab0cde5 term=2 tally=1 -43:24.078 [INFO] TestAgent.server.raft: election won: tally=1 -43:24.078 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13696 [Leader]" -43:24.078 [INFO] TestAgent.server: cluster leadership acquired -43:24.078 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13696 -43:24.079 [INFO] TestAgent.server: New leader elected: payload=test-node -43:24.080 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:24.086 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:24.086 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:24.086 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:24.087 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:24.087 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:24.087 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -43:24.087 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -43:24.088 [INFO] TestAgent.server: federation state anti-entropy synced -43:24.222 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:24.227 [INFO] TestAgent: Synced node info -43:24.341 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=2.26252ms client=127.0.0.1:60068 client_network=udp -43:24.344 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=2.03004ms client=127.0.0.1:49128 client_network=udp -43:24.347 [DEBUG] TestAgent.dns: request served from client: name=e8c6c10c-7637-369f-cab6-f68c40f751a3.query.consul. type=ANY class=IN latency=2.29626ms client=127.0.0.1:57785 client_network=udp -43:24.347 [INFO] TestAgent: Requesting shutdown -43:24.347 [INFO] TestAgent.server: shutting down server -43:24.347 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:24.347 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:24.347 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:24.347 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:24.347 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:24.347 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:24.348 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:24.348 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:24.348 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:24.348 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:24.348 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:24.348Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:24.348Z [INFO] agent.router.manager: shutting down -43:24.348 [INFO] TestAgent: consul server down -43:24.348 [INFO] TestAgent: shutdown complete -43:24.348 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13706 network=tcp -43:24.348 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13706 network=udp -43:24.348 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13685 network=tcp -43:24.850 [INFO] TestAgent: Waiting for endpoints to shut down -43:24.850 [INFO] TestAgent: Endpoints down -[WARN] freeport: 4 out of 39 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:24.106Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:24.171Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:24.187 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:211c0016-7e34-d744-b7ca-244ce6e7fa24 Address:127.0.0.1:13718}]" -43:24.188 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -43:24.198 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13718 [Follower]" leader= -43:24.210 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2022-09-15T21:43:24.211Z [INFO] agent.router: Initializing LAN area manager -43:24.214 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13699 network=tcp -43:24.215 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -43:24.215 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13699 network=udp -43:24.217 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13693 network=tcp -43:24.217 [INFO] TestAgent: started state syncer -43:24.220 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13719 network=tcp -43:24.221 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:13718) (DC: dc1)" -43:24.236 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:24.236 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13718 [Candidate]" term=2 -43:24.236 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:24.236 [DEBUG] TestAgent.server.raft: vote granted: from=211c0016-7e34-d744-b7ca-244ce6e7fa24 term=2 tally=1 -43:24.236 [INFO] TestAgent.server.raft: election won: tally=1 -43:24.236 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13718 [Leader]" -43:24.236 [INFO] TestAgent.server: cluster leadership acquired -43:24.236 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13718 -43:24.237 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:24.238 [INFO] TestAgent.server: New leader elected: payload=test-node -43:24.241 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:24.241 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:24.241 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:24.241 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:24.241 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:24.241 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -43:24.241 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -43:24.246 [INFO] TestAgent.server: federation state anti-entropy synced -43:24.571 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=1.88206ms client=127.0.0.1:35686 client_network=udp -43:24.573 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=1.67842ms client=127.0.0.1:38149 client_network=udp -43:24.577 [DEBUG] TestAgent.dns: request served from client: name=63aca9ef-becc-96b6-aba3-c1152641e991.query.consul. type=A class=IN latency=1.86174ms client=127.0.0.1:45096 client_network=udp -43:24.577 [INFO] TestAgent: Requesting shutdown -43:24.577 [INFO] TestAgent.server: shutting down server -43:24.577 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:24.577 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:24.577 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:24.577 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:24.577 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:24.577 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:24.577 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:24.577 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:24.577 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:24.578 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:24.578 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:24.578 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:24.578Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:24.578Z [INFO] agent.router.manager: shutting down -43:24.578 [INFO] TestAgent: consul server down -43:24.578 [INFO] TestAgent: shutdown complete -43:24.578 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13699 network=tcp -43:24.578 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13699 network=udp -43:24.578 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13693 network=tcp -43:25.078 [INFO] TestAgent: Waiting for endpoints to shut down -43:25.078 [INFO] TestAgent: Endpoints down -[WARN] freeport: 4 out of 11 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:24.515Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:24.536Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:24.539 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6f09e755-ac04-27ca-7f72-8e5d6212922c Address:127.0.0.1:13690}]" -43:24.540 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -43:24.541 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2022-09-15T21:43:24.541Z [INFO] agent.router: Initializing LAN area manager -43:24.541 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13741 network=udp -43:24.541 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13690 [Follower]" leader= -43:24.541 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:13690) (DC: dc1)" -43:24.542 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -43:24.542 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13741 network=tcp -43:24.543 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13742 network=tcp -43:24.543 [INFO] TestAgent: started state syncer -43:24.543 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13691 network=tcp -43:24.600 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:24.600 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13690 [Candidate]" term=2 -43:24.601 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:24.601 [DEBUG] TestAgent.server.raft: vote granted: from=6f09e755-ac04-27ca-7f72-8e5d6212922c term=2 tally=1 -43:24.601 [INFO] TestAgent.server.raft: election won: tally=1 -43:24.601 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13690 [Leader]" -43:24.601 [INFO] TestAgent.server: cluster leadership acquired -43:24.601 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13690 -43:24.601 [INFO] TestAgent.server: New leader elected: payload=test-node -43:24.603 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:24.605 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:24.605 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:24.605 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:24.605 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:24.605 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:24.605 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -43:24.605 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -43:24.607 [INFO] TestAgent.server: federation state anti-entropy synced -43:24.666 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:24.667 [INFO] TestAgent: Synced node info -43:24.667 [DEBUG] TestAgent: Node info in sync -43:24.801 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=1.81522ms client=127.0.0.1:42619 client_network=udp -43:24.804 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=1.7231ms client=127.0.0.1:55661 client_network=udp -43:24.807 [DEBUG] TestAgent.dns: request served from client: name=8219c187-621a-36b7-6599-971fbc90ce90.query.consul. type=AAAA class=IN latency=2.1986ms client=127.0.0.1:36373 client_network=udp -43:24.807 [INFO] TestAgent: Requesting shutdown -43:24.807 [INFO] TestAgent.server: shutting down server -43:24.807 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:24.807 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:24.807 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:24.807 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:24.807 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:24.807 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:24.807 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:24.807 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:24.807 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:24.807 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:24.807 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:24.807Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:24.807Z [INFO] agent.router.manager: shutting down -43:24.807 [INFO] TestAgent: consul server down -43:24.808 [INFO] TestAgent: shutdown complete -43:24.808 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13741 network=tcp -43:24.808 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13741 network=udp -43:24.808 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13742 network=tcp -43:25.308 [INFO] TestAgent: Waiting for endpoints to shut down -43:25.308 [INFO] TestAgent: Endpoints down -2022-09-15T21:43:24.428Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:24.447Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:24.451 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:df77947a-7c4a-43f6-8218-f76fa5ddcdad Address:127.0.0.1:13739}]" -43:24.452 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -43:24.453 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2022-09-15T21:43:24.453Z [INFO] agent.router: Initializing LAN area manager -43:24.453 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13739 [Follower]" leader= -43:24.453 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13720 network=udp -43:24.453 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:13739) (DC: dc1)" -43:24.453 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -43:24.453 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13720 network=tcp -43:24.454 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13700 network=tcp -43:24.454 [INFO] TestAgent: started state syncer -43:24.454 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13740 network=tcp -43:24.499 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:24.499 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13739 [Candidate]" term=2 -43:24.499 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:24.499 [DEBUG] TestAgent.server.raft: vote granted: from=df77947a-7c4a-43f6-8218-f76fa5ddcdad term=2 tally=1 -43:24.499 [INFO] TestAgent.server.raft: election won: tally=1 -43:24.499 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13739 [Leader]" -43:24.499 [INFO] TestAgent.server: cluster leadership acquired -43:24.500 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13739 -43:24.501 [INFO] TestAgent.server: New leader elected: payload=test-node -43:24.501 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:24.504 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:24.504 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:24.504 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:24.504 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:24.504 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:24.504 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -43:24.504 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -43:24.506 [INFO] TestAgent.server: federation state anti-entropy synced -43:24.822 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:24.823 [INFO] TestAgent: Synced node info -43:24.944 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=1.75168ms client=127.0.0.1:45294 client_network=udp -43:24.946 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=1.95174ms client=127.0.0.1:58726 client_network=udp -43:24.948 [DEBUG] TestAgent.dns: request served from client: name=a804c413-ebfa-4994-b4a7-233dd28e5ddf.query.consul. type=A class=IN latency=1.6305ms client=127.0.0.1:56094 client_network=udp -43:24.948 [INFO] TestAgent: Requesting shutdown -43:24.948 [INFO] TestAgent.server: shutting down server -43:24.949 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:24.949 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:24.949 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:24.949 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:24.949 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:24.949 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:24.949 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:24.949 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:24.949 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:24.949 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:24.949 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:24.949Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:24.949Z [INFO] agent.router.manager: shutting down -43:24.949 [INFO] TestAgent: consul server down -43:24.949 [INFO] TestAgent: shutdown complete -43:24.949 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13720 network=tcp -43:24.949 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13720 network=udp -43:24.949 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13700 network=tcp -43:25.450 [INFO] TestAgent: Waiting for endpoints to shut down -43:25.450 [INFO] TestAgent: Endpoints down -[WARN] freeport: 4 out of 18 pending ports are still in use; something probably didn't wait around for the port to be closed! -2022-09-15T21:43:24.774Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:24.797Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:24.802 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:15d5384d-8038-8c06-fca0-3ee515cd25fb Address:127.0.0.1:13669}]" -43:24.803 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -43:24.804 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2022-09-15T21:43:24.804Z [INFO] agent.router: Initializing LAN area manager -43:24.806 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13669 [Follower]" leader= -43:24.806 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:13669) (DC: dc1)" -43:24.806 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -43:24.823 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13692 network=tcp -43:24.823 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13692 network=udp -43:24.824 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13707 network=tcp -43:24.824 [INFO] TestAgent: started state syncer -43:24.824 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13670 network=tcp -43:24.878 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:24.878 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13669 [Candidate]" term=2 -43:24.878 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:24.878 [DEBUG] TestAgent.server.raft: vote granted: from=15d5384d-8038-8c06-fca0-3ee515cd25fb term=2 tally=1 -43:24.878 [INFO] TestAgent.server.raft: election won: tally=1 -43:24.878 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13669 [Leader]" -43:24.878 [INFO] TestAgent.server: cluster leadership acquired -43:24.878 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13669 -43:24.879 [INFO] TestAgent.server: New leader elected: payload=test-node -43:24.880 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:24.887 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:24.887 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:24.887 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:24.887 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:24.887 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:24.887 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -43:24.887 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -43:24.888 [INFO] TestAgent.server: federation state anti-entropy synced -43:25.042 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:25.044 [INFO] TestAgent: Synced node info -43:25.087 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=2.26318ms client=127.0.0.1:42282 client_network=udp -43:25.092 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=2.00582ms client=127.0.0.1:45110 client_network=udp -43:25.095 [DEBUG] TestAgent.dns: request served from client: name=c2f7fb03-d543-08d1-3513-c21ae7b053ef.query.consul. type=ANY class=IN latency=2.2492ms client=127.0.0.1:48598 client_network=udp -43:25.098 [INFO] TestAgent: Requesting shutdown -43:25.098 [INFO] TestAgent.server: shutting down server -43:25.098 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:25.098 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:25.098 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:25.098 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:25.098 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:25.098 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:25.098 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:25.098 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:25.098 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:25.098 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:25.098 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:25.099Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:25.099Z [INFO] agent.router.manager: shutting down -43:25.099 [INFO] TestAgent: consul server down -43:25.099 [INFO] TestAgent: shutdown complete -43:25.099 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13692 network=tcp -43:25.099 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13692 network=udp -43:25.099 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13707 network=tcp -43:25.599 [INFO] TestAgent: Waiting for endpoints to shut down -43:25.599 [INFO] TestAgent: Endpoints down -2022-09-15T21:43:24.805Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:24.824Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:24.827 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8b768e40-984b-715b-72de-d2b8f9cae00a Address:127.0.0.1:13668}]" -43:24.828 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -43:24.828 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13668 [Follower]" leader= -43:24.829 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2022-09-15T21:43:24.829Z [INFO] agent.router: Initializing LAN area manager -43:24.829 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:13668) (DC: dc1)" -43:24.830 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -43:24.830 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13671 network=tcp -43:24.832 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13671 network=udp -43:24.833 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13657 network=tcp -43:24.833 [INFO] TestAgent: started state syncer -43:24.834 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13704 network=tcp -43:24.871 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:24.871 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13668 [Candidate]" term=2 -43:24.872 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:24.872 [DEBUG] TestAgent.server.raft: vote granted: from=8b768e40-984b-715b-72de-d2b8f9cae00a term=2 tally=1 -43:24.872 [INFO] TestAgent.server.raft: election won: tally=1 -43:24.872 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13668 [Leader]" -43:24.872 [INFO] TestAgent.server: cluster leadership acquired -43:24.872 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13668 -43:24.873 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:24.874 [INFO] TestAgent.server: New leader elected: payload=test-node -43:24.876 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:24.876 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:24.876 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:24.876 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:24.876 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:24.876 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -43:24.876 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -43:24.877 [INFO] TestAgent.server: federation state anti-entropy synced -43:25.105 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=4.5839ms client=127.0.0.1:40154 client_network=udp -43:25.107 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=1.78074ms client=127.0.0.1:35649 client_network=udp -43:25.113 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:25.115 [DEBUG] TestAgent.dns: request served from client: name=0f042c9f-9a4d-a87b-c641-7a3597827e27.query.consul. type=AAAA class=IN latency=1.89256ms client=127.0.0.1:55680 client_network=udp -43:25.120 [INFO] TestAgent: Requesting shutdown -43:25.120 [INFO] TestAgent.server: shutting down server -43:25.120 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:25.120 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:25.120 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:25.120 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:25.120 [INFO] TestAgent: Synced node info -43:25.120 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:25.120 [DEBUG] TestAgent: Node info in sync -43:25.120 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:25.121 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:25.121 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:25.121 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:25.121 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:25.121 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:25.121Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:25.121Z [INFO] agent.router.manager: shutting down -43:25.121 [INFO] TestAgent: consul server down -43:25.121 [INFO] TestAgent: shutdown complete -43:25.121 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13671 network=tcp -43:25.121 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13671 network=udp -43:25.121 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13657 network=tcp -43:25.622 [INFO] TestAgent: Waiting for endpoints to shut down -43:25.622 [INFO] TestAgent: Endpoints down +2021-08-13T15:35:00.474Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:35:00.492Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +35:00.494 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:adb80434-0ec5-7098-e23f-ce594c2a7462 Address:127.0.0.1:24855}]" +35:00.494 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24855 [Follower]" leader= +35:00.495 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +35:00.496 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2021-08-13T15:35:00.496Z [INFO] agent.router: Initializing LAN area manager +35:00.496 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24913 network=udp +35:00.497 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +35:00.497 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:24855) (DC: dc1)" +35:00.497 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24913 network=tcp +35:00.498 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24914 network=tcp +35:00.498 [INFO] TestAgent: started state syncer +35:00.498 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24856 network=tcp +35:00.552 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +35:00.552 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24855 [Candidate]" term=2 +35:00.553 [DEBUG] TestAgent.server.raft: votes: needed=1 +35:00.553 [DEBUG] TestAgent.server.raft: vote granted: from=adb80434-0ec5-7098-e23f-ce594c2a7462 term=2 tally=1 +35:00.553 [INFO] TestAgent.server.raft: election won: tally=1 +35:00.553 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24855 [Leader]" +35:00.553 [INFO] TestAgent.server: cluster leadership acquired +35:00.553 [INFO] TestAgent.server: New leader elected: payload=test-node +35:00.553 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24855 +35:00.554 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +35:00.561 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +35:00.561 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +35:00.561 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +35:00.561 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +35:00.561 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +35:00.561 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +35:00.561 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +35:00.562 [INFO] TestAgent.server: federation state anti-entropy synced +35:00.743 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +35:00.744 [INFO] TestAgent: Synced node info +35:00.920 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=1.836645ms client=127.0.0.1:56638 client_network=udp +35:00.922 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=1.568704ms client=127.0.0.1:36369 client_network=udp +35:00.924 [DEBUG] TestAgent.dns: request served from client: name=5f0ba8c0-a30c-b126-c7fd-5bd10fdb67bd.query.consul. type=AAAA class=IN latency=1.606224ms client=127.0.0.1:35029 client_network=udp +35:00.924 [INFO] TestAgent: Requesting shutdown +35:00.924 [INFO] TestAgent.server: shutting down server +35:00.924 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +35:00.924 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +35:00.924 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +35:00.924 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +35:00.924 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +35:00.924 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +35:00.924 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +35:00.924 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +35:00.924 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +35:00.924 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +35:00.924 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:35:00.924Z [INFO] agent.router.manager: shutting down +2021-08-13T15:35:00.924Z [INFO] agent.router.manager: shutting down +35:00.924 [INFO] TestAgent: consul server down +35:00.924 [INFO] TestAgent: shutdown complete +35:00.924 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24913 network=tcp +35:00.924 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24913 network=udp +35:00.924 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24914 network=tcp +35:01.425 [INFO] TestAgent: Waiting for endpoints to shut down +35:01.425 [INFO] TestAgent: Endpoints down +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._tcp_(qType:=1)_compressed=false#01 +2021-08-13T15:35:00.780Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:35:00.854Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +35:00.858 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:aaaa42dc-18b3-ce7f-fdba-0181e4f77bd0 Address:127.0.0.1:24876}]" +35:00.859 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24876 [Follower]" leader= +35:00.866 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +35:00.867 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2021-08-13T15:35:00.867Z [INFO] agent.router: Initializing LAN area manager +35:00.868 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24857 network=udp +35:00.868 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:24876) (DC: dc1)" +35:00.868 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +35:00.868 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24857 network=tcp +35:00.870 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24858 network=tcp +35:00.871 [INFO] TestAgent: started state syncer +35:00.871 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24877 network=tcp +35:00.901 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +35:00.902 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24876 [Candidate]" term=2 +35:00.902 [DEBUG] TestAgent.server.raft: votes: needed=1 +35:00.902 [DEBUG] TestAgent.server.raft: vote granted: from=aaaa42dc-18b3-ce7f-fdba-0181e4f77bd0 term=2 tally=1 +35:00.902 [INFO] TestAgent.server.raft: election won: tally=1 +35:00.902 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24876 [Leader]" +35:00.902 [INFO] TestAgent.server: cluster leadership acquired +35:00.902 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24876 +35:00.903 [INFO] TestAgent.server: New leader elected: payload=test-node +35:00.903 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +35:00.906 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +35:00.906 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +35:00.906 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +35:00.906 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +35:00.906 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +35:00.906 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +35:00.906 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +35:00.907 [INFO] TestAgent.server: federation state anti-entropy synced +35:01.110 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +35:01.110 [INFO] TestAgent: Synced node info +35:01.206 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=2.826947ms client=127.0.0.1:56650 client_network=udp +35:01.227 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=11.535107ms client=127.0.0.1:51339 client_network=udp +35:01.230 [DEBUG] TestAgent.dns: request served from client: name=393e4ccf-3dcc-6f52-9302-bdbd469c738f.query.consul. type=A class=IN latency=2.358645ms client=127.0.0.1:35657 client_network=udp +35:01.230 [INFO] TestAgent: Requesting shutdown +35:01.230 [INFO] TestAgent.server: shutting down server +35:01.230 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +35:01.230 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +35:01.230 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +35:01.230 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +35:01.230 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +35:01.230 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +35:01.230 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +35:01.231 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +35:01.231 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +35:01.231 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +35:01.231 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:35:01.231Z [INFO] agent.router.manager: shutting down +2021-08-13T15:35:01.231Z [INFO] agent.router.manager: shutting down +35:01.232 [INFO] TestAgent: consul server down +35:01.232 [INFO] TestAgent: shutdown complete +35:01.232 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24857 network=tcp +35:01.232 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24857 network=udp +35:01.232 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24858 network=tcp +35:01.733 [INFO] TestAgent: Waiting for endpoints to shut down +35:01.733 [INFO] TestAgent: Endpoints down +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:35:00.906Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:35:00.923Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +35:00.926 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8ab8e232-0ff2-b2b1-d14d-4b779b938ce8 Address:127.0.0.1:24918}]" +35:00.926 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24918 [Follower]" leader= +35:00.927 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +35:00.928 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2021-08-13T15:35:00.928Z [INFO] agent.router: Initializing LAN area manager +35:00.928 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:24918) (DC: dc1)" +35:00.928 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +35:00.928 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24878 network=tcp +35:00.928 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24878 network=udp +35:00.929 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24879 network=tcp +35:00.929 [INFO] TestAgent: started state syncer +35:00.929 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24919 network=tcp +35:00.973 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +35:00.973 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24918 [Candidate]" term=2 +35:00.974 [DEBUG] TestAgent.server.raft: votes: needed=1 +35:00.974 [DEBUG] TestAgent.server.raft: vote granted: from=8ab8e232-0ff2-b2b1-d14d-4b779b938ce8 term=2 tally=1 +35:00.974 [INFO] TestAgent.server.raft: election won: tally=1 +35:00.974 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24918 [Leader]" +35:00.974 [INFO] TestAgent.server: cluster leadership acquired +35:00.974 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24918 +35:00.974 [INFO] TestAgent.server: New leader elected: payload=test-node +35:00.975 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +35:00.978 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +35:00.978 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +35:00.978 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +35:00.978 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +35:00.978 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +35:00.978 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +35:00.978 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +35:00.979 [INFO] TestAgent.server: federation state anti-entropy synced +35:01.375 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +35:01.383 [INFO] TestAgent: Synced node info +35:01.383 [DEBUG] TestAgent: Node info in sync +35:01.437 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=2.172565ms client=127.0.0.1:43895 client_network=udp +35:01.440 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=2.001285ms client=127.0.0.1:52390 client_network=udp +35:01.443 [DEBUG] TestAgent.dns: request served from client: name=2a0df953-3f0d-50c4-c452-d9e6fa75e481.query.consul. type=A class=IN latency=2.065585ms client=127.0.0.1:40468 client_network=udp +35:01.443 [INFO] TestAgent: Requesting shutdown +35:01.443 [INFO] TestAgent.server: shutting down server +35:01.443 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +35:01.443 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +35:01.443 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +35:01.443 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +35:01.443 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +35:01.444 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +35:01.444 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +35:01.444 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +35:01.444 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +35:01.444 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +35:01.451 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:35:01.453Z [INFO] agent.router.manager: shutting down +2021-08-13T15:35:01.453Z [INFO] agent.router.manager: shutting down +35:01.453 [INFO] TestAgent: consul server down +35:01.453 [INFO] TestAgent: shutdown complete +35:01.453 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24878 network=tcp +35:01.454 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24878 network=udp +35:01.454 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24879 network=tcp +35:01.957 [INFO] TestAgent: Waiting for endpoints to shut down +35:01.957 [INFO] TestAgent: Endpoints down +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:35:01.254Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:35:01.298Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +35:01.311 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:72554494-589f-cf6b-89df-7f49e8997dce Address:127.0.0.1:24925}]" +35:01.313 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +35:01.314 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2021-08-13T15:35:01.314Z [INFO] agent.router: Initializing LAN area manager +35:01.315 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24969 network=udp +35:01.319 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24925 [Follower]" leader= +35:01.320 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:24925) (DC: dc1)" +35:01.320 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +35:01.321 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24969 network=tcp +35:01.323 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24970 network=tcp +35:01.323 [INFO] TestAgent: started state syncer +35:01.323 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24926 network=tcp +35:01.360 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +35:01.360 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24925 [Candidate]" term=2 +35:01.360 [DEBUG] TestAgent.server.raft: votes: needed=1 +35:01.360 [DEBUG] TestAgent.server.raft: vote granted: from=72554494-589f-cf6b-89df-7f49e8997dce term=2 tally=1 +35:01.360 [INFO] TestAgent.server.raft: election won: tally=1 +35:01.360 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24925 [Leader]" +35:01.363 [INFO] TestAgent.server: cluster leadership acquired +35:01.364 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24925 +35:01.364 [INFO] TestAgent.server: New leader elected: payload=test-node +35:01.366 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +35:01.370 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +35:01.370 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +35:01.370 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +35:01.370 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +35:01.370 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +35:01.370 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +35:01.370 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +35:01.392 [INFO] TestAgent.server: federation state anti-entropy synced +35:01.525 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=2.080385ms client=127.0.0.1:44730 client_network=udp +35:01.527 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=1.858065ms client=127.0.0.1:48975 client_network=udp +35:01.530 [DEBUG] TestAgent.dns: request served from client: name=b7b47e4c-027d-0d13-7f80-5d41f9457413.query.consul. type=ANY class=IN latency=2.045324ms client=127.0.0.1:46988 client_network=udp +35:01.530 [INFO] TestAgent: Requesting shutdown +35:01.530 [INFO] TestAgent.server: shutting down server +35:01.530 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +35:01.530 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +35:01.530 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +35:01.530 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +35:01.530 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +35:01.536 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +35:01.536 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +35:01.536 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +35:01.537 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +35:01.537 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +35:01.537 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +35:01.537 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:35:01.537Z [INFO] agent.router.manager: shutting down +2021-08-13T15:35:01.537Z [INFO] agent.router.manager: shutting down +35:01.537 [INFO] TestAgent: consul server down +35:01.537 [INFO] TestAgent: shutdown complete +35:01.537 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24969 network=tcp +35:01.538 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24969 network=udp +35:01.538 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24970 network=tcp +35:02.038 [INFO] TestAgent: Waiting for endpoints to shut down +35:02.038 [INFO] TestAgent: Endpoints down +2021-08-13T15:35:01.002Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:35:01.019Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +35:01.022 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f1c9af14-b191-65fc-c433-12ac85c673fa Address:127.0.0.1:24967}]" +35:01.022 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24967 [Follower]" leader= +35:01.023 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +35:01.023 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2021-08-13T15:35:01.023Z [INFO] agent.router: Initializing LAN area manager +35:01.024 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24920 network=udp +35:01.024 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:24967) (DC: dc1)" +35:01.024 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +35:01.024 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24920 network=tcp +35:01.025 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24921 network=tcp +35:01.025 [INFO] TestAgent: started state syncer +35:01.025 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24968 network=tcp +35:01.059 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +35:01.059 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24967 [Candidate]" term=2 +35:01.059 [DEBUG] TestAgent.server.raft: votes: needed=1 +35:01.059 [DEBUG] TestAgent.server.raft: vote granted: from=f1c9af14-b191-65fc-c433-12ac85c673fa term=2 tally=1 +35:01.059 [INFO] TestAgent.server.raft: election won: tally=1 +35:01.059 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24967 [Leader]" +35:01.059 [INFO] TestAgent.server: cluster leadership acquired +35:01.060 [INFO] TestAgent.server: New leader elected: payload=test-node +35:01.060 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24967 +35:01.061 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +35:01.064 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +35:01.064 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +35:01.064 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +35:01.064 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +35:01.064 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +35:01.064 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +35:01.064 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +35:01.065 [INFO] TestAgent.server: federation state anti-entropy synced +35:01.382 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +35:01.383 [INFO] TestAgent: Synced node info +35:01.588 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=2.672947ms client=127.0.0.1:46844 client_network=udp +35:01.591 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=2.451605ms client=127.0.0.1:57070 client_network=udp +35:01.594 [DEBUG] TestAgent.dns: request served from client: name=efd2397c-9456-c648-4448-30da067d2b86.query.consul. type=AAAA class=IN latency=2.936567ms client=127.0.0.1:37449 client_network=udp +35:01.594 [INFO] TestAgent: Requesting shutdown +35:01.595 [INFO] TestAgent.server: shutting down server +35:01.595 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +35:01.595 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +35:01.595 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +35:01.595 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +35:01.595 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +35:01.595 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +35:01.595 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +35:01.595 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +35:01.595 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +35:01.595 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +35:01.595 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:35:01.595Z [INFO] agent.router.manager: shutting down +2021-08-13T15:35:01.595Z [INFO] agent.router.manager: shutting down +35:01.596 [INFO] TestAgent: consul server down +35:01.596 [INFO] TestAgent: shutdown complete +35:01.596 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24920 network=tcp +35:01.596 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24920 network=udp +35:01.596 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24921 network=tcp +35:02.099 [INFO] TestAgent: Waiting for endpoints to shut down +35:02.099 [INFO] TestAgent: Endpoints down +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._udp_(qType:=1)_compressed=true#01 +2021-08-13T15:35:01.260Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:35:01.292Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +35:01.304 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c11e6417-2382-a20e-7244-c4c75e819b18 Address:127.0.0.1:24953}]" +35:01.306 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +35:01.307 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24953 [Follower]" leader= +35:01.307 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2021-08-13T15:35:01.307Z [INFO] agent.router: Initializing LAN area manager +35:01.308 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24927 network=udp +35:01.309 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:24953) (DC: dc1)" +35:01.309 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +35:01.310 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24927 network=tcp +35:01.318 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24928 network=tcp +35:01.318 [INFO] TestAgent: started state syncer +35:01.323 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24954 network=tcp +35:01.359 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +35:01.359 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24953 [Candidate]" term=2 +35:01.359 [DEBUG] TestAgent.server.raft: votes: needed=1 +35:01.359 [DEBUG] TestAgent.server.raft: vote granted: from=c11e6417-2382-a20e-7244-c4c75e819b18 term=2 tally=1 +35:01.359 [INFO] TestAgent.server.raft: election won: tally=1 +35:01.359 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24953 [Leader]" +35:01.361 [INFO] TestAgent.server: cluster leadership acquired +35:01.361 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24953 +35:01.363 [INFO] TestAgent.server: New leader elected: payload=test-node +35:01.371 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +35:01.380 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +35:01.380 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +35:01.380 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +35:01.380 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +35:01.380 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +35:01.380 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +35:01.380 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +35:01.382 [INFO] TestAgent.server: federation state anti-entropy synced +35:01.596 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +35:01.597 [INFO] TestAgent: Synced node info +35:01.597 [DEBUG] TestAgent: Node info in sync +35:01.702 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=1.739384ms client=127.0.0.1:36118 client_network=udp +35:01.704 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=1.511244ms client=127.0.0.1:34316 client_network=udp +35:01.714 [DEBUG] TestAgent.dns: request served from client: name=0049bae9-78c4-cdfd-7d88-8c992776e453.query.consul. type=AAAA class=IN latency=1.775784ms client=127.0.0.1:57739 client_network=udp +35:01.715 [INFO] TestAgent: Requesting shutdown +35:01.715 [INFO] TestAgent.server: shutting down server +35:01.715 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +35:01.715 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +35:01.715 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +35:01.715 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +35:01.715 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +35:01.715 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +35:01.715 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +35:01.715 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +35:01.715 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +35:01.715 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +35:01.715 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:35:01.715Z [INFO] agent.router.manager: shutting down +2021-08-13T15:35:01.715Z [INFO] agent.router.manager: shutting down +35:01.716 [INFO] TestAgent: consul server down +35:01.716 [INFO] TestAgent: shutdown complete +35:01.716 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24927 network=tcp +35:01.716 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24927 network=udp +35:01.716 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24928 network=tcp +35:02.217 [INFO] TestAgent: Waiting for endpoints to shut down +35:02.217 [INFO] TestAgent: Endpoints down --- PASS: TestDNS_ServiceLookup_AnswerLimits (0.00s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{30_30_8_8_6_8_8_5_8_-5_-5} (1.00s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{0_0_0_0_0_0_0_0_0_0_0} (1.18s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{30_30_8_8_6_8_8_5_8_-5_-5} (0.93s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{30_30_8_8_6_8_8_5_8_-5_-5} (0.95s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{6_6_6_6_6_6_6_5_6_6_-5} (1.23s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{20_20_8_8_6_8_8_5_8_-5_-5} (0.94s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{20_20_8_8_6_8_8_5_8_-5_-5} (0.85s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{20_20_8_8_6_8_8_5_8_-5_-5} (1.07s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{9_9_8_8_6_8_8_5_8_8_-5} (0.88s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{9_9_8_8_6_8_8_5_8_8_-5} (0.93s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{9_9_8_8_6_8_8_5_8_8_-5} (0.93s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{8_8_8_8_6_8_8_5_8_8_-5} (0.88s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{8_8_8_8_6_8_8_5_8_8_-5} (0.97s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{8_8_8_8_6_8_8_5_8_8_-5} (1.03s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{7_7_7_7_6_7_7_5_7_7_-5} (0.93s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{7_7_7_7_6_7_7_5_7_7_-5} (1.04s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{7_7_7_7_6_7_7_5_7_7_-5} (0.89s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{6_6_6_6_6_6_6_5_6_6_-5} (0.88s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{5_5_5_5_5_5_5_5_5_5_5} (0.79s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{3_3_3_3_3_3_3_3_3_3_3} (0.97s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{6_6_6_6_6_6_6_5_6_6_-5} (1.01s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{5_5_5_5_5_5_5_5_5_5_5} (1.09s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{5_5_5_5_5_5_5_5_5_5_5} (1.01s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{4_4_4_4_4_4_4_4_4_4_4} (1.02s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{4_4_4_4_4_4_4_4_4_4_4} (0.82s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{3_3_3_3_3_3_3_3_3_3_3} (0.84s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{1_1_1_1_1_1_1_1_1_1_1} (1.01s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{4_4_4_4_4_4_4_4_4_4_4} (1.17s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{2_2_2_2_2_2_2_2_2_2_2} (0.82s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{3_3_3_3_3_3_3_3_3_3_3} (1.11s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{2_2_2_2_2_2_2_2_2_2_2} (0.87s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{2_2_2_2_2_2_2_2_2_2_2} (1.01s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{1_1_1_1_1_1_1_1_1_1_1} (0.82s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{1_1_1_1_1_1_1_1_1_1_1} (1.05s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{0_0_0_0_0_0_0_0_0_0_0} (0.86s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{0_0_0_0_0_0_0_0_0_0_0} (0.84s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{6_6_6_6_6_6_6_5_6_6_-5} (0.92s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{0_0_0_0_0_0_0_0_0_0_0} (1.07s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{30_30_8_8_6_8_8_5_8_-5_-5} (1.06s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{30_30_8_8_6_8_8_5_8_-5_-5} (0.96s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{20_20_8_8_6_8_8_5_8_-5_-5} (1.02s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{30_30_8_8_6_8_8_5_8_-5_-5} (1.16s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{9_9_8_8_6_8_8_5_8_8_-5} (0.83s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{20_20_8_8_6_8_8_5_8_-5_-5} (1.23s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{20_20_8_8_6_8_8_5_8_-5_-5} (1.18s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{9_9_8_8_6_8_8_5_8_8_-5} (0.92s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{9_9_8_8_6_8_8_5_8_8_-5} (1.11s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{8_8_8_8_6_8_8_5_8_8_-5} (1.10s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{8_8_8_8_6_8_8_5_8_8_-5} (1.15s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{7_7_7_7_6_7_7_5_7_7_-5} (0.91s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{8_8_8_8_6_8_8_5_8_8_-5} (1.15s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{7_7_7_7_6_7_7_5_7_7_-5} (1.03s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{7_7_7_7_6_7_7_5_7_7_-5} (0.86s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{6_6_6_6_6_6_6_5_6_6_-5} (0.95s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{6_6_6_6_6_6_6_5_6_6_-5} (0.77s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{3_3_3_3_3_3_3_3_3_3_3} (1.09s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{5_5_5_5_5_5_5_5_5_5_5} (0.73s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{1_1_1_1_1_1_1_1_1_1_1} (0.85s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{2_2_2_2_2_2_2_2_2_2_2} (0.90s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{5_5_5_5_5_5_5_5_5_5_5} (1.09s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{2_2_2_2_2_2_2_2_2_2_2} (0.93s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{5_5_5_5_5_5_5_5_5_5_5} (1.04s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{4_4_4_4_4_4_4_4_4_4_4} (0.75s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{4_4_4_4_4_4_4_4_4_4_4} (0.95s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{3_3_3_3_3_3_3_3_3_3_3} (0.92s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{2_2_2_2_2_2_2_2_2_2_2} (1.19s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{3_3_3_3_3_3_3_3_3_3_3} (0.97s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{4_4_4_4_4_4_4_4_4_4_4} (0.99s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{1_1_1_1_1_1_1_1_1_1_1} (1.10s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{0_0_0_0_0_0_0_0_0_0_0} (0.84s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{1_1_1_1_1_1_1_1_1_1_1} (1.12s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{0_0_0_0_0_0_0_0_0_0_0} (0.98s) +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._udp_(qType:=1)_compressed=false#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=33)_compressed=true#01 +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=33)_compressed=false#01 +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=33)_compressed=true#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=33)_compressed=false#01 +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._tcp_(qType:=33)_compressed=true#01 +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._tcp_(qType:=33)_compressed=false#01 +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._udp_(qType:=33)_compressed=true#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._udp_(qType:=33)_compressed=false#01 +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2021-08-13T15:34:44.342Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:44.359Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:44.362 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:311b0c3a-8e59-ae3b-ed02-d919a3f59116 Address:127.0.0.1:25064}]" +34:44.362 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25064 [Follower]" leader= +34:44.363 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-311b0c3a-8e59-ae3b-ed02-d919a3f59116.dc1 127.0.0.1 +34:44.364 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-311b0c3a-8e59-ae3b-ed02-d919a3f59116 127.0.0.1 +2021-08-13T15:34:44.364Z [INFO] agent.router: Initializing LAN area manager +34:44.364 [INFO] TestAgent.server: Adding LAN server: server="Node-311b0c3a-8e59-ae3b-ed02-d919a3f59116 (Addr: tcp/127.0.0.1:25064) (DC: dc1)" +34:44.364 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-311b0c3a-8e59-ae3b-ed02-d919a3f59116.dc1 area=wan +34:44.364 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25038 network=tcp +34:44.364 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25038 network=udp +34:44.365 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25039 network=tcp +34:44.365 [INFO] TestAgent: started state syncer +34:44.365 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25079 network=tcp +34:44.418 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:44.418 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25064 [Candidate]" term=2 +34:44.418 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:44.418 [DEBUG] TestAgent.server.raft: vote granted: from=311b0c3a-8e59-ae3b-ed02-d919a3f59116 term=2 tally=1 +34:44.418 [INFO] TestAgent.server.raft: election won: tally=1 +34:44.418 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25064 [Leader]" +34:44.419 [INFO] TestAgent.server: cluster leadership acquired +34:44.419 [INFO] TestAgent.server: New leader elected: payload=Node-311b0c3a-8e59-ae3b-ed02-d919a3f59116 +34:44.419 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25064 +34:44.420 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:44.422 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:44.422 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:44.422 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:44.422 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:44.422 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:44.422 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-311b0c3a-8e59-ae3b-ed02-d919a3f59116 +34:44.423 [INFO] TestAgent.server: member joined, marking health alive: member=Node-311b0c3a-8e59-ae3b-ed02-d919a3f59116 +34:44.423 [INFO] TestAgent.server: federation state anti-entropy synced +34:44.571 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=363.661µs client=127.0.0.1:40042 client_network=tcp +34:44.572 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=229.241µs client=127.0.0.1:40044 client_network=tcp +34:44.573 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=217.9µs client=127.0.0.1:43465 client_network=udp +34:44.574 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=201.401µs client=127.0.0.1:59814 client_network=udp +34:44.574 [DEBUG] TestAgent.dns: request served from client: name=01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul. type=ANY class=IN latency=276.24µs client=127.0.0.1:40046 client_network=tcp +34:44.575 [DEBUG] TestAgent.dns: request served from client: name=01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul. type=ANY class=IN latency=199.401µs client=127.0.0.1:40048 client_network=tcp +34:44.576 [DEBUG] TestAgent.dns: request served from client: name=01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul. type=ANY class=IN latency=162.72µs client=127.0.0.1:44659 client_network=udp +34:44.576 [DEBUG] TestAgent.dns: request served from client: name=01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul. type=ANY class=IN latency=152.68µs client=127.0.0.1:49696 client_network=udp +34:44.577 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=241.461µs client=127.0.0.1:40050 client_network=tcp +34:44.577 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=219.6µs client=127.0.0.1:40052 client_network=tcp +34:44.578 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=213.461µs client=127.0.0.1:42002 client_network=udp +34:44.578 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=193.001µs client=127.0.0.1:46430 client_network=udp +34:44.579 [DEBUG] TestAgent.dns: request served from client: name=01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul. type=ANY class=IN latency=215.101µs client=127.0.0.1:40054 client_network=tcp +34:44.580 [DEBUG] TestAgent.dns: request served from client: name=01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul. type=ANY class=IN latency=214.841µs client=127.0.0.1:40056 client_network=tcp +34:44.581 [DEBUG] TestAgent.dns: request served from client: name=01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul. type=ANY class=IN latency=164.78µs client=127.0.0.1:51932 client_network=udp +34:44.581 [DEBUG] TestAgent.dns: request served from client: name=01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul. type=ANY class=IN latency=174.36µs client=127.0.0.1:45295 client_network=udp +34:44.582 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=249.701µs client=127.0.0.1:40058 client_network=tcp +34:44.583 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=239.301µs client=127.0.0.1:40060 client_network=tcp +34:44.583 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=213.56µs client=127.0.0.1:35961 client_network=udp +34:44.584 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=202.261µs client=127.0.0.1:45698 client_network=udp +34:44.585 [DEBUG] TestAgent.dns: request served from client: name=01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul. type=ANY class=IN latency=220.88µs client=127.0.0.1:40062 client_network=tcp +34:44.585 [DEBUG] TestAgent.dns: request served from client: name=01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul. type=ANY class=IN latency=197.541µs client=127.0.0.1:40064 client_network=tcp +34:44.586 [DEBUG] TestAgent.dns: request served from client: name=01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul. type=ANY class=IN latency=161.54µs client=127.0.0.1:47399 client_network=udp +34:44.586 [DEBUG] TestAgent.dns: request served from client: name=01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul. type=ANY class=IN latency=152.32µs client=127.0.0.1:59972 client_network=udp +34:44.587 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=242.881µs client=127.0.0.1:40066 client_network=tcp +34:44.588 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=215.501µs client=127.0.0.1:40068 client_network=tcp +34:44.588 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=196.741µs client=127.0.0.1:51038 client_network=udp +34:44.589 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=185.62µs client=127.0.0.1:35725 client_network=udp +34:44.590 [DEBUG] TestAgent.dns: request served from client: name=01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul. type=ANY class=IN latency=208.981µs client=127.0.0.1:40070 client_network=tcp +34:44.590 [DEBUG] TestAgent.dns: request served from client: name=01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul. type=ANY class=IN latency=194.12µs client=127.0.0.1:40072 client_network=tcp +34:44.591 [DEBUG] TestAgent.dns: request served from client: name=01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul. type=ANY class=IN latency=154.24µs client=127.0.0.1:34986 client_network=udp +34:44.591 [DEBUG] TestAgent.dns: request served from client: name=01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul. type=ANY class=IN latency=152.72µs client=127.0.0.1:57057 client_network=udp +34:44.592 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=237.341µs client=127.0.0.1:40074 client_network=tcp +34:44.593 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=225.701µs client=127.0.0.1:40076 client_network=tcp +34:44.593 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=205.46µs client=127.0.0.1:46151 client_network=udp +34:44.594 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=180.08µs client=127.0.0.1:44633 client_network=udp +34:44.594 [DEBUG] TestAgent.dns: request served from client: name=01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul. type=ANY class=IN latency=201.321µs client=127.0.0.1:40078 client_network=tcp +34:44.595 [DEBUG] TestAgent.dns: request served from client: name=01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul. type=ANY class=IN latency=197.3µs client=127.0.0.1:40080 client_network=tcp +34:44.596 [DEBUG] TestAgent.dns: request served from client: name=01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul. type=ANY class=IN latency=152.08µs client=127.0.0.1:39514 client_network=udp +34:44.596 [DEBUG] TestAgent.dns: request served from client: name=01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul. type=ANY class=IN latency=175.801µs client=127.0.0.1:55841 client_network=udp +34:44.597 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=263.9µs client=127.0.0.1:40082 client_network=tcp +34:44.597 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=242.581µs client=127.0.0.1:40084 client_network=tcp +34:44.598 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=200.64µs client=127.0.0.1:37677 client_network=udp +34:44.599 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=189.341µs client=127.0.0.1:45459 client_network=udp +34:44.599 [DEBUG] TestAgent.dns: request served from client: name=01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul. type=ANY class=IN latency=210.52µs client=127.0.0.1:40086 client_network=tcp +34:44.600 [DEBUG] TestAgent.dns: request served from client: name=01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul. type=ANY class=IN latency=212.901µs client=127.0.0.1:40088 client_network=tcp +34:44.601 [DEBUG] TestAgent.dns: request served from client: name=01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul. type=ANY class=IN latency=173.44µs client=127.0.0.1:32781 client_network=udp +34:44.601 [DEBUG] TestAgent.dns: request served from client: name=01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul. type=ANY class=IN latency=164.36µs client=127.0.0.1:37650 client_network=udp +34:44.795 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:44.799 [INFO] TestAgent: Synced node info +34:44.799 [DEBUG] TestAgent: Node info in sync +34:45.259 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:45.259 [DEBUG] TestAgent: Node info in sync +34:48.561 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 +34:48.563 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=99.648991ms client=127.0.0.1:40092 client_network=tcp +34:49.104 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 +34:49.106 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=163.53412ms client=127.0.0.1:40110 client_network=tcp +34:49.503 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=105.450984ms client=127.0.0.1:48878 client_network=udp +34:49.672 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=163.495999ms client=127.0.0.1:42571 client_network=udp +34:49.810 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul. 255 1}"] records=4091/5001 size=65523/80083 +34:49.813 [DEBUG] TestAgent.dns: request served from client: name=d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul. type=ANY class=IN latency=136.067495ms client=127.0.0.1:40112 client_network=tcp +34:50.481 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul. 255 1}"] records=4091/5001 size=65523/80083 +34:50.484 [DEBUG] TestAgent.dns: request served from client: name=d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul. type=ANY class=IN latency=111.899559ms client=127.0.0.1:40114 client_network=tcp +34:51.102 [DEBUG] TestAgent.dns: request served from client: name=d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul. type=ANY class=IN latency=171.862619ms client=127.0.0.1:50442 client_network=udp +34:51.225 [DEBUG] TestAgent.dns: request served from client: name=d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul. type=ANY class=IN latency=116.382509ms client=127.0.0.1:53163 client_network=udp +34:51.376 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 +34:51.378 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=126.613674ms client=127.0.0.1:40116 client_network=tcp +34:51.892 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 +34:51.894 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=102.004157ms client=127.0.0.1:40118 client_network=tcp +34:52.221 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=102.977359ms client=127.0.0.1:54786 client_network=udp +34:52.378 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=134.636372ms client=127.0.0.1:55858 client_network=udp +34:52.493 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul. 255 1}"] records=4091/5001 size=65523/80083 +34:52.496 [DEBUG] TestAgent.dns: request served from client: name=d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul. type=ANY class=IN latency=112.25784ms client=127.0.0.1:40122 client_network=tcp +34:53.166 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul. 255 1}"] records=4091/5001 size=65523/80083 +34:53.168 [DEBUG] TestAgent.dns: request served from client: name=d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul. type=ANY class=IN latency=148.057263ms client=127.0.0.1:40124 client_network=tcp +34:53.804 [DEBUG] TestAgent.dns: request served from client: name=d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul. type=ANY class=IN latency=124.133248ms client=127.0.0.1:57067 client_network=udp +34:53.916 [DEBUG] TestAgent.dns: request served from client: name=d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul. type=ANY class=IN latency=105.250884ms client=127.0.0.1:39954 client_network=udp +34:54.052 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 +34:54.054 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=131.219404ms client=127.0.0.1:40126 client_network=tcp +34:54.551 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 +34:54.553 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=120.224239ms client=127.0.0.1:40128 client_network=tcp +34:54.892 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=110.018995ms client=127.0.0.1:52946 client_network=udp +34:54.992 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=95.00562ms client=127.0.0.1:38635 client_network=udp +34:55.185 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul. 255 1}"] records=4091/5001 size=65523/80083 +34:55.188 [DEBUG] TestAgent.dns: request served from client: name=d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul. type=ANY class=IN latency=190.450322ms client=127.0.0.1:40138 client_network=tcp +34:55.902 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul. 255 1}"] records=4091/5001 size=65523/80083 +34:55.905 [DEBUG] TestAgent.dns: request served from client: name=d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul. type=ANY class=IN latency=181.37006ms client=127.0.0.1:40140 client_network=tcp +34:56.605 [DEBUG] TestAgent.dns: request served from client: name=d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul. type=ANY class=IN latency=194.570291ms client=127.0.0.1:47083 client_network=udp +34:56.766 [DEBUG] TestAgent.dns: request served from client: name=d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul. type=ANY class=IN latency=155.16204ms client=127.0.0.1:55490 client_network=udp +34:56.879 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 +34:56.881 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=108.572652ms client=127.0.0.1:40142 client_network=tcp +34:57.537 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 +34:57.539 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=119.617198ms client=127.0.0.1:40144 client_network=tcp +34:58.001 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=117.219572ms client=127.0.0.1:38123 client_network=udp +34:58.104 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=97.652726ms client=127.0.0.1:33233 client_network=udp +34:58.251 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul. 255 1}"] records=4091/5001 size=65523/80083 +34:58.253 [DEBUG] TestAgent.dns: request served from client: name=d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul. type=ANY class=IN latency=141.377488ms client=127.0.0.1:40146 client_network=tcp +34:59.000 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul. 255 1}"] records=4091/5001 size=65523/80083 +34:59.003 [DEBUG] TestAgent.dns: request served from client: name=d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul. type=ANY class=IN latency=176.69865ms client=127.0.0.1:40150 client_network=tcp +34:59.695 [DEBUG] TestAgent.dns: request served from client: name=d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul. type=ANY class=IN latency=157.938607ms client=127.0.0.1:57139 client_network=udp +34:59.863 [DEBUG] TestAgent.dns: request served from client: name=d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul. type=ANY class=IN latency=159.974791ms client=127.0.0.1:55898 client_network=udp +35:00.012 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 +35:00.014 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=144.024534ms client=127.0.0.1:40152 client_network=tcp +35:00.546 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 +35:00.548 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=100.895974ms client=127.0.0.1:40162 client_network=tcp +35:00.962 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=140.592886ms client=127.0.0.1:45863 client_network=udp +35:01.061 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=94.396539ms client=127.0.0.1:36738 client_network=udp +35:01.169 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul. 255 1}"] records=4091/5001 size=65523/80083 +35:01.172 [DEBUG] TestAgent.dns: request served from client: name=d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul. type=ANY class=IN latency=105.726225ms client=127.0.0.1:40164 client_network=tcp +35:01.805 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul. 255 1}"] records=4091/5001 size=65523/80083 +35:01.807 [DEBUG] TestAgent.dns: request served from client: name=d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul. type=ANY class=IN latency=114.097365ms client=127.0.0.1:40166 client_network=tcp +35:02.287 [DEBUG] TestAgent.dns: request served from client: name=d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul. type=ANY class=IN latency=126.958774ms client=127.0.0.1:40932 client_network=udp +35:02.397 [DEBUG] TestAgent.dns: request served from client: name=d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul. type=ANY class=IN latency=103.169119ms client=127.0.0.1:42467 client_network=udp +35:02.552 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 +35:02.553 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=142.12121ms client=127.0.0.1:40184 client_network=tcp +35:03.070 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 +35:03.072 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=121.902263ms client=127.0.0.1:40186 client_network=tcp +35:03.476 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=119.440417ms client=127.0.0.1:45001 client_network=udp +35:03.581 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=99.726091ms client=127.0.0.1:43358 client_network=udp +35:03.694 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul. 255 1}"] records=4091/5001 size=65523/80083 +35:03.697 [DEBUG] TestAgent.dns: request served from client: name=d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul. type=ANY class=IN latency=110.658377ms client=127.0.0.1:40188 client_network=tcp +35:04.184 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul. 255 1}"] records=4091/5001 size=65523/80083 +35:04.187 [DEBUG] TestAgent.dns: request served from client: name=d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul. type=ANY class=IN latency=130.074822ms client=127.0.0.1:40190 client_network=tcp +35:04.709 [DEBUG] TestAgent.dns: request served from client: name=d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul. type=ANY class=IN latency=107.288109ms client=127.0.0.1:33914 client_network=udp +35:04.818 [DEBUG] TestAgent.dns: request served from client: name=d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul. type=ANY class=IN latency=102.325877ms client=127.0.0.1:44717 client_network=udp +35:04.823 [INFO] TestAgent: Requesting shutdown +35:04.824 [INFO] TestAgent.server: shutting down server +35:04.824 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +35:04.824 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +35:04.824 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +35:04.824 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +35:04.824 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +35:04.824 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +35:04.824 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +35:04.824 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +35:04.824 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +35:04.824 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +35:04.824 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:35:04.824Z [INFO] agent.router.manager: shutting down +2021-08-13T15:35:04.824Z [INFO] agent.router.manager: shutting down +35:04.825 [INFO] TestAgent: consul server down +35:04.825 [INFO] TestAgent: shutdown complete +35:04.825 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25038 network=tcp +35:04.825 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25038 network=udp +35:04.825 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25039 network=tcp +35:05.325 [INFO] TestAgent: Waiting for endpoints to shut down +35:05.325 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_TCP_and_UDP_Truncate (21.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=255)_compressed=true (0.01s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=255)_compressed=false (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=255)_compressed=true (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=255)_compressed=false (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._tcp_(qType:=255)_compressed=true (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._tcp_(qType:=255)_compressed=false (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._udp_(qType:=255)_compressed=true (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._udp_(qType:=255)_compressed=false (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=1)_compressed=true (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=1)_compressed=false (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=1)_compressed=true (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=1)_compressed=false (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._tcp_(qType:=1)_compressed=true (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._tcp_(qType:=1)_compressed=false (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._udp_(qType:=1)_compressed=true (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._udp_(qType:=1)_compressed=false (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=33)_compressed=true (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=33)_compressed=false (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=33)_compressed=true (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=33)_compressed=false (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._tcp_(qType:=33)_compressed=true (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._tcp_(qType:=33)_compressed=false (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._udp_(qType:=33)_compressed=true (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._udp_(qType:=33)_compressed=false (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=255)_compressed=true#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=255)_compressed=false#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=255)_compressed=true#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=255)_compressed=false#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._tcp_(qType:=255)_compressed=true#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._tcp_(qType:=255)_compressed=false#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._udp_(qType:=255)_compressed=true#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._udp_(qType:=255)_compressed=false#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=1)_compressed=true#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=1)_compressed=false#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=1)_compressed=true#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=1)_compressed=false#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._tcp_(qType:=1)_compressed=true#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._tcp_(qType:=1)_compressed=false#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._udp_(qType:=1)_compressed=true#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._udp_(qType:=1)_compressed=false#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=33)_compressed=true#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=33)_compressed=false#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=33)_compressed=true#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=33)_compressed=false#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._tcp_(qType:=33)_compressed=true#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._tcp_(qType:=33)_compressed=false#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._udp_(qType:=33)_compressed=true#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_01d6f928-7d12-98d8-3a7b-846b5a29e418.query.consul._udp_(qType:=33)_compressed=false#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=255)_compressed=true (0.48s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=255)_compressed=false (0.45s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=255)_compressed=true (0.11s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=255)_compressed=false (0.17s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._tcp_(qType:=255)_compressed=true (0.70s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._tcp_(qType:=255)_compressed=false (0.56s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._udp_(qType:=255)_compressed=true (0.18s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._udp_(qType:=255)_compressed=false (0.13s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=1)_compressed=true (0.54s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=1)_compressed=false (0.33s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=1)_compressed=true (0.13s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=1)_compressed=false (0.14s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._tcp_(qType:=1)_compressed=true (0.64s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._tcp_(qType:=1)_compressed=false (0.66s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._udp_(qType:=1)_compressed=true (0.13s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._udp_(qType:=1)_compressed=false (0.11s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=33)_compressed=true (0.51s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=33)_compressed=false (0.35s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=33)_compressed=true (0.11s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=33)_compressed=false (0.10s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._tcp_(qType:=33)_compressed=true (0.73s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._tcp_(qType:=33)_compressed=false (0.69s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._udp_(qType:=33)_compressed=true (0.20s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._udp_(qType:=33)_compressed=false (0.16s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=255)_compressed=true#01 (0.65s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=255)_compressed=false#01 (0.46s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=255)_compressed=true#01 (0.12s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=255)_compressed=false#01 (0.11s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._tcp_(qType:=255)_compressed=true#01 (0.71s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._tcp_(qType:=255)_compressed=false#01 (0.71s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._udp_(qType:=255)_compressed=true#01 (0.17s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._udp_(qType:=255)_compressed=false#01 (0.17s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=1)_compressed=true#01 (0.57s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=1)_compressed=false#01 (0.38s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=1)_compressed=true#01 (0.15s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=1)_compressed=false#01 (0.10s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._tcp_(qType:=1)_compressed=true#01 (0.63s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._tcp_(qType:=1)_compressed=false#01 (0.46s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._udp_(qType:=1)_compressed=true#01 (0.14s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._udp_(qType:=1)_compressed=false#01 (0.11s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=33)_compressed=true#01 (0.54s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=33)_compressed=false#01 (0.41s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=33)_compressed=true#01 (0.12s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=33)_compressed=false#01 (0.10s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._tcp_(qType:=33)_compressed=true#01 (0.47s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._tcp_(qType:=33)_compressed=false#01 (0.55s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._udp_(qType:=33)_compressed=true#01 (0.11s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_d6c790f8-a065-cc0c-0537-3afea5c42d42.query.consul._udp_(qType:=33)_compressed=false#01 (0.11s) PASS -ok github.com/hashicorp/consul/agent 159.125s +ok github.com/hashicorp/consul/agent 153.743s === RUN TestAE_scaleFactor === PAUSE TestAE_scaleFactor === RUN TestAE_Pause_nestedPauseResume @@ -56652,7 +56629,7 @@ === RUN TestAE_FSM/fullSyncState === RUN TestAE_FSM/fullSyncState/Paused_->_retryFullSyncState === RUN TestAE_FSM/fullSyncState/SyncFull()_error_->_retryFullSyncState -2022-09-15T21:40:25.015Z [ERROR] anti_entropy: failed to sync remote state: error=boom +2021-08-13T15:32:08.004Z [ERROR] anti_entropy: failed to sync remote state: error=boom === RUN TestAE_FSM/fullSyncState/SyncFull()_OK_->_partialSyncState === RUN TestAE_FSM/retryFullSyncState === RUN TestAE_FSM/retryFullSyncState/shutdownEvent_->_doneState @@ -56665,7 +56642,7 @@ === RUN TestAE_FSM/partialSyncState/syncFullTimerEvent_->_fullSyncState === RUN TestAE_FSM/partialSyncState/syncChangesEvent+Paused_->_partialSyncState === RUN TestAE_FSM/partialSyncState/syncChangesEvent+SyncChanges()_error_->_partialSyncState -2022-09-15T21:40:25.016Z [ERROR] anti_entropy: failed to sync changes: error=boom +2021-08-13T15:32:08.005Z [ERROR] anti_entropy: failed to sync changes: error=boom === RUN TestAE_FSM/partialSyncState/syncChangesEvent+SyncChanges()_OK_->_partialSyncState === RUN TestAE_FSM/partialSyncState/invalid_event_->_panic_ === RUN TestAE_FSM/invalid_state_->_panic_ @@ -56723,43 +56700,43 @@ === CONT TestAE_Pause_nestedPauseResume --- PASS: TestAE_Pause_nestedPauseResume (0.00s) PASS -ok github.com/hashicorp/consul/agent/ae 2.284s +ok github.com/hashicorp/consul/agent/ae 2.291s === RUN TestNew -=== RUN TestNew/no-cache -=== RUN TestNew/no-tls-configurator === RUN TestNew/no-tokens === RUN TestNew/ok === RUN TestNew/no-direct-rpc === RUN TestNew/no-config-loader +=== RUN TestNew/no-cache +=== RUN TestNew/no-tls-configurator --- PASS: TestNew (0.00s) - --- PASS: TestNew/no-cache (0.00s) - --- PASS: TestNew/no-tls-configurator (0.00s) --- PASS: TestNew/no-tokens (0.00s) --- PASS: TestNew/ok (0.00s) --- PASS: TestNew/no-direct-rpc (0.00s) --- PASS: TestNew/no-config-loader (0.00s) + --- PASS: TestNew/no-cache (0.00s) + --- PASS: TestNew/no-tls-configurator (0.00s) === RUN TestReadConfig --- PASS: TestReadConfig (0.00s) === RUN TestInitialConfiguration_disabled --- PASS: TestInitialConfiguration_disabled (0.02s) === RUN TestInitialConfiguration_cancelled -2022-09-15T21:40:37.033Z [DEBUG] TestInitialConfiguration_cancelled.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestInitialConfiguration_cancelled-auto-config090504861/auto-config.json -2022-09-15T21:40:37.033Z [INFO] TestInitialConfiguration_cancelled.auto_config: retrieving initial agent auto configuration remotely -2022-09-15T21:40:37.058Z [DEBUG] TestInitialConfiguration_cancelled.auto_config: making AutoConfig.InitialConfiguration RPC: addr=127.0.0.1:8300 -2022-09-15T21:40:37.058Z [ERROR] TestInitialConfiguration_cancelled.auto_config: AutoConfig.InitialConfiguration RPC failed: addr=127.0.0.1:8300 error="injected error" -2022-09-15T21:40:37.058Z [ERROR] TestInitialConfiguration_cancelled.auto_config: No server successfully responded to the auto-config request -2022-09-15T21:40:37.058Z [DEBUG] TestInitialConfiguration_cancelled.auto_config: making AutoConfig.InitialConfiguration RPC: addr=127.0.0.1:8300 -2022-09-15T21:40:37.059Z [ERROR] TestInitialConfiguration_cancelled.auto_config: AutoConfig.InitialConfiguration RPC failed: addr=127.0.0.1:8300 error="injected error" -2022-09-15T21:40:37.059Z [ERROR] TestInitialConfiguration_cancelled.auto_config: No server successfully responded to the auto-config request -2022-09-15T21:40:37.118Z [INFO] TestInitialConfiguration_cancelled.auto_config: interrupted during initial auto configuration: err="context deadline exceeded" +2021-08-13T15:32:19.941Z [DEBUG] TestInitialConfiguration_cancelled.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestInitialConfiguration_cancelled-auto-config688884778/auto-config.json +2021-08-13T15:32:19.941Z [INFO] TestInitialConfiguration_cancelled.auto_config: retrieving initial agent auto configuration remotely +2021-08-13T15:32:19.963Z [DEBUG] TestInitialConfiguration_cancelled.auto_config: making AutoConfig.InitialConfiguration RPC: addr=127.0.0.1:8300 +2021-08-13T15:32:19.964Z [ERROR] TestInitialConfiguration_cancelled.auto_config: AutoConfig.InitialConfiguration RPC failed: addr=127.0.0.1:8300 error="injected error" +2021-08-13T15:32:19.964Z [ERROR] TestInitialConfiguration_cancelled.auto_config: No server successfully responded to the auto-config request +2021-08-13T15:32:19.964Z [DEBUG] TestInitialConfiguration_cancelled.auto_config: making AutoConfig.InitialConfiguration RPC: addr=127.0.0.1:8300 +2021-08-13T15:32:19.964Z [ERROR] TestInitialConfiguration_cancelled.auto_config: AutoConfig.InitialConfiguration RPC failed: addr=127.0.0.1:8300 error="injected error" +2021-08-13T15:32:19.964Z [ERROR] TestInitialConfiguration_cancelled.auto_config: No server successfully responded to the auto-config request +2021-08-13T15:32:20.029Z [INFO] TestInitialConfiguration_cancelled.auto_config: interrupted during initial auto configuration: err="context deadline exceeded" mock_test.go:241: PASS: RPC(string,string,*net.TCPAddr,string,*pbautoconf.AutoConfigRequest,string) mock_test.go:242: PASS: FindLANServer() --- PASS: TestInitialConfiguration_cancelled (0.10s) === RUN TestInitialConfiguration_restored -2022-09-15T21:40:37.136Z [DEBUG] TestInitialConfiguration_restored.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestInitialConfiguration_restored-auto-config639753048/auto-config.json -2022-09-15T21:40:37.137Z [INFO] TestInitialConfiguration_restored.auto_config: read persisted configuration: path=/tmp/consul-test/TestInitialConfiguration_restored-auto-config639753048/auto-config.json -2022-09-15T21:40:37.137Z [DEBUG] TestInitialConfiguration_restored.auto_config: updating auto-config settings -2022-09-15T21:40:37.166Z [DEBUG] TestInitialConfiguration_restored.auto_config: auto-config settings were persisted to disk +2021-08-13T15:32:20.049Z [DEBUG] TestInitialConfiguration_restored.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestInitialConfiguration_restored-auto-config083820417/auto-config.json +2021-08-13T15:32:20.050Z [INFO] TestInitialConfiguration_restored.auto_config: read persisted configuration: path=/tmp/consul-test/TestInitialConfiguration_restored-auto-config083820417/auto-config.json +2021-08-13T15:32:20.050Z [DEBUG] TestInitialConfiguration_restored.auto_config: updating auto-config settings +2021-08-13T15:32:20.064Z [DEBUG] TestInitialConfiguration_restored.auto_config: auto-config settings were persisted to disk mock_test.go:243: PASS: Prepopulate(string,cache.FetchResult,string,string,string) mock_test.go:243: PASS: Prepopulate(string,cache.FetchResult,string,string,string) mock_test.go:244: PASS: AgentToken() @@ -56768,12 +56745,12 @@ mock_test.go:245: PASS: UpdateAutoTLS([]string,[]string,string,string,bool) --- PASS: TestInitialConfiguration_restored (0.06s) === RUN TestInitialConfiguration_success -2022-09-15T21:40:37.208Z [DEBUG] TestInitialConfiguration_success.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestInitialConfiguration_success-auto-config023691991/auto-config.json -2022-09-15T21:40:37.208Z [INFO] TestInitialConfiguration_success.auto_config: retrieving initial agent auto configuration remotely -2022-09-15T21:40:37.209Z [DEBUG] TestInitialConfiguration_success.auto_config: making AutoConfig.InitialConfiguration RPC: addr=127.0.0.1:8300 -2022-09-15T21:40:37.212Z [DEBUG] TestInitialConfiguration_success.auto_config: AutoConfig.InitialConfiguration RPC was successful -2022-09-15T21:40:37.212Z [DEBUG] TestInitialConfiguration_success.auto_config: updating auto-config settings -2022-09-15T21:40:37.229Z [DEBUG] TestInitialConfiguration_success.auto_config: auto-config settings were persisted to disk +2021-08-13T15:32:20.100Z [DEBUG] TestInitialConfiguration_success.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestInitialConfiguration_success-auto-config849901804/auto-config.json +2021-08-13T15:32:20.100Z [INFO] TestInitialConfiguration_success.auto_config: retrieving initial agent auto configuration remotely +2021-08-13T15:32:20.101Z [DEBUG] TestInitialConfiguration_success.auto_config: making AutoConfig.InitialConfiguration RPC: addr=127.0.0.1:8300 +2021-08-13T15:32:20.102Z [DEBUG] TestInitialConfiguration_success.auto_config: AutoConfig.InitialConfiguration RPC was successful +2021-08-13T15:32:20.102Z [DEBUG] TestInitialConfiguration_success.auto_config: updating auto-config settings +2021-08-13T15:32:20.118Z [DEBUG] TestInitialConfiguration_success.auto_config: auto-config settings were persisted to disk mock_test.go:241: PASS: RPC(string,string,*net.TCPAddr,string,*pbautoconf.AutoConfigRequest,*pbautoconf.AutoConfigResponse) mock_test.go:242: PASS: FindLANServer() mock_test.go:243: PASS: Prepopulate(string,cache.FetchResult,string,string,string) @@ -56782,29 +56759,29 @@ mock_test.go:244: PASS: UpdateAgentToken(string,token.TokenSource) mock_test.go:244: PASS: AgentToken() mock_test.go:245: PASS: UpdateAutoTLS([]string,[]string,string,string,bool) ---- PASS: TestInitialConfiguration_success (0.08s) +--- PASS: TestInitialConfiguration_success (0.05s) === RUN TestInitialConfiguration_retries -2022-09-15T21:40:37.284Z [DEBUG] TestInitialConfiguration_retries.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestInitialConfiguration_retries-auto-config578546762/auto-config.json -2022-09-15T21:40:37.284Z [INFO] TestInitialConfiguration_retries.auto_config: retrieving initial agent auto configuration remotely -2022-09-15T21:40:37.286Z [DEBUG] TestInitialConfiguration_retries.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.0.1:8300 -2022-09-15T21:40:37.286Z [ERROR] TestInitialConfiguration_retries.auto_config: AutoConfig.InitialConfiguration RPC failed: addr=198.18.0.1:8300 error="injected failure" -2022-09-15T21:40:37.286Z [DEBUG] TestInitialConfiguration_retries.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.0.2:8398 -2022-09-15T21:40:37.287Z [ERROR] TestInitialConfiguration_retries.auto_config: AutoConfig.InitialConfiguration RPC failed: addr=198.18.0.2:8398 error="injected failure" -2022-09-15T21:40:37.287Z [DEBUG] TestInitialConfiguration_retries.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.0.3:8399 -2022-09-15T21:40:37.287Z [ERROR] TestInitialConfiguration_retries.auto_config: AutoConfig.InitialConfiguration RPC failed: addr=198.18.0.3:8399 error="injected failure" -2022-09-15T21:40:37.287Z [DEBUG] TestInitialConfiguration_retries.auto_config: making AutoConfig.InitialConfiguration RPC: addr=127.0.0.1:1234 -2022-09-15T21:40:37.288Z [ERROR] TestInitialConfiguration_retries.auto_config: AutoConfig.InitialConfiguration RPC failed: addr=127.0.0.1:1234 error="injected failure" -2022-09-15T21:40:37.288Z [ERROR] TestInitialConfiguration_retries.auto_config: No server successfully responded to the auto-config request -2022-09-15T21:40:37.288Z [DEBUG] TestInitialConfiguration_retries.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.0.1:8300 -2022-09-15T21:40:37.289Z [ERROR] TestInitialConfiguration_retries.auto_config: AutoConfig.InitialConfiguration RPC failed: addr=198.18.0.1:8300 error="injected failure" -2022-09-15T21:40:37.289Z [DEBUG] TestInitialConfiguration_retries.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.0.2:8398 -2022-09-15T21:40:37.289Z [ERROR] TestInitialConfiguration_retries.auto_config: AutoConfig.InitialConfiguration RPC failed: addr=198.18.0.2:8398 error="injected failure" -2022-09-15T21:40:37.289Z [DEBUG] TestInitialConfiguration_retries.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.0.3:8399 -2022-09-15T21:40:37.289Z [ERROR] TestInitialConfiguration_retries.auto_config: AutoConfig.InitialConfiguration RPC failed: addr=198.18.0.3:8399 error="injected failure" -2022-09-15T21:40:37.289Z [DEBUG] TestInitialConfiguration_retries.auto_config: making AutoConfig.InitialConfiguration RPC: addr=127.0.0.1:1234 -2022-09-15T21:40:37.290Z [DEBUG] TestInitialConfiguration_retries.auto_config: AutoConfig.InitialConfiguration RPC was successful -2022-09-15T21:40:37.290Z [DEBUG] TestInitialConfiguration_retries.auto_config: updating auto-config settings -2022-09-15T21:40:37.305Z [DEBUG] TestInitialConfiguration_retries.auto_config: auto-config settings were persisted to disk +2021-08-13T15:32:20.149Z [DEBUG] TestInitialConfiguration_retries.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestInitialConfiguration_retries-auto-config104850779/auto-config.json +2021-08-13T15:32:20.149Z [INFO] TestInitialConfiguration_retries.auto_config: retrieving initial agent auto configuration remotely +2021-08-13T15:32:20.149Z [DEBUG] TestInitialConfiguration_retries.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.0.1:8300 +2021-08-13T15:32:20.150Z [ERROR] TestInitialConfiguration_retries.auto_config: AutoConfig.InitialConfiguration RPC failed: addr=198.18.0.1:8300 error="injected failure" +2021-08-13T15:32:20.150Z [DEBUG] TestInitialConfiguration_retries.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.0.2:8398 +2021-08-13T15:32:20.150Z [ERROR] TestInitialConfiguration_retries.auto_config: AutoConfig.InitialConfiguration RPC failed: addr=198.18.0.2:8398 error="injected failure" +2021-08-13T15:32:20.150Z [DEBUG] TestInitialConfiguration_retries.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.0.3:8399 +2021-08-13T15:32:20.150Z [ERROR] TestInitialConfiguration_retries.auto_config: AutoConfig.InitialConfiguration RPC failed: addr=198.18.0.3:8399 error="injected failure" +2021-08-13T15:32:20.150Z [DEBUG] TestInitialConfiguration_retries.auto_config: making AutoConfig.InitialConfiguration RPC: addr=127.0.0.1:1234 +2021-08-13T15:32:20.151Z [ERROR] TestInitialConfiguration_retries.auto_config: AutoConfig.InitialConfiguration RPC failed: addr=127.0.0.1:1234 error="injected failure" +2021-08-13T15:32:20.151Z [ERROR] TestInitialConfiguration_retries.auto_config: No server successfully responded to the auto-config request +2021-08-13T15:32:20.151Z [DEBUG] TestInitialConfiguration_retries.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.0.1:8300 +2021-08-13T15:32:20.151Z [ERROR] TestInitialConfiguration_retries.auto_config: AutoConfig.InitialConfiguration RPC failed: addr=198.18.0.1:8300 error="injected failure" +2021-08-13T15:32:20.151Z [DEBUG] TestInitialConfiguration_retries.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.0.2:8398 +2021-08-13T15:32:20.151Z [ERROR] TestInitialConfiguration_retries.auto_config: AutoConfig.InitialConfiguration RPC failed: addr=198.18.0.2:8398 error="injected failure" +2021-08-13T15:32:20.151Z [DEBUG] TestInitialConfiguration_retries.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.0.3:8399 +2021-08-13T15:32:20.151Z [ERROR] TestInitialConfiguration_retries.auto_config: AutoConfig.InitialConfiguration RPC failed: addr=198.18.0.3:8399 error="injected failure" +2021-08-13T15:32:20.151Z [DEBUG] TestInitialConfiguration_retries.auto_config: making AutoConfig.InitialConfiguration RPC: addr=127.0.0.1:1234 +2021-08-13T15:32:20.160Z [DEBUG] TestInitialConfiguration_retries.auto_config: AutoConfig.InitialConfiguration RPC was successful +2021-08-13T15:32:20.160Z [DEBUG] TestInitialConfiguration_retries.auto_config: updating auto-config settings +2021-08-13T15:32:20.174Z [DEBUG] TestInitialConfiguration_retries.auto_config: auto-config settings were persisted to disk mock_test.go:241: PASS: RPC(string,string,*net.TCPAddr,string,*pbautoconf.AutoConfigRequest,*pbautoconf.AutoConfigResponse) mock_test.go:241: PASS: RPC(string,string,*net.TCPAddr,string,*pbautoconf.AutoConfigRequest,*pbautoconf.AutoConfigResponse) mock_test.go:241: PASS: RPC(string,string,*net.TCPAddr,string,*pbautoconf.AutoConfigRequest,*pbautoconf.AutoConfigResponse) @@ -56819,12 +56796,12 @@ mock_test.go:245: PASS: UpdateAutoTLS([]string,[]string,string,string,bool) --- PASS: TestInitialConfiguration_retries (0.07s) === RUN TestGoRoutineManagement -2022-09-15T21:40:37.353Z [INFO] TestGoRoutineManagement.auto_config: auto-config started -2022-09-15T21:40:37.364Z [DEBUG] TestGoRoutineManagement.auto_config: stopping auto-config -2022-09-15T21:40:37.364Z [DEBUG] TestGoRoutineManagement.auto_config: auto-config has been stopped -2022-09-15T21:40:37.365Z [INFO] TestGoRoutineManagement.auto_config: auto-config started -2022-09-15T21:40:37.365Z [DEBUG] TestGoRoutineManagement.auto_config: stopping auto-config -2022-09-15T21:40:37.365Z [DEBUG] TestGoRoutineManagement.auto_config: auto-config has been stopped +2021-08-13T15:32:20.218Z [INFO] TestGoRoutineManagement.auto_config: auto-config started +2021-08-13T15:32:20.229Z [DEBUG] TestGoRoutineManagement.auto_config: stopping auto-config +2021-08-13T15:32:20.229Z [DEBUG] TestGoRoutineManagement.auto_config: auto-config has been stopped +2021-08-13T15:32:20.230Z [INFO] TestGoRoutineManagement.auto_config: auto-config started +2021-08-13T15:32:20.230Z [DEBUG] TestGoRoutineManagement.auto_config: stopping auto-config +2021-08-13T15:32:20.230Z [DEBUG] TestGoRoutineManagement.auto_config: auto-config has been stopped mock_test.go:243: PASS: Notify(string,string,*structs.DCSpecificRequest,string,string) mock_test.go:243: PASS: Notify(string,string,*cachetype.ConnectCALeafRequest,string,string) mock_test.go:244: PASS: AgentToken() @@ -56833,17 +56810,17 @@ mock_test.go:245: PASS: AutoEncryptCertNotAfter() --- PASS: TestGoRoutineManagement (0.03s) === RUN TestTokenUpdate -2022-09-15T21:40:37.385Z [DEBUG] TestTokenUpdate.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestTokenUpdate-auto-config737477132/auto-config.json -2022-09-15T21:40:37.385Z [INFO] TestTokenUpdate.auto_config: retrieving initial agent auto configuration remotely -2022-09-15T21:40:37.386Z [DEBUG] TestTokenUpdate.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.0.1:8300 -2022-09-15T21:40:37.387Z [DEBUG] TestTokenUpdate.auto_config: AutoConfig.InitialConfiguration RPC was successful -2022-09-15T21:40:37.387Z [DEBUG] TestTokenUpdate.auto_config: updating auto-config settings -2022-09-15T21:40:37.404Z [DEBUG] TestTokenUpdate.auto_config: auto-config settings were persisted to disk -2022-09-15T21:40:37.418Z [INFO] TestTokenUpdate.auto_config: auto-config started -2022-09-15T21:40:37.419Z [DEBUG] TestTokenUpdate.auto_config: handling a token update event -2022-09-15T21:40:37.419Z [DEBUG] TestTokenUpdate.auto_config: Agent token updated - resetting watches -2022-09-15T21:40:37.419Z [DEBUG] TestTokenUpdate.auto_config: stopping auto-config -2022-09-15T21:40:37.420Z [DEBUG] TestTokenUpdate.auto_config: auto-config has been stopped +2021-08-13T15:32:20.245Z [DEBUG] TestTokenUpdate.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestTokenUpdate-auto-config067299397/auto-config.json +2021-08-13T15:32:20.245Z [INFO] TestTokenUpdate.auto_config: retrieving initial agent auto configuration remotely +2021-08-13T15:32:20.245Z [DEBUG] TestTokenUpdate.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.0.1:8300 +2021-08-13T15:32:20.246Z [DEBUG] TestTokenUpdate.auto_config: AutoConfig.InitialConfiguration RPC was successful +2021-08-13T15:32:20.246Z [DEBUG] TestTokenUpdate.auto_config: updating auto-config settings +2021-08-13T15:32:20.263Z [DEBUG] TestTokenUpdate.auto_config: auto-config settings were persisted to disk +2021-08-13T15:32:20.276Z [INFO] TestTokenUpdate.auto_config: auto-config started +2021-08-13T15:32:20.276Z [DEBUG] TestTokenUpdate.auto_config: handling a token update event +2021-08-13T15:32:20.276Z [DEBUG] TestTokenUpdate.auto_config: Agent token updated - resetting watches +2021-08-13T15:32:20.277Z [DEBUG] TestTokenUpdate.auto_config: stopping auto-config +2021-08-13T15:32:20.277Z [DEBUG] TestTokenUpdate.auto_config: auto-config has been stopped mock_test.go:241: PASS: RPC(string,string,*net.TCPAddr,string,*pbautoconf.AutoConfigRequest,*pbautoconf.AutoConfigResponse) mock_test.go:242: PASS: FindLANServer() mock_test.go:243: PASS: Notify(string,string,*structs.DCSpecificRequest,string,string) @@ -56860,22 +56837,24 @@ mock_test.go:244: PASS: AgentToken() mock_test.go:245: PASS: UpdateAutoTLS([]string,[]string,string,string,bool) mock_test.go:245: PASS: AutoEncryptCertNotAfter() ---- PASS: TestTokenUpdate (0.06s) +--- PASS: TestTokenUpdate (0.05s) === RUN TestRootsUpdate -2022-09-15T21:40:37.438Z [DEBUG] TestRootsUpdate.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestRootsUpdate-auto-config073710075/auto-config.json -2022-09-15T21:40:37.438Z [INFO] TestRootsUpdate.auto_config: retrieving initial agent auto configuration remotely -2022-09-15T21:40:37.440Z [DEBUG] TestRootsUpdate.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.0.1:8300 -2022-09-15T21:40:37.442Z [DEBUG] TestRootsUpdate.auto_config: AutoConfig.InitialConfiguration RPC was successful -2022-09-15T21:40:37.442Z [DEBUG] TestRootsUpdate.auto_config: updating auto-config settings -2022-09-15T21:40:37.479Z [DEBUG] TestRootsUpdate.auto_config: auto-config settings were persisted to disk -2022-09-15T21:40:37.499Z [INFO] TestRootsUpdate.auto_config: auto-config started -2022-09-15T21:40:37.500Z [DEBUG] TestRootsUpdate.auto_config: handling a cache update event: correlation_id=roots -2022-09-15T21:40:37.500Z [DEBUG] TestRootsUpdate.auto_config: roots watch fired - updating CA certificates -2022-09-15T21:40:37.505Z [DEBUG] TestRootsUpdate.auto_config: auto-config settings were persisted to disk -2022-09-15T21:40:37.505Z [DEBUG] TestRootsUpdate.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestRootsUpdate-auto-config073710075/auto-config.json -2022-09-15T21:40:37.509Z [INFO] TestRootsUpdate.auto_config: read persisted configuration: path=/tmp/consul-test/TestRootsUpdate-auto-config073710075/auto-config.json -2022-09-15T21:40:37.509Z [DEBUG] TestRootsUpdate.auto_config: stopping auto-config -2022-09-15T21:40:37.510Z [DEBUG] TestRootsUpdate.auto_config: auto-config has been stopped +2021-08-13T15:32:20.299Z [DEBUG] TestRootsUpdate.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestRootsUpdate-auto-config414215904/auto-config.json +2021-08-13T15:32:20.299Z [INFO] TestRootsUpdate.auto_config: retrieving initial agent auto configuration remotely +2021-08-13T15:32:20.300Z [DEBUG] TestRootsUpdate.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.0.1:8300 +2021-08-13T15:32:20.300Z [DEBUG] TestRootsUpdate.auto_config: AutoConfig.InitialConfiguration RPC was successful +2021-08-13T15:32:20.300Z [DEBUG] TestRootsUpdate.auto_config: updating auto-config settings +2021-08-13T15:32:20.314Z [DEBUG] TestRootsUpdate.auto_config: auto-config settings were persisted to disk +2021-08-13T15:32:20.337Z [INFO] TestRootsUpdate.auto_config: auto-config started +2021-08-13T15:32:20.339Z [DEBUG] TestRootsUpdate.auto_config: handling a cache update event: correlation_id=roots +2021-08-13T15:32:20.339Z [DEBUG] TestRootsUpdate.auto_config: roots watch fired - updating CA certificates +2021-08-13T15:32:20.340Z [DEBUG] TestRootsUpdate.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestRootsUpdate-auto-config414215904/auto-config.json +2021-08-13T15:32:20.341Z [DEBUG] TestRootsUpdate.auto_config: auto-config settings were persisted to disk +2021-08-13T15:32:20.342Z [INFO] TestRootsUpdate.auto_config: read persisted configuration: path=/tmp/consul-test/TestRootsUpdate-auto-config414215904/auto-config.json +2021-08-13T15:32:20.367Z [DEBUG] TestRootsUpdate.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestRootsUpdate-auto-config414215904/auto-config.json +2021-08-13T15:32:20.369Z [INFO] TestRootsUpdate.auto_config: read persisted configuration: path=/tmp/consul-test/TestRootsUpdate-auto-config414215904/auto-config.json +2021-08-13T15:32:20.369Z [DEBUG] TestRootsUpdate.auto_config: stopping auto-config +2021-08-13T15:32:20.370Z [DEBUG] TestRootsUpdate.auto_config: auto-config has been stopped mock_test.go:241: PASS: RPC(string,string,*net.TCPAddr,string,*pbautoconf.AutoConfigRequest,*pbautoconf.AutoConfigResponse) mock_test.go:242: PASS: FindLANServer() mock_test.go:243: PASS: Notify(string,string,*structs.DCSpecificRequest,string,string) @@ -56893,20 +56872,20 @@ mock_test.go:245: PASS: AutoEncryptCertNotAfter() --- PASS: TestRootsUpdate (0.09s) === RUN TestCertUpdate -2022-09-15T21:40:37.527Z [DEBUG] TestCertUpdate.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestCertUpdate-auto-config154283294/auto-config.json -2022-09-15T21:40:37.527Z [INFO] TestCertUpdate.auto_config: retrieving initial agent auto configuration remotely -2022-09-15T21:40:37.528Z [DEBUG] TestCertUpdate.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.0.1:8300 -2022-09-15T21:40:37.529Z [DEBUG] TestCertUpdate.auto_config: AutoConfig.InitialConfiguration RPC was successful -2022-09-15T21:40:37.529Z [DEBUG] TestCertUpdate.auto_config: updating auto-config settings -2022-09-15T21:40:37.552Z [DEBUG] TestCertUpdate.auto_config: auto-config settings were persisted to disk -2022-09-15T21:40:37.567Z [INFO] TestCertUpdate.auto_config: auto-config started -2022-09-15T21:40:37.571Z [DEBUG] TestCertUpdate.auto_config: handling a cache update event: correlation_id=leaf -2022-09-15T21:40:37.571Z [DEBUG] TestCertUpdate.auto_config: leaf certificate watch fired - updating TLS certificate -2022-09-15T21:40:37.573Z [DEBUG] TestCertUpdate.auto_config: auto-config settings were persisted to disk -2022-09-15T21:40:37.573Z [DEBUG] TestCertUpdate.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestCertUpdate-auto-config154283294/auto-config.json -2022-09-15T21:40:37.575Z [INFO] TestCertUpdate.auto_config: read persisted configuration: path=/tmp/consul-test/TestCertUpdate-auto-config154283294/auto-config.json -2022-09-15T21:40:37.575Z [DEBUG] TestCertUpdate.auto_config: stopping auto-config -2022-09-15T21:40:37.575Z [DEBUG] TestCertUpdate.auto_config: auto-config has been stopped +2021-08-13T15:32:20.392Z [DEBUG] TestCertUpdate.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestCertUpdate-auto-config015516863/auto-config.json +2021-08-13T15:32:20.392Z [INFO] TestCertUpdate.auto_config: retrieving initial agent auto configuration remotely +2021-08-13T15:32:20.393Z [DEBUG] TestCertUpdate.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.0.1:8300 +2021-08-13T15:32:20.395Z [DEBUG] TestCertUpdate.auto_config: AutoConfig.InitialConfiguration RPC was successful +2021-08-13T15:32:20.395Z [DEBUG] TestCertUpdate.auto_config: updating auto-config settings +2021-08-13T15:32:20.409Z [DEBUG] TestCertUpdate.auto_config: auto-config settings were persisted to disk +2021-08-13T15:32:20.422Z [INFO] TestCertUpdate.auto_config: auto-config started +2021-08-13T15:32:20.423Z [DEBUG] TestCertUpdate.auto_config: handling a cache update event: correlation_id=leaf +2021-08-13T15:32:20.423Z [DEBUG] TestCertUpdate.auto_config: leaf certificate watch fired - updating TLS certificate +2021-08-13T15:32:20.424Z [DEBUG] TestCertUpdate.auto_config: auto-config settings were persisted to disk +2021-08-13T15:32:20.424Z [DEBUG] TestCertUpdate.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestCertUpdate-auto-config015516863/auto-config.json +2021-08-13T15:32:20.430Z [INFO] TestCertUpdate.auto_config: read persisted configuration: path=/tmp/consul-test/TestCertUpdate-auto-config015516863/auto-config.json +2021-08-13T15:32:20.430Z [DEBUG] TestCertUpdate.auto_config: stopping auto-config +2021-08-13T15:32:20.430Z [DEBUG] TestCertUpdate.auto_config: auto-config has been stopped mock_test.go:241: PASS: RPC(string,string,*net.TCPAddr,string,*pbautoconf.AutoConfigRequest,*pbautoconf.AutoConfigResponse) mock_test.go:242: PASS: FindLANServer() mock_test.go:243: PASS: Notify(string,string,*structs.DCSpecificRequest,string,string) @@ -56922,28 +56901,28 @@ mock_test.go:245: PASS: AutoEncryptCertNotAfter() mock_test.go:245: PASS: UpdateAutoTLS([]string,[]string,string,string,bool) mock_test.go:245: PASS: AutoEncryptCertNotAfter() ---- PASS: TestCertUpdate (0.07s) +--- PASS: TestCertUpdate (0.06s) === RUN TestFallback -2022-09-15T21:40:37.596Z [DEBUG] TestFallback.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestFallback-auto-config434319845/auto-config.json -2022-09-15T21:40:37.596Z [INFO] TestFallback.auto_config: retrieving initial agent auto configuration remotely -2022-09-15T21:40:37.597Z [DEBUG] TestFallback.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.0.1:8300 -2022-09-15T21:40:37.599Z [DEBUG] TestFallback.auto_config: AutoConfig.InitialConfiguration RPC was successful -2022-09-15T21:40:37.599Z [DEBUG] TestFallback.auto_config: updating auto-config settings -2022-09-15T21:40:37.616Z [DEBUG] TestFallback.auto_config: auto-config settings were persisted to disk -2022-09-15T21:40:37.629Z [INFO] TestFallback.auto_config: auto-config started -2022-09-15T21:40:37.638Z [DEBUG] TestFallback.auto_config: handling a cache update event: correlation_id=leaf -2022-09-15T21:40:37.638Z [DEBUG] TestFallback.auto_config: leaf certificate watch fired - updating TLS certificate -2022-09-15T21:40:37.640Z [DEBUG] TestFallback.auto_config: auto-config settings were persisted to disk -2022-09-15T21:40:37.641Z [WARN] TestFallback.auto_config: agent's client certificate has expired -2022-09-15T21:40:37.642Z [DEBUG] TestFallback.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.23.2:8300 -2022-09-15T21:40:37.644Z [DEBUG] TestFallback.auto_config: AutoConfig.InitialConfiguration RPC was successful -2022-09-15T21:40:37.645Z [DEBUG] TestFallback.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestFallback-auto-config434319845/auto-config.json -2022-09-15T21:40:37.646Z [INFO] TestFallback.auto_config: read persisted configuration: path=/tmp/consul-test/TestFallback-auto-config434319845/auto-config.json -2022-09-15T21:40:37.660Z [DEBUG] TestFallback.auto_config: auto-config settings were persisted to disk -2022-09-15T21:40:37.672Z [DEBUG] TestFallback.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestFallback-auto-config434319845/auto-config.json -2022-09-15T21:40:37.674Z [INFO] TestFallback.auto_config: read persisted configuration: path=/tmp/consul-test/TestFallback-auto-config434319845/auto-config.json -2022-09-15T21:40:37.674Z [DEBUG] TestFallback.auto_config: stopping auto-config -2022-09-15T21:40:37.679Z [DEBUG] TestFallback.auto_config: auto-config has been stopped +2021-08-13T15:32:20.449Z [DEBUG] TestFallback.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestFallback-auto-config955972370/auto-config.json +2021-08-13T15:32:20.449Z [INFO] TestFallback.auto_config: retrieving initial agent auto configuration remotely +2021-08-13T15:32:20.449Z [DEBUG] TestFallback.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.0.1:8300 +2021-08-13T15:32:20.450Z [DEBUG] TestFallback.auto_config: AutoConfig.InitialConfiguration RPC was successful +2021-08-13T15:32:20.450Z [DEBUG] TestFallback.auto_config: updating auto-config settings +2021-08-13T15:32:20.467Z [DEBUG] TestFallback.auto_config: auto-config settings were persisted to disk +2021-08-13T15:32:20.480Z [INFO] TestFallback.auto_config: auto-config started +2021-08-13T15:32:20.484Z [DEBUG] TestFallback.auto_config: handling a cache update event: correlation_id=leaf +2021-08-13T15:32:20.484Z [DEBUG] TestFallback.auto_config: leaf certificate watch fired - updating TLS certificate +2021-08-13T15:32:20.485Z [DEBUG] TestFallback.auto_config: auto-config settings were persisted to disk +2021-08-13T15:32:20.485Z [WARN] TestFallback.auto_config: agent's client certificate has expired +2021-08-13T15:32:20.486Z [DEBUG] TestFallback.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.23.2:8300 +2021-08-13T15:32:20.487Z [DEBUG] TestFallback.auto_config: AutoConfig.InitialConfiguration RPC was successful +2021-08-13T15:32:20.488Z [DEBUG] TestFallback.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestFallback-auto-config955972370/auto-config.json +2021-08-13T15:32:20.489Z [INFO] TestFallback.auto_config: read persisted configuration: path=/tmp/consul-test/TestFallback-auto-config955972370/auto-config.json +2021-08-13T15:32:20.506Z [DEBUG] TestFallback.auto_config: auto-config settings were persisted to disk +2021-08-13T15:32:20.515Z [DEBUG] TestFallback.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestFallback-auto-config955972370/auto-config.json +2021-08-13T15:32:20.517Z [INFO] TestFallback.auto_config: read persisted configuration: path=/tmp/consul-test/TestFallback-auto-config955972370/auto-config.json +2021-08-13T15:32:20.517Z [DEBUG] TestFallback.auto_config: stopping auto-config +2021-08-13T15:32:20.517Z [DEBUG] TestFallback.auto_config: auto-config has been stopped mock_test.go:241: PASS: RPC(string,string,*net.TCPAddr,string,*pbautoconf.AutoConfigRequest,*pbautoconf.AutoConfigResponse) mock_test.go:241: PASS: RPC(string,string,*net.TCPAddr,string,*pbautoconf.AutoConfigRequest,*pbautoconf.AutoConfigResponse) mock_test.go:242: PASS: FindLANServer() @@ -56968,24 +56947,24 @@ mock_test.go:245: PASS: AutoEncryptCertExpired() mock_test.go:245: PASS: UpdateAutoTLS([]string,[]string,string,string,bool) mock_test.go:245: PASS: AutoEncryptCertNotAfter() ---- PASS: TestFallback (0.11s) +--- PASS: TestFallback (0.09s) === RUN TestIntroToken -=== RUN TestIntroToken/file-empty === RUN TestIntroToken/config === RUN TestIntroToken/file +=== RUN TestIntroToken/file-empty --- PASS: TestIntroToken (0.00s) - --- PASS: TestIntroToken/file-empty (0.00s) --- PASS: TestIntroToken/config (0.00s) --- PASS: TestIntroToken/file (0.00s) + --- PASS: TestIntroToken/file-empty (0.00s) === RUN TestAutoEncrypt_generateCSR -=== RUN TestAutoEncrypt_generateCSR/dns-sans === RUN TestAutoEncrypt_generateCSR/ip-sans +=== RUN TestAutoEncrypt_generateCSR/dns-sans --- PASS: TestAutoEncrypt_generateCSR (0.00s) - --- PASS: TestAutoEncrypt_generateCSR/dns-sans (0.00s) --- PASS: TestAutoEncrypt_generateCSR/ip-sans (0.00s) + --- PASS: TestAutoEncrypt_generateCSR/dns-sans (0.00s) === RUN TestAutoEncrypt_hosts === RUN TestAutoEncrypt_hosts/split-host-port-error -2022-09-15T21:40:37.691Z [WARN] TestAutoEncrypt_hosts/split-host-port-error: error splitting host address into IP and port: address=this-is-not:a:ip:and_port error="address this-is-not:a:ip:and_port: too many colons in address" +2021-08-13T15:32:20.523Z [WARN] TestAutoEncrypt_hosts/split-host-port-error: error splitting host address into IP and port: address=this-is-not:a:ip:and_port error="address this-is-not:a:ip:and_port: too many colons in address" === RUN TestAutoEncrypt_hosts/router-override === RUN TestAutoEncrypt_hosts/various-addresses --- PASS: TestAutoEncrypt_hosts (0.00s) @@ -56993,12 +56972,12 @@ --- PASS: TestAutoEncrypt_hosts/router-override (0.00s) --- PASS: TestAutoEncrypt_hosts/various-addresses (0.00s) === RUN TestAutoEncrypt_InitialCerts -2022-09-15T21:40:37.694Z [DEBUG] TestAutoEncrypt_InitialCerts: making AutoEncrypt.Sign RPC: addr=198.18.0.1:8300 -2022-09-15T21:40:37.694Z [ERROR] TestAutoEncrypt_InitialCerts: AutoEncrypt.Sign RPC failed: addr=198.18.0.1:8300 error="injected error" -2022-09-15T21:40:37.694Z [DEBUG] TestAutoEncrypt_InitialCerts: making AutoEncrypt.Sign RPC: addr=198.18.0.2:8300 -2022-09-15T21:40:37.695Z [ERROR] TestAutoEncrypt_InitialCerts: AutoEncrypt.Sign RPC failed: addr=198.18.0.2:8300 error="injected error" -2022-09-15T21:40:37.695Z [ERROR] TestAutoEncrypt_InitialCerts: No servers successfully responded to the auto-encrypt request -2022-09-15T21:40:37.695Z [DEBUG] TestAutoEncrypt_InitialCerts: making AutoEncrypt.Sign RPC: addr=198.18.0.1:8300 +2021-08-13T15:32:20.527Z [DEBUG] TestAutoEncrypt_InitialCerts: making AutoEncrypt.Sign RPC: addr=198.18.0.1:8300 +2021-08-13T15:32:20.527Z [ERROR] TestAutoEncrypt_InitialCerts: AutoEncrypt.Sign RPC failed: addr=198.18.0.1:8300 error="injected error" +2021-08-13T15:32:20.527Z [DEBUG] TestAutoEncrypt_InitialCerts: making AutoEncrypt.Sign RPC: addr=198.18.0.2:8300 +2021-08-13T15:32:20.527Z [ERROR] TestAutoEncrypt_InitialCerts: AutoEncrypt.Sign RPC failed: addr=198.18.0.2:8300 error="injected error" +2021-08-13T15:32:20.527Z [ERROR] TestAutoEncrypt_InitialCerts: No servers successfully responded to the auto-encrypt request +2021-08-13T15:32:20.527Z [DEBUG] TestAutoEncrypt_InitialCerts: making AutoEncrypt.Sign RPC: addr=198.18.0.1:8300 mock_test.go:241: PASS: RPC(string,string,*net.TCPAddr,string,*structs.CASignRequest,*structs.SignedResponse) mock_test.go:241: PASS: RPC(string,string,*net.TCPAddr,string,*structs.CASignRequest,*structs.SignedResponse) mock_test.go:241: PASS: RPC(string,string,*net.TCPAddr,string,*structs.CASignRequest,*structs.SignedResponse) @@ -57006,8 +56985,8 @@ mock_test.go:244: PASS: AgentToken() --- PASS: TestAutoEncrypt_InitialCerts (0.01s) === RUN TestAutoEncrypt_InitialConfiguration -2022-09-15T21:40:37.714Z [DEBUG] TestAutoEncrypt_InitialConfiguration.auto_config: making AutoEncrypt.Sign RPC: addr=127.0.0.1:8300 -2022-09-15T21:40:37.715Z [INFO] TestAutoEncrypt_InitialConfiguration.auto_config: automatically upgraded to TLS +2021-08-13T15:32:20.549Z [DEBUG] TestAutoEncrypt_InitialConfiguration.auto_config: making AutoEncrypt.Sign RPC: addr=127.0.0.1:8300 +2021-08-13T15:32:20.550Z [INFO] TestAutoEncrypt_InitialConfiguration.auto_config: automatically upgraded to TLS mock_test.go:241: PASS: RPC(string,string,*net.TCPAddr,string,*structs.CASignRequest,*structs.SignedResponse) mock_test.go:242: PASS: FindLANServer() mock_test.go:243: PASS: Prepopulate(string,cache.FetchResult,string,string,string) @@ -57017,13 +56996,13 @@ mock_test.go:245: PASS: UpdateAutoTLS([]string,[]string,string,string,bool) --- PASS: TestAutoEncrypt_InitialConfiguration (0.02s) === RUN TestAutoEncrypt_TokenUpdate -2022-09-15T21:40:37.736Z [DEBUG] TestAutoEncrypt_TokenUpdate.auto_config: making AutoEncrypt.Sign RPC: addr=198.18.0.1:8300 -2022-09-15T21:40:37.739Z [INFO] TestAutoEncrypt_TokenUpdate.auto_config: automatically upgraded to TLS -2022-09-15T21:40:37.740Z [INFO] TestAutoEncrypt_TokenUpdate.auto_config: auto-config started -2022-09-15T21:40:37.741Z [DEBUG] TestAutoEncrypt_TokenUpdate.auto_config: handling a token update event -2022-09-15T21:40:37.741Z [DEBUG] TestAutoEncrypt_TokenUpdate.auto_config: Agent token updated - resetting watches -2022-09-15T21:40:37.743Z [DEBUG] TestAutoEncrypt_TokenUpdate.auto_config: stopping auto-config -2022-09-15T21:40:37.744Z [DEBUG] TestAutoEncrypt_TokenUpdate.auto_config: auto-config has been stopped +2021-08-13T15:32:20.566Z [DEBUG] TestAutoEncrypt_TokenUpdate.auto_config: making AutoEncrypt.Sign RPC: addr=198.18.0.1:8300 +2021-08-13T15:32:20.567Z [INFO] TestAutoEncrypt_TokenUpdate.auto_config: automatically upgraded to TLS +2021-08-13T15:32:20.568Z [INFO] TestAutoEncrypt_TokenUpdate.auto_config: auto-config started +2021-08-13T15:32:20.568Z [DEBUG] TestAutoEncrypt_TokenUpdate.auto_config: handling a token update event +2021-08-13T15:32:20.568Z [DEBUG] TestAutoEncrypt_TokenUpdate.auto_config: Agent token updated - resetting watches +2021-08-13T15:32:20.569Z [DEBUG] TestAutoEncrypt_TokenUpdate.auto_config: stopping auto-config +2021-08-13T15:32:20.569Z [DEBUG] TestAutoEncrypt_TokenUpdate.auto_config: auto-config has been stopped mock_test.go:241: PASS: RPC(string,string,*net.TCPAddr,string,*structs.CASignRequest,*structs.SignedResponse) mock_test.go:242: PASS: FindLANServer() mock_test.go:243: PASS: Notify(string,string,*structs.DCSpecificRequest,string,string) @@ -57040,15 +57019,15 @@ mock_test.go:244: PASS: AgentToken() mock_test.go:245: PASS: UpdateAutoTLS([]string,[]string,string,string,bool) mock_test.go:245: PASS: AutoEncryptCertNotAfter() ---- PASS: TestAutoEncrypt_TokenUpdate (0.03s) +--- PASS: TestAutoEncrypt_TokenUpdate (0.02s) === RUN TestAutoEncrypt_RootsUpdate -2022-09-15T21:40:37.762Z [DEBUG] TestAutoEncrypt_RootsUpdate.auto_config: making AutoEncrypt.Sign RPC: addr=198.18.0.1:8300 -2022-09-15T21:40:37.763Z [INFO] TestAutoEncrypt_RootsUpdate.auto_config: automatically upgraded to TLS -2022-09-15T21:40:37.764Z [INFO] TestAutoEncrypt_RootsUpdate.auto_config: auto-config started -2022-09-15T21:40:37.765Z [DEBUG] TestAutoEncrypt_RootsUpdate.auto_config: handling a cache update event: correlation_id=roots -2022-09-15T21:40:37.765Z [DEBUG] TestAutoEncrypt_RootsUpdate.auto_config: roots watch fired - updating CA certificates -2022-09-15T21:40:37.766Z [DEBUG] TestAutoEncrypt_RootsUpdate.auto_config: stopping auto-config -2022-09-15T21:40:37.766Z [DEBUG] TestAutoEncrypt_RootsUpdate.auto_config: auto-config has been stopped +2021-08-13T15:32:20.591Z [DEBUG] TestAutoEncrypt_RootsUpdate.auto_config: making AutoEncrypt.Sign RPC: addr=198.18.0.1:8300 +2021-08-13T15:32:20.593Z [INFO] TestAutoEncrypt_RootsUpdate.auto_config: automatically upgraded to TLS +2021-08-13T15:32:20.593Z [INFO] TestAutoEncrypt_RootsUpdate.auto_config: auto-config started +2021-08-13T15:32:20.595Z [DEBUG] TestAutoEncrypt_RootsUpdate.auto_config: handling a cache update event: correlation_id=roots +2021-08-13T15:32:20.595Z [DEBUG] TestAutoEncrypt_RootsUpdate.auto_config: roots watch fired - updating CA certificates +2021-08-13T15:32:20.595Z [DEBUG] TestAutoEncrypt_RootsUpdate.auto_config: stopping auto-config +2021-08-13T15:32:20.595Z [DEBUG] TestAutoEncrypt_RootsUpdate.auto_config: auto-config has been stopped mock_test.go:241: PASS: RPC(string,string,*net.TCPAddr,string,*structs.CASignRequest,*structs.SignedResponse) mock_test.go:242: PASS: FindLANServer() mock_test.go:243: PASS: Notify(string,string,*structs.DCSpecificRequest,string,string) @@ -57066,13 +57045,13 @@ mock_test.go:245: PASS: AutoEncryptCertNotAfter() --- PASS: TestAutoEncrypt_RootsUpdate (0.02s) === RUN TestAutoEncrypt_CertUpdate -2022-09-15T21:40:37.791Z [DEBUG] TestAutoEncrypt_CertUpdate.auto_config: making AutoEncrypt.Sign RPC: addr=198.18.0.1:8300 -2022-09-15T21:40:37.792Z [INFO] TestAutoEncrypt_CertUpdate.auto_config: automatically upgraded to TLS -2022-09-15T21:40:37.793Z [INFO] TestAutoEncrypt_CertUpdate.auto_config: auto-config started -2022-09-15T21:40:37.794Z [DEBUG] TestAutoEncrypt_CertUpdate.auto_config: handling a cache update event: correlation_id=leaf -2022-09-15T21:40:37.794Z [DEBUG] TestAutoEncrypt_CertUpdate.auto_config: leaf certificate watch fired - updating TLS certificate -2022-09-15T21:40:37.794Z [DEBUG] TestAutoEncrypt_CertUpdate.auto_config: stopping auto-config -2022-09-15T21:40:37.794Z [DEBUG] TestAutoEncrypt_CertUpdate.auto_config: auto-config has been stopped +2021-08-13T15:32:20.615Z [DEBUG] TestAutoEncrypt_CertUpdate.auto_config: making AutoEncrypt.Sign RPC: addr=198.18.0.1:8300 +2021-08-13T15:32:20.616Z [INFO] TestAutoEncrypt_CertUpdate.auto_config: automatically upgraded to TLS +2021-08-13T15:32:20.616Z [INFO] TestAutoEncrypt_CertUpdate.auto_config: auto-config started +2021-08-13T15:32:20.618Z [DEBUG] TestAutoEncrypt_CertUpdate.auto_config: handling a cache update event: correlation_id=leaf +2021-08-13T15:32:20.618Z [DEBUG] TestAutoEncrypt_CertUpdate.auto_config: leaf certificate watch fired - updating TLS certificate +2021-08-13T15:32:20.618Z [DEBUG] TestAutoEncrypt_CertUpdate.auto_config: stopping auto-config +2021-08-13T15:32:20.618Z [DEBUG] TestAutoEncrypt_CertUpdate.auto_config: auto-config has been stopped mock_test.go:241: PASS: RPC(string,string,*net.TCPAddr,string,*structs.CASignRequest,*structs.SignedResponse) mock_test.go:242: PASS: FindLANServer() mock_test.go:243: PASS: Notify(string,string,*structs.DCSpecificRequest,string,string) @@ -57088,17 +57067,17 @@ mock_test.go:245: PASS: AutoEncryptCertNotAfter() mock_test.go:245: PASS: UpdateAutoTLSCert(string,string) mock_test.go:245: PASS: AutoEncryptCertNotAfter() ---- PASS: TestAutoEncrypt_CertUpdate (0.03s) +--- PASS: TestAutoEncrypt_CertUpdate (0.02s) === RUN TestAutoEncrypt_Fallback -2022-09-15T21:40:37.820Z [DEBUG] TestAutoEncrypt_Fallback.auto_config: making AutoEncrypt.Sign RPC: addr=198.18.0.1:8300 -2022-09-15T21:40:37.822Z [INFO] TestAutoEncrypt_Fallback.auto_config: automatically upgraded to TLS -2022-09-15T21:40:37.823Z [INFO] TestAutoEncrypt_Fallback.auto_config: auto-config started -2022-09-15T21:40:37.827Z [DEBUG] TestAutoEncrypt_Fallback.auto_config: handling a cache update event: correlation_id=leaf -2022-09-15T21:40:37.827Z [DEBUG] TestAutoEncrypt_Fallback.auto_config: leaf certificate watch fired - updating TLS certificate -2022-09-15T21:40:37.827Z [WARN] TestAutoEncrypt_Fallback.auto_config: agent's client certificate has expired -2022-09-15T21:40:37.828Z [DEBUG] TestAutoEncrypt_Fallback.auto_config: making AutoEncrypt.Sign RPC: addr=198.18.23.2:8300 -2022-09-15T21:40:37.829Z [DEBUG] TestAutoEncrypt_Fallback.auto_config: stopping auto-config -2022-09-15T21:40:37.829Z [DEBUG] TestAutoEncrypt_Fallback.auto_config: auto-config has been stopped +2021-08-13T15:32:20.639Z [DEBUG] TestAutoEncrypt_Fallback.auto_config: making AutoEncrypt.Sign RPC: addr=198.18.0.1:8300 +2021-08-13T15:32:20.640Z [INFO] TestAutoEncrypt_Fallback.auto_config: automatically upgraded to TLS +2021-08-13T15:32:20.641Z [INFO] TestAutoEncrypt_Fallback.auto_config: auto-config started +2021-08-13T15:32:20.644Z [DEBUG] TestAutoEncrypt_Fallback.auto_config: handling a cache update event: correlation_id=leaf +2021-08-13T15:32:20.644Z [DEBUG] TestAutoEncrypt_Fallback.auto_config: leaf certificate watch fired - updating TLS certificate +2021-08-13T15:32:20.645Z [WARN] TestAutoEncrypt_Fallback.auto_config: agent's client certificate has expired +2021-08-13T15:32:20.645Z [DEBUG] TestAutoEncrypt_Fallback.auto_config: making AutoEncrypt.Sign RPC: addr=198.18.23.2:8300 +2021-08-13T15:32:20.647Z [DEBUG] TestAutoEncrypt_Fallback.auto_config: stopping auto-config +2021-08-13T15:32:20.647Z [DEBUG] TestAutoEncrypt_Fallback.auto_config: auto-config has been stopped mock_test.go:241: PASS: RPC(string,string,*net.TCPAddr,string,*structs.CASignRequest,*structs.SignedResponse) mock_test.go:241: PASS: RPC(string,string,*net.TCPAddr,string,*structs.CASignRequest,*structs.SignedResponse) mock_test.go:242: PASS: FindLANServer() @@ -57123,37 +57102,37 @@ mock_test.go:245: PASS: AutoEncryptCertExpired() mock_test.go:245: PASS: UpdateAutoTLS([]string,[]string,string,string,bool) mock_test.go:245: PASS: AutoEncryptCertNotAfter() ---- PASS: TestAutoEncrypt_Fallback (0.04s) +--- PASS: TestAutoEncrypt_Fallback (0.03s) === RUN TestTranslateConfig --- PASS: TestTranslateConfig (0.00s) === RUN TestCArootsTranslation --- PASS: TestCArootsTranslation (0.00s) PASS -ok github.com/hashicorp/consul/agent/auto-config 0.905s +ok github.com/hashicorp/consul/agent/auto-config 0.794s === RUN TestBuildAndValidate_HTTPMaxConnsPerClientExceedsRLimit === PAUSE TestBuildAndValidate_HTTPMaxConnsPerClientExceedsRLimit === RUN TestBuilder_validateEnterpriseConfigKeys +=== RUN TestBuilder_validateEnterpriseConfigKeys/autopilot.disable_upgrade_migration +=== RUN TestBuilder_validateEnterpriseConfigKeys/dns_config.prefer_namespace +=== RUN TestBuilder_validateEnterpriseConfigKeys/multi === RUN TestBuilder_validateEnterpriseConfigKeys/non_voting_server -=== RUN TestBuilder_validateEnterpriseConfigKeys/segment +=== RUN TestBuilder_validateEnterpriseConfigKeys/segments === RUN TestBuilder_validateEnterpriseConfigKeys/autopilot.redundancy_zone_tag === RUN TestBuilder_validateEnterpriseConfigKeys/autopilot.upgrade_version_tag -=== RUN TestBuilder_validateEnterpriseConfigKeys/segments -=== RUN TestBuilder_validateEnterpriseConfigKeys/autopilot.disable_upgrade_migration -=== RUN TestBuilder_validateEnterpriseConfigKeys/dns_config.prefer_namespace === RUN TestBuilder_validateEnterpriseConfigKeys/acl.msp_disable_bootstrap === RUN TestBuilder_validateEnterpriseConfigKeys/acl.tokens.managed_service_provider -=== RUN TestBuilder_validateEnterpriseConfigKeys/multi +=== RUN TestBuilder_validateEnterpriseConfigKeys/segment --- PASS: TestBuilder_validateEnterpriseConfigKeys (0.00s) + --- PASS: TestBuilder_validateEnterpriseConfigKeys/autopilot.disable_upgrade_migration (0.00s) + --- PASS: TestBuilder_validateEnterpriseConfigKeys/dns_config.prefer_namespace (0.00s) + --- PASS: TestBuilder_validateEnterpriseConfigKeys/multi (0.00s) --- PASS: TestBuilder_validateEnterpriseConfigKeys/non_voting_server (0.00s) - --- PASS: TestBuilder_validateEnterpriseConfigKeys/segment (0.00s) + --- PASS: TestBuilder_validateEnterpriseConfigKeys/segments (0.00s) --- PASS: TestBuilder_validateEnterpriseConfigKeys/autopilot.redundancy_zone_tag (0.00s) --- PASS: TestBuilder_validateEnterpriseConfigKeys/autopilot.upgrade_version_tag (0.00s) - --- PASS: TestBuilder_validateEnterpriseConfigKeys/segments (0.00s) - --- PASS: TestBuilder_validateEnterpriseConfigKeys/autopilot.disable_upgrade_migration (0.00s) - --- PASS: TestBuilder_validateEnterpriseConfigKeys/dns_config.prefer_namespace (0.00s) --- PASS: TestBuilder_validateEnterpriseConfigKeys/acl.msp_disable_bootstrap (0.00s) --- PASS: TestBuilder_validateEnterpriseConfigKeys/acl.tokens.managed_service_provider (0.00s) - --- PASS: TestBuilder_validateEnterpriseConfigKeys/multi (0.00s) + --- PASS: TestBuilder_validateEnterpriseConfigKeys/segment (0.00s) === RUN TestLoad --- PASS: TestLoad (0.02s) === RUN TestShouldParseFile @@ -57559,94 +57538,94 @@ === RUN TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_authorizer_invalid_claim_assertion === RUN TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_authorizer_ok === RUN TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_authorizer_ok ---- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases (9.00s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-advertise (0.03s) +--- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases (7.15s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-advertise (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-advertise-wan (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-advertise_and_-advertise-wan (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-bind (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-bootstrap (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-advertise_and_-advertise-wan (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-bind (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-bootstrap (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-bootstrap-expect (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-client (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-config-dir (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-config-file_json (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-client (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-config-dir (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-config-file_json (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-config-file_hcl_and_json (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-data-dir_empty (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-data-dir_non-directory (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-data-dir_non-directory (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-datacenter (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-datacenter_empty (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-datacenter_empty (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-dev (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-disable-host-node-id (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-disable-keyring-file (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-dns-port (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-domain (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-alt-domain (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-disable-keyring-file (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-dns-port (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-domain (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-alt-domain (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-alt-domain_can't_be_prefixed_by_DC (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-alt-domain_can't_be_prefixed_by_service (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-alt-domain_can_be_prefixed_by_non-keywords (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-alt-domain_can_be_prefixed_by_non-keywords (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-alt-domain_can't_be_prefixed_by_DC#01 (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-enable-script-checks (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-encrypt (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-encrypt (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-config-format_disabled,_skip_unknown_files (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-config-format=json (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-config-format=hcl (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-http-port (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-join (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-join-wan (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-log-level (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-log-json (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-http-port (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-join (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-join-wan (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-log-level (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-log-json (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:-log-rotate-max-files (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:-log-rotate-max-files (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-node (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:-log-rotate-max-files (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-node (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-node-id (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-node-meta (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-non-voting-server (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-non-voting-server (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-pid-file (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:-primary-gateway (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:-primary-gateway (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:-primary-gateway (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-protocol (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-raft-protocol (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-recursor (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-raft-protocol (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-recursor (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-rejoin (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-retry-interval (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-retry-interval-wan (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-retry-join (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-retry-join-wan (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-retry-join-wan (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-retry-max (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-retry-max-wan (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-retry-max-wan (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-serf-lan-bind (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-serf-lan-port (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-serf-lan-port (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-serf-wan-bind (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-serf-wan-port (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-server (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-server-port (0.07s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-serf-wan-port (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-server (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-server-port (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-syslog (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-ui (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-ui-dir (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-ui (0.05s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-ui-dir (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-ui-content-path (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bind_addr_any_v4 (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bind_addr_any_v4 (0.06s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bind_addr_any_v6 (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bind_addr_any_v6 (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bind_addr_any_and_advertise_set_should_not_detect (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bind_addr_any_v4 (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bind_addr_any_v4 (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bind_addr_any_v6 (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bind_addr_any_v6 (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bind_addr_any_and_advertise_set_should_not_detect (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bind_addr_any_and_advertise_set_should_not_detect (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:client_addr_and_ports_==_0 (0.04s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:client_addr_and_ports_==_0 (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:client_addr_and_ports_<_0 (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:client_addr_and_ports_<_0 (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:client_addr_and_ports_>_0 (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:client_addr_and_ports_>_0 (0.04s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:client_addr_and_ports_>_0 (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:client_addr,_addresses_and_ports_==_0 (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:client_addr,_addresses_and_ports_==_0 (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:client_addr,_addresses_and_ports_==_0 (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:client_addr,_addresses_and_ports_<_0 (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:client_addr,_addresses_and_ports_<_0 (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:client_addr,_addresses_and_ports (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:client_addr,_addresses_and_ports (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:client_template_and_ports (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:client_template_and_ports (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:client_template_and_ports (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:client,_address_template_and_ports (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:client,_address_template_and_ports (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:advertise_address_lan_template (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:advertise_address_lan_template (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:advertise_address_lan_template (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:advertise_address_wan_template (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:advertise_address_wan_template (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:advertise_address_lan_with_ports (0.03s) @@ -57654,114 +57633,114 @@ --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:advertise_address_wan_with_ports (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:advertise_address_wan_with_ports (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:allow_disabling_serf_wan_port (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:allow_disabling_serf_wan_port (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:allow_disabling_serf_wan_port (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:serf_bind_address_lan_template (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:serf_bind_address_lan_template (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:serf_bind_address_wan_template (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:serf_bind_address_wan_template (0.05s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:serf_bind_address_wan_template (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:dns_recursor_templates_with_deduplication (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:dns_recursor_templates_with_deduplication (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:dns_recursor_templates_with_deduplication (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:start_join_address_template (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:start_join_address_template (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:start_join_wan_address_template (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:start_join_wan_address_template (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:start_join_wan_address_template (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:retry_join_address_template (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:retry_join_address_template (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:retry_join_wan_address_template (0.05s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:retry_join_address_template (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:retry_join_address_template (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:retry_join_wan_address_template (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:retry_join_wan_address_template (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:min/max_ports_for_dynamic_exposed_listeners (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:min/max_ports_for_dynamic_exposed_listeners (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/defaults_for_dynamic_exposed_listeners (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:precedence:_merge_order (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:precedence:_merge_order (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:precedence:_flag_before_file (0.06s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:min/max_ports_for_dynamic_exposed_listeners (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:min/max_ports_for_dynamic_exposed_listeners (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/defaults_for_dynamic_exposed_listeners (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:precedence:_merge_order (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:precedence:_merge_order (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:precedence:_flag_before_file (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:precedence:_flag_before_file (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:raft_performance_scaling (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:raft_performance_scaling (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/Serf_Allowed_CIDRS_LAN,_multiple_values_from_flags (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:Serf_Allowed_CIDRS_LAN/WAN,_multiple_values_from_HCL/JSON (0.05s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/Serf_Allowed_CIDRS_LAN,_multiple_values_from_flags (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:Serf_Allowed_CIDRS_LAN/WAN,_multiple_values_from_HCL/JSON (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:Serf_Allowed_CIDRS_LAN/WAN,_multiple_values_from_HCL/JSON (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/Serf_Allowed_CIDRS_WAN,_multiple_values_from_flags (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/Serf_Allowed_CIDRS_WAN,_multiple_values_from_flags (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:invalid_input (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:invalid_input (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:datacenter_is_lower-cased (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:invalid_input (0.00s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:datacenter_is_lower-cased (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:datacenter_is_lower-cased (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:acl_datacenter_is_lower-cased (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:acl_datacenter_is_lower-cased (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:acl_replication_token_enables_acl_replication (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:acl_replication_token_enables_acl_replication (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:acl_enforce_version_8_is_deprecated (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:acl_replication_token_enables_acl_replication (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:acl_enforce_version_8_is_deprecated (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:acl_enforce_version_8_is_deprecated (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:advertise_address_detect_fails_v4 (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:advertise_address_detect_fails_v4 (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:advertise_address_detect_fails_v4 (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:advertise_address_detect_none_v4 (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:advertise_address_detect_none_v4 (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:advertise_address_detect_multiple_v4 (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:advertise_address_detect_multiple_v4 (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:advertise_address_detect_multiple_v4 (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:advertise_address_detect_fails_v6 (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:advertise_address_detect_fails_v6 (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:advertise_address_detect_fails_v6 (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:advertise_address_detect_none_v6 (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:advertise_address_detect_none_v6 (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:advertise_address_detect_multiple_v6 (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:advertise_address_detect_none_v6 (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:advertise_address_detect_none_v6 (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:advertise_address_detect_multiple_v6 (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:advertise_address_detect_multiple_v6 (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/ae_interval_invalid_==_0 (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/ae_interval_invalid_<_0 (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/ae_interval_invalid_==_0 (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/ae_interval_invalid_<_0 (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:acl_datacenter_invalid (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:acl_datacenter_invalid (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:autopilot.max_trailing_logs_invalid (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:autopilot.max_trailing_logs_invalid (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:autopilot.max_trailing_logs_invalid (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:autopilot.max_trailing_logs_invalid (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bind_addr_cannot_be_empty (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bind_addr_cannot_be_empty (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bind_addr_cannot_be_empty (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bind_addr_does_not_allow_multiple_addresses (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bind_addr_does_not_allow_multiple_addresses (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bind_addr_cannot_be_a_unix_socket (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bind_addr_cannot_be_a_unix_socket (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bootstrap_without_server (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bootstrap_without_server (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bootstrap-expect_without_server (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bootstrap-expect_without_server (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bootstrap-expect_invalid (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bootstrap-expect_invalid (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bootstrap-expect_and_dev_mode (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bootstrap-expect_and_dev_mode (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bootstrap-expect_without_server (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bootstrap-expect_without_server (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bootstrap-expect_invalid (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bootstrap-expect_invalid (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bootstrap-expect_and_dev_mode (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bootstrap-expect_and_dev_mode (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bootstrap-expect_and_bootstrap (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bootstrap-expect_and_bootstrap (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bootstrap-expect=1_equals_bootstrap (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bootstrap-expect=1_equals_bootstrap (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bootstrap-expect=2_warning (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bootstrap-expect=2_warning (0.08s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bootstrap-expect_>_2_but_even_warning (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bootstrap-expect_>_2_but_even_warning (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:client_mode_sets_LeaveOnTerm_and_SkipLeaveOnInt_correctly (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:client_mode_sets_LeaveOnTerm_and_SkipLeaveOnInt_correctly (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:client_does_not_allow_socket (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:client_does_not_allow_socket (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:datacenter_invalid (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:datacenter_invalid (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bootstrap-expect=1_equals_bootstrap (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bootstrap-expect=2_warning (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bootstrap-expect=2_warning (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bootstrap-expect_>_2_but_even_warning (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bootstrap-expect_>_2_but_even_warning (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:client_mode_sets_LeaveOnTerm_and_SkipLeaveOnInt_correctly (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:client_mode_sets_LeaveOnTerm_and_SkipLeaveOnInt_correctly (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:client_does_not_allow_socket (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:client_does_not_allow_socket (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:datacenter_invalid (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:datacenter_invalid (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:dns_does_not_allow_socket (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:dns_does_not_allow_socket (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:ui_and_ui_dir (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:dns_does_not_allow_socket (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:ui_and_ui_dir (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:ui_and_ui_dir (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:advertise_addr_any (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:advertise_addr_any (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:advertise_addr_wan_any (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:advertise_addr_wan_any (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:recursors_any (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:recursors_any (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:dns_config.udp_answer_limit_invalid (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:dns_config.udp_answer_limit_invalid (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:dns_config.a_record_limit_invalid (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:recursors_any (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:dns_config.udp_answer_limit_invalid (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:dns_config.udp_answer_limit_invalid (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:dns_config.a_record_limit_invalid (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:dns_config.a_record_limit_invalid (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:performance.raft_multiplier_<_0 (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:performance.raft_multiplier_<_0 (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:performance.raft_multiplier_<_0 (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:performance.raft_multiplier_<_0 (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:performance.raft_multiplier_==_0 (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:performance.raft_multiplier_==_0 (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:performance.raft_multiplier_==_0 (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:performance.raft_multiplier_>_10 (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:performance.raft_multiplier_>_10 (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:performance.raft_multiplier_>_10 (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/node_name_invalid (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:node_meta_key_too_long (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:node_meta_key_too_long (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:node_meta_key_too_long (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:node_meta_value_too_long (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:node_meta_value_too_long (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:node_meta_value_too_long (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:node_meta_too_many_keys (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:node_meta_too_many_keys (0.02s) @@ -57772,128 +57751,128 @@ --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:unique_listeners_http_vs_https (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:unique_listeners_http_vs_https (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:unique_advertise_addresses_HTTP_vs_RPC (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:unique_advertise_addresses_HTTP_vs_RPC (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:unique_advertise_addresses_RPC_vs_Serf_LAN (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:unique_advertise_addresses_HTTP_vs_RPC (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:unique_advertise_addresses_RPC_vs_Serf_LAN (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:unique_advertise_addresses_RPC_vs_Serf_LAN (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:unique_advertise_addresses_RPC_vs_Serf_WAN (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:unique_advertise_addresses_RPC_vs_Serf_WAN (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:unique_advertise_addresses_RPC_vs_Serf_WAN (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:unique_advertise_addresses_RPC_vs_Serf_WAN (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:http_use_cache_defaults_to_true (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:http_use_cache_defaults_to_true (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:http_use_cache_is_enabled_when_true (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:http_use_cache_defaults_to_true (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:http_use_cache_is_enabled_when_true (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:http_use_cache_is_enabled_when_true (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:http_use_cache_is_disabled_when_false (0.07s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:http_use_cache_is_disabled_when_false (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:sidecar_service_can't_have_ID (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:sidecar_service_can't_have_ID (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:http_use_cache_is_disabled_when_false (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:http_use_cache_is_disabled_when_false (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:sidecar_service_can't_have_ID (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:sidecar_service_can't_have_ID (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:sidecar_service_can't_have_nested_sidecar (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:sidecar_service_can't_have_nested_sidecar (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:telemetry.prefix_filter_cannot_be_empty (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:telemetry.prefix_filter_cannot_be_empty (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:telemetry.prefix_filter_must_start_with_+_or_- (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:telemetry.prefix_filter_cannot_be_empty (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:telemetry.prefix_filter_must_start_with_+_or_- (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:telemetry.prefix_filter_must_start_with_+_or_- (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:encrypt_has_invalid_key (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:encrypt_has_invalid_key (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:multiple_check_files (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:multiple_check_files (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:multiple_check_files (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:grpc_check (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:grpc_check (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:alias_check_with_no_node (0.06s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:alias_check_with_no_node (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:multiple_service_files (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:multiple_service_files (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:service_with_wrong_meta:_too_long_key (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:service_with_wrong_meta:_too_long_key (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:service_with_wrong_meta:_too_long_value (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:service_with_wrong_meta:_too_long_value (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:grpc_check (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:grpc_check (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:alias_check_with_no_node (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:alias_check_with_no_node (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:multiple_service_files (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:multiple_service_files (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:service_with_wrong_meta:_too_long_key (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:service_with_wrong_meta:_too_long_key (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:service_with_wrong_meta:_too_long_value (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:service_with_wrong_meta:_too_long_value (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:service_with_wrong_meta:_too_many_meta (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:service_with_wrong_meta:_too_many_meta (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:service_with_wrong_meta:_too_many_meta (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:translated_keys (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:translated_keys (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:ignore_snapshot_agent_sub-object (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:translated_keys (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:ignore_snapshot_agent_sub-object (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:ignore_snapshot_agent_sub-object (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:service.connectsidecar_service_with_checks_and_upstreams (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:service.connectsidecar_service_with_checks_and_upstreams (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:services.connect.sidecar_service_with_checks_and_upstreams (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:services.connect.sidecar_service_with_checks_and_upstreams (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:verify_server_hostname_implies_verify_outgoing (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:verify_server_hostname_implies_verify_outgoing (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_encrypt.allow_tls_works_implies_connect (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_encrypt.allow_tls_works_implies_connect (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_encrypt.allow_tls_works_with_verify_incoming (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_encrypt.allow_tls_works_with_verify_incoming (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_encrypt.allow_tls_works_with_verify_incoming_rpc (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_encrypt.allow_tls_works_with_verify_incoming_rpc (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_encrypt.allow_tls_warns_without_verify_incoming_or_verify_incoming_rpc (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:service.connectsidecar_service_with_checks_and_upstreams (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:service.connectsidecar_service_with_checks_and_upstreams (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:services.connect.sidecar_service_with_checks_and_upstreams (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:services.connect.sidecar_service_with_checks_and_upstreams (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:verify_server_hostname_implies_verify_outgoing (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:verify_server_hostname_implies_verify_outgoing (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_encrypt.allow_tls_works_implies_connect (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_encrypt.allow_tls_works_implies_connect (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_encrypt.allow_tls_works_with_verify_incoming (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_encrypt.allow_tls_works_with_verify_incoming (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_encrypt.allow_tls_works_with_verify_incoming_rpc (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_encrypt.allow_tls_works_with_verify_incoming_rpc (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_encrypt.allow_tls_warns_without_verify_incoming_or_verify_incoming_rpc (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_encrypt.allow_tls_warns_without_verify_incoming_or_verify_incoming_rpc (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_encrypt.allow_tls_errors_in_client_mode (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_encrypt.allow_tls_errors_in_client_mode (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_encrypt.tls_errors_in_server_mode (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_encrypt.allow_tls_errors_in_client_mode (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_encrypt.tls_errors_in_server_mode (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_encrypt.tls_errors_in_server_mode (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:test_connect_vault_provider_configuration (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:test_connect_vault_provider_configuration (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:Connect_AWS_CA_provider_configuration (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:Connect_AWS_CA_provider_configuration (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:Connect_AWS_CA_provider_TTL_validation (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:Connect_AWS_CA_provider_TTL_validation (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:Connect_AWS_CA_provider_EC_key_length_validation (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:Connect_AWS_CA_provider_EC_key_length_validation (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:connect.enable_mesh_gateway_wan_federation_requires_connect.enabled (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:test_connect_vault_provider_configuration (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:test_connect_vault_provider_configuration (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:Connect_AWS_CA_provider_configuration (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:Connect_AWS_CA_provider_configuration (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:Connect_AWS_CA_provider_TTL_validation (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:Connect_AWS_CA_provider_TTL_validation (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:Connect_AWS_CA_provider_EC_key_length_validation (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:Connect_AWS_CA_provider_EC_key_length_validation (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:connect.enable_mesh_gateway_wan_federation_requires_connect.enabled (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:connect.enable_mesh_gateway_wan_federation_requires_connect.enabled (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:connect.enable_mesh_gateway_wan_federation_cannot_use_-join-wan (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:connect.enable_mesh_gateway_wan_federation_cannot_use_-join-wan (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:connect.enable_mesh_gateway_wan_federation_cannot_use_-retry-join-wan (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:connect.enable_mesh_gateway_wan_federation_cannot_use_-retry-join-wan (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:connect.enable_mesh_gateway_wan_federation_cannot_use_-retry-join-wan (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:connect.enable_mesh_gateway_wan_federation_requires_server_mode (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:connect.enable_mesh_gateway_wan_federation_requires_server_mode (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:connect.enable_mesh_gateway_wan_federation_requires_server_mode (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:connect.enable_mesh_gateway_wan_federation_requires_no_slashes_in_node_names (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:connect.enable_mesh_gateway_wan_federation_requires_no_slashes_in_node_names (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:primary_gateways_requires_server_mode (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:primary_gateways_requires_server_mode (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:primary_gateways_only_works_in_a_secondary_datacenter (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:primary_gateways_only_works_in_a_secondary_datacenter (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:connect.enable_mesh_gateway_wan_federation_in_secondary_with_primary_gateways_configured (0.08s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:connect.enable_mesh_gateway_wan_federation_in_secondary_with_primary_gateways_configured (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:connect.enable_mesh_gateway_wan_federation_requires_no_slashes_in_node_names (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:primary_gateways_requires_server_mode (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:primary_gateways_requires_server_mode (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:primary_gateways_only_works_in_a_secondary_datacenter (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:primary_gateways_only_works_in_a_secondary_datacenter (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:connect.enable_mesh_gateway_wan_federation_in_secondary_with_primary_gateways_configured (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:connect.enable_mesh_gateway_wan_federation_in_secondary_with_primary_gateways_configured (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:ConfigEntry_bootstrap_doesn't_parse (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:ConfigEntry_bootstrap_doesn't_parse (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:ConfigEntry_bootstrap_unknown_kind (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:ConfigEntry_bootstrap_unknown_kind (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:ConfigEntry_bootstrap_invalid_service-defaults (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:ConfigEntry_bootstrap_invalid_service-defaults (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:ConfigEntry_bootstrap_proxy-defaults_(snake-case) (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:ConfigEntry_bootstrap_proxy-defaults_(snake-case) (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:ConfigEntry_bootstrap_unknown_kind (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:ConfigEntry_bootstrap_unknown_kind (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:ConfigEntry_bootstrap_invalid_service-defaults (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:ConfigEntry_bootstrap_invalid_service-defaults (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:ConfigEntry_bootstrap_proxy-defaults_(snake-case) (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:ConfigEntry_bootstrap_proxy-defaults_(snake-case) (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:ConfigEntry_bootstrap_proxy-defaults_(camel-case) (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:ConfigEntry_bootstrap_proxy-defaults_(camel-case) (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:ConfigEntry_bootstrap_service-defaults_(snake-case) (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:ConfigEntry_bootstrap_service-defaults_(snake-case) (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:ConfigEntry_bootstrap_service-defaults_(camel-case) (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:ConfigEntry_bootstrap_service-defaults_(camel-case) (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:ConfigEntry_bootstrap_service-router_(snake-case) (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:ConfigEntry_bootstrap_service-router_(snake-case) (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/default_limits (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:ConfigEntry_bootstrap_service-defaults_(snake-case) (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:ConfigEntry_bootstrap_service-defaults_(camel-case) (0.05s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:ConfigEntry_bootstrap_service-defaults_(camel-case) (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:ConfigEntry_bootstrap_service-router_(snake-case) (0.07s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:ConfigEntry_bootstrap_service-router_(snake-case) (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/default_limits (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_and_auto_encrypt_error (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_and_auto_encrypt_error (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_not_allowed_for_servers (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_not_allowed_for_servers (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_and_auto_encrypt_error (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_not_allowed_for_servers (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_not_allowed_for_servers (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_tls_not_enabled (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_tls_not_enabled (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_server_tls_not_enabled (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_server_tls_not_enabled (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_no_intro_token (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_no_intro_token (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_no_server_addresses (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_no_server_addresses (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_server_tls_not_enabled (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_no_intro_token (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_no_intro_token (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_no_server_addresses (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_no_server_addresses (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_client (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_client (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_authorizer_client_not_allowed (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_authorizer_client_not_allowed (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_authorizer_invalid_config (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_authorizer_invalid_config (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_authorizer_invalid_config_2 (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_authorizer_client_not_allowed (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_authorizer_invalid_config (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_authorizer_invalid_config (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_authorizer_invalid_config_2 (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_authorizer_invalid_config_2 (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_authorizer_require_token_replication_in_secondary (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_authorizer_require_token_replication_in_secondary (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_authorizer_require_token_replication_in_secondary (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_authorizer_invalid_claim_assertion (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_authorizer_invalid_claim_assertion (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_authorizer_invalid_claim_assertion (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_authorizer_ok (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_authorizer_ok (0.03s) === RUN TestNewBuilder_InvalidConfigFormat @@ -57980,30 +57959,30 @@ === RUN Test_UIPathBuilder --- PASS: Test_UIPathBuilder (0.00s) === RUN TestConnectCAConfiguration -=== RUN TestConnectCAConfiguration/cluster-id-override -=== RUN TestConnectCAConfiguration/cluster-id-non-uuid === RUN TestConnectCAConfiguration/provider-override === RUN TestConnectCAConfiguration/other-config === RUN TestConnectCAConfiguration/defaults ---- PASS: TestConnectCAConfiguration (0.00s) - --- PASS: TestConnectCAConfiguration/cluster-id-override (0.00s) - --- PASS: TestConnectCAConfiguration/cluster-id-non-uuid (0.00s) +=== RUN TestConnectCAConfiguration/cluster-id-override +=== RUN TestConnectCAConfiguration/cluster-id-non-uuid +--- PASS: TestConnectCAConfiguration (0.01s) --- PASS: TestConnectCAConfiguration/provider-override (0.00s) --- PASS: TestConnectCAConfiguration/other-config (0.00s) --- PASS: TestConnectCAConfiguration/defaults (0.00s) + --- PASS: TestConnectCAConfiguration/cluster-id-override (0.00s) + --- PASS: TestConnectCAConfiguration/cluster-id-non-uuid (0.00s) === RUN TestSegments segment_oss_test.go:13: DM-skipped --- SKIP: TestSegments (0.00s) === CONT TestBuildAndValidate_HTTPMaxConnsPerClientExceedsRLimit === CONT TestConfigDecodeBytes --- PASS: TestConfigDecodeBytes (0.00s) ---- PASS: TestBuildAndValidate_HTTPMaxConnsPerClientExceedsRLimit (0.01s) +--- PASS: TestBuildAndValidate_HTTPMaxConnsPerClientExceedsRLimit (0.02s) PASS -ok github.com/hashicorp/consul/agent/config 9.161s +ok github.com/hashicorp/consul/agent/config 7.310s === RUN TestCollectHostInfo --- PASS: TestCollectHostInfo (0.00s) PASS -ok github.com/hashicorp/consul/agent/debug 0.028s +ok github.com/hashicorp/consul/agent/debug 0.017s ? github.com/hashicorp/consul/agent/dns [no test files] ? github.com/hashicorp/consul/agent/exec [no test files] === RUN TestAgentAntiEntropy_Services @@ -58050,985 +58029,995 @@ === RUN TestAliasNotifications_local === PAUSE TestAliasNotifications_local === CONT TestAgentAntiEntropy_Services -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestAgentAntiEntropy_NodeInfo -=== CONT TestAgent_AliasCheck_ServiceNotification +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestAliasNotifications_local -=== CONT TestAgent_AliasCheck +=== CONT TestState_Notify === CONT TestAgentAntiEntropy_Checks === CONT TestAgentAntiEntropy_Check_DeferSync -=== CONT TestState_Notify --- PASS: TestState_Notify (0.00s) +=== CONT TestAgent_sendCoordinate === CONT TestAgent_UpdateCheck_DiscardOutput +=== CONT TestAgent_AliasCheck_ServiceNotification [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range ---- PASS: TestAgent_AliasCheck_ServiceNotification (0.06s) +--- PASS: TestAgent_AliasCheck_ServiceNotification (0.01s) === CONT TestAgentAntiEntropy_Checks_ACLDeny ---- PASS: TestAgent_AliasCheck (0.06s) +=== CONT TestAgent_sendCoordinate + state_test.go:1985: 128 5 100ms +2021-08-13T15:32:38.184Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:38.253Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:38.255 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e9e27e1d-a3c7-2e9f-f4b3-21307cab8617 Address:127.0.0.1:25363}]" +32:38.256 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e9e27e1d-a3c7-2e9f-f4b3-21307cab8617.dc1 127.0.0.1 +32:38.257 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e9e27e1d-a3c7-2e9f-f4b3-21307cab8617 127.0.0.1 +2021-08-13T15:32:38.257Z [INFO] agent.router: Initializing LAN area manager +32:38.259 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25363 [Follower]" leader= +32:38.265 [INFO] TestAgent.server: Adding LAN server: server="Node-e9e27e1d-a3c7-2e9f-f4b3-21307cab8617 (Addr: tcp/127.0.0.1:25363) (DC: dc1)" +32:38.265 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e9e27e1d-a3c7-2e9f-f4b3-21307cab8617.dc1 area=wan +32:38.329 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:38.329 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25363 [Candidate]" term=2 +32:38.329 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:38.329 [DEBUG] TestAgent.server.raft: vote granted: from=e9e27e1d-a3c7-2e9f-f4b3-21307cab8617 term=2 tally=1 +32:38.329 [INFO] TestAgent.server.raft: election won: tally=1 +32:38.329 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25363 [Leader]" +32:38.338 [INFO] TestAgent.server: cluster leadership acquired +32:38.338 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25363 +32:38.340 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:38.341 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25358 network=udp +32:38.341 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25358 network=tcp +32:38.343 [INFO] TestAgent.server: New leader elected: payload=Node-e9e27e1d-a3c7-2e9f-f4b3-21307cab8617 +32:38.347 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25359 network=tcp +32:38.348 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25364 network=tcp +32:38.349 [INFO] TestAgent: started state syncer +32:38.353 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:38.353 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:38.353 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:38.353 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:38.353 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:38.353 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e9e27e1d-a3c7-2e9f-f4b3-21307cab8617 +32:38.353 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e9e27e1d-a3c7-2e9f-f4b3-21307cab8617 +32:38.354 [INFO] TestAgent: Synced node info +32:38.354 [DEBUG] TestAgent: Node info in sync +32:38.356 [INFO] TestAgent.server: federation state anti-entropy synced +32:38.376 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:38.376 [DEBUG] TestAgent: Node info in sync +32:38.377 [INFO] TestAgent: Synced check: check=web +32:38.377 [INFO] TestAgent: Requesting shutdown +32:38.377 [INFO] TestAgent.server: shutting down server +32:38.377 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:38.377 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:38.377 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:38.377 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:38.377 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:38.377 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:38.377 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:38.377 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:38.377 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:38.377 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:38.377 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:38.378Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:38.378Z [INFO] agent.router.manager: shutting down +32:38.378 [INFO] TestAgent: consul server down +32:38.378 [INFO] TestAgent: shutdown complete +32:38.378 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25358 network=tcp +32:38.378 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25358 network=udp +32:38.378 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25359 network=tcp +32:38.878 [INFO] TestAgent: Waiting for endpoints to shut down +32:38.878 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_UpdateCheck_DiscardOutput (0.83s) === CONT TestAgentAntiEntropy_RemovingServiceAndCheck -2022-09-15T21:40:52.349Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:40:52.411Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -40:52.430 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c2fe6431-51f8-933b-9974-da9ab6a80efd Address:127.0.0.1:29426}]" -40:52.432 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c2fe6431-51f8-933b-9974-da9ab6a80efd.dc1 127.0.0.1 -40:52.446 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c2fe6431-51f8-933b-9974-da9ab6a80efd 127.0.0.1 -2022-09-15T21:40:52.446Z [INFO] agent.router: Initializing LAN area manager -40:52.447 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29421 network=udp -40:52.447 [INFO] TestAgent.server: Adding LAN server: server="Node-c2fe6431-51f8-933b-9974-da9ab6a80efd (Addr: tcp/127.0.0.1:29426) (DC: dc1)" -40:52.447 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c2fe6431-51f8-933b-9974-da9ab6a80efd.dc1 area=wan -40:52.447 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29421 network=tcp -40:52.448 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29422 network=tcp -40:52.448 [INFO] TestAgent: started state syncer -40:52.448 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29427 network=tcp -40:52.454 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29426 [Follower]" leader= -40:52.513 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -40:52.513 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29426 [Candidate]" term=2 -40:52.514 [DEBUG] TestAgent.server.raft: votes: needed=1 -40:52.514 [DEBUG] TestAgent.server.raft: vote granted: from=c2fe6431-51f8-933b-9974-da9ab6a80efd term=2 tally=1 -40:52.514 [INFO] TestAgent.server.raft: election won: tally=1 -40:52.514 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29426 [Leader]" -40:52.514 [INFO] TestAgent.server: cluster leadership acquired -40:52.514 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29426 -40:52.515 [INFO] TestAgent.server: New leader elected: payload=Node-c2fe6431-51f8-933b-9974-da9ab6a80efd -40:52.517 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -40:52.529 [INFO] TestAgent: Synced node info -40:52.529 [DEBUG] TestAgent: Node info in sync -40:52.531 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -40:52.531 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -40:52.531 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -40:52.531 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -40:52.531 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -40:52.531 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c2fe6431-51f8-933b-9974-da9ab6a80efd -40:52.531 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c2fe6431-51f8-933b-9974-da9ab6a80efd -40:52.532 [INFO] TestAgent.server: federation state anti-entropy synced -40:52.540 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -40:52.541 [INFO] TestAgent: Synced node info -40:52.541 [INFO] TestAgent: Synced check: check=redis -40:52.542 [INFO] TestAgent: Synced check: check=web -40:52.542 [INFO] TestAgent: Synced check: check=cache -40:52.543 [INFO] TestAgent: Deregistered check: check=lb -40:52.543 [DEBUG] TestAgent: Check in sync: check=mysql -40:52.543 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -40:52.544 [INFO] TestAgent: Synced node info -40:52.544 [DEBUG] TestAgent: Check in sync: check=mysql -40:52.544 [INFO] TestAgent: Deregistered check: check=redis -40:52.544 [DEBUG] TestAgent: Check in sync: check=web -40:52.544 [DEBUG] TestAgent: Check in sync: check=cache -40:52.544 [INFO] TestAgent: Requesting shutdown -40:52.545 [INFO] TestAgent.server: shutting down server -40:52.545 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -40:52.545 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -40:52.545 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -40:52.545 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -40:52.545 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -40:52.545 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -40:52.545 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -40:52.545 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -40:52.545 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -40:52.545 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -40:52.545 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:40:52.545Z [INFO] agent.router.manager: shutting down -2022-09-15T21:40:52.545Z [INFO] agent.router.manager: shutting down -40:52.545 [INFO] TestAgent: consul server down -40:52.545 [INFO] TestAgent: shutdown complete -40:52.545 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29421 network=tcp -40:52.545 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29421 network=udp -40:52.545 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29422 network=tcp -40:53.050 [INFO] TestAgent: Waiting for endpoints to shut down -40:53.050 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentAntiEntropy_Checks (0.89s) +2021-08-13T15:32:38.216Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:32:38.216Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:38.241Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:32:38.241Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:38.257 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0e2b2f6b-5ac2-f851-75fb-0c1d8b714d14 Address:127.0.0.1:25370}]" +32:38.258 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0e2b2f6b-5ac2-f851-75fb-0c1d8b714d14.dc1 127.0.0.1 +32:38.259 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25370 [Follower]" leader= +32:38.332 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:38.332 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25370 [Candidate]" term=2 +32:38.336 [WARN] TestAgent.server.raft: unable to get address for sever, using fallback address: id=0e2b2f6b-5ac2-f851-75fb-0c1d8b714d14 fallback=127.0.0.1:25370 error="Could not find address for server id 0e2b2f6b-5ac2-f851-75fb-0c1d8b714d14" +32:38.336 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:38.336 [DEBUG] TestAgent.server.raft: vote granted: from=0e2b2f6b-5ac2-f851-75fb-0c1d8b714d14 term=2 tally=1 +32:38.336 [INFO] TestAgent.server.raft: election won: tally=1 +32:38.336 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25370 [Leader]" +32:38.343 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0e2b2f6b-5ac2-f851-75fb-0c1d8b714d14 127.0.0.1 +2021-08-13T15:32:38.343Z [INFO] agent.router: Initializing LAN area manager +32:38.345 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25365 network=udp +32:38.353 [INFO] TestAgent.server: Adding LAN server: server="Node-0e2b2f6b-5ac2-f851-75fb-0c1d8b714d14 (Addr: tcp/127.0.0.1:25370) (DC: dc1)" +32:38.353 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0e2b2f6b-5ac2-f851-75fb-0c1d8b714d14.dc1 area=wan +32:38.353 [INFO] TestAgent.server: cluster leadership acquired +32:38.354 [INFO] TestAgent.server: initializing acls +32:38.354 [INFO] TestAgent.server: Created ACL 'global-management' policy +32:38.354 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +32:38.354 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25365 network=tcp +32:38.354 [INFO] TestAgent.server: New leader elected: payload=Node-0e2b2f6b-5ac2-f851-75fb-0c1d8b714d14 +32:38.356 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25366 network=tcp +32:38.356 [INFO] TestAgent: started state syncer +32:38.356 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25371 network=tcp +32:38.356 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +32:38.356 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="ACL not found" +32:38.357 [INFO] TestAgent.server: Created ACL anonymous token from configuration +32:38.357 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +32:38.357 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +32:38.357 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-0e2b2f6b-5ac2-f851-75fb-0c1d8b714d14 +32:38.357 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-0e2b2f6b-5ac2-f851-75fb-0c1d8b714d14.dc1 +32:38.357 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-0e2b2f6b-5ac2-f851-75fb-0c1d8b714d14.dc1 area=wan +32:38.358 [INFO] TestAgent.server: Updating LAN server: server="Node-0e2b2f6b-5ac2-f851-75fb-0c1d8b714d14 (Addr: tcp/127.0.0.1:25370) (DC: dc1)" +32:38.359 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:38.361 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:38.361 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:38.361 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:38.361 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:38.361 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:38.361 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0e2b2f6b-5ac2-f851-75fb-0c1d8b714d14 +32:38.361 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0e2b2f6b-5ac2-f851-75fb-0c1d8b714d14 +32:38.362 [INFO] TestAgent.server: federation state anti-entropy synced +32:38.362 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0e2b2f6b-5ac2-f851-75fb-0c1d8b714d14 +32:38.362 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0e2b2f6b-5ac2-f851-75fb-0c1d8b714d14 +32:38.381 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-0e2b2f6b-5ac2-f851-75fb-0c1d8b714d14 +32:38.381 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-0e2b2f6b-5ac2-f851-75fb-0c1d8b714d14 +32:38.383 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth +32:38.383 [WARN] TestAgent: Node info update blocked by ACLs: node=0e2b2f6b-5ac2-f851-75fb-0c1d8b714d14 accessorID=00000000-0000-0000-0000-000000000002 +32:38.384 [INFO] TestAgent: Synced service: service=api +32:38.384 [INFO] TestAgent: Synced service: service=mysql +32:38.385 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth +32:38.385 [WARN] TestAgent: Node info update blocked by ACLs: node=0e2b2f6b-5ac2-f851-75fb-0c1d8b714d14 accessorID=00000000-0000-0000-0000-000000000002 +32:38.385 [INFO] TestAgent: Synced service: service=api +32:38.386 [INFO] TestAgent: Synced service: service=mysql +32:38.386 [WARN] TestAgent: Check registration blocked by ACLs: check=mysql-check accessorID= +32:38.387 [INFO] TestAgent: Synced check: check=api-check +32:38.387 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=api-check +32:38.387 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth +32:38.387 [WARN] TestAgent: Node info update blocked by ACLs: node=0e2b2f6b-5ac2-f851-75fb-0c1d8b714d14 accessorID=00000000-0000-0000-0000-000000000002 +32:38.387 [INFO] TestAgent: Synced service: service=api +32:38.388 [INFO] TestAgent: Synced service: service=mysql +32:38.388 [WARN] TestAgent: Check registration blocked by ACLs: check=mysql-check accessorID= +32:38.389 [INFO] TestAgent: Deregistered check: check=api-check +32:38.389 [INFO] TestAgent: Requesting shutdown +32:38.389 [INFO] TestAgent.server: shutting down server +32:38.389 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +32:38.389 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +32:38.389 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:38.389 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:38.389 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:38.389 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:38.389 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:38.389 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:38.389 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +32:38.389 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:38.389 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:38.389 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +32:38.389 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:38.389 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:38.389 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:38.389Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:38.389Z [INFO] agent.router.manager: shutting down +32:38.389 [INFO] TestAgent: consul server down +32:38.389 [INFO] TestAgent: shutdown complete +32:38.389 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25365 network=tcp +32:38.389 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25365 network=udp +32:38.390 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25366 network=tcp +32:38.890 [INFO] TestAgent: Waiting for endpoints to shut down +32:38.891 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentAntiEntropy_Checks_ACLDeny (0.82s) +=== CONT TestAgent_AddCheckFailure +--- PASS: TestAgent_AddCheckFailure (0.01s) +=== CONT TestAgent_AliasCheck +--- PASS: TestAgent_AliasCheck (0.03s) === CONT TestAgentAntiEntropy_EnableTagOverride -2022-09-15T21:40:52.346Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:40:52.453Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -40:52.496 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:40e4a748-2192-161a-0510-9bf59fe950b5 Address:127.0.0.1:29412}]" -40:52.497 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1b1fd3a1-d007-26be-0221-cbfc064510d5.dc1 127.0.0.1 -40:52.498 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29412 [Follower]" leader= -40:52.503 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1b1fd3a1-d007-26be-0221-cbfc064510d5 127.0.0.1 -2022-09-15T21:40:52.503Z [INFO] agent.router: Initializing LAN area manager -40:52.508 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1b1fd3a1-d007-26be-0221-cbfc064510d5.dc1 area=wan -40:52.509 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29407 network=udp -40:52.510 [INFO] TestAgent.server: Adding LAN server: server="Node-1b1fd3a1-d007-26be-0221-cbfc064510d5 (Addr: tcp/127.0.0.1:29412) (DC: dc1)" -40:52.512 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29407 network=tcp -40:52.513 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29408 network=tcp -40:52.513 [INFO] TestAgent: started state syncer -40:52.513 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29413 network=tcp -40:52.540 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -40:52.540 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29412 [Candidate]" term=2 -40:52.540 [DEBUG] TestAgent.server.raft: votes: needed=1 -40:52.540 [DEBUG] TestAgent.server.raft: vote granted: from=40e4a748-2192-161a-0510-9bf59fe950b5 term=2 tally=1 -40:52.540 [INFO] TestAgent.server.raft: election won: tally=1 -40:52.540 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29412 [Leader]" -40:52.541 [INFO] TestAgent.server: cluster leadership acquired -40:52.541 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29412 -40:52.542 [INFO] TestAgent.server: New leader elected: payload=Node-1b1fd3a1-d007-26be-0221-cbfc064510d5 -40:52.542 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -40:52.545 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -40:52.545 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -40:52.545 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -40:52.545 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -40:52.545 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -40:52.545 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1b1fd3a1-d007-26be-0221-cbfc064510d5 -40:52.545 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1b1fd3a1-d007-26be-0221-cbfc064510d5 -40:52.546 [INFO] TestAgent.server: federation state anti-entropy synced -40:52.585 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -40:52.585 [INFO] TestAgent: Synced node info -40:52.586 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -40:52.587 [INFO] TestAgent: Synced node info -40:52.587 [INFO] TestAgent: Requesting shutdown -40:52.587 [INFO] TestAgent.server: shutting down server -40:52.587 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -40:52.587 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -40:52.587 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -40:52.587 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -40:52.587 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -40:52.587 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -40:52.587 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -40:52.587 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -40:52.587 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -40:52.587 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -40:52.587 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -40:52.587 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:40:52.587Z [INFO] agent.router.manager: shutting down -2022-09-15T21:40:52.587Z [INFO] agent.router.manager: shutting down -40:52.588 [INFO] TestAgent: consul server down -40:52.588 [INFO] TestAgent: shutdown complete -40:52.588 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29407 network=tcp -40:52.588 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29407 network=udp -40:52.588 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29408 network=tcp -40:53.089 [INFO] TestAgent: Waiting for endpoints to shut down -40:53.089 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentAntiEntropy_NodeInfo (0.93s) +2021-08-13T15:32:38.127Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:38.237Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:38.240 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:40e4a748-2192-161a-0510-9bf59fe950b5 Address:127.0.0.1:25328}]" +32:38.241 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a997edef-956b-9563-432e-eba4bdea1735.dc1 127.0.0.1 +32:38.243 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25328 [Follower]" leader= +32:38.252 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a997edef-956b-9563-432e-eba4bdea1735 127.0.0.1 +2021-08-13T15:32:38.252Z [INFO] agent.router: Initializing LAN area manager +32:38.252 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25323 network=udp +32:38.253 [INFO] TestAgent.server: Adding LAN server: server="Node-a997edef-956b-9563-432e-eba4bdea1735 (Addr: tcp/127.0.0.1:25328) (DC: dc1)" +32:38.253 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a997edef-956b-9563-432e-eba4bdea1735.dc1 area=wan +32:38.254 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25323 network=tcp +32:38.254 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25324 network=tcp +32:38.255 [INFO] TestAgent: started state syncer +32:38.255 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25329 network=tcp +32:38.331 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:38.331 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25328 [Candidate]" term=2 +32:38.331 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:38.331 [DEBUG] TestAgent.server.raft: vote granted: from=40e4a748-2192-161a-0510-9bf59fe950b5 term=2 tally=1 +32:38.331 [INFO] TestAgent.server.raft: election won: tally=1 +32:38.331 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25328 [Leader]" +32:38.332 [INFO] TestAgent.server: cluster leadership acquired +32:38.332 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25328 +32:38.338 [INFO] TestAgent.server: New leader elected: payload=Node-a997edef-956b-9563-432e-eba4bdea1735 +32:38.343 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:38.348 [INFO] TestAgent: Synced node info +32:38.353 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:38.353 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:38.353 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:38.353 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:38.353 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:38.353 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a997edef-956b-9563-432e-eba4bdea1735 +32:38.353 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a997edef-956b-9563-432e-eba4bdea1735 +32:38.358 [INFO] TestAgent.server: federation state anti-entropy synced +32:38.488 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:38.488 [INFO] TestAgent: Synced node info +32:38.489 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:38.489 [INFO] TestAgent: Synced node info +32:38.489 [INFO] TestAgent: Requesting shutdown +32:38.489 [INFO] TestAgent.server: shutting down server +32:38.489 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:38.489 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:38.489 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:38.489 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:38.489 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:38.489 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:38.489 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:38.489 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:38.489 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:38.489 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:38.489 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:38.490Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:38.490Z [INFO] agent.router.manager: shutting down +32:38.490 [INFO] TestAgent: consul server down +32:38.490 [INFO] TestAgent: shutdown complete +32:38.490 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25323 network=tcp +32:38.490 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25323 network=udp +32:38.490 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25324 network=tcp +32:38.991 [INFO] TestAgent: Waiting for endpoints to shut down +32:38.991 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentAntiEntropy_NodeInfo (0.94s) === CONT TestAgentAntiEntropy_Services_ACLDeny -2022-09-15T21:40:52.256Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:40:52.360Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -40:52.447 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f1d08d56-087f-2fde-14b6-df526a9e6348 Address:127.0.0.1:29405}]" -40:52.448 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f1d08d56-087f-2fde-14b6-df526a9e6348.dc1 127.0.0.1 -40:52.448 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f1d08d56-087f-2fde-14b6-df526a9e6348 127.0.0.1 -2022-09-15T21:40:52.449Z [INFO] agent.router: Initializing LAN area manager -40:52.449 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29400 network=udp -40:52.450 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29405 [Follower]" leader= -40:52.450 [INFO] TestAgent.server: Adding LAN server: server="Node-f1d08d56-087f-2fde-14b6-df526a9e6348 (Addr: tcp/127.0.0.1:29405) (DC: dc1)" -40:52.450 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f1d08d56-087f-2fde-14b6-df526a9e6348.dc1 area=wan -40:52.450 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29400 network=tcp -40:52.451 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29401 network=tcp -40:52.451 [INFO] TestAgent: started state syncer -40:52.451 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29406 network=tcp -40:52.494 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -40:52.494 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29405 [Candidate]" term=2 -40:52.498 [DEBUG] TestAgent.server.raft: votes: needed=1 -40:52.498 [DEBUG] TestAgent.server.raft: vote granted: from=f1d08d56-087f-2fde-14b6-df526a9e6348 term=2 tally=1 -40:52.498 [INFO] TestAgent.server.raft: election won: tally=1 -40:52.498 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29405 [Leader]" -40:52.499 [INFO] TestAgent.server: cluster leadership acquired -40:52.499 [INFO] TestAgent.server: New leader elected: payload=Node-f1d08d56-087f-2fde-14b6-df526a9e6348 -40:52.499 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29405 -40:52.501 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -40:52.516 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -40:52.516 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -40:52.516 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -40:52.516 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -40:52.516 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -40:52.516 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f1d08d56-087f-2fde-14b6-df526a9e6348 -40:52.516 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f1d08d56-087f-2fde-14b6-df526a9e6348 -40:52.518 [INFO] TestAgent.server: federation state anti-entropy synced -40:52.626 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -40:52.627 [INFO] TestAgent: Synced node info -40:52.627 [INFO] TestAgent: Synced service: service=redis -40:52.628 [INFO] TestAgent: Synced service: service=web -40:52.628 [INFO] TestAgent: Synced service: service=api -40:52.629 [INFO] TestAgent: Synced service: service=cache -40:52.634 [INFO] TestAgent: Deregistered service: service=lb -40:52.634 [DEBUG] TestAgent: Service in sync: service=mysql -40:52.634 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -40:52.635 [INFO] TestAgent: Synced node info -40:52.635 [DEBUG] TestAgent: Service in sync: service=mysql -40:52.635 [DEBUG] TestAgent: Service in sync: service=redis -40:52.635 [DEBUG] TestAgent: Service in sync: service=web -40:52.636 [INFO] TestAgent: Deregistered service: service=api -40:52.636 [DEBUG] TestAgent: Service in sync: service=cache -40:52.636 [INFO] TestAgent: Requesting shutdown -40:52.636 [INFO] TestAgent.server: shutting down server -40:52.636 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -40:52.636 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -40:52.636 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -40:52.636 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -40:52.636 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -40:52.636 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -40:52.636 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -40:52.636 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -40:52.636 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -40:52.636 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -40:52.636 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -40:52.636 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:40:52.636Z [INFO] agent.router.manager: shutting down -2022-09-15T21:40:52.637Z [INFO] agent.router.manager: shutting down -40:52.637 [INFO] TestAgent: consul server down -40:52.637 [INFO] TestAgent: shutdown complete -40:52.637 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29400 network=tcp -40:52.637 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29400 network=udp -40:52.637 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29401 network=tcp -40:53.138 [INFO] TestAgent: Waiting for endpoints to shut down -40:53.138 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentAntiEntropy_Services (0.98s) +2021-08-13T15:32:38.215Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:38.354Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:38.358 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:59d7593f-c34b-0783-0d69-59b31a68a045 Address:127.0.0.1:25356}]" +32:38.358 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25356 [Follower]" leader= +32:38.359 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-59d7593f-c34b-0783-0d69-59b31a68a045.dc1 127.0.0.1 +32:38.360 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-59d7593f-c34b-0783-0d69-59b31a68a045 127.0.0.1 +2021-08-13T15:32:38.360Z [INFO] agent.router: Initializing LAN area manager +32:38.360 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25351 network=udp +32:38.360 [INFO] TestAgent.server: Adding LAN server: server="Node-59d7593f-c34b-0783-0d69-59b31a68a045 (Addr: tcp/127.0.0.1:25356) (DC: dc1)" +32:38.360 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-59d7593f-c34b-0783-0d69-59b31a68a045.dc1 area=wan +32:38.361 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25351 network=tcp +32:38.361 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25352 network=tcp +32:38.362 [INFO] TestAgent: started state syncer +32:38.362 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25357 network=tcp +32:38.405 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:38.405 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25356 [Candidate]" term=2 +32:38.405 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:38.405 [DEBUG] TestAgent.server.raft: vote granted: from=59d7593f-c34b-0783-0d69-59b31a68a045 term=2 tally=1 +32:38.405 [INFO] TestAgent.server.raft: election won: tally=1 +32:38.405 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25356 [Leader]" +32:38.406 [INFO] TestAgent.server: cluster leadership acquired +32:38.406 [INFO] TestAgent.server: New leader elected: payload=Node-59d7593f-c34b-0783-0d69-59b31a68a045 +32:38.406 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25356 +32:38.407 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:38.410 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:38.410 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:38.410 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:38.410 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:38.410 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:38.410 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-59d7593f-c34b-0783-0d69-59b31a68a045 +32:38.410 [INFO] TestAgent.server: member joined, marking health alive: member=Node-59d7593f-c34b-0783-0d69-59b31a68a045 +32:38.410 [INFO] TestAgent.server: federation state anti-entropy synced +32:38.471 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:38.472 [INFO] TestAgent: Synced node info +32:38.472 [DEBUG] TestAgent: Node info in sync +32:38.567 [INFO] TestAgent: Requesting shutdown +32:38.567 [INFO] TestAgent.server: shutting down server +32:38.567 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:38.567 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:38.567 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:38.568 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:38.568 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:38.568 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:38.568 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:38.568 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:38.568 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:38.568 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:38.568 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:38.568Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:38.568Z [INFO] agent.router.manager: shutting down +32:38.568 [INFO] TestAgent: consul server down +32:38.568 [INFO] TestAgent: shutdown complete +32:38.568 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25351 network=tcp +32:38.568 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25351 network=udp +32:38.568 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25352 network=tcp +32:39.068 [INFO] TestAgent: Waiting for endpoints to shut down +32:39.068 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_sendCoordinate (1.02s) === CONT TestAgentAntiEntropy_Services_WithChecks -2022-09-15T21:40:52.295Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:40:52.322Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -40:52.326 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0b828656-9b29-0220-82dc-855fe8b30c32 Address:127.0.0.1:29419}]" -40:52.334 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29419 [Follower]" leader= -40:52.359 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0b828656-9b29-0220-82dc-855fe8b30c32.dc1 127.0.0.1 -40:52.360 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0b828656-9b29-0220-82dc-855fe8b30c32 127.0.0.1 -2022-09-15T21:40:52.360Z [INFO] agent.router: Initializing LAN area manager -40:52.360 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29414 network=udp -40:52.363 [INFO] TestAgent.server: Adding LAN server: server="Node-0b828656-9b29-0220-82dc-855fe8b30c32 (Addr: tcp/127.0.0.1:29419) (DC: dc1)" -40:52.363 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0b828656-9b29-0220-82dc-855fe8b30c32.dc1 area=wan -40:52.366 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29414 network=tcp -40:52.367 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29415 network=tcp -40:52.367 [INFO] TestAgent: started state syncer -40:52.367 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29420 network=tcp -40:52.390 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -40:52.390 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29419 [Candidate]" term=2 -40:52.411 [DEBUG] TestAgent.server.raft: votes: needed=1 -40:52.411 [DEBUG] TestAgent.server.raft: vote granted: from=0b828656-9b29-0220-82dc-855fe8b30c32 term=2 tally=1 -40:52.411 [INFO] TestAgent.server.raft: election won: tally=1 -40:52.411 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29419 [Leader]" -40:52.412 [INFO] TestAgent.server: cluster leadership acquired -40:52.412 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29419 -40:52.412 [INFO] TestAgent.server: New leader elected: payload=Node-0b828656-9b29-0220-82dc-855fe8b30c32 -40:52.422 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -40:52.464 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -40:52.464 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -40:52.464 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -40:52.464 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -40:52.464 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -40:52.464 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0b828656-9b29-0220-82dc-855fe8b30c32 -40:52.464 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0b828656-9b29-0220-82dc-855fe8b30c32 -40:52.465 [INFO] TestAgent.server: federation state anti-entropy synced -40:52.511 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -40:52.512 [INFO] TestAgent: Synced node info -40:52.636 [INFO] TestAgent: Requesting shutdown -40:52.636 [INFO] TestAgent.server: shutting down server -40:52.636 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -40:52.636 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -40:52.636 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -40:52.636 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -40:52.636 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -40:52.637 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -40:52.637 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -40:52.637 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -40:52.637 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -40:52.637 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -40:52.637 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:40:52.637Z [INFO] agent.router.manager: shutting down -2022-09-15T21:40:52.637Z [INFO] agent.router.manager: shutting down -40:52.637 [INFO] TestAgent: consul server down -40:52.637 [INFO] TestAgent: shutdown complete -40:52.637 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29414 network=tcp -40:52.637 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29414 network=udp -40:52.637 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29415 network=tcp -40:53.137 [INFO] TestAgent: Waiting for endpoints to shut down -40:53.138 [INFO] TestAgent: Endpoints down ---- PASS: TestAliasNotifications_local (0.98s) -=== CONT TestAgent_ServiceWatchCh -2022-09-15T21:40:52.502Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:40:52.502Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:40:52.548Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:40:52.548Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -40:52.551 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:43fa4ec0-c24a-373d-4659-7fbe022d6cbc Address:127.0.0.1:29447}]" -40:52.551 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29447 [Follower]" leader= -40:52.552 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-43fa4ec0-c24a-373d-4659-7fbe022d6cbc.dc1 127.0.0.1 -40:52.553 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-43fa4ec0-c24a-373d-4659-7fbe022d6cbc 127.0.0.1 -2022-09-15T21:40:52.553Z [INFO] agent.router: Initializing LAN area manager -40:52.553 [INFO] TestAgent.server: Adding LAN server: server="Node-43fa4ec0-c24a-373d-4659-7fbe022d6cbc (Addr: tcp/127.0.0.1:29447) (DC: dc1)" -40:52.553 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-43fa4ec0-c24a-373d-4659-7fbe022d6cbc.dc1 area=wan -40:52.553 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29442 network=udp -40:52.553 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29442 network=tcp -40:52.558 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29443 network=tcp -40:52.558 [INFO] TestAgent: started state syncer -40:52.558 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29448 network=tcp -40:52.604 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -40:52.614 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -40:52.614 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29447 [Candidate]" term=2 -40:52.614 [DEBUG] TestAgent.server.raft: votes: needed=1 -40:52.614 [DEBUG] TestAgent.server.raft: vote granted: from=43fa4ec0-c24a-373d-4659-7fbe022d6cbc term=2 tally=1 -40:52.614 [INFO] TestAgent.server.raft: election won: tally=1 -40:52.614 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29447 [Leader]" -40:52.614 [INFO] TestAgent.server: cluster leadership acquired -40:52.615 [INFO] TestAgent.server: New leader elected: payload=Node-43fa4ec0-c24a-373d-4659-7fbe022d6cbc -40:52.615 [INFO] TestAgent.server: initializing acls -40:52.615 [INFO] TestAgent.server: Created ACL 'global-management' policy -40:52.615 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -40:52.616 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -40:52.617 [INFO] TestAgent.server: Created ACL anonymous token from configuration -40:52.617 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -40:52.617 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -40:52.617 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-43fa4ec0-c24a-373d-4659-7fbe022d6cbc -40:52.617 [INFO] TestAgent.server: Updating LAN server: server="Node-43fa4ec0-c24a-373d-4659-7fbe022d6cbc (Addr: tcp/127.0.0.1:29447) (DC: dc1)" -40:52.617 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-43fa4ec0-c24a-373d-4659-7fbe022d6cbc.dc1 -40:52.618 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-43fa4ec0-c24a-373d-4659-7fbe022d6cbc.dc1 area=wan -40:52.619 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -40:52.621 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -40:52.621 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -40:52.621 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -40:52.621 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -40:52.621 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -40:52.622 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-43fa4ec0-c24a-373d-4659-7fbe022d6cbc -40:52.622 [INFO] TestAgent.server: member joined, marking health alive: member=Node-43fa4ec0-c24a-373d-4659-7fbe022d6cbc -40:52.623 [INFO] TestAgent.server: federation state anti-entropy synced -40:52.623 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-43fa4ec0-c24a-373d-4659-7fbe022d6cbc -40:52.668 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-43fa4ec0-c24a-373d-4659-7fbe022d6cbc -40:52.668 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-43fa4ec0-c24a-373d-4659-7fbe022d6cbc -40:52.670 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth -40:52.670 [WARN] TestAgent: Node info update blocked by ACLs: node=43fa4ec0-c24a-373d-4659-7fbe022d6cbc accessorID=00000000-0000-0000-0000-000000000002 -40:52.671 [INFO] TestAgent: Synced service: service=mysql -40:52.672 [INFO] TestAgent: Synced service: service=api -40:52.672 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth -40:52.672 [WARN] TestAgent: Node info update blocked by ACLs: node=43fa4ec0-c24a-373d-4659-7fbe022d6cbc accessorID=00000000-0000-0000-0000-000000000002 -40:52.673 [INFO] TestAgent: Synced service: service=mysql -40:52.674 [INFO] TestAgent: Synced service: service=api -40:52.675 [WARN] TestAgent: Check registration blocked by ACLs: check=mysql-check accessorID= -40:52.676 [INFO] TestAgent: Synced check: check=api-check -40:52.676 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=api-check -40:52.676 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth -40:52.676 [WARN] TestAgent: Node info update blocked by ACLs: node=43fa4ec0-c24a-373d-4659-7fbe022d6cbc accessorID=00000000-0000-0000-0000-000000000002 -40:52.676 [INFO] TestAgent: Synced service: service=mysql -40:52.677 [INFO] TestAgent: Synced service: service=api -40:52.677 [WARN] TestAgent: Check registration blocked by ACLs: check=mysql-check accessorID= -40:52.678 [INFO] TestAgent: Deregistered check: check=api-check -40:52.678 [INFO] TestAgent: Requesting shutdown -40:52.678 [INFO] TestAgent.server: shutting down server -40:52.678 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -40:52.678 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -40:52.678 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -40:52.678 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -40:52.678 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -40:52.678 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -40:52.678 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -40:52.678 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -40:52.678 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -40:52.678 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -40:52.678 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -40:52.678 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -40:52.678 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -40:52.678 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -40:52.678 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -40:52.678 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -2022-09-15T21:40:52.679Z [INFO] agent.router.manager: shutting down -2022-09-15T21:40:52.679Z [INFO] agent.router.manager: shutting down -40:52.679 [INFO] TestAgent: consul server down -40:52.679 [INFO] TestAgent: shutdown complete -40:52.679 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29442 network=tcp -40:52.679 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29442 network=udp -40:52.679 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29443 network=tcp -40:53.179 [INFO] TestAgent: Waiting for endpoints to shut down -40:53.179 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentAntiEntropy_Checks_ACLDeny (0.96s) -=== CONT TestAgent_sendCoordinate -2022-09-15T21:40:52.360Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:40:52.408Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -40:52.426 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4b912a30-ca83-f6e5-7f55-0590107bebc5 Address:127.0.0.1:29454}]" -40:52.436 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4b912a30-ca83-f6e5-7f55-0590107bebc5.dc1 127.0.0.1 -40:52.438 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4b912a30-ca83-f6e5-7f55-0590107bebc5 127.0.0.1 -2022-09-15T21:40:52.438Z [INFO] agent.router: Initializing LAN area manager -40:52.449 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29454 [Follower]" leader= -40:52.458 [INFO] TestAgent.server: Adding LAN server: server="Node-4b912a30-ca83-f6e5-7f55-0590107bebc5 (Addr: tcp/127.0.0.1:29454) (DC: dc1)" -40:52.458 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4b912a30-ca83-f6e5-7f55-0590107bebc5.dc1 area=wan -40:52.494 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29449 network=tcp -40:52.495 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29449 network=udp -40:52.496 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29450 network=tcp -40:52.496 [INFO] TestAgent: started state syncer -40:52.496 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29455 network=tcp -40:52.507 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -40:52.507 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29454 [Candidate]" term=2 -40:52.508 [DEBUG] TestAgent.server.raft: votes: needed=1 -40:52.508 [DEBUG] TestAgent.server.raft: vote granted: from=4b912a30-ca83-f6e5-7f55-0590107bebc5 term=2 tally=1 -40:52.508 [INFO] TestAgent.server.raft: election won: tally=1 -40:52.508 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29454 [Leader]" -40:52.508 [INFO] TestAgent.server: cluster leadership acquired -40:52.511 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29454 -40:52.512 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -40:52.514 [INFO] TestAgent.server: New leader elected: payload=Node-4b912a30-ca83-f6e5-7f55-0590107bebc5 -40:52.514 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -40:52.514 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -40:52.514 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -40:52.514 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -40:52.514 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -40:52.514 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4b912a30-ca83-f6e5-7f55-0590107bebc5 -40:52.515 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4b912a30-ca83-f6e5-7f55-0590107bebc5 -40:52.517 [INFO] TestAgent.server: federation state anti-entropy synced -40:52.686 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -40:52.687 [INFO] TestAgent: Synced node info -40:52.688 [INFO] TestAgent: Deregistered service: service=deleted-check-service -40:52.688 [INFO] TestAgent: Deregistered check: check=lb -40:52.688 [INFO] TestAgent: Requesting shutdown -40:52.688 [INFO] TestAgent.server: shutting down server -40:52.688 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -40:52.688 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -40:52.688 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -40:52.688 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -40:52.688 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -40:52.688 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -40:52.688 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -40:52.688 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -40:52.688 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -40:52.688 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -40:52.688 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -40:52.688 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:40:52.689Z [INFO] agent.router.manager: shutting down -2022-09-15T21:40:52.689Z [INFO] agent.router.manager: shutting down -40:52.689 [INFO] TestAgent: consul server down -40:52.689 [INFO] TestAgent: shutdown complete -40:52.689 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29449 network=tcp -40:52.689 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29449 network=udp -40:52.689 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29450 network=tcp -40:53.206 [INFO] TestAgent: Waiting for endpoints to shut down -40:53.206 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentAntiEntropy_RemovingServiceAndCheck (0.99s) -=== CONT TestAgentAntiEntropy_Services_ConnectProxy -2022-09-15T21:40:52.410Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:40:52.503Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -40:52.516 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:37576172-fd4a-34b6-f25f-485311f28d36 Address:127.0.0.1:29440}]" -40:52.516 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29440 [Follower]" leader= -40:52.519 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-37576172-fd4a-34b6-f25f-485311f28d36.dc1 127.0.0.1 -40:52.520 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-37576172-fd4a-34b6-f25f-485311f28d36 127.0.0.1 -2022-09-15T21:40:52.520Z [INFO] agent.router: Initializing LAN area manager -40:52.521 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29435 network=udp -40:52.521 [INFO] TestAgent.server: Adding LAN server: server="Node-37576172-fd4a-34b6-f25f-485311f28d36 (Addr: tcp/127.0.0.1:29440) (DC: dc1)" -40:52.521 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-37576172-fd4a-34b6-f25f-485311f28d36.dc1 area=wan -40:52.526 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29435 network=tcp -40:52.527 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29436 network=tcp -40:52.527 [INFO] TestAgent: started state syncer -40:52.527 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29441 network=tcp -40:52.575 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -40:52.576 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29440 [Candidate]" term=2 -40:52.576 [DEBUG] TestAgent.server.raft: votes: needed=1 -40:52.576 [DEBUG] TestAgent.server.raft: vote granted: from=37576172-fd4a-34b6-f25f-485311f28d36 term=2 tally=1 -40:52.576 [INFO] TestAgent.server.raft: election won: tally=1 -40:52.576 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29440 [Leader]" -40:52.576 [INFO] TestAgent.server: cluster leadership acquired -40:52.576 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29440 -40:52.578 [INFO] TestAgent.server: New leader elected: payload=Node-37576172-fd4a-34b6-f25f-485311f28d36 -40:52.578 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -40:52.580 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -40:52.580 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -40:52.580 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -40:52.580 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -40:52.580 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -40:52.580 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-37576172-fd4a-34b6-f25f-485311f28d36 -40:52.580 [INFO] TestAgent.server: member joined, marking health alive: member=Node-37576172-fd4a-34b6-f25f-485311f28d36 -40:52.582 [INFO] TestAgent.server: federation state anti-entropy synced -40:52.751 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -40:52.752 [INFO] TestAgent: Synced node info -40:52.752 [DEBUG] TestAgent: Node info in sync -40:52.817 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -40:52.817 [DEBUG] TestAgent: Node info in sync -40:52.818 [INFO] TestAgent: Synced check: check=web -40:52.818 [INFO] TestAgent: Requesting shutdown -40:52.818 [INFO] TestAgent.server: shutting down server -40:52.818 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -40:52.818 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -40:52.818 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -40:52.818 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -40:52.818 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -40:52.818 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -40:52.818 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -40:52.818 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -40:52.818 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -40:52.818 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -40:52.818 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:40:52.818Z [INFO] agent.router.manager: shutting down -2022-09-15T21:40:52.818Z [INFO] agent.router.manager: shutting down -40:52.818 [INFO] TestAgent: consul server down -40:52.818 [INFO] TestAgent: shutdown complete -40:52.818 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29435 network=tcp -40:52.818 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29435 network=udp -40:52.818 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29436 network=tcp -40:53.319 [INFO] TestAgent: Waiting for endpoints to shut down -40:53.319 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_UpdateCheck_DiscardOutput (1.16s) +2021-08-13T15:32:38.195Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:38.237Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:38.243 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a0757d76-0347-6006-dfab-7cb8cfb8617a Address:127.0.0.1:25342}]" +32:38.244 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25342 [Follower]" leader= +32:38.246 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a0757d76-0347-6006-dfab-7cb8cfb8617a.dc1 127.0.0.1 +32:38.247 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a0757d76-0347-6006-dfab-7cb8cfb8617a 127.0.0.1 +2021-08-13T15:32:38.251Z [INFO] agent.router: Initializing LAN area manager +32:38.329 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25337 network=udp +32:38.330 [INFO] TestAgent.server: Adding LAN server: server="Node-a0757d76-0347-6006-dfab-7cb8cfb8617a (Addr: tcp/127.0.0.1:25342) (DC: dc1)" +32:38.330 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a0757d76-0347-6006-dfab-7cb8cfb8617a.dc1 area=wan +32:38.330 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25337 network=tcp +32:38.337 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25338 network=tcp +32:38.337 [INFO] TestAgent: started state syncer +32:38.337 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25343 network=tcp +32:38.339 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:38.339 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25342 [Candidate]" term=2 +32:38.339 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:38.339 [DEBUG] TestAgent.server.raft: vote granted: from=a0757d76-0347-6006-dfab-7cb8cfb8617a term=2 tally=1 +32:38.339 [INFO] TestAgent.server.raft: election won: tally=1 +32:38.339 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25342 [Leader]" +32:38.342 [INFO] TestAgent.server: cluster leadership acquired +32:38.346 [INFO] TestAgent.server: New leader elected: payload=Node-a0757d76-0347-6006-dfab-7cb8cfb8617a +32:38.346 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25342 +32:38.357 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:38.364 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:38.364 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:38.364 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:38.364 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:38.364 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:38.364 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a0757d76-0347-6006-dfab-7cb8cfb8617a +32:38.364 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a0757d76-0347-6006-dfab-7cb8cfb8617a +32:38.367 [INFO] TestAgent.server: federation state anti-entropy synced +32:38.401 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:38.401 [INFO] TestAgent: Synced node info +32:38.401 [DEBUG] TestAgent: Node info in sync +32:38.572 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:38.572 [INFO] TestAgent: Synced node info +32:38.573 [INFO] TestAgent: Deregistered check: check=lb +32:38.573 [DEBUG] TestAgent: Check in sync: check=mysql +32:38.573 [INFO] TestAgent: Synced check: check=redis +32:38.574 [INFO] TestAgent: Synced check: check=web +32:38.574 [INFO] TestAgent: Synced check: check=cache +32:38.575 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:38.575 [INFO] TestAgent: Synced node info +32:38.575 [DEBUG] TestAgent: Check in sync: check=mysql +32:38.575 [INFO] TestAgent: Deregistered check: check=redis +32:38.575 [DEBUG] TestAgent: Check in sync: check=web +32:38.575 [DEBUG] TestAgent: Check in sync: check=cache +32:38.576 [INFO] TestAgent: Requesting shutdown +32:38.576 [INFO] TestAgent.server: shutting down server +32:38.576 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:38.576 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:38.576 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:38.576 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:38.576 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:38.576 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:38.576 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:38.576 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:38.576 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:38.576 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:38.576 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:38.576Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:38.576Z [INFO] agent.router.manager: shutting down +32:38.576 [INFO] TestAgent: consul server down +32:38.576 [INFO] TestAgent: shutdown complete +32:38.576 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25337 network=tcp +32:38.576 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25337 network=udp +32:38.576 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25338 network=tcp +32:39.077 [INFO] TestAgent: Waiting for endpoints to shut down +32:39.077 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentAntiEntropy_Checks (1.03s) === CONT TestAgent_CheckTokens ---- PASS: TestAgent_CheckTokens (0.04s) -=== CONT TestAgent_AddCheckFailure ---- PASS: TestAgent_AddCheckFailure (0.02s) +--- PASS: TestAgent_CheckTokens (0.01s) === CONT TestAgent_ServiceTokens ---- PASS: TestAgent_ServiceTokens (0.02s) -=== CONT TestAgent_sendCoordinate - state_test.go:1985: 128 5 100ms -2022-09-15T21:40:53.127Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:40:53.127Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:40:53.167Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2022-09-15T21:40:53.167Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -40:53.169 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a30d636d-a517-12e7-c49b-d2e584d5ceaf Address:127.0.0.1:29468}]" -40:53.172 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29468 [Follower]" leader= -40:53.179 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a30d636d-a517-12e7-c49b-d2e584d5ceaf.dc1 127.0.0.1 -40:53.179 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a30d636d-a517-12e7-c49b-d2e584d5ceaf 127.0.0.1 -2022-09-15T21:40:53.179Z [INFO] agent.router: Initializing LAN area manager -40:53.180 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29463 network=udp -40:53.180 [INFO] TestAgent.server: Adding LAN server: server="Node-a30d636d-a517-12e7-c49b-d2e584d5ceaf (Addr: tcp/127.0.0.1:29468) (DC: dc1)" -40:53.180 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a30d636d-a517-12e7-c49b-d2e584d5ceaf.dc1 area=wan -40:53.180 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29463 network=tcp -40:53.181 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29464 network=tcp -40:53.181 [INFO] TestAgent: started state syncer -40:53.181 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29469 network=tcp -40:53.230 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -40:53.246 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -40:53.246 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29468 [Candidate]" term=2 -40:53.246 [DEBUG] TestAgent.server.raft: votes: needed=1 -40:53.246 [DEBUG] TestAgent.server.raft: vote granted: from=a30d636d-a517-12e7-c49b-d2e584d5ceaf term=2 tally=1 -40:53.246 [INFO] TestAgent.server.raft: election won: tally=1 -40:53.246 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29468 [Leader]" -40:53.246 [INFO] TestAgent.server: cluster leadership acquired -40:53.247 [INFO] TestAgent.server: initializing acls -40:53.247 [INFO] TestAgent.server: Created ACL 'global-management' policy -40:53.247 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -40:53.248 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -40:53.249 [INFO] TestAgent.server: Created ACL anonymous token from configuration -40:53.249 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -40:53.249 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -40:53.249 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-a30d636d-a517-12e7-c49b-d2e584d5ceaf -40:53.250 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-a30d636d-a517-12e7-c49b-d2e584d5ceaf.dc1 -40:53.252 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -40:53.254 [INFO] TestAgent.server: New leader elected: payload=Node-a30d636d-a517-12e7-c49b-d2e584d5ceaf -40:53.254 [INFO] TestAgent.server: Updating LAN server: server="Node-a30d636d-a517-12e7-c49b-d2e584d5ceaf (Addr: tcp/127.0.0.1:29468) (DC: dc1)" -40:53.255 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-a30d636d-a517-12e7-c49b-d2e584d5ceaf.dc1 area=wan -40:53.261 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -40:53.261 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -40:53.261 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -40:53.261 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -40:53.261 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -40:53.262 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a30d636d-a517-12e7-c49b-d2e584d5ceaf -40:53.262 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a30d636d-a517-12e7-c49b-d2e584d5ceaf -40:53.263 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a30d636d-a517-12e7-c49b-d2e584d5ceaf -40:53.264 [INFO] TestAgent.server: federation state anti-entropy synced -40:53.274 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth -40:53.274 [WARN] TestAgent: Node info update blocked by ACLs: node=a30d636d-a517-12e7-c49b-d2e584d5ceaf accessorID=00000000-0000-0000-0000-000000000002 -40:53.288 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-a30d636d-a517-12e7-c49b-d2e584d5ceaf -40:53.289 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-a30d636d-a517-12e7-c49b-d2e584d5ceaf -40:53.290 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth -40:53.290 [WARN] TestAgent: Node info update blocked by ACLs: node=a30d636d-a517-12e7-c49b-d2e584d5ceaf accessorID=00000000-0000-0000-0000-000000000002 -40:53.291 [INFO] TestAgent: Synced service: service=api -40:53.291 [WARN] TestAgent: Service registration blocked by ACLs: service=mysql accessorID= -40:53.292 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth -40:53.292 [WARN] TestAgent: Node info update blocked by ACLs: node=a30d636d-a517-12e7-c49b-d2e584d5ceaf accessorID=00000000-0000-0000-0000-000000000002 -40:53.292 [WARN] TestAgent: Service registration blocked by ACLs: service=mysql accessorID= -40:53.293 [INFO] TestAgent: Deregistered service: service=api -40:53.293 [INFO] TestAgent: Requesting shutdown -40:53.293 [INFO] TestAgent.server: shutting down server -40:53.293 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -40:53.293 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -40:53.293 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -40:53.293 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -40:53.293 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -40:53.293 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -40:53.293 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -40:53.293 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -40:53.293 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -40:53.293 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -40:53.293 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -40:53.293 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -40:53.293 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -40:53.293 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -40:53.293 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:40:53.294Z [INFO] agent.router.manager: shutting down -2022-09-15T21:40:53.294Z [INFO] agent.router.manager: shutting down -40:53.294 [INFO] TestAgent: consul server down -40:53.294 [INFO] TestAgent: shutdown complete -40:53.294 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29463 network=tcp -40:53.294 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29463 network=udp -40:53.294 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29464 network=tcp -40:53.794 [INFO] TestAgent: Waiting for endpoints to shut down -40:53.794 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentAntiEntropy_Services_ACLDeny (0.70s) -2022-09-15T21:40:53.069Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:40:53.090Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -40:53.097 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0aa1c046-a57a-7262-ffcd-833590924202 Address:127.0.0.1:29461}]" -40:53.097 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29461 [Follower]" leader= -40:53.098 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0aa1c046-a57a-7262-ffcd-833590924202.dc1 127.0.0.1 -40:53.100 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0aa1c046-a57a-7262-ffcd-833590924202 127.0.0.1 -2022-09-15T21:40:53.100Z [INFO] agent.router: Initializing LAN area manager -40:53.106 [INFO] TestAgent.server: Adding LAN server: server="Node-0aa1c046-a57a-7262-ffcd-833590924202 (Addr: tcp/127.0.0.1:29461) (DC: dc1)" -40:53.106 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0aa1c046-a57a-7262-ffcd-833590924202.dc1 area=wan -40:53.107 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29456 network=udp -40:53.108 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29456 network=tcp -40:53.112 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29457 network=tcp -40:53.113 [INFO] TestAgent: started state syncer -40:53.113 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29462 network=tcp -40:53.165 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -40:53.165 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29461 [Candidate]" term=2 -40:53.165 [DEBUG] TestAgent.server.raft: votes: needed=1 -40:53.165 [DEBUG] TestAgent.server.raft: vote granted: from=0aa1c046-a57a-7262-ffcd-833590924202 term=2 tally=1 -40:53.165 [INFO] TestAgent.server.raft: election won: tally=1 -40:53.165 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29461 [Leader]" -40:53.165 [INFO] TestAgent.server: cluster leadership acquired -40:53.166 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29461 -40:53.167 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -40:53.169 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -40:53.170 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -40:53.170 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -40:53.172 [INFO] TestAgent.server: New leader elected: payload=Node-0aa1c046-a57a-7262-ffcd-833590924202 -40:53.173 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -40:53.173 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -40:53.173 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0aa1c046-a57a-7262-ffcd-833590924202 -40:53.173 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0aa1c046-a57a-7262-ffcd-833590924202 -40:53.203 [INFO] TestAgent.server: federation state anti-entropy synced -40:53.211 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -40:53.214 [INFO] TestAgent: Synced node info -40:53.214 [DEBUG] TestAgent: Node info in sync -40:53.354 [DEBUG] TestAgent: Node info in sync -40:53.355 [INFO] TestAgent: Synced service: service=svc_id1 -40:53.355 [INFO] TestAgent: Synced service: service=svc_id2 -40:53.356 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -40:53.357 [INFO] TestAgent: Synced node info -40:53.357 [INFO] TestAgent: Synced service: service=svc_id1 -40:53.358 [INFO] TestAgent: Synced service: service=svc_id2 -40:53.359 [INFO] TestAgent: Requesting shutdown -40:53.359 [INFO] TestAgent.server: shutting down server -40:53.359 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -40:53.359 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -40:53.359 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -40:53.359 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -40:53.359 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -40:53.359 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -40:53.359 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -40:53.359 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -40:53.359 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -40:53.359 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -40:53.359 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:40:53.359Z [INFO] agent.router.manager: shutting down -2022-09-15T21:40:53.359Z [INFO] agent.router.manager: shutting down -40:53.360 [INFO] TestAgent: consul server down -40:53.360 [INFO] TestAgent: shutdown complete -40:53.360 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29456 network=tcp -40:53.360 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29456 network=udp -40:53.360 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29457 network=tcp -40:53.860 [INFO] TestAgent: Waiting for endpoints to shut down -40:53.860 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentAntiEntropy_EnableTagOverride (0.81s) -2022-09-15T21:40:53.171Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:40:53.219Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -40:53.225 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:81747b36-366e-40d0-1ff3-7bc827fa3c79 Address:127.0.0.1:29475}]" -40:53.226 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29475 [Follower]" leader= -40:53.226 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-81747b36-366e-40d0-1ff3-7bc827fa3c79.dc1 127.0.0.1 -40:53.227 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-81747b36-366e-40d0-1ff3-7bc827fa3c79 127.0.0.1 -2022-09-15T21:40:53.227Z [INFO] agent.router: Initializing LAN area manager -40:53.228 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29470 network=udp -40:53.228 [INFO] TestAgent.server: Adding LAN server: server="Node-81747b36-366e-40d0-1ff3-7bc827fa3c79 (Addr: tcp/127.0.0.1:29475) (DC: dc1)" -40:53.228 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-81747b36-366e-40d0-1ff3-7bc827fa3c79.dc1 area=wan -40:53.228 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29470 network=tcp -40:53.229 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29471 network=tcp -40:53.229 [INFO] TestAgent: started state syncer -40:53.229 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29476 network=tcp -40:53.295 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -40:53.295 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29475 [Candidate]" term=2 -40:53.295 [DEBUG] TestAgent.server.raft: votes: needed=1 -40:53.295 [DEBUG] TestAgent.server.raft: vote granted: from=81747b36-366e-40d0-1ff3-7bc827fa3c79 term=2 tally=1 -40:53.295 [INFO] TestAgent.server.raft: election won: tally=1 -40:53.295 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29475 [Leader]" -40:53.295 [INFO] TestAgent.server: cluster leadership acquired -40:53.296 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29475 -40:53.296 [INFO] TestAgent.server: New leader elected: payload=Node-81747b36-366e-40d0-1ff3-7bc827fa3c79 -40:53.297 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -40:53.299 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -40:53.299 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -40:53.299 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -40:53.299 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -40:53.299 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -40:53.300 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-81747b36-366e-40d0-1ff3-7bc827fa3c79 -40:53.300 [INFO] TestAgent.server: member joined, marking health alive: member=Node-81747b36-366e-40d0-1ff3-7bc827fa3c79 -40:53.301 [INFO] TestAgent.server: federation state anti-entropy synced -40:53.308 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -40:53.309 [INFO] TestAgent: Synced node info -40:53.354 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -40:53.354 [DEBUG] TestAgent: Node info in sync -40:53.357 [INFO] TestAgent: Synced service: service=mysql -40:53.357 [DEBUG] TestAgent: Check in sync: check=mysql -40:53.357 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -40:53.357 [DEBUG] TestAgent: Node info in sync -40:53.357 [INFO] TestAgent: Synced service: service=mysql -40:53.361 [INFO] TestAgent: Synced service: service=redis -40:53.361 [DEBUG] TestAgent: Check in sync: check=redis:1 -40:53.361 [DEBUG] TestAgent: Check in sync: check=redis:2 -40:53.361 [DEBUG] TestAgent: Check in sync: check=mysql -40:53.361 [INFO] TestAgent: Requesting shutdown -40:53.361 [INFO] TestAgent.server: shutting down server -40:53.361 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -40:53.361 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -40:53.361 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -40:53.361 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -40:53.361 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -40:53.361 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -40:53.361 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -40:53.361 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -40:53.361 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -40:53.361 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -40:53.361 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2022-09-15T21:40:53.361Z [INFO] agent.router.manager: shutting down -2022-09-15T21:40:53.361Z [INFO] agent.router.manager: shutting down -40:53.362 [INFO] TestAgent: consul server down -40:53.362 [INFO] TestAgent: shutdown complete -40:53.362 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29470 network=tcp -40:53.362 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29470 network=udp -40:53.362 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29471 network=tcp -40:53.862 [INFO] TestAgent: Waiting for endpoints to shut down -40:53.862 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentAntiEntropy_Services_WithChecks (0.72s) -2022-09-15T21:40:53.210Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:40:53.238Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -40:53.260 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:39252b7a-12fc-cabb-df47-b97554481768 Address:127.0.0.1:29482}]" -40:53.261 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-39252b7a-12fc-cabb-df47-b97554481768.dc1 127.0.0.1 -40:53.263 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-39252b7a-12fc-cabb-df47-b97554481768 127.0.0.1 -2022-09-15T21:40:53.263Z [INFO] agent.router: Initializing LAN area manager -40:53.263 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29482 [Follower]" leader= -40:53.263 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29477 network=udp -40:53.264 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-39252b7a-12fc-cabb-df47-b97554481768.dc1 area=wan -40:53.264 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29477 network=tcp -40:53.264 [INFO] TestAgent.server: Adding LAN server: server="Node-39252b7a-12fc-cabb-df47-b97554481768 (Addr: tcp/127.0.0.1:29482) (DC: dc1)" -40:53.265 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29478 network=tcp -40:53.265 [INFO] TestAgent: started state syncer -40:53.265 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29483 network=tcp -40:53.308 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -40:53.308 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29482 [Candidate]" term=2 -40:53.308 [DEBUG] TestAgent.server.raft: votes: needed=1 -40:53.308 [DEBUG] TestAgent.server.raft: vote granted: from=39252b7a-12fc-cabb-df47-b97554481768 term=2 tally=1 -40:53.308 [INFO] TestAgent.server.raft: election won: tally=1 -40:53.308 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29482 [Leader]" -40:53.309 [INFO] TestAgent.server: cluster leadership acquired -40:53.309 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29482 -40:53.309 [INFO] TestAgent.server: New leader elected: payload=Node-39252b7a-12fc-cabb-df47-b97554481768 -40:53.310 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -40:53.313 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -40:53.313 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -40:53.313 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -40:53.313 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -40:53.313 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -40:53.313 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-39252b7a-12fc-cabb-df47-b97554481768 -40:53.313 [INFO] TestAgent.server: member joined, marking health alive: member=Node-39252b7a-12fc-cabb-df47-b97554481768 -40:53.314 [INFO] TestAgent.server: federation state anti-entropy synced -40:53.475 [INFO] TestAgent: Requesting shutdown -40:53.475 [INFO] TestAgent.server: shutting down server -40:53.475 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -40:53.475 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -40:53.475 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -40:53.475 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -40:53.475 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -40:53.475 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -40:53.475 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -40:53.476 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -40:53.476 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -40:53.476 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -40:53.476 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -40:53.476 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:40:53.476Z [INFO] agent.router.manager: shutting down -2022-09-15T21:40:53.476Z [INFO] agent.router.manager: shutting down -40:53.476 [INFO] TestAgent: consul server down -40:53.476 [INFO] TestAgent: shutdown complete -40:53.476 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29477 network=tcp -40:53.476 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29477 network=udp -40:53.476 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29478 network=tcp -40:53.976 [INFO] TestAgent: Waiting for endpoints to shut down -40:53.976 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_ServiceWatchCh (0.84s) -2022-09-15T21:40:53.230Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:40:53.272Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -40:53.275 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:84ac7eba-efaf-5b98-0537-f0f6943d35b9 Address:127.0.0.1:29489}]" -40:53.276 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29489 [Follower]" leader= -40:53.276 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-84ac7eba-efaf-5b98-0537-f0f6943d35b9.dc1 127.0.0.1 -40:53.277 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-84ac7eba-efaf-5b98-0537-f0f6943d35b9 127.0.0.1 -2022-09-15T21:40:53.277Z [INFO] agent.router: Initializing LAN area manager -40:53.278 [INFO] TestAgent.server: Adding LAN server: server="Node-84ac7eba-efaf-5b98-0537-f0f6943d35b9 (Addr: tcp/127.0.0.1:29489) (DC: dc1)" -40:53.278 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-84ac7eba-efaf-5b98-0537-f0f6943d35b9.dc1 area=wan -40:53.278 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29484 network=tcp -40:53.278 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29484 network=udp -40:53.279 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29485 network=tcp -40:53.279 [INFO] TestAgent: started state syncer -40:53.279 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29490 network=tcp -40:53.342 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -40:53.342 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29489 [Candidate]" term=2 -40:53.342 [DEBUG] TestAgent.server.raft: votes: needed=1 -40:53.342 [DEBUG] TestAgent.server.raft: vote granted: from=84ac7eba-efaf-5b98-0537-f0f6943d35b9 term=2 tally=1 -40:53.342 [INFO] TestAgent.server.raft: election won: tally=1 -40:53.342 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29489 [Leader]" -40:53.342 [INFO] TestAgent.server: cluster leadership acquired -40:53.343 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29489 -40:53.343 [INFO] TestAgent.server: New leader elected: payload=Node-84ac7eba-efaf-5b98-0537-f0f6943d35b9 -40:53.345 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -40:53.354 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -40:53.354 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -40:53.354 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -40:53.354 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -40:53.354 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -40:53.354 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-84ac7eba-efaf-5b98-0537-f0f6943d35b9 -40:53.354 [INFO] TestAgent.server: member joined, marking health alive: member=Node-84ac7eba-efaf-5b98-0537-f0f6943d35b9 -40:53.356 [INFO] TestAgent.server: federation state anti-entropy synced -40:53.490 [INFO] TestAgent: Requesting shutdown -40:53.490 [INFO] TestAgent.server: shutting down server -40:53.490 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -40:53.490 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -40:53.490 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -40:53.490 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -40:53.490 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -40:53.490 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -40:53.490 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -40:53.491 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -40:53.491 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -40:53.491 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -40:53.491 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -40:53.491 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:40:53.491Z [INFO] agent.router.manager: shutting down -2022-09-15T21:40:53.491Z [INFO] agent.router.manager: shutting down -40:53.491 [INFO] TestAgent: consul server down -40:53.491 [INFO] TestAgent: shutdown complete -40:53.491 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29484 network=tcp -40:53.491 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29484 network=udp -40:53.491 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29485 network=tcp -40:53.992 [INFO] TestAgent: Waiting for endpoints to shut down -40:53.992 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_sendCoordinate (0.81s) -2022-09-15T21:40:53.243Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:40:53.273Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -40:53.275 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c0da0d7b-86ff-201d-33e1-1c6db24d7b4c Address:127.0.0.1:29496}]" -40:53.275 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29496 [Follower]" leader= -40:53.276 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c0da0d7b-86ff-201d-33e1-1c6db24d7b4c.dc1 127.0.0.1 -40:53.277 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c0da0d7b-86ff-201d-33e1-1c6db24d7b4c 127.0.0.1 -2022-09-15T21:40:53.277Z [INFO] agent.router: Initializing LAN area manager -40:53.278 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29491 network=udp -40:53.278 [INFO] TestAgent.server: Adding LAN server: server="Node-c0da0d7b-86ff-201d-33e1-1c6db24d7b4c (Addr: tcp/127.0.0.1:29496) (DC: dc1)" -40:53.278 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c0da0d7b-86ff-201d-33e1-1c6db24d7b4c.dc1 area=wan -40:53.278 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29491 network=tcp -40:53.279 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29492 network=tcp -40:53.279 [INFO] TestAgent: started state syncer -40:53.279 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29497 network=tcp -40:53.325 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -40:53.325 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29496 [Candidate]" term=2 -40:53.325 [DEBUG] TestAgent.server.raft: votes: needed=1 -40:53.325 [DEBUG] TestAgent.server.raft: vote granted: from=c0da0d7b-86ff-201d-33e1-1c6db24d7b4c term=2 tally=1 -40:53.325 [INFO] TestAgent.server.raft: election won: tally=1 -40:53.325 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29496 [Leader]" -40:53.326 [INFO] TestAgent.server: cluster leadership acquired -40:53.327 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29496 -40:53.328 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -40:53.330 [INFO] TestAgent.server: New leader elected: payload=Node-c0da0d7b-86ff-201d-33e1-1c6db24d7b4c -40:53.341 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -40:53.341 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -40:53.341 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -40:53.341 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -40:53.341 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -40:53.342 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c0da0d7b-86ff-201d-33e1-1c6db24d7b4c -40:53.342 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c0da0d7b-86ff-201d-33e1-1c6db24d7b4c -40:53.345 [INFO] TestAgent.server: federation state anti-entropy synced -40:53.545 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -40:53.545 [INFO] TestAgent: Synced node info -40:53.546 [INFO] TestAgent: Synced service: service=web-proxy -40:53.547 [INFO] TestAgent: Synced service: service=cache-proxy -40:53.547 [INFO] TestAgent: Deregistered service: service=lb-proxy -40:53.547 [DEBUG] TestAgent: Service in sync: service=mysql-proxy -40:53.548 [INFO] TestAgent: Synced service: service=redis-proxy -40:53.549 [ERROR] TestAgent.proxycfg: watch error: service_id=redis-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" -40:53.549 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -40:53.549 [ERROR] TestAgent.proxycfg: watch error: service_id=mysql-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" -40:53.549 [DEBUG] TestAgent: Node info in sync -40:53.549 [DEBUG] TestAgent: Service in sync: service=web-proxy -40:53.552 [INFO] TestAgent: Deregistered service: service=cache-proxy -40:53.552 [DEBUG] TestAgent: Service in sync: service=mysql-proxy -40:53.552 [DEBUG] TestAgent: Service in sync: service=redis-proxy -40:53.552 [INFO] TestAgent: Requesting shutdown -40:53.552 [INFO] TestAgent.server: shutting down server -40:53.552 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -40:53.552 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -40:53.552 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -40:53.552 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -40:53.552 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -40:53.552 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -40:53.552 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -40:53.552 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -40:53.552 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -40:53.552 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -40:53.552 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -40:53.552 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:40:53.552Z [INFO] agent.router.manager: shutting down -2022-09-15T21:40:53.552Z [INFO] agent.router.manager: shutting down -40:53.553 [INFO] TestAgent: consul server down -40:53.553 [INFO] TestAgent: shutdown complete -40:53.553 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29491 network=tcp -40:53.553 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29491 network=udp -40:53.553 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29492 network=tcp -40:54.054 [INFO] TestAgent: Waiting for endpoints to shut down -40:54.054 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentAntiEntropy_Services_ConnectProxy (0.85s) -2022-09-15T21:40:52.300Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:40:52.352Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -40:52.356 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fec711cb-c433-3a5d-70e5-484179b06a17 Address:127.0.0.1:29433}]" -40:52.357 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29433 [Follower]" leader= -40:52.406 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fec711cb-c433-3a5d-70e5-484179b06a17.dc1 127.0.0.1 -40:52.407 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fec711cb-c433-3a5d-70e5-484179b06a17 127.0.0.1 -2022-09-15T21:40:52.407Z [INFO] agent.router: Initializing LAN area manager -40:52.408 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29428 network=udp -40:52.408 [INFO] TestAgent.server: Adding LAN server: server="Node-fec711cb-c433-3a5d-70e5-484179b06a17 (Addr: tcp/127.0.0.1:29433) (DC: dc1)" -40:52.408 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fec711cb-c433-3a5d-70e5-484179b06a17.dc1 area=wan -40:52.408 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29428 network=tcp -40:52.409 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29429 network=tcp -40:52.409 [INFO] TestAgent: started state syncer -40:52.411 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29434 network=tcp -40:52.418 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -40:52.418 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29433 [Candidate]" term=2 -40:52.419 [DEBUG] TestAgent.server.raft: votes: needed=1 -40:52.419 [DEBUG] TestAgent.server.raft: vote granted: from=fec711cb-c433-3a5d-70e5-484179b06a17 term=2 tally=1 -40:52.419 [INFO] TestAgent.server.raft: election won: tally=1 -40:52.419 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29433 [Leader]" -40:52.419 [INFO] TestAgent.server: cluster leadership acquired -40:52.420 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29433 -40:52.420 [INFO] TestAgent.server: New leader elected: payload=Node-fec711cb-c433-3a5d-70e5-484179b06a17 -40:52.427 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -40:52.461 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -40:52.461 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -40:52.461 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -40:52.461 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -40:52.461 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -40:52.461 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fec711cb-c433-3a5d-70e5-484179b06a17 -40:52.461 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fec711cb-c433-3a5d-70e5-484179b06a17 -40:52.498 [INFO] TestAgent.server: federation state anti-entropy synced -40:52.550 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -40:52.551 [INFO] TestAgent: Synced node info -40:52.551 [INFO] TestAgent: Synced check: check=web -40:52.758 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -40:52.758 [DEBUG] TestAgent: Node info in sync -40:52.758 [DEBUG] TestAgent: Check in sync: check=web -40:53.227 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -40:53.227 [DEBUG] TestAgent: Node info in sync -40:53.228 [INFO] TestAgent: Synced check: check=web -40:53.228 [DEBUG] TestAgent: Node info in sync -40:53.228 [DEBUG] TestAgent: Check in sync: check=web -40:53.311 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -40:53.312 [INFO] TestAgent: Synced node info -40:53.312 [INFO] TestAgent: Synced check: check=web -40:53.313 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -40:53.314 [INFO] TestAgent: Synced node info -40:53.314 [DEBUG] TestAgent: Check in sync: check=web -40:53.938 [DEBUG] TestAgent: Node info in sync -40:53.939 [INFO] TestAgent: Synced check: check=web -40:53.956 [INFO] TestAgent: Requesting shutdown -40:53.956 [INFO] TestAgent.server: shutting down server -40:53.956 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -40:53.957 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -40:53.957 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -40:53.957 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -40:53.957 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -40:53.957 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -40:53.957 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -40:53.957 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -40:53.957 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -40:53.957 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -40:53.957 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:40:53.957Z [INFO] agent.router.manager: shutting down -2022-09-15T21:40:53.957Z [INFO] agent.router.manager: shutting down -40:53.957 [INFO] TestAgent: consul server down -40:53.957 [INFO] TestAgent: shutdown complete -40:53.957 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29428 network=tcp -40:53.957 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29428 network=udp -40:53.957 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29429 network=tcp -40:54.462 [INFO] TestAgent: Waiting for endpoints to shut down -40:54.462 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentAntiEntropy_Check_DeferSync (2.30s) +--- PASS: TestAgent_ServiceTokens (0.01s) +=== CONT TestAgent_ServiceWatchCh +2021-08-13T15:32:38.127Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:38.159Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:38.165 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4d614605-5eb1-bfcc-9e82-f5d89a934e8b Address:127.0.0.1:25335}]" +32:38.166 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25335 [Follower]" leader= +32:38.171 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4d614605-5eb1-bfcc-9e82-f5d89a934e8b.dc1 127.0.0.1 +32:38.173 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4d614605-5eb1-bfcc-9e82-f5d89a934e8b 127.0.0.1 +2021-08-13T15:32:38.173Z [INFO] agent.router: Initializing LAN area manager +32:38.173 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25330 network=udp +32:38.174 [INFO] TestAgent.server: Adding LAN server: server="Node-4d614605-5eb1-bfcc-9e82-f5d89a934e8b (Addr: tcp/127.0.0.1:25335) (DC: dc1)" +32:38.174 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4d614605-5eb1-bfcc-9e82-f5d89a934e8b.dc1 area=wan +32:38.174 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25330 network=tcp +32:38.176 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25331 network=tcp +32:38.176 [INFO] TestAgent: started state syncer +32:38.176 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25336 network=tcp +32:38.226 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:38.238 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25335 [Candidate]" term=2 +32:38.238 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:38.238 [DEBUG] TestAgent.server.raft: vote granted: from=4d614605-5eb1-bfcc-9e82-f5d89a934e8b term=2 tally=1 +32:38.238 [INFO] TestAgent.server.raft: election won: tally=1 +32:38.238 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25335 [Leader]" +32:38.238 [INFO] TestAgent.server: cluster leadership acquired +32:38.238 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25335 +32:38.239 [INFO] TestAgent.server: New leader elected: payload=Node-4d614605-5eb1-bfcc-9e82-f5d89a934e8b +32:38.244 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:38.343 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:38.343 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:38.343 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:38.343 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:38.343 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:38.343 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4d614605-5eb1-bfcc-9e82-f5d89a934e8b +32:38.343 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4d614605-5eb1-bfcc-9e82-f5d89a934e8b +32:38.350 [INFO] TestAgent.server: federation state anti-entropy synced +32:38.589 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:38.589 [INFO] TestAgent: Synced node info +32:38.675 [INFO] TestAgent: Requesting shutdown +32:38.675 [INFO] TestAgent.server: shutting down server +32:38.675 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:38.675 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:38.675 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:38.675 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:38.675 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:38.675 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:38.675 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:38.675 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:38.675 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:38.675 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:38.675 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:38.676Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:38.676Z [INFO] agent.router.manager: shutting down +32:38.676 [INFO] TestAgent: consul server down +32:38.676 [INFO] TestAgent: shutdown complete +32:38.676 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25330 network=tcp +32:38.676 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25330 network=udp +32:38.676 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25331 network=tcp +32:39.176 [INFO] TestAgent: Waiting for endpoints to shut down +32:39.176 [INFO] TestAgent: Endpoints down +--- PASS: TestAliasNotifications_local (1.12s) +=== CONT TestAgentAntiEntropy_Services_ConnectProxy +2021-08-13T15:32:38.144Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:38.223Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:38.228 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:078dc75a-4d73-bb20-afa3-3d3d76a5a313 Address:127.0.0.1:25321}]" +32:38.229 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-078dc75a-4d73-bb20-afa3-3d3d76a5a313.dc1 127.0.0.1 +32:38.230 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-078dc75a-4d73-bb20-afa3-3d3d76a5a313 127.0.0.1 +2021-08-13T15:32:38.230Z [INFO] agent.router: Initializing LAN area manager +32:38.230 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25316 network=udp +32:38.231 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25321 [Follower]" leader= +32:38.235 [INFO] TestAgent.server: Adding LAN server: server="Node-078dc75a-4d73-bb20-afa3-3d3d76a5a313 (Addr: tcp/127.0.0.1:25321) (DC: dc1)" +32:38.235 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-078dc75a-4d73-bb20-afa3-3d3d76a5a313.dc1 area=wan +32:38.235 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25316 network=tcp +32:38.236 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25317 network=tcp +32:38.236 [INFO] TestAgent: started state syncer +32:38.239 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25322 network=tcp +32:38.341 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:38.341 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25321 [Candidate]" term=2 +32:38.343 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:38.343 [DEBUG] TestAgent.server.raft: vote granted: from=078dc75a-4d73-bb20-afa3-3d3d76a5a313 term=2 tally=1 +32:38.343 [INFO] TestAgent.server.raft: election won: tally=1 +32:38.343 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25321 [Leader]" +32:38.343 [INFO] TestAgent.server: cluster leadership acquired +32:38.343 [INFO] TestAgent.server: New leader elected: payload=Node-078dc75a-4d73-bb20-afa3-3d3d76a5a313 +32:38.355 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25321 +32:38.357 [INFO] TestAgent: Synced node info +32:38.358 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:38.361 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:38.361 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:38.361 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:38.361 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:38.361 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:38.361 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-078dc75a-4d73-bb20-afa3-3d3d76a5a313 +32:38.361 [INFO] TestAgent.server: member joined, marking health alive: member=Node-078dc75a-4d73-bb20-afa3-3d3d76a5a313 +32:38.362 [INFO] TestAgent.server: federation state anti-entropy synced +32:38.765 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:38.765 [INFO] TestAgent: Synced node info +32:38.765 [DEBUG] TestAgent: Service in sync: service=mysql +32:38.766 [INFO] TestAgent: Synced service: service=redis +32:38.766 [INFO] TestAgent: Synced service: service=web +32:38.766 [INFO] TestAgent: Synced service: service=api +32:38.767 [INFO] TestAgent: Synced service: service=cache +32:38.768 [INFO] TestAgent: Deregistered service: service=lb +32:38.768 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:38.768 [INFO] TestAgent: Synced node info +32:38.768 [DEBUG] TestAgent: Service in sync: service=redis +32:38.768 [DEBUG] TestAgent: Service in sync: service=web +32:38.769 [INFO] TestAgent: Deregistered service: service=api +32:38.769 [DEBUG] TestAgent: Service in sync: service=cache +32:38.769 [DEBUG] TestAgent: Service in sync: service=mysql +32:38.769 [INFO] TestAgent: Requesting shutdown +32:38.769 [INFO] TestAgent.server: shutting down server +32:38.769 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:38.769 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:38.769 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:38.769 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:38.769 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:38.769 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:38.769 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:38.769 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:38.769 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:38.769 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:38.769 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:38.770Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:38.770Z [INFO] agent.router.manager: shutting down +32:38.770 [INFO] TestAgent: consul server down +32:38.770 [INFO] TestAgent: shutdown complete +32:38.770 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25316 network=tcp +32:38.770 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25316 network=udp +32:38.770 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25317 network=tcp +32:39.270 [INFO] TestAgent: Waiting for endpoints to shut down +32:39.270 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentAntiEntropy_Services (1.22s) +2021-08-13T15:32:38.894Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:38.919Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:38.925 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:088ad1e0-03c9-b3b3-58e1-eea9f2511a37 Address:127.0.0.1:25377}]" +32:38.926 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-088ad1e0-03c9-b3b3-58e1-eea9f2511a37.dc1 127.0.0.1 +32:38.927 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-088ad1e0-03c9-b3b3-58e1-eea9f2511a37 127.0.0.1 +2021-08-13T15:32:38.927Z [INFO] agent.router: Initializing LAN area manager +32:38.927 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25377 [Follower]" leader= +32:38.930 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-088ad1e0-03c9-b3b3-58e1-eea9f2511a37.dc1 area=wan +32:38.930 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25372 network=udp +32:38.930 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25372 network=tcp +32:38.931 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25373 network=tcp +32:38.931 [INFO] TestAgent: started state syncer +32:38.931 [INFO] TestAgent.server: Adding LAN server: server="Node-088ad1e0-03c9-b3b3-58e1-eea9f2511a37 (Addr: tcp/127.0.0.1:25377) (DC: dc1)" +32:38.931 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25378 network=tcp +32:38.990 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:38.990 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25377 [Candidate]" term=2 +32:38.990 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:38.990 [DEBUG] TestAgent.server.raft: vote granted: from=088ad1e0-03c9-b3b3-58e1-eea9f2511a37 term=2 tally=1 +32:38.990 [INFO] TestAgent.server.raft: election won: tally=1 +32:38.990 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25377 [Leader]" +32:38.990 [INFO] TestAgent.server: cluster leadership acquired +32:38.990 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25377 +32:38.991 [INFO] TestAgent.server: New leader elected: payload=Node-088ad1e0-03c9-b3b3-58e1-eea9f2511a37 +32:38.991 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:38.994 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:38.994 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:38.994 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:38.994 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:38.994 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:38.994 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-088ad1e0-03c9-b3b3-58e1-eea9f2511a37 +32:38.994 [INFO] TestAgent.server: member joined, marking health alive: member=Node-088ad1e0-03c9-b3b3-58e1-eea9f2511a37 +32:38.995 [INFO] TestAgent.server: federation state anti-entropy synced +32:39.048 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:39.049 [INFO] TestAgent: Synced node info +32:39.141 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:39.141 [INFO] TestAgent: Synced node info +32:39.142 [INFO] TestAgent: Deregistered service: service=deleted-check-service +32:39.142 [INFO] TestAgent: Deregistered check: check=lb +32:39.142 [INFO] TestAgent: Requesting shutdown +32:39.142 [INFO] TestAgent.server: shutting down server +32:39.142 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:39.142 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:39.142 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:39.142 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:39.142 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:39.142 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:39.142 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:39.142 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:39.142 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:39.142 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:39.142 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:39.143Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:39.143Z [INFO] agent.router.manager: shutting down +32:39.143 [INFO] TestAgent: consul server down +32:39.143 [INFO] TestAgent: shutdown complete +32:39.143 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25372 network=tcp +32:39.143 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25372 network=udp +32:39.143 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25373 network=tcp +32:39.643 [INFO] TestAgent: Waiting for endpoints to shut down +32:39.643 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentAntiEntropy_RemovingServiceAndCheck (0.77s) +2021-08-13T15:32:39.008Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:32:39.008Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:39.035Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2021-08-13T15:32:39.035Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:39.038 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ba6c4f66-a896-706e-0471-cc1a494a5b0f Address:127.0.0.1:25391}]" +32:39.038 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25391 [Follower]" leader= +32:39.039 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ba6c4f66-a896-706e-0471-cc1a494a5b0f.dc1 127.0.0.1 +32:39.039 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ba6c4f66-a896-706e-0471-cc1a494a5b0f 127.0.0.1 +2021-08-13T15:32:39.039Z [INFO] agent.router: Initializing LAN area manager +32:39.040 [INFO] TestAgent.server: Adding LAN server: server="Node-ba6c4f66-a896-706e-0471-cc1a494a5b0f (Addr: tcp/127.0.0.1:25391) (DC: dc1)" +32:39.040 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ba6c4f66-a896-706e-0471-cc1a494a5b0f.dc1 area=wan +32:39.040 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25386 network=tcp +32:39.040 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25386 network=udp +32:39.041 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25387 network=tcp +32:39.041 [INFO] TestAgent: started state syncer +32:39.041 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25392 network=tcp +32:39.090 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +32:39.105 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:39.105 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25391 [Candidate]" term=2 +32:39.106 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:39.106 [DEBUG] TestAgent.server.raft: vote granted: from=ba6c4f66-a896-706e-0471-cc1a494a5b0f term=2 tally=1 +32:39.106 [INFO] TestAgent.server.raft: election won: tally=1 +32:39.106 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25391 [Leader]" +32:39.107 [INFO] TestAgent.server: cluster leadership acquired +32:39.108 [INFO] TestAgent.server: initializing acls +32:39.109 [INFO] TestAgent.server: New leader elected: payload=Node-ba6c4f66-a896-706e-0471-cc1a494a5b0f +32:39.109 [INFO] TestAgent.server: Created ACL 'global-management' policy +32:39.110 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +32:39.112 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +32:39.113 [INFO] TestAgent.server: Created ACL anonymous token from configuration +32:39.113 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +32:39.113 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +32:39.114 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-ba6c4f66-a896-706e-0471-cc1a494a5b0f +32:39.114 [INFO] TestAgent.server: Updating LAN server: server="Node-ba6c4f66-a896-706e-0471-cc1a494a5b0f (Addr: tcp/127.0.0.1:25391) (DC: dc1)" +32:39.114 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-ba6c4f66-a896-706e-0471-cc1a494a5b0f.dc1 +32:39.115 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:39.116 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-ba6c4f66-a896-706e-0471-cc1a494a5b0f.dc1 area=wan +32:39.118 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:39.118 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:39.118 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:39.118 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:39.118 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:39.118 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ba6c4f66-a896-706e-0471-cc1a494a5b0f +32:39.118 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ba6c4f66-a896-706e-0471-cc1a494a5b0f +32:39.118 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ba6c4f66-a896-706e-0471-cc1a494a5b0f +32:39.120 [INFO] TestAgent.server: federation state anti-entropy synced +32:39.137 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-ba6c4f66-a896-706e-0471-cc1a494a5b0f +32:39.137 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-ba6c4f66-a896-706e-0471-cc1a494a5b0f +32:39.139 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth +32:39.139 [WARN] TestAgent: Node info update blocked by ACLs: node=ba6c4f66-a896-706e-0471-cc1a494a5b0f accessorID=00000000-0000-0000-0000-000000000002 +32:39.139 [WARN] TestAgent: Service registration blocked by ACLs: service=mysql accessorID= +32:39.141 [INFO] TestAgent: Synced service: service=api +32:39.142 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth +32:39.142 [WARN] TestAgent: Node info update blocked by ACLs: node=ba6c4f66-a896-706e-0471-cc1a494a5b0f accessorID=00000000-0000-0000-0000-000000000002 +32:39.142 [WARN] TestAgent: Service registration blocked by ACLs: service=mysql accessorID= +32:39.142 [INFO] TestAgent: Deregistered service: service=api +32:39.142 [INFO] TestAgent: Requesting shutdown +32:39.142 [INFO] TestAgent.server: shutting down server +32:39.142 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +32:39.142 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +32:39.142 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:39.142 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:39.142 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:39.142 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:39.142 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:39.143 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:39.143 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:39.143 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +32:39.143 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +32:39.143 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +32:39.143 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:39.143 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:39.143 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:39.143 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:39.143Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:39.143Z [INFO] agent.router.manager: shutting down +32:39.143 [INFO] TestAgent: consul server down +32:39.143 [INFO] TestAgent: shutdown complete +32:39.143 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25386 network=tcp +32:39.143 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25386 network=udp +32:39.143 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25387 network=tcp +32:39.644 [INFO] TestAgent: Waiting for endpoints to shut down +32:39.644 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentAntiEntropy_Services_ACLDeny (0.65s) +2021-08-13T15:32:38.947Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:38.962Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:38.964 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:74b86e20-5ab3-8c47-1ee5-35aa5bd99cd9 Address:127.0.0.1:25384}]" +32:38.964 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25384 [Follower]" leader= +32:38.965 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-74b86e20-5ab3-8c47-1ee5-35aa5bd99cd9.dc1 127.0.0.1 +32:38.966 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-74b86e20-5ab3-8c47-1ee5-35aa5bd99cd9 127.0.0.1 +2021-08-13T15:32:38.966Z [INFO] agent.router: Initializing LAN area manager +32:38.966 [INFO] TestAgent.server: Adding LAN server: server="Node-74b86e20-5ab3-8c47-1ee5-35aa5bd99cd9 (Addr: tcp/127.0.0.1:25384) (DC: dc1)" +32:38.966 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-74b86e20-5ab3-8c47-1ee5-35aa5bd99cd9.dc1 area=wan +32:38.966 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25379 network=udp +32:38.967 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25379 network=tcp +32:38.967 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25380 network=tcp +32:38.968 [INFO] TestAgent: started state syncer +32:38.968 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25385 network=tcp +32:39.003 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:39.003 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25384 [Candidate]" term=2 +32:39.003 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:39.003 [DEBUG] TestAgent.server.raft: vote granted: from=74b86e20-5ab3-8c47-1ee5-35aa5bd99cd9 term=2 tally=1 +32:39.003 [INFO] TestAgent.server.raft: election won: tally=1 +32:39.003 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25384 [Leader]" +32:39.004 [INFO] TestAgent.server: cluster leadership acquired +32:39.004 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25384 +32:39.005 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:39.007 [INFO] TestAgent.server: New leader elected: payload=Node-74b86e20-5ab3-8c47-1ee5-35aa5bd99cd9 +32:39.007 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:39.007 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:39.007 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:39.007 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:39.007 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:39.007 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-74b86e20-5ab3-8c47-1ee5-35aa5bd99cd9 +32:39.007 [INFO] TestAgent.server: member joined, marking health alive: member=Node-74b86e20-5ab3-8c47-1ee5-35aa5bd99cd9 +32:39.008 [INFO] TestAgent.server: federation state anti-entropy synced +32:39.204 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:39.204 [INFO] TestAgent: Synced node info +32:39.204 [DEBUG] TestAgent: Node info in sync +32:39.244 [DEBUG] TestAgent: Node info in sync +32:39.244 [INFO] TestAgent: Synced service: service=svc_id1 +32:39.245 [INFO] TestAgent: Synced service: service=svc_id2 +32:39.246 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:39.246 [INFO] TestAgent: Synced node info +32:39.247 [INFO] TestAgent: Synced service: service=svc_id1 +32:39.247 [INFO] TestAgent: Synced service: service=svc_id2 +32:39.248 [INFO] TestAgent: Requesting shutdown +32:39.248 [INFO] TestAgent.server: shutting down server +32:39.248 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:39.248 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:39.248 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:39.248 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:39.248 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:39.248 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:39.248 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:39.248 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:39.248 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:39.248 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:39.248 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:39.248Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:39.248Z [INFO] agent.router.manager: shutting down +32:39.248 [INFO] TestAgent: consul server down +32:39.248 [INFO] TestAgent: shutdown complete +32:39.248 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25379 network=tcp +32:39.248 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25379 network=udp +32:39.248 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25380 network=tcp +32:39.748 [INFO] TestAgent: Waiting for endpoints to shut down +32:39.748 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentAntiEntropy_EnableTagOverride (0.82s) +2021-08-13T15:32:39.128Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:39.144Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:39.147 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5762e3ee-10e9-9233-3dc0-7a77c36707d8 Address:127.0.0.1:25405}]" +32:39.147 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25405 [Follower]" leader= +32:39.149 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5762e3ee-10e9-9233-3dc0-7a77c36707d8.dc1 127.0.0.1 +32:39.150 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5762e3ee-10e9-9233-3dc0-7a77c36707d8 127.0.0.1 +2021-08-13T15:32:39.150Z [INFO] agent.router: Initializing LAN area manager +32:39.150 [INFO] TestAgent.server: Adding LAN server: server="Node-5762e3ee-10e9-9233-3dc0-7a77c36707d8 (Addr: tcp/127.0.0.1:25405) (DC: dc1)" +32:39.150 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5762e3ee-10e9-9233-3dc0-7a77c36707d8.dc1 area=wan +32:39.150 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25400 network=udp +32:39.150 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25400 network=tcp +32:39.151 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25401 network=tcp +32:39.151 [INFO] TestAgent: started state syncer +32:39.151 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25406 network=tcp +32:39.215 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:39.215 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25405 [Candidate]" term=2 +32:39.215 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:39.215 [DEBUG] TestAgent.server.raft: vote granted: from=5762e3ee-10e9-9233-3dc0-7a77c36707d8 term=2 tally=1 +32:39.215 [INFO] TestAgent.server.raft: election won: tally=1 +32:39.215 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25405 [Leader]" +32:39.216 [INFO] TestAgent.server: cluster leadership acquired +32:39.216 [INFO] TestAgent.server: New leader elected: payload=Node-5762e3ee-10e9-9233-3dc0-7a77c36707d8 +32:39.216 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25405 +32:39.217 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:39.219 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:39.219 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:39.219 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:39.219 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:39.219 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:39.219 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5762e3ee-10e9-9233-3dc0-7a77c36707d8 +32:39.219 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5762e3ee-10e9-9233-3dc0-7a77c36707d8 +32:39.220 [INFO] TestAgent.server: federation state anti-entropy synced +32:39.348 [INFO] TestAgent: Requesting shutdown +32:39.348 [INFO] TestAgent.server: shutting down server +32:39.348 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:39.348 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:39.348 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:39.348 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:39.348 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:39.348 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:39.348 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:39.348 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:39.348 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:39.348 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +32:39.348 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:39.348 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:39.348Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:39.349Z [INFO] agent.router.manager: shutting down +32:39.349 [INFO] TestAgent: consul server down +32:39.349 [INFO] TestAgent: shutdown complete +32:39.349 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25400 network=tcp +32:39.349 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25400 network=udp +32:39.349 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25401 network=tcp +32:39.849 [INFO] TestAgent: Waiting for endpoints to shut down +32:39.849 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_ServiceWatchCh (0.75s) +2021-08-13T15:32:39.201Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:39.216Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:39.218 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4542aa97-6a40-1285-9895-f575e65945fb Address:127.0.0.1:25412}]" +32:39.219 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25412 [Follower]" leader= +32:39.219 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4542aa97-6a40-1285-9895-f575e65945fb.dc1 127.0.0.1 +32:39.220 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4542aa97-6a40-1285-9895-f575e65945fb 127.0.0.1 +2021-08-13T15:32:39.220Z [INFO] agent.router: Initializing LAN area manager +32:39.220 [INFO] TestAgent.server: Adding LAN server: server="Node-4542aa97-6a40-1285-9895-f575e65945fb (Addr: tcp/127.0.0.1:25412) (DC: dc1)" +32:39.220 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4542aa97-6a40-1285-9895-f575e65945fb.dc1 area=wan +32:39.220 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25407 network=udp +32:39.221 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25407 network=tcp +32:39.221 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25408 network=tcp +32:39.221 [INFO] TestAgent: started state syncer +32:39.222 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25413 network=tcp +32:39.287 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:39.287 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25412 [Candidate]" term=2 +32:39.287 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:39.287 [DEBUG] TestAgent.server.raft: vote granted: from=4542aa97-6a40-1285-9895-f575e65945fb term=2 tally=1 +32:39.287 [INFO] TestAgent.server.raft: election won: tally=1 +32:39.287 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25412 [Leader]" +32:39.287 [INFO] TestAgent.server: cluster leadership acquired +32:39.287 [INFO] TestAgent.server: New leader elected: payload=Node-4542aa97-6a40-1285-9895-f575e65945fb +32:39.288 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25412 +32:39.289 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:39.291 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:39.291 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:39.291 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:39.291 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:39.291 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:39.291 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4542aa97-6a40-1285-9895-f575e65945fb +32:39.291 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4542aa97-6a40-1285-9895-f575e65945fb +32:39.292 [INFO] TestAgent.server: federation state anti-entropy synced +32:39.404 [ERROR] TestAgent.proxycfg: watch error: service_id=mysql-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" +32:39.404 [ERROR] TestAgent.proxycfg: watch error: service_id=mysql-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" +32:39.405 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:39.406 [INFO] TestAgent: Synced node info +32:39.406 [DEBUG] TestAgent: Service in sync: service=mysql-proxy +32:39.407 [INFO] TestAgent: Synced service: service=redis-proxy +32:39.408 [INFO] TestAgent: Synced service: service=web-proxy +32:39.408 [INFO] TestAgent: Synced service: service=cache-proxy +32:39.409 [INFO] TestAgent: Deregistered service: service=lb-proxy +32:39.410 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:39.411 [ERROR] TestAgent.proxycfg: watch error: service_id=mysql-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" +32:39.411 [ERROR] TestAgent.proxycfg: watch error: service_id=web-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" +32:39.411 [ERROR] TestAgent.proxycfg: watch error: service_id=redis-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" +32:39.411 [DEBUG] TestAgent: Node info in sync +32:39.411 [DEBUG] TestAgent: Service in sync: service=mysql-proxy +32:39.411 [DEBUG] TestAgent: Service in sync: service=redis-proxy +32:39.411 [DEBUG] TestAgent: Service in sync: service=web-proxy +32:39.411 [INFO] TestAgent: Deregistered service: service=cache-proxy +32:39.412 [INFO] TestAgent: Requesting shutdown +32:39.412 [ERROR] TestAgent.proxycfg: watch error: service_id=mysql-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" +32:39.412 [INFO] TestAgent.server: shutting down server +32:39.412 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:39.412 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:39.412 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:39.412 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:39.412 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:39.412 [ERROR] TestAgent.proxycfg: watch error: service_id=redis-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" +32:39.412 [ERROR] TestAgent.proxycfg: watch error: service_id=web-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" +32:39.412 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:39.413 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:39.413 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +32:39.413 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +32:39.413 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:39.413 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:39.413 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +2021-08-13T15:32:39.413Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:39.413Z [INFO] agent.router.manager: shutting down +32:39.413 [INFO] TestAgent: consul server down +32:39.413 [INFO] TestAgent: shutdown complete +32:39.413 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25407 network=tcp +32:39.413 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25407 network=udp +32:39.413 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25408 network=tcp +32:39.913 [INFO] TestAgent: Waiting for endpoints to shut down +32:39.913 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentAntiEntropy_Services_ConnectProxy (0.74s) +2021-08-13T15:32:39.084Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:39.099Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:39.104 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bab6a354-6313-4ff7-ac90-471b0d52a407 Address:127.0.0.1:25398}]" +32:39.106 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25398 [Follower]" leader= +32:39.108 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bab6a354-6313-4ff7-ac90-471b0d52a407.dc1 127.0.0.1 +32:39.115 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bab6a354-6313-4ff7-ac90-471b0d52a407 127.0.0.1 +2021-08-13T15:32:39.115Z [INFO] agent.router: Initializing LAN area manager +32:39.116 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25393 network=udp +32:39.119 [INFO] TestAgent.server: Adding LAN server: server="Node-bab6a354-6313-4ff7-ac90-471b0d52a407 (Addr: tcp/127.0.0.1:25398) (DC: dc1)" +32:39.119 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bab6a354-6313-4ff7-ac90-471b0d52a407.dc1 area=wan +32:39.119 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25393 network=tcp +32:39.120 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25394 network=tcp +32:39.120 [INFO] TestAgent: started state syncer +32:39.120 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25399 network=tcp +32:39.161 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:39.161 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25398 [Candidate]" term=2 +32:39.161 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:39.161 [DEBUG] TestAgent.server.raft: vote granted: from=bab6a354-6313-4ff7-ac90-471b0d52a407 term=2 tally=1 +32:39.161 [INFO] TestAgent.server.raft: election won: tally=1 +32:39.161 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25398 [Leader]" +32:39.161 [INFO] TestAgent.server: cluster leadership acquired +32:39.161 [INFO] TestAgent.server: New leader elected: payload=Node-bab6a354-6313-4ff7-ac90-471b0d52a407 +32:39.161 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25398 +32:39.162 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:39.165 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:39.165 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:39.165 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:39.165 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:39.165 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:39.165 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bab6a354-6313-4ff7-ac90-471b0d52a407 +32:39.165 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bab6a354-6313-4ff7-ac90-471b0d52a407 +32:39.166 [INFO] TestAgent.server: federation state anti-entropy synced +32:39.415 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:39.415 [INFO] TestAgent: Synced node info +32:39.416 [INFO] TestAgent: Synced service: service=mysql +32:39.416 [DEBUG] TestAgent: Check in sync: check=mysql +32:39.416 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:39.416 [DEBUG] TestAgent: Node info in sync +32:39.417 [INFO] TestAgent: Synced service: service=mysql +32:39.418 [INFO] TestAgent: Synced service: service=redis +32:39.418 [DEBUG] TestAgent: Check in sync: check=mysql +32:39.418 [DEBUG] TestAgent: Check in sync: check=redis:1 +32:39.418 [DEBUG] TestAgent: Check in sync: check=redis:2 +32:39.418 [INFO] TestAgent: Requesting shutdown +32:39.418 [INFO] TestAgent.server: shutting down server +32:39.418 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:39.418 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:39.418 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:39.418 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:39.418 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:39.418 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:39.418 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:39.418 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +32:39.418 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:39.418 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +32:39.418 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:39.418 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2021-08-13T15:32:39.418Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:39.418Z [INFO] agent.router.manager: shutting down +32:39.418 [INFO] TestAgent: consul server down +32:39.418 [INFO] TestAgent: shutdown complete +32:39.418 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25393 network=tcp +32:39.418 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25393 network=udp +32:39.418 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25394 network=tcp +32:39.918 [INFO] TestAgent: Waiting for endpoints to shut down +32:39.918 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentAntiEntropy_Services_WithChecks (0.85s) +2021-08-13T15:32:38.208Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:38.338Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:38.352 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6fcf34cc-059d-4922-c53d-1bec981ac834 Address:127.0.0.1:25349}]" +32:38.355 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25349 [Follower]" leader= +32:38.358 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6fcf34cc-059d-4922-c53d-1bec981ac834.dc1 127.0.0.1 +32:38.359 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6fcf34cc-059d-4922-c53d-1bec981ac834 127.0.0.1 +2021-08-13T15:32:38.359Z [INFO] agent.router: Initializing LAN area manager +32:38.360 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25344 network=udp +32:38.360 [INFO] TestAgent.server: Adding LAN server: server="Node-6fcf34cc-059d-4922-c53d-1bec981ac834 (Addr: tcp/127.0.0.1:25349) (DC: dc1)" +32:38.360 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6fcf34cc-059d-4922-c53d-1bec981ac834.dc1 area=wan +32:38.360 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25344 network=tcp +32:38.361 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25345 network=tcp +32:38.361 [INFO] TestAgent: started state syncer +32:38.362 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25350 network=tcp +32:38.417 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:38.417 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25349 [Candidate]" term=2 +32:38.417 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:38.417 [DEBUG] TestAgent.server.raft: vote granted: from=6fcf34cc-059d-4922-c53d-1bec981ac834 term=2 tally=1 +32:38.417 [INFO] TestAgent.server.raft: election won: tally=1 +32:38.417 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25349 [Leader]" +32:38.417 [INFO] TestAgent.server: cluster leadership acquired +32:38.417 [INFO] TestAgent.server: New leader elected: payload=Node-6fcf34cc-059d-4922-c53d-1bec981ac834 +32:38.417 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25349 +32:38.419 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:38.421 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:38.421 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:38.421 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:38.421 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:38.421 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:38.421 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6fcf34cc-059d-4922-c53d-1bec981ac834 +32:38.421 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6fcf34cc-059d-4922-c53d-1bec981ac834 +32:38.422 [INFO] TestAgent.server: federation state anti-entropy synced +32:38.643 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:38.643 [INFO] TestAgent: Synced node info +32:38.644 [INFO] TestAgent: Synced check: check=web +32:38.678 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:38.678 [DEBUG] TestAgent: Node info in sync +32:38.678 [DEBUG] TestAgent: Check in sync: check=web +32:38.678 [DEBUG] TestAgent: Node info in sync +32:38.678 [DEBUG] TestAgent: Check in sync: check=web +32:38.971 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:38.971 [DEBUG] TestAgent: Node info in sync +32:38.971 [DEBUG] TestAgent: Check in sync: check=web +32:39.266 [DEBUG] TestAgent: Node info in sync +32:39.267 [INFO] TestAgent: Synced check: check=web +32:39.298 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:39.299 [INFO] TestAgent: Synced node info +32:39.299 [INFO] TestAgent: Synced check: check=web +32:39.300 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:39.300 [INFO] TestAgent: Synced node info +32:39.300 [DEBUG] TestAgent: Check in sync: check=web +32:39.559 [DEBUG] TestAgent: Node info in sync +32:39.560 [INFO] TestAgent: Synced check: check=web +32:39.578 [INFO] TestAgent: Requesting shutdown +32:39.578 [INFO] TestAgent.server: shutting down server +32:39.578 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:39.578 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:39.578 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:39.578 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:39.579 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:39.579 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:39.579 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:39.579 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:39.579 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:39.579 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:39.579 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:39.579Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:39.579Z [INFO] agent.router.manager: shutting down +32:39.579 [INFO] TestAgent: consul server down +32:39.579 [INFO] TestAgent: shutdown complete +32:39.579 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25344 network=tcp +32:39.579 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25344 network=udp +32:39.579 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25345 network=tcp +32:40.079 [INFO] TestAgent: Waiting for endpoints to shut down +32:40.079 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentAntiEntropy_Check_DeferSync (2.03s) PASS -ok github.com/hashicorp/consul/agent/local 2.457s +ok github.com/hashicorp/consul/agent/local 2.096s === RUN TestBuild === RUN TestBuild/no_version === RUN TestBuild/bad_version @@ -59052,7 +59041,7 @@ === RUN TestIsConsulServer_Optional --- PASS: TestIsConsulServer_Optional (0.00s) PASS -ok github.com/hashicorp/consul/agent/metadata 0.041s +ok github.com/hashicorp/consul/agent/metadata 0.016s ? github.com/hashicorp/consul/agent/mock [no test files] === RUN TestPeekForTLS_not_TLS === RUN TestPeekForTLS_not_TLS/tcp_rpc_type_byte_0 @@ -59081,7 +59070,7 @@ === RUN TestPeekForTLS_actual_TLS/tcp_rpc_type_byte_5 === RUN TestPeekForTLS_actual_TLS/tcp_rpc_type_byte_6 === RUN TestPeekForTLS_actual_TLS/tcp_rpc_type_byte_7 ---- PASS: TestPeekForTLS_actual_TLS (0.08s) +--- PASS: TestPeekForTLS_actual_TLS (0.07s) --- PASS: TestPeekForTLS_actual_TLS/tcp_rpc_type_byte_0 (0.03s) --- PASS: TestPeekForTLS_actual_TLS/tcp_rpc_type_byte_1 (0.01s) --- PASS: TestPeekForTLS_actual_TLS/tcp_rpc_type_byte_2 (0.01s) @@ -59091,7 +59080,7 @@ --- PASS: TestPeekForTLS_actual_TLS/tcp_rpc_type_byte_6 (0.01s) --- PASS: TestPeekForTLS_actual_TLS/tcp_rpc_type_byte_7 (0.01s) PASS -ok github.com/hashicorp/consul/agent/pool 0.115s +ok github.com/hashicorp/consul/agent/pool 0.082s === RUN TestManagerInternal_cycleServer --- PASS: TestManagerInternal_cycleServer (0.00s) === RUN TestManagerInternal_getServerList @@ -59125,11 +59114,11 @@ === RUN TestRouter_Routing --- PASS: TestRouter_Routing (0.00s) === RUN TestRouter_Routing_Offline -2022-09-15T21:40:49.202Z [DEBUG] TestRouter_Routing_Offline.router.manager: pinging server failed: server="node2.dc1 (Addr: tcp/127.0.0.3:8300) (DC: dc1)" error= -2022-09-15T21:40:49.202Z [DEBUG] TestRouter_Routing_Offline.router.manager: pinging server failed: server="node3.dc1 (Addr: tcp/127.0.0.4:8300) (DC: dc1)" error= -2022-09-15T21:40:49.202Z [DEBUG] TestRouter_Routing_Offline.router.manager: pinging server failed: server="node4.dc1 (Addr: tcp/127.0.0.5:8300) (DC: dc1)" error= -2022-09-15T21:40:49.202Z [DEBUG] TestRouter_Routing_Offline.router.manager: pinging server failed: server="node1.dc1 (Addr: tcp/127.0.0.2:8300) (DC: dc1)" error= -2022-09-15T21:40:49.202Z [DEBUG] TestRouter_Routing_Offline.router.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.777Z [DEBUG] TestRouter_Routing_Offline.router.manager: pinging server failed: server="node4.dc1 (Addr: tcp/127.0.0.5:8300) (DC: dc1)" error= +2021-08-13T15:32:32.777Z [DEBUG] TestRouter_Routing_Offline.router.manager: pinging server failed: server="node2.dc1 (Addr: tcp/127.0.0.3:8300) (DC: dc1)" error= +2021-08-13T15:32:32.777Z [DEBUG] TestRouter_Routing_Offline.router.manager: pinging server failed: server="node3.dc1 (Addr: tcp/127.0.0.4:8300) (DC: dc1)" error= +2021-08-13T15:32:32.777Z [DEBUG] TestRouter_Routing_Offline.router.manager: pinging server failed: server="node1.dc1 (Addr: tcp/127.0.0.2:8300) (DC: dc1)" error= +2021-08-13T15:32:32.777Z [DEBUG] TestRouter_Routing_Offline.router.manager: No healthy servers during rebalance, aborting --- PASS: TestRouter_Routing_Offline (0.00s) === RUN TestRouter_GetDatacenters --- PASS: TestRouter_GetDatacenters (0.00s) @@ -59140,10528 +59129,10552 @@ === RUN TestRouter_GetDatacenterMaps --- PASS: TestRouter_GetDatacenterMaps (0.00s) === RUN TestRouter_FindLANServer -2022-09-15T21:40:49.204Z [INFO] TestRouter_FindLANServer.router: Initializing LAN area manager +2021-08-13T15:32:32.778Z [INFO] TestRouter_FindLANServer.router: Initializing LAN area manager --- PASS: TestRouter_FindLANServer (0.00s) === RUN TestServers_AddServer --- PASS: TestServers_AddServer (0.00s) === RUN TestServers_IsOffline -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.205Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.204Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.778Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.778Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.778Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.778Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.778Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.778Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.778Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.778Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.779Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2021-08-13T15:32:32.778Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.778Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting --- PASS: TestServers_IsOffline (0.00s) === RUN TestServers_FindServer -2022-09-15T21:40:49.205Z [WARN] TestServers_FindServer.manager: No servers available -2022-09-15T21:40:49.205Z [DEBUG] TestServers_FindServer.manager: cycled away from server: server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.205Z [DEBUG] TestServers_FindServer.manager: cycled away from server: server="s2 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [WARN] TestServers_FindServer.manager: No servers available +2021-08-13T15:32:32.779Z [DEBUG] TestServers_FindServer.manager: cycled away from server: server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.779Z [DEBUG] TestServers_FindServer.manager: cycled away from server: server="s2 (Addr: /) (DC: )" --- PASS: TestServers_FindServer (0.00s) === RUN TestServers_New --- PASS: TestServers_New (0.00s) === RUN TestServers_NotifyFailedServer -2022-09-15T21:40:49.205Z [DEBUG] TestServers_NotifyFailedServer.manager: cycled away from server: server="s1 (Addr: /) (DC: )" -2022-09-15T21:40:49.205Z [DEBUG] TestServers_NotifyFailedServer.manager: cycled away from server: server="s2 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_NotifyFailedServer.manager: cycled away from server: server="s1 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_NotifyFailedServer.manager: cycled away from server: server="s2 (Addr: /) (DC: )" --- PASS: TestServers_NotifyFailedServer (0.00s) === RUN TestServers_NumServers --- PASS: TestServers_NumServers (0.00s) === RUN TestServers_RebalanceServers -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.206Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s98 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s80 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s93 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s09 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s40 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.207Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s86 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s14 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s47 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s16 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.208Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s39 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s54 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s69 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s57 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s73 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s97 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.209Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.216Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.217Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s18 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.218Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.219Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s90 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s01 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.220Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.221Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.222Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.226Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.226Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.226Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.226Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.226Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.226Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.226Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.227Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.228Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s07 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.229Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.230Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.230Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.230Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.230Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.230Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.230Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s78 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s30 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.231Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s41 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s54 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s57 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.232Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s37 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.233Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.234Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s79 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.235Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s37 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.236Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.237Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.238Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s45 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.239Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.240Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s84 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.241Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s31 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.242Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.243Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.244Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.263Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.263Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.263Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.263Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.263Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.263Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.263Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.263Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.263Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.263Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.263Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.263Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.263Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.263Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.263Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.263Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.263Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.263Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.263Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.263Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s80 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.264Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s99 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s66 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.265Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s06 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.266Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s71 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s22 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s48 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s14 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s95 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.267Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s17 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.268Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s46 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s76 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.269Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s95 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s23 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s81 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.270Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s62 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s84 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s14 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.271Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.272Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.273Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.274Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.275Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.275Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.275Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.275Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.275Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.275Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.275Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.275Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.275Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.275Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.275Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.275Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.275Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.275Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.275Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.275Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.275Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.275Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.275Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.275Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.275Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.275Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.276Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s03 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.277Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.278Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.279Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.280Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.281Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s55 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s63 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.282Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.283Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s27 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.284Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.285Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s42 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s94 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s09 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s87 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.286Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.287Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.288Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s92 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.289Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.290Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.290Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.290Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.290Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.290Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.290Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.290Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.290Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.290Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.290Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.290Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.290Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.290Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.290Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.290Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.290Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.290Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.290Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.290Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.290Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.290Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.290Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.290Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.290Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.290Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.290Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.290Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.290Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.290Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.290Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.290Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.290Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.290Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.290Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.290Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.290Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.290Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.290Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.290Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.291Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.291Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.291Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.291Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.291Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.291Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.291Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.291Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.291Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.291Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.291Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.291Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.291Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.291Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.291Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.291Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s70 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.291Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s88 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.291Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s56 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.291Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.291Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.291Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.291Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.291Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.291Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.292Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.293Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s11 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s67 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s84 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.294Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.295Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s81 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s66 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.296Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s01 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.297Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.298Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.299Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s64 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.300Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s66 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.301Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s22 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.302Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.303Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.304Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s78 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s77 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.305Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s33 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.306Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.307Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.307Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.307Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.307Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.307Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.307Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.307Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.307Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.307Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.307Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.307Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.307Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.307Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.307Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.307Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.307Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.307Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.307Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.307Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.307Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.307Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.307Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.307Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.307Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.307Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.307Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.307Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.307Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.307Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.307Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.307Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.307Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.308Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.308Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.308Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.308Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.308Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.308Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.308Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.308Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.308Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.308Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.308Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.308Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.308Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.308Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.308Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.308Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.308Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.308Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.308Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.308Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.308Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.308Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.308Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.308Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.308Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.308Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.308Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.308Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.308Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.308Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.308Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.308Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.308Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.308Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.308Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.308Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.308Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.308Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.308Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s11 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s40 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.309Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.310Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.311Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s67 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s71 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s68 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.312Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s29 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.313Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.314Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.314Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.314Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.314Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.314Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.314Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.314Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.314Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.314Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.314Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.314Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.314Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.314Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.314Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.314Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.314Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.314Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.314Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.314Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.314Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.314Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.314Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.314Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.314Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.314Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.314Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.314Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.314Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.314Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.314Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.314Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.314Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.314Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.314Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.314Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.315Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.315Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.315Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.315Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.315Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.315Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.315Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.315Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s77 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.316Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.317Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.320Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s25 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.320Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s77 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.320Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s93 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.320Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s23 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.320Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s70 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.320Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.320Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.320Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.320Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.320Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.320Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.320Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.320Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.320Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.320Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.320Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.320Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.320Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.320Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.320Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.320Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.320Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.320Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.320Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.320Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.320Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.320Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.320Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.320Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.320Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.320Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.320Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.320Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.320Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.320Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.320Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.320Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.321Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.322Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.322Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.322Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.322Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.322Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.322Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.322Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.322Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.322Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.322Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.322Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.322Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.322Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.322Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.322Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.322Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.322Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s30 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.322Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s03 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.322Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s70 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.322Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s98 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.322Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s69 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.322Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.322Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.322Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.322Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.322Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.322Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.322Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.322Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.322Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.322Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.322Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.322Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.322Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.322Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.322Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.322Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.322Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.322Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.323Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s35 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s95 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.324Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s53 (Addr: /) (DC: )" error= -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.325Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.326Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2022-09-15T21:40:49.327Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2022-09-15T21:40:49.328Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" ---- PASS: TestServers_RebalanceServers (0.16s) +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s53 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.780Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.781Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s29 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s99 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s32 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s98 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s22 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.782Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.783Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s91 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.784Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s38 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s04 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s32 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.785Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s00 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s80 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s22 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.786Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s32 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s31 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s64 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.787Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.788Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.789Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.789Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.789Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.789Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.789Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.789Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.789Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.789Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.789Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.789Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.789Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.789Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.789Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.789Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.789Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.789Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.789Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.789Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.790Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s32 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.791Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s90 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.792Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.793Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s82 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s74 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.794Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s95 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s20 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.795Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s06 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.796Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s30 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.797Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s38 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.798Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s09 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.799Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.800Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s04 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.801Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s90 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s96 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.802Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s69 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s52 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s85 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s14 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.803Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.804Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.805Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.806Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.807Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.808Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s64 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s02 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s97 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s84 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s14 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.809Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.810Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.811Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s24 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s30 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.812Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.813Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s15 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.814Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.815Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.816Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s60 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s56 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.817Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s08 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s91 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.818Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.819Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s37 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s57 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.820Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s28 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.821Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.822Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s80 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s87 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s40 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.823Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.824Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s47 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.825Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s49 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.826Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.827Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.828Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s21 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s03 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s37 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s69 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s29 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s59 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s51 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.829Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s27 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s60 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.830Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s63 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s35 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s45 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s90 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s87 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s34 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s57 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s67 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s25 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.831Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s14 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.832Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.833Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.834Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s60 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s72 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.835Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s00 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s41 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s04 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s51 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s06 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.836Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s53 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s28 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.837Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.838Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.839Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s24 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s23 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s06 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s87 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.840Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.841Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s63 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.842Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s67 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s81 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.843Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s66 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s77 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.844Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s24 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s74 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s48 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s99 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.845Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s12 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s87 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s78 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s27 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.846Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.847Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.848Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s05 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s00 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s19 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s93 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s07 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s63 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s33 (Addr: /) (DC: )" error= +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:32.849Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:32.850Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:33.612Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:33.612Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:33.612Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:33.612Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:33.612Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:33.612Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:33.612Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:33.612Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:33.612Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:33.612Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:33.612Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:33.612Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:33.612Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:33.612Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:33.612Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:33.612Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:33.612Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:33.612Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:33.613Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s94 (Addr: /) (DC: )" error= +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s83 (Addr: /) (DC: )" error= +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:33.614Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:33.615Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s63 (Addr: /) (DC: )" error= +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:33.616Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:33.617Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s64 (Addr: /) (DC: )" error= +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:33.618Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:33.619Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:33.620Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:33.621Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:33.622Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:33.623Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:33.624Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2021-08-13T15:32:33.625Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2021-08-13T15:32:33.626Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2021-08-13T15:32:33.626Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:33.626Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2021-08-13T15:32:33.626Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:33.626Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2021-08-13T15:32:33.626Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2021-08-13T15:32:33.626Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2021-08-13T15:32:33.626Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:33.626Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2021-08-13T15:32:33.626Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2021-08-13T15:32:33.626Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2021-08-13T15:32:33.626Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2021-08-13T15:32:33.626Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2021-08-13T15:32:33.626Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2021-08-13T15:32:33.626Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +--- PASS: TestServers_RebalanceServers (0.85s) === RUN TestServers_RebalanceServers_AvoidFailed -2022-09-15T21:40:49.372Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.372Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.372Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.372Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.372Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.372Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.372Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2022-09-15T21:40:49.372Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.372Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.372Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.372Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.372Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2022-09-15T21:40:49.372Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.372Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.372Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.372Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2022-09-15T21:40:49.372Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.372Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.372Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.372Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.372Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.372Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2022-09-15T21:40:49.372Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.372Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.372Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.372Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2022-09-15T21:40:49.372Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.372Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.372Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.372Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.372Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2022-09-15T21:40:49.372Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2022-09-15T21:40:49.371Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.370Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.369Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.369Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.369Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2022-09-15T21:40:49.369Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.369Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.369Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2022-09-15T21:40:49.369Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2022-09-15T21:40:49.369Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2022-09-15T21:40:49.369Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.631Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.631Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2021-08-13T15:32:33.631Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.631Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2021-08-13T15:32:33.631Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.631Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.631Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.631Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.631Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.631Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2021-08-13T15:32:33.631Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.631Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.631Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.631Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2021-08-13T15:32:33.631Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.631Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2021-08-13T15:32:33.631Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.631Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.631Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.631Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.631Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2021-08-13T15:32:33.631Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.631Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.631Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2021-08-13T15:32:33.631Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.631Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.631Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.631Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.631Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.631Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.631Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.631Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.631Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.631Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.631Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.631Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.631Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.631Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2021-08-13T15:32:33.631Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.631Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2021-08-13T15:32:33.631Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.630Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2021-08-13T15:32:33.629Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" --- PASS: TestServers_RebalanceServers_AvoidFailed (0.00s) === RUN TestManager_RemoveServer -2022-09-15T21:40:49.373Z [DEBUG] TestManager_RemoveServer.manager: Rebalanced servers, new active server: number_of_servers=19 active_server="s02 (Addr: /) (DC: )" -2022-09-15T21:40:49.373Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2022-09-15T21:40:49.373Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2022-09-15T21:40:49.373Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2022-09-15T21:40:49.373Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2022-09-15T21:40:49.373Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.373Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2022-09-15T21:40:49.373Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.373Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2022-09-15T21:40:49.373Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.375Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2022-09-15T21:40:49.375Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.375Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2022-09-15T21:40:49.375Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.375Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2022-09-15T21:40:49.375Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.375Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2022-09-15T21:40:49.375Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.375Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2022-09-15T21:40:49.375Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.375Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2022-09-15T21:40:49.375Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.375Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2022-09-15T21:40:49.375Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.375Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2022-09-15T21:40:49.375Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2022-09-15T21:40:49.375Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2022-09-15T21:40:49.375Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:33.632Z [DEBUG] TestManager_RemoveServer.manager: Rebalanced servers, new active server: number_of_servers=19 active_server="s14 (Addr: /) (DC: )" +2021-08-13T15:32:33.632Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2021-08-13T15:32:33.632Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2021-08-13T15:32:33.632Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2021-08-13T15:32:33.632Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2021-08-13T15:32:33.632Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:33.632Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2021-08-13T15:32:33.632Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:33.632Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2021-08-13T15:32:33.632Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:33.632Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2021-08-13T15:32:33.632Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:33.632Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2021-08-13T15:32:33.632Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:33.632Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2021-08-13T15:32:33.632Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:33.632Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2021-08-13T15:32:33.632Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:33.632Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2021-08-13T15:32:33.632Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:33.632Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2021-08-13T15:32:33.632Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:33.632Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2021-08-13T15:32:33.632Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:33.632Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2021-08-13T15:32:33.632Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2021-08-13T15:32:33.632Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2021-08-13T15:32:33.632Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s18 (Addr: /) (DC: )" --- PASS: TestManager_RemoveServer (0.00s) PASS -ok github.com/hashicorp/consul/agent/router 0.197s +ok github.com/hashicorp/consul/agent/router 0.881s === RUN TestStructs_ACLCaches === RUN TestStructs_ACLCaches/New === RUN TestStructs_ACLCaches/New/Valid_Sizes @@ -69807,7 +69820,7 @@ === RUN TestConfigEntries_ListRelatedServices_AndACLs/router:_some/can_read_test === RUN TestConfigEntries_ListRelatedServices_AndACLs/router:_some/cannot_write_test === RUN TestConfigEntries_ListRelatedServices_AndACLs/router:_some/can_write_test_(with_foo:read_and_bar:read) ---- PASS: TestConfigEntries_ListRelatedServices_AndACLs (0.00s) +--- PASS: TestConfigEntries_ListRelatedServices_AndACLs (0.01s) --- PASS: TestConfigEntries_ListRelatedServices_AndACLs/resolver:_self (0.00s) --- PASS: TestConfigEntries_ListRelatedServices_AndACLs/resolver:_self/deny (0.00s) --- PASS: TestConfigEntries_ListRelatedServices_AndACLs/resolver:_self/can_read_test (0.00s) @@ -70265,7 +70278,7 @@ === RUN TestDecodeConfigEntry/ingress-gateway:_kitchen_sink_(camel_case) === RUN TestDecodeConfigEntry/terminating-gateway:_kitchen_sink_(snake_case) === RUN TestDecodeConfigEntry/terminating-gateway:_kitchen_sink_(camel_case) ---- PASS: TestDecodeConfigEntry (0.01s) +--- PASS: TestDecodeConfigEntry (0.02s) --- PASS: TestDecodeConfigEntry/proxy-defaults:_extra_fields_or_typo_(snake_case) (0.00s) --- PASS: TestDecodeConfigEntry/proxy-defaults:_extra_fields_or_typo_(camel_case) (0.00s) --- PASS: TestDecodeConfigEntry/proxy-defaults_(snake_case) (0.00s) @@ -70289,13 +70302,13 @@ === RUN TestServiceConfigResponse_MsgPack --- PASS: TestServiceConfigResponse_MsgPack (0.00s) === RUN TestConfigEntryResponseMarshalling +=== RUN TestConfigEntryResponseMarshalling/service-default_entry === RUN TestConfigEntryResponseMarshalling/nil_entry === RUN TestConfigEntryResponseMarshalling/proxy-default_entry -=== RUN TestConfigEntryResponseMarshalling/service-default_entry --- PASS: TestConfigEntryResponseMarshalling (0.00s) + --- PASS: TestConfigEntryResponseMarshalling/service-default_entry (0.00s) --- PASS: TestConfigEntryResponseMarshalling/nil_entry (0.00s) --- PASS: TestConfigEntryResponseMarshalling/proxy-default_entry (0.00s) - --- PASS: TestConfigEntryResponseMarshalling/service-default_entry (0.00s) === RUN TestCAConfiguration_GetCommonConfig === RUN TestCAConfiguration_GetCommonConfig/basic_defaults === RUN TestCAConfiguration_GetCommonConfig/basic_defaults_after_encoding_fun @@ -70387,23 +70400,23 @@ --- PASS: TestValidateMeshGatewayMode/remote_(constant) (0.00s) --- PASS: TestValidateMeshGatewayMode/remote_(explicit) (0.00s) === RUN TestIntention_ACLs -=== RUN TestIntention_ACLs/allow-wildcard-write -=== RUN TestIntention_ACLs/allow-wildcard-read -=== RUN TestIntention_ACLs/all-denied -=== RUN TestIntention_ACLs/deny-write-read-dest === RUN TestIntention_ACLs/deny-write-read-source === RUN TestIntention_ACLs/allow-write-with-dest-write === RUN TestIntention_ACLs/deny-write-with-source-write === RUN TestIntention_ACLs/deny-wildcard-write-allow-read +=== RUN TestIntention_ACLs/allow-wildcard-write +=== RUN TestIntention_ACLs/allow-wildcard-read +=== RUN TestIntention_ACLs/all-denied +=== RUN TestIntention_ACLs/deny-write-read-dest --- PASS: TestIntention_ACLs (0.00s) - --- PASS: TestIntention_ACLs/allow-wildcard-write (0.00s) - --- PASS: TestIntention_ACLs/allow-wildcard-read (0.00s) - --- PASS: TestIntention_ACLs/all-denied (0.00s) - --- PASS: TestIntention_ACLs/deny-write-read-dest (0.00s) --- PASS: TestIntention_ACLs/deny-write-read-source (0.00s) --- PASS: TestIntention_ACLs/allow-write-with-dest-write (0.00s) --- PASS: TestIntention_ACLs/deny-write-with-source-write (0.00s) --- PASS: TestIntention_ACLs/deny-wildcard-write-allow-read (0.00s) + --- PASS: TestIntention_ACLs/allow-wildcard-write (0.00s) + --- PASS: TestIntention_ACLs/allow-wildcard-read (0.00s) + --- PASS: TestIntention_ACLs/all-denied (0.00s) + --- PASS: TestIntention_ACLs/deny-write-read-dest (0.00s) === RUN TestIntentionValidate === RUN TestIntentionValidate/long_description === RUN TestIntentionValidate/no_action_set @@ -70449,7 +70462,6 @@ --- PASS: TestServiceDefinitionValidate (0.00s) --- PASS: TestServiceDefinitionValidate/valid (0.00s) === RUN TestStructs_FilterFieldConfigurations -=== RUN TestStructs_FilterFieldConfigurations/Intention === RUN TestStructs_FilterFieldConfigurations/Node === RUN TestStructs_FilterFieldConfigurations/NodeService === RUN TestStructs_FilterFieldConfigurations/ServiceNode @@ -70457,8 +70469,8 @@ === RUN TestStructs_FilterFieldConfigurations/CheckServiceNode === RUN TestStructs_FilterFieldConfigurations/NodeInfo === RUN TestStructs_FilterFieldConfigurations/api.AgentService +=== RUN TestStructs_FilterFieldConfigurations/Intention --- PASS: TestStructs_FilterFieldConfigurations (0.01s) - --- PASS: TestStructs_FilterFieldConfigurations/Intention (0.00s) --- PASS: TestStructs_FilterFieldConfigurations/Node (0.00s) --- PASS: TestStructs_FilterFieldConfigurations/NodeService (0.00s) --- PASS: TestStructs_FilterFieldConfigurations/ServiceNode (0.00s) @@ -70466,6 +70478,7 @@ --- PASS: TestStructs_FilterFieldConfigurations/CheckServiceNode (0.00s) --- PASS: TestStructs_FilterFieldConfigurations/NodeInfo (0.00s) --- PASS: TestStructs_FilterFieldConfigurations/api.AgentService (0.00s) + --- PASS: TestStructs_FilterFieldConfigurations/Intention (0.00s) === RUN TestEncodeDecode --- PASS: TestEncodeDecode (0.00s) === RUN TestStructs_Implements @@ -70483,71 +70496,71 @@ === RUN TestStructs_ServiceNode_Conversions --- PASS: TestStructs_ServiceNode_Conversions (0.00s) === RUN TestStructs_NodeService_ValidateMeshGateway +=== RUN TestStructs_NodeService_ValidateMeshGateway/sidecar-service +=== RUN TestStructs_NodeService_ValidateMeshGateway/proxy-destination-name === RUN TestStructs_NodeService_ValidateMeshGateway/proxy-destination-id === RUN TestStructs_NodeService_ValidateMeshGateway/proxy-local-address === RUN TestStructs_NodeService_ValidateMeshGateway/proxy-local-port === RUN TestStructs_NodeService_ValidateMeshGateway/proxy-upstreams === RUN TestStructs_NodeService_ValidateMeshGateway/valid === RUN TestStructs_NodeService_ValidateMeshGateway/zero-port -=== RUN TestStructs_NodeService_ValidateMeshGateway/sidecar-service -=== RUN TestStructs_NodeService_ValidateMeshGateway/proxy-destination-name --- PASS: TestStructs_NodeService_ValidateMeshGateway (0.00s) + --- PASS: TestStructs_NodeService_ValidateMeshGateway/sidecar-service (0.00s) + --- PASS: TestStructs_NodeService_ValidateMeshGateway/proxy-destination-name (0.00s) --- PASS: TestStructs_NodeService_ValidateMeshGateway/proxy-destination-id (0.00s) --- PASS: TestStructs_NodeService_ValidateMeshGateway/proxy-local-address (0.00s) --- PASS: TestStructs_NodeService_ValidateMeshGateway/proxy-local-port (0.00s) --- PASS: TestStructs_NodeService_ValidateMeshGateway/proxy-upstreams (0.00s) --- PASS: TestStructs_NodeService_ValidateMeshGateway/valid (0.00s) --- PASS: TestStructs_NodeService_ValidateMeshGateway/zero-port (0.00s) - --- PASS: TestStructs_NodeService_ValidateMeshGateway/sidecar-service (0.00s) - --- PASS: TestStructs_NodeService_ValidateMeshGateway/proxy-destination-name (0.00s) === RUN TestStructs_NodeService_ValidateTerminatingGateway +=== RUN TestStructs_NodeService_ValidateTerminatingGateway/valid +=== RUN TestStructs_NodeService_ValidateTerminatingGateway/sidecar-service === RUN TestStructs_NodeService_ValidateTerminatingGateway/proxy-destination-name === RUN TestStructs_NodeService_ValidateTerminatingGateway/proxy-destination-id === RUN TestStructs_NodeService_ValidateTerminatingGateway/proxy-local-address === RUN TestStructs_NodeService_ValidateTerminatingGateway/proxy-local-port === RUN TestStructs_NodeService_ValidateTerminatingGateway/proxy-upstreams -=== RUN TestStructs_NodeService_ValidateTerminatingGateway/valid -=== RUN TestStructs_NodeService_ValidateTerminatingGateway/sidecar-service --- PASS: TestStructs_NodeService_ValidateTerminatingGateway (0.00s) + --- PASS: TestStructs_NodeService_ValidateTerminatingGateway/valid (0.00s) + --- PASS: TestStructs_NodeService_ValidateTerminatingGateway/sidecar-service (0.00s) --- PASS: TestStructs_NodeService_ValidateTerminatingGateway/proxy-destination-name (0.00s) --- PASS: TestStructs_NodeService_ValidateTerminatingGateway/proxy-destination-id (0.00s) --- PASS: TestStructs_NodeService_ValidateTerminatingGateway/proxy-local-address (0.00s) --- PASS: TestStructs_NodeService_ValidateTerminatingGateway/proxy-local-port (0.00s) --- PASS: TestStructs_NodeService_ValidateTerminatingGateway/proxy-upstreams (0.00s) - --- PASS: TestStructs_NodeService_ValidateTerminatingGateway/valid (0.00s) - --- PASS: TestStructs_NodeService_ValidateTerminatingGateway/sidecar-service (0.00s) === RUN TestStructs_NodeService_ValidateIngressGateway +=== RUN TestStructs_NodeService_ValidateIngressGateway/proxy-local-address +=== RUN TestStructs_NodeService_ValidateIngressGateway/proxy-local-port === RUN TestStructs_NodeService_ValidateIngressGateway/proxy-upstreams === RUN TestStructs_NodeService_ValidateIngressGateway/valid === RUN TestStructs_NodeService_ValidateIngressGateway/sidecar-service === RUN TestStructs_NodeService_ValidateIngressGateway/proxy-destination-name === RUN TestStructs_NodeService_ValidateIngressGateway/proxy-destination-id -=== RUN TestStructs_NodeService_ValidateIngressGateway/proxy-local-address -=== RUN TestStructs_NodeService_ValidateIngressGateway/proxy-local-port --- PASS: TestStructs_NodeService_ValidateIngressGateway (0.00s) + --- PASS: TestStructs_NodeService_ValidateIngressGateway/proxy-local-address (0.00s) + --- PASS: TestStructs_NodeService_ValidateIngressGateway/proxy-local-port (0.00s) --- PASS: TestStructs_NodeService_ValidateIngressGateway/proxy-upstreams (0.00s) --- PASS: TestStructs_NodeService_ValidateIngressGateway/valid (0.00s) --- PASS: TestStructs_NodeService_ValidateIngressGateway/sidecar-service (0.00s) --- PASS: TestStructs_NodeService_ValidateIngressGateway/proxy-destination-name (0.00s) --- PASS: TestStructs_NodeService_ValidateIngressGateway/proxy-destination-id (0.00s) - --- PASS: TestStructs_NodeService_ValidateIngressGateway/proxy-local-address (0.00s) - --- PASS: TestStructs_NodeService_ValidateIngressGateway/proxy-local-port (0.00s) === RUN TestStructs_NodeService_ValidateExposeConfig +=== RUN TestStructs_NodeService_ValidateExposeConfig/valid === RUN TestStructs_NodeService_ValidateExposeConfig/empty_path === RUN TestStructs_NodeService_ValidateExposeConfig/invalid_port_negative === RUN TestStructs_NodeService_ValidateExposeConfig/invalid_port_too_large === RUN TestStructs_NodeService_ValidateExposeConfig/duplicate_paths === RUN TestStructs_NodeService_ValidateExposeConfig/duplicate_ports === RUN TestStructs_NodeService_ValidateExposeConfig/protocol_not_supported -=== RUN TestStructs_NodeService_ValidateExposeConfig/valid --- PASS: TestStructs_NodeService_ValidateExposeConfig (0.00s) + --- PASS: TestStructs_NodeService_ValidateExposeConfig/valid (0.00s) --- PASS: TestStructs_NodeService_ValidateExposeConfig/empty_path (0.00s) --- PASS: TestStructs_NodeService_ValidateExposeConfig/invalid_port_negative (0.00s) --- PASS: TestStructs_NodeService_ValidateExposeConfig/invalid_port_too_large (0.00s) --- PASS: TestStructs_NodeService_ValidateExposeConfig/duplicate_paths (0.00s) --- PASS: TestStructs_NodeService_ValidateExposeConfig/duplicate_ports (0.00s) --- PASS: TestStructs_NodeService_ValidateExposeConfig/protocol_not_supported (0.00s) - --- PASS: TestStructs_NodeService_ValidateExposeConfig/valid (0.00s) === RUN TestStructs_NodeService_ValidateConnectProxy === RUN TestStructs_NodeService_ValidateConnectProxy/valid === RUN TestStructs_NodeService_ValidateConnectProxy/connect-proxy:_no_Proxy.DestinationServiceName @@ -70607,6 +70620,10 @@ === RUN TestStructs_DirEntry_Clone --- PASS: TestStructs_DirEntry_Clone (0.00s) === RUN TestStructs_ValidateServiceAndNodeMetadata +=== RUN TestStructs_ValidateServiceAndNodeMetadata/invalid_key +=== RUN TestStructs_ValidateServiceAndNodeMetadata/invalid_key/ValidateNodeMetadata +=== RUN TestStructs_ValidateServiceAndNodeMetadata/invalid_key/ValidateServiceMetadata_-_typical +=== RUN TestStructs_ValidateServiceAndNodeMetadata/invalid_key/ValidateServiceMetadata_-_mesh-gateway === RUN TestStructs_ValidateServiceAndNodeMetadata/too_many_keys === RUN TestStructs_ValidateServiceAndNodeMetadata/too_many_keys/ValidateNodeMetadata === RUN TestStructs_ValidateServiceAndNodeMetadata/too_many_keys/ValidateServiceMetadata_-_typical @@ -70627,11 +70644,11 @@ === RUN TestStructs_ValidateServiceAndNodeMetadata/should_succeed/ValidateNodeMetadata === RUN TestStructs_ValidateServiceAndNodeMetadata/should_succeed/ValidateServiceMetadata_-_typical === RUN TestStructs_ValidateServiceAndNodeMetadata/should_succeed/ValidateServiceMetadata_-_mesh-gateway -=== RUN TestStructs_ValidateServiceAndNodeMetadata/invalid_key -=== RUN TestStructs_ValidateServiceAndNodeMetadata/invalid_key/ValidateNodeMetadata -=== RUN TestStructs_ValidateServiceAndNodeMetadata/invalid_key/ValidateServiceMetadata_-_typical -=== RUN TestStructs_ValidateServiceAndNodeMetadata/invalid_key/ValidateServiceMetadata_-_mesh-gateway --- PASS: TestStructs_ValidateServiceAndNodeMetadata (0.00s) + --- PASS: TestStructs_ValidateServiceAndNodeMetadata/invalid_key (0.00s) + --- PASS: TestStructs_ValidateServiceAndNodeMetadata/invalid_key/ValidateNodeMetadata (0.00s) + --- PASS: TestStructs_ValidateServiceAndNodeMetadata/invalid_key/ValidateServiceMetadata_-_typical (0.00s) + --- PASS: TestStructs_ValidateServiceAndNodeMetadata/invalid_key/ValidateServiceMetadata_-_mesh-gateway (0.00s) --- PASS: TestStructs_ValidateServiceAndNodeMetadata/too_many_keys (0.00s) --- PASS: TestStructs_ValidateServiceAndNodeMetadata/too_many_keys/ValidateNodeMetadata (0.00s) --- PASS: TestStructs_ValidateServiceAndNodeMetadata/too_many_keys/ValidateServiceMetadata_-_typical (0.00s) @@ -70652,10 +70669,6 @@ --- PASS: TestStructs_ValidateServiceAndNodeMetadata/should_succeed/ValidateNodeMetadata (0.00s) --- PASS: TestStructs_ValidateServiceAndNodeMetadata/should_succeed/ValidateServiceMetadata_-_typical (0.00s) --- PASS: TestStructs_ValidateServiceAndNodeMetadata/should_succeed/ValidateServiceMetadata_-_mesh-gateway (0.00s) - --- PASS: TestStructs_ValidateServiceAndNodeMetadata/invalid_key (0.00s) - --- PASS: TestStructs_ValidateServiceAndNodeMetadata/invalid_key/ValidateNodeMetadata (0.00s) - --- PASS: TestStructs_ValidateServiceAndNodeMetadata/invalid_key/ValidateServiceMetadata_-_typical (0.00s) - --- PASS: TestStructs_ValidateServiceAndNodeMetadata/invalid_key/ValidateServiceMetadata_-_mesh-gateway (0.00s) === RUN TestStructs_validateMetaPair --- PASS: TestStructs_validateMetaPair (0.00s) === RUN TestSpecificServiceRequest_CacheInfo @@ -70697,31 +70710,31 @@ --- PASS: TestNode_BestAddress/address (0.00s) --- PASS: TestNode_BestAddress/wan-address (0.00s) === RUN TestNodeService_BestAddress +=== RUN TestNodeService_BestAddress/service-wan-address-node-lan === RUN TestNodeService_BestAddress/no-address === RUN TestNodeService_BestAddress/service-address === RUN TestNodeService_BestAddress/service-wan-address === RUN TestNodeService_BestAddress/service-wan-address-default-port -=== RUN TestNodeService_BestAddress/service-wan-address-node-lan --- PASS: TestNodeService_BestAddress (0.00s) + --- PASS: TestNodeService_BestAddress/service-wan-address-node-lan (0.00s) --- PASS: TestNodeService_BestAddress/no-address (0.00s) --- PASS: TestNodeService_BestAddress/service-address (0.00s) --- PASS: TestNodeService_BestAddress/service-wan-address (0.00s) --- PASS: TestNodeService_BestAddress/service-wan-address-default-port (0.00s) - --- PASS: TestNodeService_BestAddress/service-wan-address-node-lan (0.00s) === RUN TestCheckServiceNode_BestAddress +=== RUN TestCheckServiceNode_BestAddress/node-address === RUN TestCheckServiceNode_BestAddress/node-wan-address === RUN TestCheckServiceNode_BestAddress/service-address === RUN TestCheckServiceNode_BestAddress/service-wan-address === RUN TestCheckServiceNode_BestAddress/service-wan-address-default-port === RUN TestCheckServiceNode_BestAddress/service-wan-address-node-lan -=== RUN TestCheckServiceNode_BestAddress/node-address --- PASS: TestCheckServiceNode_BestAddress (0.00s) + --- PASS: TestCheckServiceNode_BestAddress/node-address (0.00s) --- PASS: TestCheckServiceNode_BestAddress/node-wan-address (0.00s) --- PASS: TestCheckServiceNode_BestAddress/service-address (0.00s) --- PASS: TestCheckServiceNode_BestAddress/service-wan-address (0.00s) --- PASS: TestCheckServiceNode_BestAddress/service-wan-address-default-port (0.00s) --- PASS: TestCheckServiceNode_BestAddress/service-wan-address-node-lan (0.00s) - --- PASS: TestCheckServiceNode_BestAddress/node-address (0.00s) === RUN TestNodeService_JSON_Marshal --- PASS: TestNodeService_JSON_Marshal (0.00s) === RUN TestServiceNode_JSON_Marshal @@ -70773,7 +70786,7 @@ === RUN TestGatewayService_IsSame --- PASS: TestGatewayService_IsSame (0.00s) PASS -ok github.com/hashicorp/consul/agent/structs 0.106s +ok github.com/hashicorp/consul/agent/structs 0.102s ? github.com/hashicorp/consul/agent/systemd [no test files] === RUN TestStore_RegularTokens === PAUSE TestStore_RegularTokens @@ -70785,9 +70798,13 @@ === RUN TestStore_RegularTokens/set_user_-_config === PAUSE TestStore_RegularTokens/set_user_-_config === RUN TestStore_RegularTokens/set_user_-_api +=== CONT TestStore_Notify === PAUSE TestStore_RegularTokens/set_user_-_api === RUN TestStore_RegularTokens/set_agent_-_config +=== CONT TestStore_AgentMasterToken === PAUSE TestStore_RegularTokens/set_agent_-_config +--- PASS: TestStore_AgentMasterToken (0.00s) +--- PASS: TestStore_Notify (0.00s) === RUN TestStore_RegularTokens/set_agent_-_api === PAUSE TestStore_RegularTokens/set_agent_-_api === RUN TestStore_RegularTokens/set_user_and_agent @@ -70803,522 +70820,518 @@ === RUN TestStore_RegularTokens/set_all === PAUSE TestStore_RegularTokens/set_all === CONT TestStore_RegularTokens/set_user_-_config -=== CONT TestStore_Notify ---- PASS: TestStore_Notify (0.00s) -=== CONT TestStore_AgentMasterToken ---- PASS: TestStore_AgentMasterToken (0.00s) === CONT TestStore_RegularTokens/set_all -=== CONT TestStore_RegularTokens/set_master_-_api -=== CONT TestStore_RegularTokens/set_master_-_config +=== CONT TestStore_RegularTokens/set_user_and_agent === CONT TestStore_RegularTokens/set_repl_-_api === CONT TestStore_RegularTokens/set_repl_-_config -=== CONT TestStore_RegularTokens/set_user_and_agent +=== CONT TestStore_RegularTokens/set_user_-_api === CONT TestStore_RegularTokens/set_agent_-_api === CONT TestStore_RegularTokens/set_agent_-_config -=== CONT TestStore_RegularTokens/set_user_-_api +=== CONT TestStore_RegularTokens/set_master_-_config +=== CONT TestStore_RegularTokens/set_master_-_api --- PASS: TestStore_RegularTokens (0.00s) --- PASS: TestStore_RegularTokens/set_user_-_config (0.00s) --- PASS: TestStore_RegularTokens/set_all (0.00s) - --- PASS: TestStore_RegularTokens/set_master_-_api (0.00s) - --- PASS: TestStore_RegularTokens/set_master_-_config (0.00s) - --- PASS: TestStore_RegularTokens/set_repl_-_api (0.00s) --- PASS: TestStore_RegularTokens/set_repl_-_config (0.00s) + --- PASS: TestStore_RegularTokens/set_user_-_api (0.00s) + --- PASS: TestStore_RegularTokens/set_repl_-_api (0.00s) --- PASS: TestStore_RegularTokens/set_user_and_agent (0.00s) --- PASS: TestStore_RegularTokens/set_agent_-_api (0.00s) + --- PASS: TestStore_RegularTokens/set_master_-_config (0.00s) --- PASS: TestStore_RegularTokens/set_agent_-_config (0.00s) - --- PASS: TestStore_RegularTokens/set_user_-_api (0.00s) + --- PASS: TestStore_RegularTokens/set_master_-_api (0.00s) PASS -ok github.com/hashicorp/consul/agent/token 0.025s +ok github.com/hashicorp/consul/agent/token 0.008s === RUN TestClustersFromSnapshot === RUN TestClustersFromSnapshot/envoy-1.14.5 === RUN TestClustersFromSnapshot/envoy-1.14.5/defaults -2022-09-15T21:41:00.605Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/defaults: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.266Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/defaults: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/custom-local-app -2022-09-15T21:41:00.614Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/custom-local-app: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.274Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/custom-local-app: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/custom-local-app-typed -2022-09-15T21:41:00.622Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/custom-local-app-typed: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.281Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/custom-local-app-typed: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/custom-upstream -2022-09-15T21:41:00.627Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/custom-upstream: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.285Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/custom-upstream: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/custom-upstream-default-chain -2022-09-15T21:41:00.632Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/custom-upstream-default-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.289Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/custom-upstream-default-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/custom-upstream-typed -2022-09-15T21:41:00.645Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/custom-upstream-typed: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.293Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/custom-upstream-typed: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/custom-upstream-ignores-tls -2022-09-15T21:41:00.654Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/custom-upstream-ignores-tls: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.297Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/custom-upstream-ignores-tls: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/custom-timeouts -2022-09-15T21:41:00.659Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/custom-timeouts: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.300Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/custom-timeouts: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/custom-limits-max-connections-only -2022-09-15T21:41:00.663Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/custom-limits-max-connections-only: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.304Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/custom-limits-max-connections-only: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/custom-limits-set-to-zero -2022-09-15T21:41:00.668Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/custom-limits-set-to-zero: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.308Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/custom-limits-set-to-zero: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/custom-limits -2022-09-15T21:41:00.674Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/custom-limits: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.312Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/custom-limits: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain -2022-09-15T21:41:00.679Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.350Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-external-sni -2022-09-15T21:41:00.689Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-external-sni: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.354Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-external-sni: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-and-overrides -2022-09-15T21:41:00.693Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-and-overrides: generating cluster for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.358Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-and-overrides: generating cluster for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-and-failover -2022-09-15T21:41:00.698Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-and-failover: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.362Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-and-failover: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-remote-gateway -2022-09-15T21:41:00.704Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.366Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered -2022-09-15T21:41:00.709Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.370Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway -2022-09-15T21:41:00.713Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.374Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered -2022-09-15T21:41:00.718Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.378Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-local-gateway -2022-09-15T21:41:00.723Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.382Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered -2022-09-15T21:41:00.727Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.387Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-local-gateway -2022-09-15T21:41:00.737Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.391Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered -2022-09-15T21:41:00.745Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.395Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/splitter-with-resolver-redirect -2022-09-15T21:41:00.749Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/splitter-with-resolver-redirect: generating cluster for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2022-09-15T21:41:00.749Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/splitter-with-resolver-redirect: generating cluster for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.398Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/splitter-with-resolver-redirect: generating cluster for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.398Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/splitter-with-resolver-redirect: generating cluster for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/expose-paths-local-app-paths === RUN TestClustersFromSnapshot/envoy-1.14.5/expose-paths-new-cluster-http2 === RUN TestClustersFromSnapshot/envoy-1.14.5/expose-paths-grpc-new-cluster-http1 === RUN TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway -2022-09-15T21:41:00.758Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway +2021-08-13T15:32:54.405Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway === RUN TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway-using-federation-states -2022-09-15T21:41:00.761Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway-using-federation-states.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway +2021-08-13T15:32:54.408Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway-using-federation-states.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway === RUN TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway-no-services === RUN TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway-service-subsets -2022-09-15T21:41:00.771Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway-service-subsets.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway +2021-08-13T15:32:54.414Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway-service-subsets.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway === RUN TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway-ignore-extra-resolvers -2022-09-15T21:41:00.777Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway-ignore-extra-resolvers.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway +2021-08-13T15:32:54.418Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway-ignore-extra-resolvers.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway === RUN TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway-service-timeouts -2022-09-15T21:41:00.782Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway-service-timeouts.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway +2021-08-13T15:32:54.423Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway-service-timeouts.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway === RUN TestClustersFromSnapshot/envoy-1.14.5/ingress-gateway -2022-09-15T21:41:00.788Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.429Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/ingress-gateway-no-services === RUN TestClustersFromSnapshot/envoy-1.14.5/ingress-with-chain -2022-09-15T21:41:00.794Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.434Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/ingress-with-chain-external-sni -2022-09-15T21:41:00.798Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-chain-external-sni: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.437Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-chain-external-sni: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/ingress-with-chain-and-overrides -2022-09-15T21:41:00.805Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-chain-and-overrides: generating cluster for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.440Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-chain-and-overrides: generating cluster for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/ingress-with-chain-and-failover -2022-09-15T21:41:00.812Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-chain-and-failover: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.443Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-chain-and-failover: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-remote-gateway -2022-09-15T21:41:00.817Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.446Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-remote-gateway-triggered -2022-09-15T21:41:00.821Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.449Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-remote-gateway -2022-09-15T21:41:00.825Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.452Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered -2022-09-15T21:41:00.828Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.456Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-local-gateway -2022-09-15T21:41:00.832Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.458Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-local-gateway-triggered -2022-09-15T21:41:00.835Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.462Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-local-gateway -2022-09-15T21:41:00.840Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.465Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered -2022-09-15T21:41:00.843Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.468Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/ingress-splitter-with-resolver-redirect -2022-09-15T21:41:00.846Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-splitter-with-resolver-redirect: generating cluster for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2022-09-15T21:41:00.846Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-splitter-with-resolver-redirect: generating cluster for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.472Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-splitter-with-resolver-redirect: generating cluster for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.472Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-splitter-with-resolver-redirect: generating cluster for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway -2022-09-15T21:41:00.859Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api -2022-09-15T21:41:00.859Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db +2021-08-13T15:32:54.479Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api +2021-08-13T15:32:54.479Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db === RUN TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-no-services === RUN TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-service-subsets -2022-09-15T21:41:00.871Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-service-subsets.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api -2022-09-15T21:41:00.871Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db -2022-09-15T21:41:00.872Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=cache +2021-08-13T15:32:54.486Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=cache +2021-08-13T15:32:54.487Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-service-subsets.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api +2021-08-13T15:32:54.487Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db === RUN TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-hostname-service-subsets -2022-09-15T21:41:00.878Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-hostname-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db -2022-09-15T21:41:00.879Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-hostname-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=cache -2022-09-15T21:41:00.879Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-hostname-service-subsets.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api +2021-08-13T15:32:54.491Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-hostname-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db +2021-08-13T15:32:54.491Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-hostname-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=cache +2021-08-13T15:32:54.492Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-hostname-service-subsets.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api === RUN TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-ignore-extra-resolvers -2022-09-15T21:41:00.889Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-ignore-extra-resolvers.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api -2022-09-15T21:41:00.889Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-ignore-extra-resolvers.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db +2021-08-13T15:32:54.497Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-ignore-extra-resolvers.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api +2021-08-13T15:32:54.497Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-ignore-extra-resolvers.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db === RUN TestClustersFromSnapshot/envoy-1.14.5/ingress-multiple-listeners-duplicate-service -2022-09-15T21:41:00.895Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-multiple-listeners-duplicate-service: generating cluster for: cluster=foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2022-09-15T21:41:00.895Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-multiple-listeners-duplicate-service: generating cluster for: cluster=bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.501Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-multiple-listeners-duplicate-service: generating cluster for: cluster=foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.501Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-multiple-listeners-duplicate-service: generating cluster for: cluster=bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6 === RUN TestClustersFromSnapshot/envoy-1.13.6/defaults -2022-09-15T21:41:00.903Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/defaults: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.505Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/defaults: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/custom-local-app -2022-09-15T21:41:00.907Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/custom-local-app: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.509Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/custom-local-app: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/custom-local-app-typed -2022-09-15T21:41:00.912Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/custom-local-app-typed: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.512Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/custom-local-app-typed: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/custom-upstream -2022-09-15T21:41:00.916Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/custom-upstream: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.518Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/custom-upstream: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/custom-upstream-default-chain -2022-09-15T21:41:00.920Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/custom-upstream-default-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.522Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/custom-upstream-default-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/custom-upstream-typed -2022-09-15T21:41:00.925Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/custom-upstream-typed: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.526Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/custom-upstream-typed: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/custom-upstream-ignores-tls -2022-09-15T21:41:00.929Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/custom-upstream-ignores-tls: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.529Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/custom-upstream-ignores-tls: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/custom-timeouts -2022-09-15T21:41:00.933Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/custom-timeouts: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.533Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/custom-timeouts: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/custom-limits-max-connections-only -2022-09-15T21:41:00.938Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/custom-limits-max-connections-only: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.536Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/custom-limits-max-connections-only: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/custom-limits-set-to-zero -2022-09-15T21:41:00.945Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/custom-limits-set-to-zero: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.540Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/custom-limits-set-to-zero: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/custom-limits -2022-09-15T21:41:00.953Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/custom-limits: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.544Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/custom-limits: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain -2022-09-15T21:41:00.959Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.548Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-external-sni -2022-09-15T21:41:00.963Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-external-sni: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.551Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-external-sni: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-and-overrides -2022-09-15T21:41:00.968Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-and-overrides: generating cluster for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.557Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-and-overrides: generating cluster for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-and-failover -2022-09-15T21:41:00.975Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-and-failover: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.562Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-and-failover: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-remote-gateway -2022-09-15T21:41:00.980Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.565Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered -2022-09-15T21:41:00.985Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.569Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway -2022-09-15T21:41:00.989Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.573Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered -2022-09-15T21:41:00.994Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.576Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-local-gateway -2022-09-15T21:41:00.998Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.580Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered -2022-09-15T21:41:01.002Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.583Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-local-gateway -2022-09-15T21:41:01.010Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.587Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered -2022-09-15T21:41:01.014Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.590Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/splitter-with-resolver-redirect -2022-09-15T21:41:01.019Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/splitter-with-resolver-redirect: generating cluster for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2022-09-15T21:41:01.019Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/splitter-with-resolver-redirect: generating cluster for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.594Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/splitter-with-resolver-redirect: generating cluster for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.594Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/splitter-with-resolver-redirect: generating cluster for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/expose-paths-local-app-paths === RUN TestClustersFromSnapshot/envoy-1.13.6/expose-paths-new-cluster-http2 === RUN TestClustersFromSnapshot/envoy-1.13.6/expose-paths-grpc-new-cluster-http1 === RUN TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway -2022-09-15T21:41:01.027Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway +2021-08-13T15:32:54.603Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway === RUN TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-using-federation-states -2022-09-15T21:41:01.032Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-using-federation-states.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway +2021-08-13T15:32:54.606Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-using-federation-states.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway === RUN TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-no-services === RUN TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-service-subsets -2022-09-15T21:41:01.040Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-service-subsets.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway +2021-08-13T15:32:54.612Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-service-subsets.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway === RUN TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-ignore-extra-resolvers -2022-09-15T21:41:01.045Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-ignore-extra-resolvers.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway +2021-08-13T15:32:54.617Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-ignore-extra-resolvers.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway === RUN TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-service-timeouts -2022-09-15T21:41:01.050Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-service-timeouts.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway +2021-08-13T15:32:54.621Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-service-timeouts.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway === RUN TestClustersFromSnapshot/envoy-1.13.6/ingress-gateway -2022-09-15T21:41:01.056Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.625Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/ingress-gateway-no-services === RUN TestClustersFromSnapshot/envoy-1.13.6/ingress-with-chain -2022-09-15T21:41:01.066Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.630Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/ingress-with-chain-external-sni -2022-09-15T21:41:01.072Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-chain-external-sni: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.633Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-chain-external-sni: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/ingress-with-chain-and-overrides -2022-09-15T21:41:01.076Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-chain-and-overrides: generating cluster for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.635Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-chain-and-overrides: generating cluster for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/ingress-with-chain-and-failover -2022-09-15T21:41:01.079Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-chain-and-failover: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.638Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-chain-and-failover: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-remote-gateway -2022-09-15T21:41:01.083Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.641Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-remote-gateway-triggered -2022-09-15T21:41:01.087Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.646Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-remote-gateway -2022-09-15T21:41:01.091Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.648Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered -2022-09-15T21:41:01.095Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.651Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-local-gateway -2022-09-15T21:41:01.098Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.654Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-local-gateway-triggered -2022-09-15T21:41:01.102Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.657Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-local-gateway -2022-09-15T21:41:01.106Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.660Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered -2022-09-15T21:41:01.111Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.662Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/ingress-splitter-with-resolver-redirect -2022-09-15T21:41:01.115Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-splitter-with-resolver-redirect: generating cluster for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2022-09-15T21:41:01.115Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-splitter-with-resolver-redirect: generating cluster for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.665Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-splitter-with-resolver-redirect: generating cluster for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.665Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-splitter-with-resolver-redirect: generating cluster for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway -2022-09-15T21:41:01.120Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api -2022-09-15T21:41:01.120Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db +2021-08-13T15:32:54.669Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api +2021-08-13T15:32:54.669Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db === RUN TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-no-services === RUN TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-service-subsets -2022-09-15T21:41:01.130Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-service-subsets.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api -2022-09-15T21:41:01.130Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db -2022-09-15T21:41:01.131Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=cache +2021-08-13T15:32:54.675Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-service-subsets.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api +2021-08-13T15:32:54.675Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db +2021-08-13T15:32:54.676Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=cache === RUN TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-hostname-service-subsets -2022-09-15T21:41:01.138Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-hostname-service-subsets.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api -2022-09-15T21:41:01.138Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-hostname-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db -2022-09-15T21:41:01.139Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-hostname-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=cache +2021-08-13T15:32:54.680Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-hostname-service-subsets.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api +2021-08-13T15:32:54.681Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-hostname-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db +2021-08-13T15:32:54.681Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-hostname-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=cache === RUN TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-ignore-extra-resolvers -2022-09-15T21:41:01.145Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-ignore-extra-resolvers.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api -2022-09-15T21:41:01.145Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-ignore-extra-resolvers.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db +2021-08-13T15:32:54.688Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-ignore-extra-resolvers.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api +2021-08-13T15:32:54.688Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-ignore-extra-resolvers.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db === RUN TestClustersFromSnapshot/envoy-1.13.6/ingress-multiple-listeners-duplicate-service -2022-09-15T21:41:01.151Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-multiple-listeners-duplicate-service: generating cluster for: cluster=foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2022-09-15T21:41:01.152Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-multiple-listeners-duplicate-service: generating cluster for: cluster=bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.693Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-multiple-listeners-duplicate-service: generating cluster for: cluster=foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.693Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-multiple-listeners-duplicate-service: generating cluster for: cluster=bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7 === RUN TestClustersFromSnapshot/envoy-1.12.7/defaults -2022-09-15T21:41:01.156Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/defaults: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.697Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/defaults: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/custom-local-app -2022-09-15T21:41:01.161Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/custom-local-app: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.701Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/custom-local-app: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/custom-local-app-typed -2022-09-15T21:41:01.166Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/custom-local-app-typed: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.705Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/custom-local-app-typed: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/custom-upstream -2022-09-15T21:41:01.171Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/custom-upstream: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.708Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/custom-upstream: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/custom-upstream-default-chain -2022-09-15T21:41:01.176Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/custom-upstream-default-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.712Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/custom-upstream-default-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/custom-upstream-typed -2022-09-15T21:41:01.181Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/custom-upstream-typed: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.716Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/custom-upstream-typed: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/custom-upstream-ignores-tls -2022-09-15T21:41:01.206Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/custom-upstream-ignores-tls: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.722Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/custom-upstream-ignores-tls: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/custom-timeouts -2022-09-15T21:41:01.211Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/custom-timeouts: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.728Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/custom-timeouts: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/custom-limits-max-connections-only -2022-09-15T21:41:01.216Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/custom-limits-max-connections-only: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.732Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/custom-limits-max-connections-only: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/custom-limits-set-to-zero -2022-09-15T21:41:01.221Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/custom-limits-set-to-zero: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.736Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/custom-limits-set-to-zero: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/custom-limits -2022-09-15T21:41:01.225Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/custom-limits: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.740Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/custom-limits: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-chain -2022-09-15T21:41:01.230Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.744Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-external-sni -2022-09-15T21:41:01.234Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-external-sni: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.748Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-external-sni: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-and-overrides -2022-09-15T21:41:01.239Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-and-overrides: generating cluster for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.752Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-and-overrides: generating cluster for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-and-failover -2022-09-15T21:41:01.243Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-and-failover: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.756Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-and-failover: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-remote-gateway -2022-09-15T21:41:01.251Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.760Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered -2022-09-15T21:41:01.256Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.764Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway -2022-09-15T21:41:01.260Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.768Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered -2022-09-15T21:41:01.265Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.773Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-local-gateway -2022-09-15T21:41:01.269Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.777Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered -2022-09-15T21:41:01.277Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.780Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-local-gateway -2022-09-15T21:41:01.282Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.784Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered -2022-09-15T21:41:01.287Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.788Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/splitter-with-resolver-redirect -2022-09-15T21:41:01.292Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/splitter-with-resolver-redirect: generating cluster for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2022-09-15T21:41:01.292Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/splitter-with-resolver-redirect: generating cluster for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.791Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/splitter-with-resolver-redirect: generating cluster for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.791Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/splitter-with-resolver-redirect: generating cluster for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/expose-paths-local-app-paths === RUN TestClustersFromSnapshot/envoy-1.12.7/expose-paths-new-cluster-http2 === RUN TestClustersFromSnapshot/envoy-1.12.7/expose-paths-grpc-new-cluster-http1 === RUN TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway -2022-09-15T21:41:01.324Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway +2021-08-13T15:32:54.798Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway === RUN TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-using-federation-states -2022-09-15T21:41:01.329Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-using-federation-states.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway +2021-08-13T15:32:54.801Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-using-federation-states.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway === RUN TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-no-services === RUN TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-service-subsets -2022-09-15T21:41:01.337Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-service-subsets.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway +2021-08-13T15:32:54.807Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-service-subsets.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway === RUN TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-ignore-extra-resolvers -2022-09-15T21:41:01.343Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-ignore-extra-resolvers.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway +2021-08-13T15:32:54.813Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-ignore-extra-resolvers.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway === RUN TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-service-timeouts -2022-09-15T21:41:01.349Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-service-timeouts.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway +2021-08-13T15:32:54.817Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-service-timeouts.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway === RUN TestClustersFromSnapshot/envoy-1.12.7/ingress-gateway -2022-09-15T21:41:01.354Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.821Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/ingress-gateway-no-services === RUN TestClustersFromSnapshot/envoy-1.12.7/ingress-with-chain -2022-09-15T21:41:01.360Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.826Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/ingress-with-chain-external-sni -2022-09-15T21:41:01.363Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-chain-external-sni: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.829Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-chain-external-sni: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/ingress-with-chain-and-overrides -2022-09-15T21:41:01.366Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-chain-and-overrides: generating cluster for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.832Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-chain-and-overrides: generating cluster for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/ingress-with-chain-and-failover -2022-09-15T21:41:01.370Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-chain-and-failover: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.834Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-chain-and-failover: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-remote-gateway -2022-09-15T21:41:01.373Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.837Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-remote-gateway-triggered -2022-09-15T21:41:01.377Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.840Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-remote-gateway -2022-09-15T21:41:01.381Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.843Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered -2022-09-15T21:41:01.386Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.845Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-local-gateway -2022-09-15T21:41:01.389Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.848Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-local-gateway-triggered -2022-09-15T21:41:01.393Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.851Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-local-gateway -2022-09-15T21:41:01.396Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.856Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered -2022-09-15T21:41:01.400Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.861Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/ingress-splitter-with-resolver-redirect -2022-09-15T21:41:01.403Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-splitter-with-resolver-redirect: generating cluster for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul -2022-09-15T21:41:01.403Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-splitter-with-resolver-redirect: generating cluster for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.865Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-splitter-with-resolver-redirect: generating cluster for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.865Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-splitter-with-resolver-redirect: generating cluster for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway -2022-09-15T21:41:01.408Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api -2022-09-15T21:41:01.408Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db +2021-08-13T15:32:54.869Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api +2021-08-13T15:32:54.869Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db === RUN TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-no-services === RUN TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-service-subsets -2022-09-15T21:41:01.415Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db -2022-09-15T21:41:01.416Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=cache -2022-09-15T21:41:01.417Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-service-subsets.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api +2021-08-13T15:32:54.875Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-service-subsets.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api +2021-08-13T15:32:54.876Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db +2021-08-13T15:32:54.876Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=cache === RUN TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-hostname-service-subsets -2022-09-15T21:41:01.422Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-hostname-service-subsets.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api -2022-09-15T21:41:01.423Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-hostname-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db -2022-09-15T21:41:01.423Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-hostname-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=cache +2021-08-13T15:32:54.882Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-hostname-service-subsets.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api +2021-08-13T15:32:54.882Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-hostname-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db +2021-08-13T15:32:54.883Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-hostname-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=cache === RUN TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-ignore-extra-resolvers -2022-09-15T21:41:01.429Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-ignore-extra-resolvers.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api -2022-09-15T21:41:01.429Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-ignore-extra-resolvers.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db +2021-08-13T15:32:54.888Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-ignore-extra-resolvers.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api +2021-08-13T15:32:54.889Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-ignore-extra-resolvers.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db === RUN TestClustersFromSnapshot/envoy-1.12.7/ingress-multiple-listeners-duplicate-service -2022-09-15T21:41:01.435Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-multiple-listeners-duplicate-service: generating cluster for: cluster=foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2022-09-15T21:41:01.435Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-multiple-listeners-duplicate-service: generating cluster for: cluster=bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.893Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-multiple-listeners-duplicate-service: generating cluster for: cluster=foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.893Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-multiple-listeners-duplicate-service: generating cluster for: cluster=bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2 === RUN TestClustersFromSnapshot/envoy-1.11.2/defaults -2022-09-15T21:41:01.442Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/defaults: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.897Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/defaults: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/custom-local-app -2022-09-15T21:41:01.449Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/custom-local-app: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.901Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/custom-local-app: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/custom-local-app-typed -2022-09-15T21:41:01.453Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/custom-local-app-typed: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.905Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/custom-local-app-typed: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/custom-upstream -2022-09-15T21:41:01.458Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/custom-upstream: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.911Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/custom-upstream: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/custom-upstream-default-chain -2022-09-15T21:41:01.462Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/custom-upstream-default-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.915Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/custom-upstream-default-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/custom-upstream-typed -2022-09-15T21:41:01.467Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/custom-upstream-typed: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.919Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/custom-upstream-typed: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/custom-upstream-ignores-tls -2022-09-15T21:41:01.471Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/custom-upstream-ignores-tls: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.922Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/custom-upstream-ignores-tls: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/custom-timeouts -2022-09-15T21:41:01.475Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/custom-timeouts: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.926Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/custom-timeouts: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/custom-limits-max-connections-only -2022-09-15T21:41:01.479Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/custom-limits-max-connections-only: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.930Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/custom-limits-max-connections-only: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/custom-limits-set-to-zero -2022-09-15T21:41:01.484Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/custom-limits-set-to-zero: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.933Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/custom-limits-set-to-zero: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/custom-limits -2022-09-15T21:41:01.488Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/custom-limits: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.937Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/custom-limits: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain -2022-09-15T21:41:01.492Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.941Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-external-sni -2022-09-15T21:41:01.506Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-external-sni: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.944Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-external-sni: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-overrides -2022-09-15T21:41:01.512Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-overrides: generating cluster for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.949Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-overrides: generating cluster for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-failover -2022-09-15T21:41:01.517Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-failover: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.954Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-failover: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-remote-gateway -2022-09-15T21:41:01.521Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.958Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered -2022-09-15T21:41:01.525Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.962Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway -2022-09-15T21:41:01.530Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.966Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered -2022-09-15T21:41:01.534Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.970Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-local-gateway -2022-09-15T21:41:01.539Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.974Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered -2022-09-15T21:41:01.543Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.978Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-local-gateway -2022-09-15T21:41:01.547Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.982Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered -2022-09-15T21:41:01.551Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.986Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/splitter-with-resolver-redirect -2022-09-15T21:41:01.556Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/splitter-with-resolver-redirect: generating cluster for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2022-09-15T21:41:01.556Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/splitter-with-resolver-redirect: generating cluster for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.990Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/splitter-with-resolver-redirect: generating cluster for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:54.990Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/splitter-with-resolver-redirect: generating cluster for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/expose-paths-local-app-paths === RUN TestClustersFromSnapshot/envoy-1.11.2/expose-paths-new-cluster-http2 === RUN TestClustersFromSnapshot/envoy-1.11.2/expose-paths-grpc-new-cluster-http1 === RUN TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway -2022-09-15T21:41:01.566Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway +2021-08-13T15:32:54.999Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway === RUN TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway-using-federation-states -2022-09-15T21:41:01.570Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway-using-federation-states.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway +2021-08-13T15:32:55.002Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway-using-federation-states.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway === RUN TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway-no-services === RUN TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway-service-subsets -2022-09-15T21:41:01.578Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway-service-subsets.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway +2021-08-13T15:32:55.008Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway-service-subsets.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway === RUN TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway-ignore-extra-resolvers -2022-09-15T21:41:01.583Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway-ignore-extra-resolvers.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway +2021-08-13T15:32:55.012Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway-ignore-extra-resolvers.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway === RUN TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway-service-timeouts -2022-09-15T21:41:01.588Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway-service-timeouts.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway +2021-08-13T15:32:55.016Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway-service-timeouts.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway === RUN TestClustersFromSnapshot/envoy-1.11.2/ingress-gateway -2022-09-15T21:41:01.593Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.020Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/ingress-gateway-no-services === RUN TestClustersFromSnapshot/envoy-1.11.2/ingress-with-chain -2022-09-15T21:41:01.598Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.025Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/ingress-with-chain-external-sni -2022-09-15T21:41:01.602Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-chain-external-sni: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.028Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-chain-external-sni: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/ingress-with-chain-and-overrides -2022-09-15T21:41:01.605Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-chain-and-overrides: generating cluster for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.031Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-chain-and-overrides: generating cluster for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/ingress-with-chain-and-failover -2022-09-15T21:41:01.609Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-chain-and-failover: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.035Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-chain-and-failover: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-remote-gateway -2022-09-15T21:41:01.619Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.038Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-remote-gateway-triggered -2022-09-15T21:41:01.623Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.041Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-remote-gateway -2022-09-15T21:41:01.626Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.044Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered -2022-09-15T21:41:01.630Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.047Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-local-gateway -2022-09-15T21:41:01.633Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.050Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-local-gateway-triggered -2022-09-15T21:41:01.637Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.053Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-local-gateway -2022-09-15T21:41:01.640Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.056Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered -2022-09-15T21:41:01.644Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.059Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/ingress-splitter-with-resolver-redirect -2022-09-15T21:41:01.648Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-splitter-with-resolver-redirect: generating cluster for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2022-09-15T21:41:01.648Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-splitter-with-resolver-redirect: generating cluster for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.062Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-splitter-with-resolver-redirect: generating cluster for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.062Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-splitter-with-resolver-redirect: generating cluster for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway -2022-09-15T21:41:01.652Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api -2022-09-15T21:41:01.652Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db +2021-08-13T15:32:55.066Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api +2021-08-13T15:32:55.066Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db === RUN TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-no-services === RUN TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-service-subsets -2022-09-15T21:41:01.660Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-service-subsets.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api -2022-09-15T21:41:01.660Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db -2022-09-15T21:41:01.661Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=cache +2021-08-13T15:32:55.080Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=cache +2021-08-13T15:32:55.081Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-service-subsets.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api +2021-08-13T15:32:55.081Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db === RUN TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-hostname-service-subsets -2022-09-15T21:41:01.666Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-hostname-service-subsets.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api -2022-09-15T21:41:01.667Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-hostname-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db -2022-09-15T21:41:01.667Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-hostname-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=cache +2021-08-13T15:32:55.087Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-hostname-service-subsets.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api +2021-08-13T15:32:55.088Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-hostname-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db +2021-08-13T15:32:55.088Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-hostname-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=cache === RUN TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-ignore-extra-resolvers -2022-09-15T21:41:01.679Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-ignore-extra-resolvers.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api -2022-09-15T21:41:01.679Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-ignore-extra-resolvers.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db +2021-08-13T15:32:55.094Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-ignore-extra-resolvers.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api +2021-08-13T15:32:55.094Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-ignore-extra-resolvers.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db === RUN TestClustersFromSnapshot/envoy-1.11.2/ingress-multiple-listeners-duplicate-service -2022-09-15T21:41:01.684Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-multiple-listeners-duplicate-service: generating cluster for: cluster=foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2022-09-15T21:41:01.684Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-multiple-listeners-duplicate-service: generating cluster for: cluster=bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul ---- PASS: TestClustersFromSnapshot (1.11s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5 (0.32s) +2021-08-13T15:32:55.099Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-multiple-listeners-duplicate-service: generating cluster for: cluster=foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.099Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-multiple-listeners-duplicate-service: generating cluster for: cluster=bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +--- PASS: TestClustersFromSnapshot (0.86s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5 (0.26s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/defaults (0.03s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/custom-local-app (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/custom-local-app-typed (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/custom-local-app-typed (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/custom-upstream (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/custom-upstream-default-chain (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/custom-upstream-typed (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/custom-upstream-typed (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/custom-upstream-ignores-tls (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/custom-timeouts (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/custom-limits-max-connections-only (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/custom-limits-set-to-zero (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/custom-limits (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/custom-limits-set-to-zero (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/custom-limits (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain (0.04s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-external-sni (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-and-overrides (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-and-failover (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-and-failover (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-remote-gateway (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-local-gateway (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-local-gateway (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/splitter-with-resolver-redirect (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-local-gateway (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/splitter-with-resolver-redirect (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/expose-paths-local-app-paths (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/expose-paths-new-cluster-http2 (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/expose-paths-grpc-new-cluster-http1 (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway-using-federation-states (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway-no-services (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway-service-subsets (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway-ignore-extra-resolvers (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway-service-subsets (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway-ignore-extra-resolvers (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway-service-timeouts (0.01s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-gateway (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-gateway-no-services (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-with-chain (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-with-chain-external-sni (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-with-chain-and-overrides (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-with-chain-and-failover (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-with-chain-external-sni (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-with-chain-and-overrides (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-with-chain-and-failover (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-remote-gateway (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-remote-gateway-triggered (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-remote-gateway (0.00s) @@ -71327,15 +71340,15 @@ --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-local-gateway-triggered (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-local-gateway (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-splitter-with-resolver-redirect (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-splitter-with-resolver-redirect (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-no-services (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-service-subsets (0.01s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-hostname-service-subsets (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-ignore-extra-resolvers (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-ignore-extra-resolvers (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-multiple-listeners-duplicate-service (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6 (0.26s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/defaults (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6 (0.19s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/defaults (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/custom-local-app (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/custom-local-app-typed (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/custom-upstream (0.00s) @@ -71344,34 +71357,34 @@ --- PASS: TestClustersFromSnapshot/envoy-1.13.6/custom-upstream-ignores-tls (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/custom-timeouts (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/custom-limits-max-connections-only (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/custom-limits-set-to-zero (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/custom-limits (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/custom-limits-set-to-zero (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/custom-limits (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-external-sni (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-and-overrides (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-and-failover (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-and-failover (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-remote-gateway (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-local-gateway (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-local-gateway (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-local-gateway (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/splitter-with-resolver-redirect (0.01s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/expose-paths-local-app-paths (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/expose-paths-new-cluster-http2 (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/expose-paths-grpc-new-cluster-http1 (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-using-federation-states (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-using-federation-states (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-no-services (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-service-subsets (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-ignore-extra-resolvers (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-service-timeouts (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-service-subsets (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-ignore-extra-resolvers (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-service-timeouts (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/ingress-gateway (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/ingress-gateway-no-services (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/ingress-with-chain (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/ingress-with-chain-external-sni (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/ingress-with-chain (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/ingress-with-chain-external-sni (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/ingress-with-chain-and-overrides (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/ingress-with-chain-and-failover (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-remote-gateway (0.00s) @@ -71382,21 +71395,21 @@ --- PASS: TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-local-gateway-triggered (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-local-gateway (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/ingress-splitter-with-resolver-redirect (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/ingress-splitter-with-resolver-redirect (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-no-services (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-service-subsets (0.01s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-hostname-service-subsets (0.01s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-ignore-extra-resolvers (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/ingress-multiple-listeners-duplicate-service (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7 (0.28s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/ingress-multiple-listeners-duplicate-service (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7 (0.20s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/defaults (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/custom-local-app (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/custom-local-app (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/custom-local-app-typed (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/custom-upstream (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/custom-upstream-default-chain (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/custom-upstream-typed (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/custom-upstream-ignores-tls (0.02s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/custom-upstream-default-chain (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/custom-upstream-typed (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/custom-upstream-ignores-tls (0.01s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/custom-timeouts (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/custom-limits-max-connections-only (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/custom-limits-set-to-zero (0.00s) @@ -71404,25 +71417,25 @@ --- PASS: TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-chain (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-external-sni (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-and-overrides (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-and-failover (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-remote-gateway (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-and-failover (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-remote-gateway (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-local-gateway (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-local-gateway (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-local-gateway (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/splitter-with-resolver-redirect (0.02s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/splitter-with-resolver-redirect (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/expose-paths-local-app-paths (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/expose-paths-new-cluster-http2 (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/expose-paths-grpc-new-cluster-http1 (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-using-federation-states (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-using-federation-states (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-no-services (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-service-subsets (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-service-subsets (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-ignore-extra-resolvers (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-service-timeouts (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-service-timeouts (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/ingress-gateway (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/ingress-gateway-no-services (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/ingress-with-chain (0.00s) @@ -71438,15 +71451,15 @@ --- PASS: TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-local-gateway (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/ingress-splitter-with-resolver-redirect (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-no-services (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-service-subsets (0.01s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-hostname-service-subsets (0.01s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-ignore-extra-resolvers (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/ingress-multiple-listeners-duplicate-service (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2 (0.25s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/defaults (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/custom-local-app (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/ingress-multiple-listeners-duplicate-service (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2 (0.21s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/defaults (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/custom-local-app (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.11.2/custom-local-app-typed (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.11.2/custom-upstream (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.11.2/custom-upstream-default-chain (0.00s) @@ -71456,9 +71469,9 @@ --- PASS: TestClustersFromSnapshot/envoy-1.11.2/custom-limits-max-connections-only (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.11.2/custom-limits-set-to-zero (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.11.2/custom-limits (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-external-sni (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-overrides (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-external-sni (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-overrides (0.01s) --- PASS: TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-failover (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-remote-gateway (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered (0.00s) @@ -71476,7 +71489,7 @@ --- PASS: TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway-using-federation-states (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway-no-services (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway-service-subsets (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway-ignore-extra-resolvers (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway-ignore-extra-resolvers (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway-service-timeouts (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.11.2/ingress-gateway (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.11.2/ingress-gateway-no-services (0.00s) @@ -71484,7 +71497,7 @@ --- PASS: TestClustersFromSnapshot/envoy-1.11.2/ingress-with-chain-external-sni (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.11.2/ingress-with-chain-and-overrides (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.11.2/ingress-with-chain-and-failover (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-remote-gateway (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-remote-gateway (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-remote-gateway-triggered (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-remote-gateway (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered (0.00s) @@ -71572,302 +71585,302 @@ === RUN Test_endpointsFromSnapshot === RUN Test_endpointsFromSnapshot/envoy-1.14.5 === RUN Test_endpointsFromSnapshot/envoy-1.14.5/defaults -2022-09-15T21:41:01.695Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/defaults: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.108Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/defaults: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/mesh-gateway === RUN Test_endpointsFromSnapshot/envoy-1.14.5/mesh-gateway-using-federation-states === RUN Test_endpointsFromSnapshot/envoy-1.14.5/mesh-gateway-no-services === RUN Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-chain -2022-09-15T21:41:01.709Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-chain: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.121Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-chain: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-external-sni -2022-09-15T21:41:01.712Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-external-sni: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.124Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-external-sni: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-and-overrides -2022-09-15T21:41:01.716Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-and-overrides: generating endpoints for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.127Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-and-overrides: generating endpoints for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-and-failover -2022-09-15T21:41:01.719Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-and-failover: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.131Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-and-failover: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-remote-gateway -2022-09-15T21:41:01.722Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.134Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered -2022-09-15T21:41:01.726Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.137Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway -2022-09-15T21:41:01.729Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.140Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered -2022-09-15T21:41:01.734Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.143Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-local-gateway -2022-09-15T21:41:01.753Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.145Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered -2022-09-15T21:41:01.757Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.148Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-local-gateway -2022-09-15T21:41:01.761Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.151Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered -2022-09-15T21:41:01.765Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.154Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-default-chain-and-custom-cluster -2022-09-15T21:41:01.770Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-default-chain-and-custom-cluster: generating endpoints for: cluster=myservice +2021-08-13T15:32:55.157Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-default-chain-and-custom-cluster: generating endpoints for: cluster=myservice === RUN Test_endpointsFromSnapshot/envoy-1.14.5/splitter-with-resolver-redirect -2022-09-15T21:41:01.774Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/splitter-with-resolver-redirect: generating endpoints for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul -2022-09-15T21:41:01.774Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/splitter-with-resolver-redirect: generating endpoints for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.160Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/splitter-with-resolver-redirect: generating endpoints for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.160Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/splitter-with-resolver-redirect: generating endpoints for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/mesh-gateway-service-subsets === RUN Test_endpointsFromSnapshot/envoy-1.14.5/mesh-gateway-default-service-subset === RUN Test_endpointsFromSnapshot/envoy-1.14.5/ingress-gateway -2022-09-15T21:41:01.804Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.178Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/ingress-gateway-no-services === RUN Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-chain -2022-09-15T21:41:01.810Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-chain: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.183Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-chain: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-chain-external-sni -2022-09-15T21:41:01.821Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-chain-external-sni: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.185Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-chain-external-sni: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-chain-and-overrides -2022-09-15T21:41:01.830Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-chain-and-overrides: generating endpoints for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.188Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-chain-and-overrides: generating endpoints for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-chain-and-failover -2022-09-15T21:41:01.835Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-chain-and-failover: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.191Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-chain-and-failover: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-remote-gateway -2022-09-15T21:41:01.841Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.195Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-remote-gateway-triggered -2022-09-15T21:41:01.848Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.198Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-remote-gateway -2022-09-15T21:41:01.852Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.202Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered -2022-09-15T21:41:01.856Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.205Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-local-gateway -2022-09-15T21:41:01.859Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.208Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-local-gateway-triggered -2022-09-15T21:41:01.863Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.211Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-local-gateway -2022-09-15T21:41:01.867Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.215Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered -2022-09-15T21:41:01.874Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.218Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/ingress-splitter-with-resolver-redirect -2022-09-15T21:41:01.878Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-splitter-with-resolver-redirect: generating endpoints for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2022-09-15T21:41:01.878Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-splitter-with-resolver-redirect: generating endpoints for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.221Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-splitter-with-resolver-redirect: generating endpoints for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.221Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-splitter-with-resolver-redirect: generating endpoints for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/terminating-gateway === RUN Test_endpointsFromSnapshot/envoy-1.14.5/terminating-gateway-no-services === RUN Test_endpointsFromSnapshot/envoy-1.14.5/terminating-gateway-service-subsets === RUN Test_endpointsFromSnapshot/envoy-1.14.5/terminating-gateway-default-service-subset === RUN Test_endpointsFromSnapshot/envoy-1.14.5/ingress-multiple-listeners-duplicate-service -2022-09-15T21:41:01.898Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-multiple-listeners-duplicate-service: generating endpoints for: cluster=foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2022-09-15T21:41:01.898Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-multiple-listeners-duplicate-service: generating endpoints for: cluster=bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.242Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-multiple-listeners-duplicate-service: generating endpoints for: cluster=foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.242Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-multiple-listeners-duplicate-service: generating endpoints for: cluster=bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6 === RUN Test_endpointsFromSnapshot/envoy-1.13.6/defaults -2022-09-15T21:41:01.902Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/defaults: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.246Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/defaults: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/mesh-gateway === RUN Test_endpointsFromSnapshot/envoy-1.13.6/mesh-gateway-using-federation-states === RUN Test_endpointsFromSnapshot/envoy-1.13.6/mesh-gateway-no-services === RUN Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-chain -2022-09-15T21:41:01.917Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-chain: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.259Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-chain: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-external-sni -2022-09-15T21:41:01.929Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-external-sni: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.262Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-external-sni: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-and-overrides -2022-09-15T21:41:01.938Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-and-overrides: generating endpoints for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.265Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-and-overrides: generating endpoints for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-and-failover -2022-09-15T21:41:01.941Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-and-failover: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.267Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-and-failover: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-remote-gateway -2022-09-15T21:41:01.946Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.270Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered -2022-09-15T21:41:01.953Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.273Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway -2022-09-15T21:41:01.956Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.276Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered -2022-09-15T21:41:01.960Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.278Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-local-gateway -2022-09-15T21:41:01.963Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.281Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered -2022-09-15T21:41:01.966Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.284Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-local-gateway -2022-09-15T21:41:01.969Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.286Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered -2022-09-15T21:41:01.972Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.289Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-default-chain-and-custom-cluster -2022-09-15T21:41:01.976Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-default-chain-and-custom-cluster: generating endpoints for: cluster=myservice +2021-08-13T15:32:55.292Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-default-chain-and-custom-cluster: generating endpoints for: cluster=myservice === RUN Test_endpointsFromSnapshot/envoy-1.13.6/splitter-with-resolver-redirect -2022-09-15T21:41:01.979Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/splitter-with-resolver-redirect: generating endpoints for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2022-09-15T21:41:01.979Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/splitter-with-resolver-redirect: generating endpoints for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.296Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/splitter-with-resolver-redirect: generating endpoints for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.296Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/splitter-with-resolver-redirect: generating endpoints for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/mesh-gateway-service-subsets === RUN Test_endpointsFromSnapshot/envoy-1.13.6/mesh-gateway-default-service-subset === RUN Test_endpointsFromSnapshot/envoy-1.13.6/ingress-gateway -2022-09-15T21:41:02.001Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.312Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/ingress-gateway-no-services === RUN Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-chain -2022-09-15T21:41:02.007Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-chain: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.317Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-chain: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-chain-external-sni -2022-09-15T21:41:02.010Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-chain-external-sni: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.319Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-chain-external-sni: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-chain-and-overrides -2022-09-15T21:41:02.014Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-chain-and-overrides: generating endpoints for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.322Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-chain-and-overrides: generating endpoints for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-chain-and-failover -2022-09-15T21:41:02.018Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-chain-and-failover: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.325Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-chain-and-failover: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-remote-gateway -2022-09-15T21:41:02.021Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.328Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-remote-gateway-triggered -2022-09-15T21:41:02.025Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.331Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-remote-gateway -2022-09-15T21:41:02.028Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.333Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered -2022-09-15T21:41:02.031Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.336Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-local-gateway -2022-09-15T21:41:02.035Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.339Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-local-gateway-triggered -2022-09-15T21:41:02.038Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.342Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-local-gateway -2022-09-15T21:41:02.042Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.345Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered -2022-09-15T21:41:02.045Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.350Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/ingress-splitter-with-resolver-redirect -2022-09-15T21:41:02.048Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-splitter-with-resolver-redirect: generating endpoints for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2022-09-15T21:41:02.048Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-splitter-with-resolver-redirect: generating endpoints for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.353Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-splitter-with-resolver-redirect: generating endpoints for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.353Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-splitter-with-resolver-redirect: generating endpoints for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/terminating-gateway === RUN Test_endpointsFromSnapshot/envoy-1.13.6/terminating-gateway-no-services === RUN Test_endpointsFromSnapshot/envoy-1.13.6/terminating-gateway-service-subsets === RUN Test_endpointsFromSnapshot/envoy-1.13.6/terminating-gateway-default-service-subset === RUN Test_endpointsFromSnapshot/envoy-1.13.6/ingress-multiple-listeners-duplicate-service -2022-09-15T21:41:02.071Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-multiple-listeners-duplicate-service: generating endpoints for: cluster=foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2022-09-15T21:41:02.071Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-multiple-listeners-duplicate-service: generating endpoints for: cluster=bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.369Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-multiple-listeners-duplicate-service: generating endpoints for: cluster=foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.369Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-multiple-listeners-duplicate-service: generating endpoints for: cluster=bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7 === RUN Test_endpointsFromSnapshot/envoy-1.12.7/defaults -2022-09-15T21:41:02.075Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/defaults: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.372Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/defaults: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/mesh-gateway === RUN Test_endpointsFromSnapshot/envoy-1.12.7/mesh-gateway-using-federation-states === RUN Test_endpointsFromSnapshot/envoy-1.12.7/mesh-gateway-no-services === RUN Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-chain -2022-09-15T21:41:02.089Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-chain: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.385Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-chain: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-external-sni -2022-09-15T21:41:02.092Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-external-sni: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.388Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-external-sni: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-and-overrides -2022-09-15T21:41:02.095Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-and-overrides: generating endpoints for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.390Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-and-overrides: generating endpoints for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-and-failover -2022-09-15T21:41:02.099Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-and-failover: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.393Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-and-failover: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-remote-gateway -2022-09-15T21:41:02.103Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.396Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered -2022-09-15T21:41:02.106Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.399Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway -2022-09-15T21:41:02.109Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.403Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered -2022-09-15T21:41:02.113Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.406Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-local-gateway -2022-09-15T21:41:02.116Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.409Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered -2022-09-15T21:41:02.119Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.412Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-local-gateway -2022-09-15T21:41:02.123Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.415Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered -2022-09-15T21:41:02.133Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.418Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-default-chain-and-custom-cluster -2022-09-15T21:41:02.139Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-default-chain-and-custom-cluster: generating endpoints for: cluster=myservice +2021-08-13T15:32:55.421Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-default-chain-and-custom-cluster: generating endpoints for: cluster=myservice === RUN Test_endpointsFromSnapshot/envoy-1.12.7/splitter-with-resolver-redirect -2022-09-15T21:41:02.142Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/splitter-with-resolver-redirect: generating endpoints for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2022-09-15T21:41:02.142Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/splitter-with-resolver-redirect: generating endpoints for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.424Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/splitter-with-resolver-redirect: generating endpoints for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.424Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/splitter-with-resolver-redirect: generating endpoints for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/mesh-gateway-service-subsets === RUN Test_endpointsFromSnapshot/envoy-1.12.7/mesh-gateway-default-service-subset === RUN Test_endpointsFromSnapshot/envoy-1.12.7/ingress-gateway -2022-09-15T21:41:02.158Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.439Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/ingress-gateway-no-services === RUN Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-chain -2022-09-15T21:41:02.166Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-chain: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.444Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-chain: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-chain-external-sni -2022-09-15T21:41:02.169Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-chain-external-sni: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.447Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-chain-external-sni: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-chain-and-overrides -2022-09-15T21:41:02.172Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-chain-and-overrides: generating endpoints for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.449Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-chain-and-overrides: generating endpoints for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-chain-and-failover -2022-09-15T21:41:02.175Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-chain-and-failover: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.454Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-chain-and-failover: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-remote-gateway -2022-09-15T21:41:02.179Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.457Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-remote-gateway-triggered -2022-09-15T21:41:02.182Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.460Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-remote-gateway -2022-09-15T21:41:02.185Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.462Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered -2022-09-15T21:41:02.189Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.465Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-local-gateway -2022-09-15T21:41:02.194Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.468Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-local-gateway-triggered -2022-09-15T21:41:02.198Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.470Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-local-gateway -2022-09-15T21:41:02.201Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.473Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered -2022-09-15T21:41:02.205Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.476Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/ingress-splitter-with-resolver-redirect -2022-09-15T21:41:02.227Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-splitter-with-resolver-redirect: generating endpoints for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2022-09-15T21:41:02.227Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-splitter-with-resolver-redirect: generating endpoints for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.478Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-splitter-with-resolver-redirect: generating endpoints for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.478Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-splitter-with-resolver-redirect: generating endpoints for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/terminating-gateway === RUN Test_endpointsFromSnapshot/envoy-1.12.7/terminating-gateway-no-services === RUN Test_endpointsFromSnapshot/envoy-1.12.7/terminating-gateway-service-subsets === RUN Test_endpointsFromSnapshot/envoy-1.12.7/terminating-gateway-default-service-subset === RUN Test_endpointsFromSnapshot/envoy-1.12.7/ingress-multiple-listeners-duplicate-service -2022-09-15T21:41:02.246Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-multiple-listeners-duplicate-service: generating endpoints for: cluster=foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2022-09-15T21:41:02.246Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-multiple-listeners-duplicate-service: generating endpoints for: cluster=bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.493Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-multiple-listeners-duplicate-service: generating endpoints for: cluster=foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.493Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-multiple-listeners-duplicate-service: generating endpoints for: cluster=bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2 === RUN Test_endpointsFromSnapshot/envoy-1.11.2/defaults -2022-09-15T21:41:02.250Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/defaults: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.496Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/defaults: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/mesh-gateway === RUN Test_endpointsFromSnapshot/envoy-1.11.2/mesh-gateway-using-federation-states === RUN Test_endpointsFromSnapshot/envoy-1.11.2/mesh-gateway-no-services === RUN Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-chain -2022-09-15T21:41:02.265Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-chain: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.510Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-chain: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-external-sni -2022-09-15T21:41:02.268Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-external-sni: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.512Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-external-sni: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-overrides -2022-09-15T21:41:02.271Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-overrides: generating endpoints for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.515Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-overrides: generating endpoints for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-failover -2022-09-15T21:41:02.278Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-failover: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.518Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-failover: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-remote-gateway -2022-09-15T21:41:02.307Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.521Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered -2022-09-15T21:41:02.324Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.523Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway -2022-09-15T21:41:02.328Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.526Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered -2022-09-15T21:41:02.350Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.529Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-local-gateway -2022-09-15T21:41:02.354Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.531Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered -2022-09-15T21:41:02.357Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.534Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-local-gateway -2022-09-15T21:41:02.362Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.537Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered -2022-09-15T21:41:02.365Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.539Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-default-chain-and-custom-cluster -2022-09-15T21:41:02.369Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-default-chain-and-custom-cluster: generating endpoints for: cluster=myservice +2021-08-13T15:32:55.542Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-default-chain-and-custom-cluster: generating endpoints for: cluster=myservice === RUN Test_endpointsFromSnapshot/envoy-1.11.2/splitter-with-resolver-redirect -2022-09-15T21:41:02.372Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/splitter-with-resolver-redirect: generating endpoints for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2022-09-15T21:41:02.372Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/splitter-with-resolver-redirect: generating endpoints for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.545Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/splitter-with-resolver-redirect: generating endpoints for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.545Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/splitter-with-resolver-redirect: generating endpoints for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/mesh-gateway-service-subsets === RUN Test_endpointsFromSnapshot/envoy-1.11.2/mesh-gateway-default-service-subset === RUN Test_endpointsFromSnapshot/envoy-1.11.2/ingress-gateway -2022-09-15T21:41:02.392Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.561Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/ingress-gateway-no-services === RUN Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-chain -2022-09-15T21:41:02.403Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-chain: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.566Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-chain: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-chain-external-sni -2022-09-15T21:41:02.407Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-chain-external-sni: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.569Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-chain-external-sni: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-chain-and-overrides -2022-09-15T21:41:02.414Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-chain-and-overrides: generating endpoints for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.571Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-chain-and-overrides: generating endpoints for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-chain-and-failover -2022-09-15T21:41:02.418Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-chain-and-failover: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.574Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-chain-and-failover: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-remote-gateway -2022-09-15T21:41:02.422Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.577Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-remote-gateway-triggered -2022-09-15T21:41:02.425Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.580Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-remote-gateway -2022-09-15T21:41:02.429Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.582Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered -2022-09-15T21:41:02.432Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.585Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-local-gateway -2022-09-15T21:41:02.435Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.587Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-local-gateway-triggered -2022-09-15T21:41:02.439Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.590Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-local-gateway -2022-09-15T21:41:02.442Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.593Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered -2022-09-15T21:41:02.446Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.595Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/ingress-splitter-with-resolver-redirect -2022-09-15T21:41:02.449Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-splitter-with-resolver-redirect: generating endpoints for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2022-09-15T21:41:02.449Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-splitter-with-resolver-redirect: generating endpoints for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.600Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-splitter-with-resolver-redirect: generating endpoints for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.600Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-splitter-with-resolver-redirect: generating endpoints for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/terminating-gateway === RUN Test_endpointsFromSnapshot/envoy-1.11.2/terminating-gateway-no-services === RUN Test_endpointsFromSnapshot/envoy-1.11.2/terminating-gateway-service-subsets === RUN Test_endpointsFromSnapshot/envoy-1.11.2/terminating-gateway-default-service-subset === RUN Test_endpointsFromSnapshot/envoy-1.11.2/ingress-multiple-listeners-duplicate-service -2022-09-15T21:41:02.466Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-multiple-listeners-duplicate-service: generating endpoints for: cluster=foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2022-09-15T21:41:02.466Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-multiple-listeners-duplicate-service: generating endpoints for: cluster=bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul ---- PASS: Test_endpointsFromSnapshot (0.78s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5 (0.21s) +2021-08-13T15:32:55.614Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-multiple-listeners-duplicate-service: generating endpoints for: cluster=foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:55.614Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-multiple-listeners-duplicate-service: generating endpoints for: cluster=bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +--- PASS: Test_endpointsFromSnapshot (0.51s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5 (0.14s) --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/defaults (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/mesh-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/mesh-gateway-using-federation-states (0.00s) @@ -71880,45 +71893,45 @@ --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-local-gateway (0.02s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-local-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-local-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-default-chain-and-custom-cluster (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/splitter-with-resolver-redirect (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/mesh-gateway-service-subsets (0.01s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/mesh-gateway-default-service-subset (0.02s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/ingress-gateway (0.01s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/mesh-gateway-default-service-subset (0.01s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/ingress-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/ingress-gateway-no-services (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-chain (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-chain-external-sni (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-chain-and-overrides (0.01s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-chain-and-overrides (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-chain-and-failover (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-remote-gateway (0.01s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-remote-gateway-triggered (0.01s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-remote-gateway (0.00s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-remote-gateway-triggered (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-remote-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-local-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-local-gateway-triggered (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-local-gateway (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered (0.01s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/ingress-splitter-with-resolver-redirect (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/terminating-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/terminating-gateway-no-services (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/terminating-gateway-service-subsets (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/terminating-gateway-default-service-subset (0.00s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/terminating-gateway-service-subsets (0.01s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/terminating-gateway-default-service-subset (0.01s) --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/ingress-multiple-listeners-duplicate-service (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6 (0.17s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6 (0.13s) --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/defaults (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/mesh-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/mesh-gateway-using-federation-states (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/mesh-gateway-no-services (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-chain (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-external-sni (0.01s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-and-overrides (0.01s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-external-sni (0.00s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-and-overrides (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-and-failover (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-remote-gateway (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered (0.01s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-local-gateway (0.00s) @@ -71926,7 +71939,7 @@ --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-local-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-default-chain-and-custom-cluster (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/splitter-with-resolver-redirect (0.00s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/splitter-with-resolver-redirect (0.01s) --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/mesh-gateway-service-subsets (0.01s) --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/mesh-gateway-default-service-subset (0.01s) --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/ingress-gateway (0.00s) @@ -71947,9 +71960,9 @@ --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/terminating-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/terminating-gateway-no-services (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/terminating-gateway-service-subsets (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/terminating-gateway-default-service-subset (0.01s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/terminating-gateway-default-service-subset (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/ingress-multiple-listeners-duplicate-service (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7 (0.17s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7 (0.12s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/defaults (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/mesh-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/mesh-gateway-using-federation-states (0.00s) @@ -71965,12 +71978,12 @@ --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-local-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-local-gateway (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered (0.01s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-default-chain-and-custom-cluster (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/splitter-with-resolver-redirect (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/mesh-gateway-service-subsets (0.01s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/mesh-gateway-default-service-subset (0.01s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/ingress-gateway (0.01s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/ingress-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/ingress-gateway-no-services (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-chain (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-chain-external-sni (0.00s) @@ -71984,25 +71997,25 @@ --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-local-gateway-triggered (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-local-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/ingress-splitter-with-resolver-redirect (0.02s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/ingress-splitter-with-resolver-redirect (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/terminating-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/terminating-gateway-no-services (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/terminating-gateway-service-subsets (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/terminating-gateway-default-service-subset (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/ingress-multiple-listeners-duplicate-service (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2 (0.22s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2 (0.12s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/defaults (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/mesh-gateway (0.00s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/mesh-gateway (0.01s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/mesh-gateway-using-federation-states (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/mesh-gateway-no-services (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-chain (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-external-sni (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-overrides (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-failover (0.02s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-remote-gateway (0.01s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered (0.02s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-failover (0.00s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-remote-gateway (0.00s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered (0.02s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-local-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-local-gateway (0.00s) @@ -72012,10 +72025,10 @@ --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/mesh-gateway-service-subsets (0.01s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/mesh-gateway-default-service-subset (0.01s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/ingress-gateway (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/ingress-gateway-no-services (0.01s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/ingress-gateway-no-services (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-chain (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-chain-external-sni (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-chain-and-overrides (0.01s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-chain-and-overrides (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-chain-and-failover (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-remote-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-remote-gateway-triggered (0.00s) @@ -72032,15 +72045,15 @@ --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/terminating-gateway-default-service-subset (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/ingress-multiple-listeners-duplicate-service (0.00s) === RUN TestDetermineEnvoyVersionFromNode +=== RUN TestDetermineEnvoyVersionFromNode/user_agent_build_version_with_user_agent === RUN TestDetermineEnvoyVersionFromNode/empty === RUN TestDetermineEnvoyVersionFromNode/only_build_version === RUN TestDetermineEnvoyVersionFromNode/user_agent_build_version_but_no_user_agent -=== RUN TestDetermineEnvoyVersionFromNode/user_agent_build_version_with_user_agent --- PASS: TestDetermineEnvoyVersionFromNode (0.00s) + --- PASS: TestDetermineEnvoyVersionFromNode/user_agent_build_version_with_user_agent (0.00s) --- PASS: TestDetermineEnvoyVersionFromNode/empty (0.00s) --- PASS: TestDetermineEnvoyVersionFromNode/only_build_version (0.00s) --- PASS: TestDetermineEnvoyVersionFromNode/user_agent_build_version_but_no_user_agent (0.00s) - --- PASS: TestDetermineEnvoyVersionFromNode/user_agent_build_version_with_user_agent (0.00s) === RUN TestFirstHealthyTarget === RUN TestFirstHealthyTarget/all-ok.default.dc1_and_[] === RUN TestFirstHealthyTarget/all-warn.default.dc1_and_[] @@ -72079,7 +72092,7 @@ === RUN TestListenersFromSnapshot/envoy-1.14.5/custom-upstream === RUN TestListenersFromSnapshot/envoy-1.14.5/custom-upstream-typed === RUN TestListenersFromSnapshot/envoy-1.14.5/custom-upstream-typed-ignored-with-disco-chain -2022-09-15T21:41:02.548Z [WARN] TestListenersFromSnapshot/envoy-1.14.5/custom-upstream-typed-ignored-with-disco-chain: ignoring escape hatch setting because already configured for: discovery chain=db upstream=db config=envoy_listener_json +2021-08-13T15:32:55.673Z [WARN] TestListenersFromSnapshot/envoy-1.14.5/custom-upstream-typed-ignored-with-disco-chain: ignoring escape hatch setting because already configured for: discovery chain=db upstream=db config=envoy_listener_json === RUN TestListenersFromSnapshot/envoy-1.14.5/splitter-with-resolver-redirect === RUN TestListenersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain === RUN TestListenersFromSnapshot/envoy-1.14.5/connect-proxy-with-http-chain @@ -72110,7 +72123,7 @@ === RUN TestListenersFromSnapshot/envoy-1.14.5/terminating-gateway-service-subsets === RUN TestListenersFromSnapshot/envoy-1.14.5/ingress-http-multiple-services === RUN TestListenersFromSnapshot/envoy-1.14.5/terminating-gateway-no-api-cert -2022-09-15T21:41:02.774Z [ERROR] TestListenersFromSnapshot/envoy-1.14.5/terminating-gateway-no-api-cert.terminating_gateway: no client certificate available for linked service, skipping filter chain creation: service=api error= +2021-08-13T15:32:55.810Z [ERROR] TestListenersFromSnapshot/envoy-1.14.5/terminating-gateway-no-api-cert.terminating_gateway: no client certificate available for linked service, skipping filter chain creation: service=api error= === RUN TestListenersFromSnapshot/envoy-1.14.5/ingress-with-tls-listener === RUN TestListenersFromSnapshot/envoy-1.13.6 === RUN TestListenersFromSnapshot/envoy-1.13.6/defaults @@ -72125,7 +72138,7 @@ === RUN TestListenersFromSnapshot/envoy-1.13.6/custom-upstream === RUN TestListenersFromSnapshot/envoy-1.13.6/custom-upstream-typed === RUN TestListenersFromSnapshot/envoy-1.13.6/custom-upstream-typed-ignored-with-disco-chain -2022-09-15T21:41:02.852Z [WARN] TestListenersFromSnapshot/envoy-1.13.6/custom-upstream-typed-ignored-with-disco-chain: ignoring escape hatch setting because already configured for: discovery chain=db upstream=db config=envoy_listener_json +2021-08-13T15:32:55.875Z [WARN] TestListenersFromSnapshot/envoy-1.13.6/custom-upstream-typed-ignored-with-disco-chain: ignoring escape hatch setting because already configured for: discovery chain=db upstream=db config=envoy_listener_json === RUN TestListenersFromSnapshot/envoy-1.13.6/splitter-with-resolver-redirect === RUN TestListenersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain === RUN TestListenersFromSnapshot/envoy-1.13.6/connect-proxy-with-http-chain @@ -72156,7 +72169,7 @@ === RUN TestListenersFromSnapshot/envoy-1.13.6/terminating-gateway-service-subsets === RUN TestListenersFromSnapshot/envoy-1.13.6/ingress-http-multiple-services === RUN TestListenersFromSnapshot/envoy-1.13.6/terminating-gateway-no-api-cert -2022-09-15T21:41:03.033Z [ERROR] TestListenersFromSnapshot/envoy-1.13.6/terminating-gateway-no-api-cert.terminating_gateway: no client certificate available for linked service, skipping filter chain creation: service=api error= +2021-08-13T15:32:56.023Z [ERROR] TestListenersFromSnapshot/envoy-1.13.6/terminating-gateway-no-api-cert.terminating_gateway: no client certificate available for linked service, skipping filter chain creation: service=api error= === RUN TestListenersFromSnapshot/envoy-1.13.6/ingress-with-tls-listener === RUN TestListenersFromSnapshot/envoy-1.12.7 === RUN TestListenersFromSnapshot/envoy-1.12.7/defaults @@ -72171,7 +72184,7 @@ === RUN TestListenersFromSnapshot/envoy-1.12.7/custom-upstream === RUN TestListenersFromSnapshot/envoy-1.12.7/custom-upstream-typed === RUN TestListenersFromSnapshot/envoy-1.12.7/custom-upstream-typed-ignored-with-disco-chain -2022-09-15T21:41:03.139Z [WARN] TestListenersFromSnapshot/envoy-1.12.7/custom-upstream-typed-ignored-with-disco-chain: ignoring escape hatch setting because already configured for: discovery chain=db upstream=db config=envoy_listener_json +2021-08-13T15:32:56.090Z [WARN] TestListenersFromSnapshot/envoy-1.12.7/custom-upstream-typed-ignored-with-disco-chain: ignoring escape hatch setting because already configured for: discovery chain=db upstream=db config=envoy_listener_json === RUN TestListenersFromSnapshot/envoy-1.12.7/splitter-with-resolver-redirect === RUN TestListenersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain === RUN TestListenersFromSnapshot/envoy-1.12.7/connect-proxy-with-http-chain @@ -72202,7 +72215,7 @@ === RUN TestListenersFromSnapshot/envoy-1.12.7/terminating-gateway-service-subsets === RUN TestListenersFromSnapshot/envoy-1.12.7/ingress-http-multiple-services === RUN TestListenersFromSnapshot/envoy-1.12.7/terminating-gateway-no-api-cert -2022-09-15T21:41:03.363Z [ERROR] TestListenersFromSnapshot/envoy-1.12.7/terminating-gateway-no-api-cert.terminating_gateway: no client certificate available for linked service, skipping filter chain creation: service=api error= +2021-08-13T15:32:56.291Z [ERROR] TestListenersFromSnapshot/envoy-1.12.7/terminating-gateway-no-api-cert.terminating_gateway: no client certificate available for linked service, skipping filter chain creation: service=api error= === RUN TestListenersFromSnapshot/envoy-1.12.7/ingress-with-tls-listener === RUN TestListenersFromSnapshot/envoy-1.11.2 === RUN TestListenersFromSnapshot/envoy-1.11.2/defaults @@ -72217,7 +72230,7 @@ === RUN TestListenersFromSnapshot/envoy-1.11.2/custom-upstream === RUN TestListenersFromSnapshot/envoy-1.11.2/custom-upstream-typed === RUN TestListenersFromSnapshot/envoy-1.11.2/custom-upstream-typed-ignored-with-disco-chain -2022-09-15T21:41:03.441Z [WARN] TestListenersFromSnapshot/envoy-1.11.2/custom-upstream-typed-ignored-with-disco-chain: ignoring escape hatch setting because already configured for: discovery chain=db upstream=db config=envoy_listener_json +2021-08-13T15:32:56.358Z [WARN] TestListenersFromSnapshot/envoy-1.11.2/custom-upstream-typed-ignored-with-disco-chain: ignoring escape hatch setting because already configured for: discovery chain=db upstream=db config=envoy_listener_json === RUN TestListenersFromSnapshot/envoy-1.11.2/splitter-with-resolver-redirect === RUN TestListenersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain === RUN TestListenersFromSnapshot/envoy-1.11.2/connect-proxy-with-http-chain @@ -72248,81 +72261,81 @@ === RUN TestListenersFromSnapshot/envoy-1.11.2/terminating-gateway-service-subsets === RUN TestListenersFromSnapshot/envoy-1.11.2/ingress-http-multiple-services === RUN TestListenersFromSnapshot/envoy-1.11.2/terminating-gateway-no-api-cert -2022-09-15T21:41:03.621Z [ERROR] TestListenersFromSnapshot/envoy-1.11.2/terminating-gateway-no-api-cert.terminating_gateway: no client certificate available for linked service, skipping filter chain creation: service=api error= +2021-08-13T15:32:56.513Z [ERROR] TestListenersFromSnapshot/envoy-1.11.2/terminating-gateway-no-api-cert.terminating_gateway: no client certificate available for linked service, skipping filter chain creation: service=api error= === RUN TestListenersFromSnapshot/envoy-1.11.2/ingress-with-tls-listener ---- PASS: TestListenersFromSnapshot (1.16s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5 (0.31s) +--- PASS: TestListenersFromSnapshot (0.90s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5 (0.20s) --- PASS: TestListenersFromSnapshot/envoy-1.14.5/defaults (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/listener-bind-address (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/listener-bind-port (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/listener-bind-address-port (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/listener-bind-address (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/listener-bind-port (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/listener-bind-address-port (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.14.5/http-public-listener (0.01s) --- PASS: TestListenersFromSnapshot/envoy-1.14.5/http-upstream (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/custom-public-listener (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/custom-public-listener-typed (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/custom-public-listener-ignores-tls (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/custom-upstream (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/custom-upstream-typed (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/custom-public-listener (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/custom-public-listener-typed (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/custom-public-listener-ignores-tls (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/custom-upstream (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/custom-upstream-typed (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.14.5/custom-upstream-typed-ignored-with-disco-chain (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/splitter-with-resolver-redirect (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/splitter-with-resolver-redirect (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.14.5/connect-proxy-with-http-chain (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/connect-proxy-with-http2-chain (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/connect-proxy-with-http2-chain (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.14.5/connect-proxy-with-grpc-chain (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-external-sni (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-and-overrides (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-remote-gateway (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-local-gateway (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/expose-paths-local-app-paths (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/expose-paths-new-cluster-http2 (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/mesh-gateway (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-external-sni (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-and-overrides (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-remote-gateway (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-local-gateway (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/expose-paths-local-app-paths (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/expose-paths-new-cluster-http2 (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/mesh-gateway (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.14.5/mesh-gateway-using-federation-states (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/mesh-gateway-no-services (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/mesh-gateway-tagged-addresses (0.02s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/mesh-gateway-no-services (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/mesh-gateway-tagged-addresses (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.14.5/mesh-gateway-custom-addresses (0.01s) --- PASS: TestListenersFromSnapshot/envoy-1.14.5/ingress-gateway (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/ingress-gateway-bind-addrs (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/ingress-gateway-bind-addrs (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.14.5/ingress-gateway-no-services (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/ingress-with-chain-external-sni (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/ingress-with-chain-external-sni (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.14.5/ingress-with-chain-and-overrides (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-remote-gateway (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-local-gateway (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.14.5/ingress-splitter-with-resolver-redirect (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.14.5/terminating-gateway (0.01s) --- PASS: TestListenersFromSnapshot/envoy-1.14.5/terminating-gateway-no-services (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/terminating-gateway-custom-and-tagged-addresses (0.02s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/terminating-gateway-custom-and-tagged-addresses (0.01s) --- PASS: TestListenersFromSnapshot/envoy-1.14.5/terminating-gateway-service-subsets (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/ingress-http-multiple-services (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/ingress-http-multiple-services (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.14.5/terminating-gateway-no-api-cert (0.01s) --- PASS: TestListenersFromSnapshot/envoy-1.14.5/ingress-with-tls-listener (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.13.6 (0.26s) + --- PASS: TestListenersFromSnapshot/envoy-1.13.6 (0.21s) --- PASS: TestListenersFromSnapshot/envoy-1.13.6/defaults (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.13.6/listener-bind-address (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.13.6/listener-bind-port (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.13.6/listener-bind-address-port (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.13.6/http-public-listener (0.01s) --- PASS: TestListenersFromSnapshot/envoy-1.13.6/http-upstream (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.13.6/custom-public-listener (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.13.6/custom-public-listener-typed (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.13.6/custom-public-listener-ignores-tls (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.13.6/custom-public-listener (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.13.6/custom-public-listener-typed (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.13.6/custom-public-listener-ignores-tls (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.13.6/custom-upstream (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.13.6/custom-upstream-typed (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.13.6/custom-upstream-typed (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.13.6/custom-upstream-typed-ignored-with-disco-chain (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.13.6/splitter-with-resolver-redirect (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.13.6/splitter-with-resolver-redirect (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.13.6/connect-proxy-with-http-chain (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.13.6/connect-proxy-with-http-chain (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.13.6/connect-proxy-with-http2-chain (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.13.6/connect-proxy-with-grpc-chain (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.13.6/connect-proxy-with-grpc-chain (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-external-sni (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-and-overrides (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-remote-gateway (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-remote-gateway (0.01s) --- PASS: TestListenersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-local-gateway (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.13.6/expose-paths-local-app-paths (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.13.6/expose-paths-new-cluster-http2 (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.13.6/expose-paths-local-app-paths (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.13.6/expose-paths-new-cluster-http2 (0.01s) --- PASS: TestListenersFromSnapshot/envoy-1.13.6/mesh-gateway (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.13.6/mesh-gateway-using-federation-states (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.13.6/mesh-gateway-no-services (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.13.6/mesh-gateway-tagged-addresses (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.13.6/mesh-gateway-tagged-addresses (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.13.6/mesh-gateway-custom-addresses (0.01s) --- PASS: TestListenersFromSnapshot/envoy-1.13.6/ingress-gateway (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.13.6/ingress-gateway-bind-addrs (0.00s) @@ -72333,57 +72346,57 @@ --- PASS: TestListenersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-local-gateway (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.13.6/ingress-splitter-with-resolver-redirect (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.13.6/terminating-gateway (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.13.6/terminating-gateway-no-services (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.13.6/terminating-gateway-no-services (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.13.6/terminating-gateway-custom-and-tagged-addresses (0.01s) --- PASS: TestListenersFromSnapshot/envoy-1.13.6/terminating-gateway-service-subsets (0.01s) --- PASS: TestListenersFromSnapshot/envoy-1.13.6/ingress-http-multiple-services (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.13.6/terminating-gateway-no-api-cert (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.13.6/ingress-with-tls-listener (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.12.7 (0.33s) - --- PASS: TestListenersFromSnapshot/envoy-1.12.7/defaults (0.02s) + --- PASS: TestListenersFromSnapshot/envoy-1.13.6/ingress-with-tls-listener (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.12.7 (0.27s) + --- PASS: TestListenersFromSnapshot/envoy-1.12.7/defaults (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/listener-bind-address (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/listener-bind-port (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/listener-bind-address-port (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/http-public-listener (0.01s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/http-upstream (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.12.7/custom-public-listener (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.12.7/custom-public-listener-typed (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.12.7/custom-public-listener-ignores-tls (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.12.7/custom-upstream (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.12.7/custom-upstream-typed (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.12.7/custom-upstream-typed-ignored-with-disco-chain (0.02s) + --- PASS: TestListenersFromSnapshot/envoy-1.12.7/custom-public-listener (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.12.7/custom-public-listener-typed (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.12.7/custom-public-listener-ignores-tls (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.12.7/custom-upstream (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.12.7/custom-upstream-typed (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.12.7/custom-upstream-typed-ignored-with-disco-chain (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/splitter-with-resolver-redirect (0.01s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain (0.01s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/connect-proxy-with-http-chain (0.01s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/connect-proxy-with-http2-chain (0.01s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/connect-proxy-with-grpc-chain (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-external-sni (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-external-sni (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-and-overrides (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-remote-gateway (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-local-gateway (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.12.7/expose-paths-local-app-paths (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-remote-gateway (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-local-gateway (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.12.7/expose-paths-local-app-paths (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/expose-paths-new-cluster-http2 (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.12.7/mesh-gateway (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.12.7/mesh-gateway (0.02s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/mesh-gateway-using-federation-states (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.12.7/mesh-gateway-no-services (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.12.7/mesh-gateway-tagged-addresses (0.02s) - --- PASS: TestListenersFromSnapshot/envoy-1.12.7/mesh-gateway-custom-addresses (0.03s) - --- PASS: TestListenersFromSnapshot/envoy-1.12.7/ingress-gateway (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.12.7/ingress-gateway-bind-addrs (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.12.7/ingress-gateway-no-services (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.12.7/mesh-gateway-no-services (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.12.7/mesh-gateway-tagged-addresses (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.12.7/mesh-gateway-custom-addresses (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.12.7/ingress-gateway (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.12.7/ingress-gateway-bind-addrs (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.12.7/ingress-gateway-no-services (0.01s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/ingress-with-chain-external-sni (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/ingress-with-chain-and-overrides (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-remote-gateway (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-local-gateway (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.12.7/ingress-splitter-with-resolver-redirect (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-local-gateway (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.12.7/ingress-splitter-with-resolver-redirect (0.01s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/terminating-gateway (0.01s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/terminating-gateway-no-services (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.12.7/terminating-gateway-custom-and-tagged-addresses (0.02s) + --- PASS: TestListenersFromSnapshot/envoy-1.12.7/terminating-gateway-custom-and-tagged-addresses (0.01s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/terminating-gateway-service-subsets (0.01s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/ingress-http-multiple-services (0.01s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/terminating-gateway-no-api-cert (0.01s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/ingress-with-tls-listener (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.11.2 (0.26s) + --- PASS: TestListenersFromSnapshot/envoy-1.11.2 (0.22s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/defaults (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/listener-bind-address (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/listener-bind-port (0.00s) @@ -72391,11 +72404,11 @@ --- PASS: TestListenersFromSnapshot/envoy-1.11.2/http-public-listener (0.01s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/http-upstream (0.01s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/custom-public-listener (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.11.2/custom-public-listener-typed (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.11.2/custom-public-listener-ignores-tls (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.11.2/custom-upstream (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.11.2/custom-upstream-typed (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.11.2/custom-upstream-typed-ignored-with-disco-chain (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.11.2/custom-public-listener-typed (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.11.2/custom-public-listener-ignores-tls (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.11.2/custom-upstream (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.11.2/custom-upstream-typed (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.11.2/custom-upstream-typed-ignored-with-disco-chain (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/splitter-with-resolver-redirect (0.01s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/connect-proxy-with-http-chain (0.01s) @@ -72403,17 +72416,17 @@ --- PASS: TestListenersFromSnapshot/envoy-1.11.2/connect-proxy-with-grpc-chain (0.01s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-external-sni (0.01s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-overrides (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-remote-gateway (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-remote-gateway (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-local-gateway (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.11.2/expose-paths-local-app-paths (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.11.2/expose-paths-new-cluster-http2 (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.11.2/expose-paths-local-app-paths (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.11.2/expose-paths-new-cluster-http2 (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/mesh-gateway (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/mesh-gateway-using-federation-states (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/mesh-gateway-no-services (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/mesh-gateway-tagged-addresses (0.01s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/mesh-gateway-custom-addresses (0.01s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/ingress-gateway (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.11.2/ingress-gateway-bind-addrs (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.11.2/ingress-gateway-bind-addrs (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/ingress-gateway-no-services (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/ingress-with-chain-external-sni (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/ingress-with-chain-and-overrides (0.00s) @@ -72500,8 +72513,8 @@ === RUN TestRoutesFromSnapshot/envoy-1.11.2/ingress-with-grpc-router === RUN TestRoutesFromSnapshot/envoy-1.11.2/ingress-with-chain-and-router === RUN TestRoutesFromSnapshot/envoy-1.11.2/ingress-http-multiple-services ---- PASS: TestRoutesFromSnapshot (0.25s) - --- PASS: TestRoutesFromSnapshot/envoy-1.14.5 (0.06s) +--- PASS: TestRoutesFromSnapshot (0.21s) + --- PASS: TestRoutesFromSnapshot/envoy-1.14.5 (0.05s) --- PASS: TestRoutesFromSnapshot/envoy-1.14.5/defaults-no-chain (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.14.5/connect-proxy-with-chain (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-external-sni (0.00s) @@ -72519,7 +72532,7 @@ --- PASS: TestRoutesFromSnapshot/envoy-1.14.5/ingress-with-grpc-router (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.14.5/ingress-with-chain-and-router (0.01s) --- PASS: TestRoutesFromSnapshot/envoy-1.14.5/ingress-http-multiple-services (0.00s) - --- PASS: TestRoutesFromSnapshot/envoy-1.13.6 (0.06s) + --- PASS: TestRoutesFromSnapshot/envoy-1.13.6 (0.05s) --- PASS: TestRoutesFromSnapshot/envoy-1.13.6/defaults-no-chain (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.13.6/connect-proxy-with-chain (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-external-sni (0.00s) @@ -72528,7 +72541,7 @@ --- PASS: TestRoutesFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-and-splitter (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.13.6/connect-proxy-with-grpc-router (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-and-router (0.01s) - --- PASS: TestRoutesFromSnapshot/envoy-1.13.6/ingress-defaults-no-chain (0.01s) + --- PASS: TestRoutesFromSnapshot/envoy-1.13.6/ingress-defaults-no-chain (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.13.6/ingress-with-chain (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.13.6/ingress-with-chain-external-sni (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.13.6/ingress-with-chain-and-overrides (0.00s) @@ -72537,14 +72550,14 @@ --- PASS: TestRoutesFromSnapshot/envoy-1.13.6/ingress-with-grpc-router (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.13.6/ingress-with-chain-and-router (0.01s) --- PASS: TestRoutesFromSnapshot/envoy-1.13.6/ingress-http-multiple-services (0.00s) - --- PASS: TestRoutesFromSnapshot/envoy-1.12.7 (0.06s) + --- PASS: TestRoutesFromSnapshot/envoy-1.12.7 (0.05s) --- PASS: TestRoutesFromSnapshot/envoy-1.12.7/defaults-no-chain (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.12.7/connect-proxy-with-chain (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-external-sni (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-and-overrides (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.12.7/splitter-with-resolver-redirect (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-and-splitter (0.00s) - --- PASS: TestRoutesFromSnapshot/envoy-1.12.7/connect-proxy-with-grpc-router (0.01s) + --- PASS: TestRoutesFromSnapshot/envoy-1.12.7/connect-proxy-with-grpc-router (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-and-router (0.01s) --- PASS: TestRoutesFromSnapshot/envoy-1.12.7/ingress-defaults-no-chain (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.12.7/ingress-with-chain (0.00s) @@ -72555,7 +72568,7 @@ --- PASS: TestRoutesFromSnapshot/envoy-1.12.7/ingress-with-grpc-router (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.12.7/ingress-with-chain-and-router (0.01s) --- PASS: TestRoutesFromSnapshot/envoy-1.12.7/ingress-http-multiple-services (0.00s) - --- PASS: TestRoutesFromSnapshot/envoy-1.11.2 (0.06s) + --- PASS: TestRoutesFromSnapshot/envoy-1.11.2 (0.05s) --- PASS: TestRoutesFromSnapshot/envoy-1.11.2/defaults-no-chain (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.11.2/connect-proxy-with-chain (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-external-sni (0.00s) @@ -72574,39 +72587,39 @@ --- PASS: TestRoutesFromSnapshot/envoy-1.11.2/ingress-with-chain-and-router (0.01s) --- PASS: TestRoutesFromSnapshot/envoy-1.11.2/ingress-http-multiple-services (0.00s) === RUN TestServer_StreamAggregatedResources_BasicProtocol -2022-09-15T21:41:03.895Z [DEBUG] TestServer_StreamAggregatedResources_BasicProtocol.envoy: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2022-09-15T21:41:03.897Z [DEBUG] TestServer_StreamAggregatedResources_BasicProtocol.envoy: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2022-09-15T21:41:03.923Z [DEBUG] TestServer_StreamAggregatedResources_BasicProtocol.envoy: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2022-09-15T21:41:03.924Z [DEBUG] TestServer_StreamAggregatedResources_BasicProtocol.envoy: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2022-09-15T21:41:03.946Z [DEBUG] TestServer_StreamAggregatedResources_BasicProtocol.envoy: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2022-09-15T21:41:03.947Z [DEBUG] TestServer_StreamAggregatedResources_BasicProtocol.envoy: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul ---- PASS: TestServer_StreamAggregatedResources_BasicProtocol (0.08s) +2021-08-13T15:32:56.743Z [DEBUG] TestServer_StreamAggregatedResources_BasicProtocol.envoy: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:56.745Z [DEBUG] TestServer_StreamAggregatedResources_BasicProtocol.envoy: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:56.770Z [DEBUG] TestServer_StreamAggregatedResources_BasicProtocol.envoy: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:56.770Z [DEBUG] TestServer_StreamAggregatedResources_BasicProtocol.envoy: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:56.785Z [DEBUG] TestServer_StreamAggregatedResources_BasicProtocol.envoy: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:56.785Z [DEBUG] TestServer_StreamAggregatedResources_BasicProtocol.envoy: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +--- PASS: TestServer_StreamAggregatedResources_BasicProtocol (0.06s) === RUN TestServer_StreamAggregatedResources_ACLEnforcement server_test.go:396: DM-skipped --- SKIP: TestServer_StreamAggregatedResources_ACLEnforcement (0.00s) === RUN TestServer_StreamAggregatedResources_ACLTokenDeleted_StreamTerminatedDuringDiscoveryRequest -2022-09-15T21:41:03.972Z [DEBUG] TestServer_StreamAggregatedResources_ACLTokenDeleted_StreamTerminatedDuringDiscoveryRequest.envoy: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2022-09-15T21:41:03.973Z [DEBUG] TestServer_StreamAggregatedResources_ACLTokenDeleted_StreamTerminatedDuringDiscoveryRequest.envoy: Error handling ADS stream: error="rpc error: code = Unauthenticated desc = unauthenticated: ACL not found" ---- PASS: TestServer_StreamAggregatedResources_ACLTokenDeleted_StreamTerminatedDuringDiscoveryRequest (0.02s) +2021-08-13T15:32:56.801Z [DEBUG] TestServer_StreamAggregatedResources_ACLTokenDeleted_StreamTerminatedDuringDiscoveryRequest.envoy: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:56.803Z [DEBUG] TestServer_StreamAggregatedResources_ACLTokenDeleted_StreamTerminatedDuringDiscoveryRequest.envoy: Error handling ADS stream: error="rpc error: code = Unauthenticated desc = unauthenticated: ACL not found" +--- PASS: TestServer_StreamAggregatedResources_ACLTokenDeleted_StreamTerminatedDuringDiscoveryRequest (0.01s) === RUN TestServer_StreamAggregatedResources_ACLTokenDeleted_StreamTerminatedInBackground -2022-09-15T21:41:03.987Z [DEBUG] TestServer_StreamAggregatedResources_ACLTokenDeleted_StreamTerminatedInBackground.envoy: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2022-09-15T21:41:04.090Z [DEBUG] TestServer_StreamAggregatedResources_ACLTokenDeleted_StreamTerminatedInBackground.envoy: Error handling ADS stream: error="rpc error: code = Unauthenticated desc = unauthenticated: ACL not found" +2021-08-13T15:32:56.816Z [DEBUG] TestServer_StreamAggregatedResources_ACLTokenDeleted_StreamTerminatedInBackground.envoy: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2021-08-13T15:32:56.919Z [DEBUG] TestServer_StreamAggregatedResources_ACLTokenDeleted_StreamTerminatedInBackground.envoy: Error handling ADS stream: error="rpc error: code = Unauthenticated desc = unauthenticated: ACL not found" --- PASS: TestServer_StreamAggregatedResources_ACLTokenDeleted_StreamTerminatedInBackground (0.12s) === RUN TestServer_Check === RUN TestServer_Check/auth_allowed -2022-09-15T21:41:04.090Z [DEBUG] TestServer_Check/auth_allowed.envoy: Connect AuthZ ALLOWED: source=spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/web destination=spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/db reason="default allow" +2021-08-13T15:32:56.920Z [DEBUG] TestServer_Check/auth_allowed.envoy: Connect AuthZ ALLOWED: source=spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/web destination=spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/db reason="default allow" === RUN TestServer_Check/auth_denied -2022-09-15T21:41:04.091Z [DEBUG] TestServer_Check/auth_denied.envoy: Connect AuthZ DENIED: source=spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/web destination=spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/db reason="default deny" +2021-08-13T15:32:56.920Z [DEBUG] TestServer_Check/auth_denied.envoy: Connect AuthZ DENIED: source=spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/web destination=spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/db reason="default deny" === RUN TestServer_Check/no_source === RUN TestServer_Check/no_dest === RUN TestServer_Check/dest_invalid_format -2022-09-15T21:41:04.091Z [DEBUG] TestServer_Check/dest_invalid_format.envoy: Connect AuthZ DENIED: bad destination URI: source=spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/web destination=not-a-spiffe-id +2021-08-13T15:32:56.920Z [DEBUG] TestServer_Check/dest_invalid_format.envoy: Connect AuthZ DENIED: bad destination URI: source=spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/web destination=not-a-spiffe-id === RUN TestServer_Check/dest_not_a_service_URI -2022-09-15T21:41:04.091Z [DEBUG] TestServer_Check/dest_not_a_service_URI.envoy: Connect AuthZ DENIED: bad destination service ID: source=spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/web destination=spiffe://trust-domain.consul +2021-08-13T15:32:56.920Z [DEBUG] TestServer_Check/dest_not_a_service_URI.envoy: Connect AuthZ DENIED: bad destination service ID: source=spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/web destination=spiffe://trust-domain.consul === RUN TestServer_Check/ACL_not_got_permission_for_authz_call -2022-09-15T21:41:04.091Z [DEBUG] TestServer_Check/ACL_not_got_permission_for_authz_call.envoy: Connect AuthZ failed ACL check: error="Permission denied" source=spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/web dest=spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/db +2021-08-13T15:32:56.920Z [DEBUG] TestServer_Check/ACL_not_got_permission_for_authz_call.envoy: Connect AuthZ failed ACL check: error="Permission denied" source=spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/web dest=spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/db === RUN TestServer_Check/Random_error_running_authz -2022-09-15T21:41:04.092Z [DEBUG] TestServer_Check/Random_error_running_authz.envoy: Connect AuthZ failed: error="gremlin attack" source=spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/web destination=spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/db +2021-08-13T15:32:56.921Z [DEBUG] TestServer_Check/Random_error_running_authz.envoy: Connect AuthZ failed: error="gremlin attack" source=spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/web destination=spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/db --- PASS: TestServer_Check (0.00s) --- PASS: TestServer_Check/auth_allowed (0.00s) --- PASS: TestServer_Check/auth_denied (0.00s) @@ -72619,7 +72632,7 @@ === RUN TestServer_StreamAggregatedResources_IngressEmptyResponse --- PASS: TestServer_StreamAggregatedResources_IngressEmptyResponse (0.01s) PASS -ok github.com/hashicorp/consul/agent/xds 3.598s +ok github.com/hashicorp/consul/agent/xds 2.749s ? github.com/hashicorp/consul/agent/xds/proxysupport [no test files] ? github.com/hashicorp/consul/command [no test files] ? github.com/hashicorp/consul/command/acl [no test files] @@ -72629,93 +72642,94 @@ === PAUSE TestAgentTokensCommand === CONT TestAgentTokensCommand_noTabs === CONT TestAgentTokensCommand ---- PASS: TestAgentTokensCommand_noTabs (0.00s) [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestAgentTokensCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:41:06.190Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:06.227Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:06.231 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:915ab2cb-d463-a566-35df-91a51ac63e0c Address:127.0.0.1:27363}]" -41:06.232 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-915ab2cb-d463-a566-35df-91a51ac63e0c.dc1 127.0.0.1 -41:06.232 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:27363 [Follower]" leader= -41:06.233 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-915ab2cb-d463-a566-35df-91a51ac63e0c 127.0.0.1 -2022-09-15T21:41:06.233Z [INFO] agent.router: Initializing LAN area manager -41:06.234 [INFO] TestAgent.server: Adding LAN server: server="Node-915ab2cb-d463-a566-35df-91a51ac63e0c (Addr: tcp/127.0.0.1:27363) (DC: dc1)" -41:06.234 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-915ab2cb-d463-a566-35df-91a51ac63e0c.dc1 area=wan -41:06.234 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27358 network=tcp -41:06.235 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27358 network=udp -41:06.236 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:27359 network=tcp -41:06.236 [INFO] TestAgent: started state syncer -41:06.236 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:27364 network=tcp -41:06.285 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:06.285 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:27363 [Candidate]" term=2 -41:06.285 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -41:06.285 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:06.285 [DEBUG] TestAgent.server.raft: vote granted: from=915ab2cb-d463-a566-35df-91a51ac63e0c term=2 tally=1 -41:06.285 [INFO] TestAgent.server.raft: election won: tally=1 -41:06.285 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:27363 [Leader]" -41:06.285 [INFO] TestAgent.server: cluster leadership acquired -41:06.285 [INFO] TestAgent.server: initializing acls -41:06.286 [INFO] TestAgent.server: New leader elected: payload=Node-915ab2cb-d463-a566-35df-91a51ac63e0c -41:06.286 [INFO] TestAgent.server: Created ACL 'global-management' policy -41:06.286 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -41:06.287 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -41:06.287 [INFO] TestAgent.server: Created ACL anonymous token from configuration -41:06.287 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -41:06.287 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -41:06.288 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-915ab2cb-d463-a566-35df-91a51ac63e0c -41:06.288 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-915ab2cb-d463-a566-35df-91a51ac63e0c.dc1 -41:06.289 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:06.290 [INFO] TestAgent.server: Updating LAN server: server="Node-915ab2cb-d463-a566-35df-91a51ac63e0c (Addr: tcp/127.0.0.1:27363) (DC: dc1)" -41:06.295 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-915ab2cb-d463-a566-35df-91a51ac63e0c.dc1 area=wan -41:06.323 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:06.323 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:06.323 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:06.323 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:06.323 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:06.323 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-915ab2cb-d463-a566-35df-91a51ac63e0c -41:06.323 [INFO] TestAgent.server: member joined, marking health alive: member=Node-915ab2cb-d463-a566-35df-91a51ac63e0c -41:06.324 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-915ab2cb-d463-a566-35df-91a51ac63e0c -41:06.324 [INFO] TestAgent.server: federation state anti-entropy synced -41:06.366 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:54990 latency=2.46578ms -41:06.367 [INFO] TestAgent: Updated agent's ACL token: token=default -41:06.367 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/token/default from=127.0.0.1:54992 latency=106.66µs -41:06.368 [INFO] TestAgent: Updated agent's ACL token: token=agent -41:06.368 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/token/agent from=127.0.0.1:54994 latency=110.36µs -41:06.369 [INFO] TestAgent: Updated agent's ACL token: token=agent_master -41:06.369 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/token/agent_master from=127.0.0.1:54996 latency=82.78µs -41:06.370 [INFO] TestAgent: Updated agent's ACL token: token=replication -41:06.370 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/token/replication from=127.0.0.1:54998 latency=63.1µs -41:06.371 [INFO] TestAgent: Requesting shutdown -41:06.371 [INFO] TestAgent.server: shutting down server -41:06.371 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -41:06.371 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -41:06.371 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:06.371 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:06.371 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:06.371 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:06.371 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:06.371 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:06.371 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:06.371 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:06.371 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -41:06.371 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -41:06.371 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:06.371 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:06.371 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -41:06.371 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2022-09-15T21:41:06.371Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:06.371Z [INFO] agent.router.manager: shutting down -41:06.371 [INFO] TestAgent: consul server down -41:06.371 [INFO] TestAgent: shutdown complete -41:06.371 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27358 network=tcp -41:06.371 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27358 network=udp -41:06.372 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:27359 network=tcp -41:06.872 [INFO] TestAgent: Waiting for endpoints to shut down -41:06.872 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentTokensCommand (0.76s) +2021-08-13T15:32:55.222Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:32:55.240Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +32:55.245 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a420caef-9485-b1dd-0c5b-41327abd69ab Address:127.0.0.1:13069}]" +32:55.246 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a420caef-9485-b1dd-0c5b-41327abd69ab.dc1 127.0.0.1 +32:55.247 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13069 [Follower]" leader= +32:55.248 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a420caef-9485-b1dd-0c5b-41327abd69ab 127.0.0.1 +2021-08-13T15:32:55.249Z [INFO] agent.router: Initializing LAN area manager +32:55.249 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a420caef-9485-b1dd-0c5b-41327abd69ab.dc1 area=wan +32:55.250 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13064 network=tcp +32:55.250 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13064 network=udp +32:55.250 [INFO] TestAgent.server: Adding LAN server: server="Node-a420caef-9485-b1dd-0c5b-41327abd69ab (Addr: tcp/127.0.0.1:13069) (DC: dc1)" +32:55.251 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13065 network=tcp +32:55.251 [INFO] TestAgent: started state syncer +32:55.251 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13070 network=tcp +32:55.291 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +32:55.291 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13069 [Candidate]" term=2 +32:55.292 [DEBUG] TestAgent.server.raft: votes: needed=1 +32:55.292 [DEBUG] TestAgent.server.raft: vote granted: from=a420caef-9485-b1dd-0c5b-41327abd69ab term=2 tally=1 +32:55.292 [INFO] TestAgent.server.raft: election won: tally=1 +32:55.292 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13069 [Leader]" +32:55.292 [INFO] TestAgent.server: cluster leadership acquired +32:55.292 [INFO] TestAgent.server: New leader elected: payload=Node-a420caef-9485-b1dd-0c5b-41327abd69ab +32:55.292 [INFO] TestAgent.server: initializing acls +32:55.293 [INFO] TestAgent.server: Created ACL 'global-management' policy +32:55.293 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +32:55.294 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +32:55.294 [INFO] TestAgent.server: Created ACL anonymous token from configuration +32:55.294 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +32:55.294 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +32:55.294 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-a420caef-9485-b1dd-0c5b-41327abd69ab +32:55.294 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-a420caef-9485-b1dd-0c5b-41327abd69ab.dc1 +32:55.295 [INFO] TestAgent.server: Updating LAN server: server="Node-a420caef-9485-b1dd-0c5b-41327abd69ab (Addr: tcp/127.0.0.1:13069) (DC: dc1)" +32:55.295 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-a420caef-9485-b1dd-0c5b-41327abd69ab.dc1 area=wan +32:55.296 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +32:55.320 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +32:55.320 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +32:55.320 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +32:55.320 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +32:55.320 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +32:55.320 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a420caef-9485-b1dd-0c5b-41327abd69ab +32:55.320 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a420caef-9485-b1dd-0c5b-41327abd69ab +32:55.321 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a420caef-9485-b1dd-0c5b-41327abd69ab +32:55.321 [INFO] TestAgent.server: federation state anti-entropy synced +32:55.380 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +32:55.380 [INFO] TestAgent: Synced node info +32:55.380 [DEBUG] TestAgent: Node info in sync +32:55.557 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:34398 latency=1.950044ms +32:55.558 [INFO] TestAgent: Updated agent's ACL token: token=default +32:55.558 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/token/default from=127.0.0.1:34400 latency=87.12µs +32:55.559 [INFO] TestAgent: Updated agent's ACL token: token=agent +32:55.559 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/token/agent from=127.0.0.1:34402 latency=77.221µs +32:55.560 [INFO] TestAgent: Updated agent's ACL token: token=agent_master +32:55.560 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/token/agent_master from=127.0.0.1:34404 latency=71.7µs +32:55.560 [INFO] TestAgent: Updated agent's ACL token: token=replication +32:55.560 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/token/replication from=127.0.0.1:34406 latency=40.4µs +32:55.561 [INFO] TestAgent: Requesting shutdown +32:55.561 [INFO] TestAgent.server: shutting down server +32:55.561 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +32:55.561 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +32:55.561 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +32:55.561 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +32:55.561 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +32:55.561 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +32:55.561 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +32:55.561 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +32:55.561 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +32:55.561 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +32:55.561 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +32:55.561 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +32:55.561 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +32:55.561 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +32:55.561 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:32:55.561Z [INFO] agent.router.manager: shutting down +2021-08-13T15:32:55.561Z [INFO] agent.router.manager: shutting down +32:55.561 [INFO] TestAgent: consul server down +32:55.561 [INFO] TestAgent: shutdown complete +32:55.561 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13064 network=tcp +32:55.561 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13064 network=udp +32:55.561 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13065 network=tcp +32:56.062 [INFO] TestAgent: Waiting for endpoints to shut down +32:56.062 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentTokensCommand (0.93s) PASS -ok github.com/hashicorp/consul/command/acl/agenttokens 0.850s +ok github.com/hashicorp/consul/command/acl/agenttokens 1.000s ? github.com/hashicorp/consul/command/acl/authmethod [no test files] === RUN TestAuthMethodCreateCommand_noTabs === PAUSE TestAuthMethodCreateCommand_noTabs @@ -72728,456 +72742,470 @@ === RUN TestAuthMethodCreateCommand_config === PAUSE TestAuthMethodCreateCommand_config === CONT TestAuthMethodCreateCommand_noTabs ---- PASS: TestAuthMethodCreateCommand_noTabs (0.00s) -=== CONT TestAuthMethodCreateCommand_config +=== CONT TestAuthMethodCreateCommand_k8s [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +=== CONT TestAuthMethodCreateCommand_config === CONT TestAuthMethodCreateCommand_JSON -=== CONT TestAuthMethodCreateCommand_k8s === CONT TestAuthMethodCreateCommand +--- PASS: TestAuthMethodCreateCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -=== RUN TestAuthMethodCreateCommand_config/config_file -=== RUN TestAuthMethodCreateCommand_config/config_std-in -=== RUN TestAuthMethodCreateCommand_config/config_string -=== RUN TestAuthMethodCreateCommand_k8s/k8s_host_required -=== RUN TestAuthMethodCreateCommand_k8s/k8s_ca_cert_required -=== RUN TestAuthMethodCreateCommand_k8s/k8s_jwt_required -=== RUN TestAuthMethodCreateCommand_k8s/create_k8s -=== RUN TestAuthMethodCreateCommand_k8s/create_k8s_with_cert_file -=== RUN TestAuthMethodCreateCommand_JSON/type_required -=== RUN TestAuthMethodCreateCommand_JSON/create_testing -=== RUN TestAuthMethodCreateCommand_JSON/create_testing_with_max_token_ttl -=== RUN TestAuthMethodCreateCommand_JSON/create_testing_with_token_type_global === RUN TestAuthMethodCreateCommand/type_required === RUN TestAuthMethodCreateCommand/name_required === RUN TestAuthMethodCreateCommand/invalid_type === RUN TestAuthMethodCreateCommand/create_testing === RUN TestAuthMethodCreateCommand/create_testing_with_max_token_ttl === RUN TestAuthMethodCreateCommand/create_testing_with_token_type_global -2022-09-15T21:41:07.564Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:07.661Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:07.669 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:63f56690-0a0f-88bf-8d7f-f6ac4a63958d Address:127.0.0.1:25321}]" -41:07.674 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25321 [Follower]" leader= -41:07.683 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-63f56690-0a0f-88bf-8d7f-f6ac4a63958d.dc1 127.0.0.1 -41:07.684 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-63f56690-0a0f-88bf-8d7f-f6ac4a63958d 127.0.0.1 -2022-09-15T21:41:07.684Z [INFO] agent.router: Initializing LAN area manager -41:07.684 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25316 network=udp -41:07.685 [INFO] TestAgent.server: Adding LAN server: server="Node-63f56690-0a0f-88bf-8d7f-f6ac4a63958d (Addr: tcp/127.0.0.1:25321) (DC: dc1)" -41:07.685 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-63f56690-0a0f-88bf-8d7f-f6ac4a63958d.dc1 area=wan -41:07.685 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25316 network=tcp -41:07.686 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25317 network=tcp -41:07.686 [INFO] TestAgent: started state syncer -41:07.686 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25322 network=tcp -41:07.738 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:07.738 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25321 [Candidate]" term=2 -41:07.738 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:07.738 [DEBUG] TestAgent.server.raft: vote granted: from=63f56690-0a0f-88bf-8d7f-f6ac4a63958d term=2 tally=1 -41:07.738 [INFO] TestAgent.server.raft: election won: tally=1 -41:07.738 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25321 [Leader]" -41:07.738 [INFO] TestAgent.server: cluster leadership acquired -41:07.739 [INFO] TestAgent.server: initializing acls -41:07.739 [INFO] TestAgent.server: Created ACL 'global-management' policy -41:07.739 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -41:07.740 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -41:07.741 [INFO] TestAgent.server: Created ACL anonymous token from configuration -41:07.741 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -41:07.741 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -41:07.741 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-63f56690-0a0f-88bf-8d7f-f6ac4a63958d -41:07.742 [INFO] TestAgent.server: New leader elected: payload=Node-63f56690-0a0f-88bf-8d7f-f6ac4a63958d -41:07.742 [INFO] TestAgent.server: Updating LAN server: server="Node-63f56690-0a0f-88bf-8d7f-f6ac4a63958d (Addr: tcp/127.0.0.1:25321) (DC: dc1)" -41:07.755 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-63f56690-0a0f-88bf-8d7f-f6ac4a63958d.dc1 -41:07.758 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-63f56690-0a0f-88bf-8d7f-f6ac4a63958d.dc1 area=wan -41:07.760 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:07.763 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:07.763 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:07.763 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:07.763 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:07.763 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:07.763 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-63f56690-0a0f-88bf-8d7f-f6ac4a63958d -41:07.763 [INFO] TestAgent.server: member joined, marking health alive: member=Node-63f56690-0a0f-88bf-8d7f-f6ac4a63958d -41:07.766 [INFO] TestAgent.server: federation state anti-entropy synced -41:07.767 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-63f56690-0a0f-88bf-8d7f-f6ac4a63958d -41:07.774 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:34790 latency=1.64098ms -41:07.777 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-5abaedcb-6ec5-963b-9d69-6a6a2e4ac22b from=127.0.0.1:34792 latency=474.44µs -41:07.781 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:34794 latency=1.04778ms -41:07.782 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-c9fae17f-b94b-abbf-b007-bc763dbef72e from=127.0.0.1:34792 latency=385.16µs -41:07.787 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:34796 latency=2.34828ms -41:07.789 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-9899f813-d295-50ca-ba8c-4fc2cc9552de from=127.0.0.1:34792 latency=347.16µs -41:07.789 [INFO] TestAgent: Requesting shutdown -41:07.789 [INFO] TestAgent.server: shutting down server -41:07.789 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -41:07.789 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:07.789 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:07.789 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:07.789 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:07.789 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -41:07.790 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:07.790 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:07.790 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:07.790 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:07.790 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -41:07.790 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:07.790 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:07.790 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:07.790 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -41:07.790 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:07.790Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:07.790Z [INFO] agent.router.manager: shutting down -41:07.790 [INFO] TestAgent: consul server down -41:07.790 [INFO] TestAgent: shutdown complete -41:07.790 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25316 network=tcp -41:07.790 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25316 network=udp -41:07.790 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25317 network=tcp -41:08.291 [INFO] TestAgent: Waiting for endpoints to shut down -41:08.291 [INFO] TestAgent: Endpoints down ---- PASS: TestAuthMethodCreateCommand_config (0.84s) - --- PASS: TestAuthMethodCreateCommand_config/config_file (0.01s) - --- PASS: TestAuthMethodCreateCommand_config/config_std-in (0.01s) - --- PASS: TestAuthMethodCreateCommand_config/config_string (0.01s) -2022-09-15T21:41:07.578Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:07.665Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:07.672 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9762af2f-286a-2d1b-c9ad-b9da123701c7 Address:127.0.0.1:25335}]" -41:07.673 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9762af2f-286a-2d1b-c9ad-b9da123701c7.dc1 127.0.0.1 -41:07.674 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25335 [Follower]" leader= -41:07.686 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9762af2f-286a-2d1b-c9ad-b9da123701c7 127.0.0.1 -2022-09-15T21:41:07.686Z [INFO] agent.router: Initializing LAN area manager -41:07.687 [INFO] TestAgent.server: Adding LAN server: server="Node-9762af2f-286a-2d1b-c9ad-b9da123701c7 (Addr: tcp/127.0.0.1:25335) (DC: dc1)" -41:07.687 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9762af2f-286a-2d1b-c9ad-b9da123701c7.dc1 area=wan -41:07.687 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25330 network=tcp -41:07.687 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25330 network=udp -41:07.688 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25331 network=tcp -41:07.688 [INFO] TestAgent: started state syncer -41:07.688 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25336 network=tcp -41:07.710 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:07.710 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25335 [Candidate]" term=2 -41:07.710 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:07.710 [DEBUG] TestAgent.server.raft: vote granted: from=9762af2f-286a-2d1b-c9ad-b9da123701c7 term=2 tally=1 -41:07.710 [INFO] TestAgent.server.raft: election won: tally=1 -41:07.710 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25335 [Leader]" -41:07.710 [INFO] TestAgent.server: cluster leadership acquired -41:07.710 [INFO] TestAgent.server: initializing acls -41:07.711 [INFO] TestAgent.server: Created ACL 'global-management' policy -41:07.711 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -41:07.711 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -41:07.712 [INFO] TestAgent.server: Created ACL anonymous token from configuration -41:07.712 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -41:07.712 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -41:07.712 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-9762af2f-286a-2d1b-c9ad-b9da123701c7 -41:07.712 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-9762af2f-286a-2d1b-c9ad-b9da123701c7.dc1 -41:07.713 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:07.713 [INFO] TestAgent.server: New leader elected: payload=Node-9762af2f-286a-2d1b-c9ad-b9da123701c7 -41:07.713 [INFO] TestAgent.server: Updating LAN server: server="Node-9762af2f-286a-2d1b-c9ad-b9da123701c7 (Addr: tcp/127.0.0.1:25335) (DC: dc1)" -41:07.713 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-9762af2f-286a-2d1b-c9ad-b9da123701c7.dc1 area=wan -41:07.721 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:07.721 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:07.721 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:07.721 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:07.721 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:07.721 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9762af2f-286a-2d1b-c9ad-b9da123701c7 -41:07.721 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9762af2f-286a-2d1b-c9ad-b9da123701c7 -41:07.726 [INFO] TestAgent.server: federation state anti-entropy synced -41:07.726 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9762af2f-286a-2d1b-c9ad-b9da123701c7 -41:07.891 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:57092 latency=3.6046ms -41:07.894 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-65e36ea6-18bc-d5c2-458b-5bc1cf940252 from=127.0.0.1:57094 latency=392.44µs -41:07.915 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:57096 latency=13.6102ms -41:07.917 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-88b0b14f-b447-06ee-de68-295b3b5dcb64 from=127.0.0.1:57094 latency=597.14µs -41:07.919 [INFO] TestAgent: Requesting shutdown -41:07.919 [INFO] TestAgent.server: shutting down server -41:07.919 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:07.919 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:07.919 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:07.919 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -41:07.919 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -41:07.919 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:07.919 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:07.919 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:07.919 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:07.919 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:07.919 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:07.919 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:07.919 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:07.919 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -41:07.919 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -41:07.920 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:07.920Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:07.920Z [INFO] agent.router.manager: shutting down -41:07.920 [INFO] TestAgent: consul server down -41:07.920 [INFO] TestAgent: shutdown complete -41:07.920 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25330 network=tcp -41:07.920 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25330 network=udp -41:07.920 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25331 network=tcp -41:08.421 [INFO] TestAgent: Waiting for endpoints to shut down -41:08.421 [INFO] TestAgent: Endpoints down ---- PASS: TestAuthMethodCreateCommand_k8s (0.97s) +=== RUN TestAuthMethodCreateCommand_JSON/type_required +=== RUN TestAuthMethodCreateCommand_JSON/create_testing +=== RUN TestAuthMethodCreateCommand_JSON/create_testing_with_max_token_ttl +=== RUN TestAuthMethodCreateCommand_JSON/create_testing_with_token_type_global +=== RUN TestAuthMethodCreateCommand_k8s/k8s_host_required +=== RUN TestAuthMethodCreateCommand_k8s/k8s_ca_cert_required +=== RUN TestAuthMethodCreateCommand_k8s/k8s_jwt_required +=== RUN TestAuthMethodCreateCommand_k8s/create_k8s +=== RUN TestAuthMethodCreateCommand_k8s/create_k8s_with_cert_file +=== RUN TestAuthMethodCreateCommand_config/config_file +=== RUN TestAuthMethodCreateCommand_config/config_std-in +=== RUN TestAuthMethodCreateCommand_config/config_string +2021-08-13T15:33:06.168Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:06.192Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:06.200 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ba19dc2c-4c22-1a7c-9828-e85c238a0be5 Address:127.0.0.1:14111}]" +33:06.208 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14111 [Follower]" leader= +33:06.212 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ba19dc2c-4c22-1a7c-9828-e85c238a0be5.dc1 127.0.0.1 +33:06.230 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ba19dc2c-4c22-1a7c-9828-e85c238a0be5 127.0.0.1 +2021-08-13T15:33:06.230Z [INFO] agent.router: Initializing LAN area manager +33:06.230 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14106 network=udp +33:06.230 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ba19dc2c-4c22-1a7c-9828-e85c238a0be5.dc1 area=wan +33:06.230 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14106 network=tcp +33:06.231 [INFO] TestAgent.server: Adding LAN server: server="Node-ba19dc2c-4c22-1a7c-9828-e85c238a0be5 (Addr: tcp/127.0.0.1:14111) (DC: dc1)" +33:06.232 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14107 network=tcp +33:06.232 [INFO] TestAgent: started state syncer +33:06.232 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14112 network=tcp +33:06.248 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:06.248 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14111 [Candidate]" term=2 +33:06.248 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:06.248 [DEBUG] TestAgent.server.raft: vote granted: from=ba19dc2c-4c22-1a7c-9828-e85c238a0be5 term=2 tally=1 +33:06.248 [INFO] TestAgent.server.raft: election won: tally=1 +33:06.248 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14111 [Leader]" +33:06.249 [INFO] TestAgent.server: cluster leadership acquired +33:06.249 [INFO] TestAgent.server: New leader elected: payload=Node-ba19dc2c-4c22-1a7c-9828-e85c238a0be5 +33:06.249 [INFO] TestAgent.server: initializing acls +33:06.249 [INFO] TestAgent.server: Created ACL 'global-management' policy +33:06.249 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +33:06.250 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +33:06.250 [INFO] TestAgent.server: Created ACL anonymous token from configuration +33:06.250 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +33:06.250 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +33:06.250 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-ba19dc2c-4c22-1a7c-9828-e85c238a0be5 +33:06.250 [INFO] TestAgent.server: Updating LAN server: server="Node-ba19dc2c-4c22-1a7c-9828-e85c238a0be5 (Addr: tcp/127.0.0.1:14111) (DC: dc1)" +33:06.250 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-ba19dc2c-4c22-1a7c-9828-e85c238a0be5.dc1 +33:06.251 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-ba19dc2c-4c22-1a7c-9828-e85c238a0be5.dc1 area=wan +33:06.252 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:06.267 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:06.267 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:06.267 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:06.267 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:06.267 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:06.267 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ba19dc2c-4c22-1a7c-9828-e85c238a0be5 +33:06.267 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ba19dc2c-4c22-1a7c-9828-e85c238a0be5 +33:06.267 [INFO] TestAgent.server: federation state anti-entropy synced +33:06.271 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ba19dc2c-4c22-1a7c-9828-e85c238a0be5 +33:06.368 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/auth-method from=127.0.0.1:44140 error="Invalid Auth Method: Type should be one of: [jwt kubernetes testing]" +33:06.368 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:44140 latency=620.201µs +33:06.371 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:44142 latency=1.050022ms +33:06.373 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-81773f91-38fe-c64f-3731-cee1d187650e from=127.0.0.1:44144 latency=437.261µs +33:06.376 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:44146 latency=836.742µs +33:06.377 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-86582fa3-366d-bd7d-ee0f-8ca354c8da94 from=127.0.0.1:44144 latency=264.621µs +33:06.528 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:06.529 [INFO] TestAgent: Synced node info +33:06.529 [DEBUG] TestAgent: Node info in sync +33:06.532 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:44148 latency=863.742µs +33:06.533 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-8cab2d7e-dce1-b38d-7b4e-db3d0b7bbce6 from=127.0.0.1:44144 latency=344.361µs +33:06.534 [INFO] TestAgent: Requesting shutdown +33:06.534 [INFO] TestAgent.server: shutting down server +33:06.534 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +33:06.534 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +33:06.534 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:06.534 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:06.534 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:06.534 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:06.534 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:06.534 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:06.534 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:06.535 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +33:06.535 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +33:06.535 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:06.535 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:06.535 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:06.535 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:06.535Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:06.535Z [INFO] agent.router.manager: shutting down +33:06.535 [INFO] TestAgent: consul server down +33:06.535 [INFO] TestAgent: shutdown complete +33:06.535 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14106 network=tcp +33:06.535 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14106 network=udp +33:06.535 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14107 network=tcp +33:07.035 [INFO] TestAgent: Waiting for endpoints to shut down +33:07.035 [INFO] TestAgent: Endpoints down +--- PASS: TestAuthMethodCreateCommand (0.94s) + --- PASS: TestAuthMethodCreateCommand/type_required (0.00s) + --- PASS: TestAuthMethodCreateCommand/name_required (0.00s) + --- PASS: TestAuthMethodCreateCommand/invalid_type (0.00s) + --- PASS: TestAuthMethodCreateCommand/create_testing (0.01s) + --- PASS: TestAuthMethodCreateCommand/create_testing_with_max_token_ttl (0.00s) + --- PASS: TestAuthMethodCreateCommand/create_testing_with_token_type_global (0.16s) +2021-08-13T15:33:06.164Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:06.193Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:06.204 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e7059fb7-e115-5e88-3004-489843ba7605 Address:127.0.0.1:14104}]" +33:06.204 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14104 [Follower]" leader= +33:06.231 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e7059fb7-e115-5e88-3004-489843ba7605.dc1 127.0.0.1 +33:06.233 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e7059fb7-e115-5e88-3004-489843ba7605 127.0.0.1 +2021-08-13T15:33:06.233Z [INFO] agent.router: Initializing LAN area manager +33:06.234 [INFO] TestAgent.server: Adding LAN server: server="Node-e7059fb7-e115-5e88-3004-489843ba7605 (Addr: tcp/127.0.0.1:14104) (DC: dc1)" +33:06.234 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e7059fb7-e115-5e88-3004-489843ba7605.dc1 area=wan +33:06.234 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14099 network=tcp +33:06.234 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14099 network=udp +33:06.235 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14100 network=tcp +33:06.235 [INFO] TestAgent: started state syncer +33:06.235 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14105 network=tcp +33:06.246 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:06.246 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14104 [Candidate]" term=2 +33:06.246 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:06.246 [DEBUG] TestAgent.server.raft: vote granted: from=e7059fb7-e115-5e88-3004-489843ba7605 term=2 tally=1 +33:06.246 [INFO] TestAgent.server.raft: election won: tally=1 +33:06.246 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14104 [Leader]" +33:06.246 [INFO] TestAgent.server: cluster leadership acquired +33:06.246 [INFO] TestAgent.server: initializing acls +33:06.246 [INFO] TestAgent.server: Created ACL 'global-management' policy +33:06.246 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +33:06.247 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +33:06.247 [INFO] TestAgent.server: Created ACL anonymous token from configuration +33:06.247 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +33:06.247 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +33:06.248 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-e7059fb7-e115-5e88-3004-489843ba7605 +33:06.248 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-e7059fb7-e115-5e88-3004-489843ba7605.dc1 +33:06.248 [INFO] TestAgent.server: New leader elected: payload=Node-e7059fb7-e115-5e88-3004-489843ba7605 +33:06.248 [INFO] TestAgent.server: Updating LAN server: server="Node-e7059fb7-e115-5e88-3004-489843ba7605 (Addr: tcp/127.0.0.1:14104) (DC: dc1)" +33:06.248 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-e7059fb7-e115-5e88-3004-489843ba7605.dc1 area=wan +33:06.249 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:06.264 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:06.264 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:06.264 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:06.264 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:06.264 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:06.264 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e7059fb7-e115-5e88-3004-489843ba7605 +33:06.264 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e7059fb7-e115-5e88-3004-489843ba7605 +33:06.266 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e7059fb7-e115-5e88-3004-489843ba7605 +33:06.268 [INFO] TestAgent.server: federation state anti-entropy synced +33:06.398 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:06.398 [INFO] TestAgent: Synced node info +33:06.532 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:37904 latency=771.962µs +33:06.534 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-4623041c-93d9-982a-cb40-b09dc0a6b95a from=127.0.0.1:37906 latency=282.641µs +33:06.537 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:37908 latency=876.042µs +33:06.538 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-c53a24b7-e018-5ab7-61b2-c0bbe6e30ce1 from=127.0.0.1:37906 latency=164.701µs +33:06.541 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:37910 latency=771.321µs +33:06.542 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-4464faad-026b-accd-2299-78b31fa46a23 from=127.0.0.1:37906 latency=279.2µs +33:06.542 [INFO] TestAgent: Requesting shutdown +33:06.542 [INFO] TestAgent.server: shutting down server +33:06.542 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +33:06.542 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +33:06.542 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:06.542 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:06.542 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:06.542 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:06.542 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:06.542 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:06.542 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +33:06.542 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:06.542 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +33:06.542 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:06.542 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:06.542 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:06.543 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:06.543Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:06.543Z [INFO] agent.router.manager: shutting down +33:06.543 [INFO] TestAgent: consul server down +33:06.543 [INFO] TestAgent: shutdown complete +33:06.543 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14099 network=tcp +33:06.543 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14099 network=udp +33:06.543 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14100 network=tcp +33:07.043 [INFO] TestAgent: Waiting for endpoints to shut down +33:07.043 [INFO] TestAgent: Endpoints down +--- PASS: TestAuthMethodCreateCommand_JSON (0.95s) + --- PASS: TestAuthMethodCreateCommand_JSON/type_required (0.00s) + --- PASS: TestAuthMethodCreateCommand_JSON/create_testing (0.01s) + --- PASS: TestAuthMethodCreateCommand_JSON/create_testing_with_max_token_ttl (0.00s) + --- PASS: TestAuthMethodCreateCommand_JSON/create_testing_with_token_type_global (0.00s) +2021-08-13T15:33:06.163Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:06.187Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:06.192 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:09c1fd5b-e8f5-4359-4c82-4680c784115d Address:127.0.0.1:14090}]" +33:06.193 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14090 [Follower]" leader= +33:06.193 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-09c1fd5b-e8f5-4359-4c82-4680c784115d.dc1 127.0.0.1 +33:06.208 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-09c1fd5b-e8f5-4359-4c82-4680c784115d 127.0.0.1 +2021-08-13T15:33:06.208Z [INFO] agent.router: Initializing LAN area manager +33:06.209 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14085 network=udp +33:06.228 [INFO] TestAgent.server: Adding LAN server: server="Node-09c1fd5b-e8f5-4359-4c82-4680c784115d (Addr: tcp/127.0.0.1:14090) (DC: dc1)" +33:06.228 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-09c1fd5b-e8f5-4359-4c82-4680c784115d.dc1 area=wan +33:06.229 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14085 network=tcp +33:06.230 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14086 network=tcp +33:06.230 [INFO] TestAgent: started state syncer +33:06.230 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14091 network=tcp +33:06.234 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:06.234 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14090 [Candidate]" term=2 +33:06.235 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:06.235 [DEBUG] TestAgent.server.raft: vote granted: from=09c1fd5b-e8f5-4359-4c82-4680c784115d term=2 tally=1 +33:06.235 [INFO] TestAgent.server.raft: election won: tally=1 +33:06.235 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14090 [Leader]" +33:06.235 [INFO] TestAgent.server: cluster leadership acquired +33:06.236 [INFO] TestAgent.server: New leader elected: payload=Node-09c1fd5b-e8f5-4359-4c82-4680c784115d +33:06.236 [INFO] TestAgent.server: initializing acls +33:06.236 [INFO] TestAgent.server: Created ACL 'global-management' policy +33:06.236 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +33:06.237 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +33:06.237 [INFO] TestAgent.server: Created ACL anonymous token from configuration +33:06.237 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +33:06.238 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +33:06.238 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-09c1fd5b-e8f5-4359-4c82-4680c784115d +33:06.238 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-09c1fd5b-e8f5-4359-4c82-4680c784115d.dc1 +33:06.239 [INFO] TestAgent.server: Updating LAN server: server="Node-09c1fd5b-e8f5-4359-4c82-4680c784115d (Addr: tcp/127.0.0.1:14090) (DC: dc1)" +33:06.239 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-09c1fd5b-e8f5-4359-4c82-4680c784115d.dc1 area=wan +33:06.239 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:06.265 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:06.265 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:06.265 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:06.265 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:06.265 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:06.265 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-09c1fd5b-e8f5-4359-4c82-4680c784115d +33:06.265 [INFO] TestAgent.server: member joined, marking health alive: member=Node-09c1fd5b-e8f5-4359-4c82-4680c784115d +33:06.268 [INFO] TestAgent.server: federation state anti-entropy synced +33:06.268 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-09c1fd5b-e8f5-4359-4c82-4680c784115d +33:06.528 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:06.529 [INFO] TestAgent: Synced node info +33:06.598 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:47638 latency=2.893047ms +33:06.599 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-88d63b34-787c-f34e-8123-906c7347ece3 from=127.0.0.1:47640 latency=310.401µs +33:06.610 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:47642 latency=7.781918ms +33:06.612 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-085b7a5a-a89f-c00d-f990-2b01778fae5b from=127.0.0.1:47640 latency=496.581µs +33:06.613 [INFO] TestAgent: Requesting shutdown +33:06.613 [INFO] TestAgent.server: shutting down server +33:06.614 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +33:06.614 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +33:06.614 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:06.614 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:06.614 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:06.614 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:06.614 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:06.614 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +33:06.614 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:06.614 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:06.614 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:06.614 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +33:06.614 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:06.614 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:06.614 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:06.614Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:06.614Z [INFO] agent.router.manager: shutting down +33:06.614 [INFO] TestAgent: consul server down +33:06.614 [INFO] TestAgent: shutdown complete +33:06.614 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14085 network=tcp +33:06.614 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14085 network=udp +33:06.614 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14086 network=tcp +33:07.114 [INFO] TestAgent: Waiting for endpoints to shut down +33:07.114 [INFO] TestAgent: Endpoints down +--- PASS: TestAuthMethodCreateCommand_k8s (1.02s) --- PASS: TestAuthMethodCreateCommand_k8s/k8s_host_required (0.00s) --- PASS: TestAuthMethodCreateCommand_k8s/k8s_ca_cert_required (0.00s) --- PASS: TestAuthMethodCreateCommand_k8s/k8s_jwt_required (0.00s) --- PASS: TestAuthMethodCreateCommand_k8s/create_k8s (0.01s) - --- PASS: TestAuthMethodCreateCommand_k8s/create_k8s_with_cert_file (0.02s) -2022-09-15T21:41:07.552Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:07.633Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:07.641 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fda90cf3-111b-4a8f-bd5a-0df4f0c677f9 Address:127.0.0.1:25328}]" -41:07.642 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25328 [Follower]" leader= -41:07.651 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fda90cf3-111b-4a8f-bd5a-0df4f0c677f9.dc1 127.0.0.1 -41:07.652 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fda90cf3-111b-4a8f-bd5a-0df4f0c677f9 127.0.0.1 -2022-09-15T21:41:07.652Z [INFO] agent.router: Initializing LAN area manager -41:07.653 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25323 network=udp -41:07.654 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25323 network=tcp -41:07.658 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fda90cf3-111b-4a8f-bd5a-0df4f0c677f9.dc1 area=wan -41:07.659 [INFO] TestAgent.server: Adding LAN server: server="Node-fda90cf3-111b-4a8f-bd5a-0df4f0c677f9 (Addr: tcp/127.0.0.1:25328) (DC: dc1)" -41:07.667 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25324 network=tcp -41:07.667 [INFO] TestAgent: started state syncer -41:07.674 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25329 network=tcp -41:07.698 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:07.698 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25328 [Candidate]" term=2 -41:07.698 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:07.698 [DEBUG] TestAgent.server.raft: vote granted: from=fda90cf3-111b-4a8f-bd5a-0df4f0c677f9 term=2 tally=1 -41:07.698 [INFO] TestAgent.server.raft: election won: tally=1 -41:07.698 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25328 [Leader]" -41:07.698 [INFO] TestAgent.server: cluster leadership acquired -41:07.698 [INFO] TestAgent.server: initializing acls -41:07.699 [INFO] TestAgent.server: Created ACL 'global-management' policy -41:07.699 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -41:07.699 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -41:07.700 [INFO] TestAgent.server: Created ACL anonymous token from configuration -41:07.700 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -41:07.700 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -41:07.700 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-fda90cf3-111b-4a8f-bd5a-0df4f0c677f9 -41:07.700 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-fda90cf3-111b-4a8f-bd5a-0df4f0c677f9.dc1 -41:07.701 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:07.701 [INFO] TestAgent.server: New leader elected: payload=Node-fda90cf3-111b-4a8f-bd5a-0df4f0c677f9 -41:07.701 [INFO] TestAgent.server: Updating LAN server: server="Node-fda90cf3-111b-4a8f-bd5a-0df4f0c677f9 (Addr: tcp/127.0.0.1:25328) (DC: dc1)" -41:07.702 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-fda90cf3-111b-4a8f-bd5a-0df4f0c677f9.dc1 area=wan -41:07.723 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:07.723 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:07.723 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:07.723 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:07.723 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:07.723 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fda90cf3-111b-4a8f-bd5a-0df4f0c677f9 -41:07.723 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fda90cf3-111b-4a8f-bd5a-0df4f0c677f9 -41:07.724 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fda90cf3-111b-4a8f-bd5a-0df4f0c677f9 -41:07.724 [INFO] TestAgent.server: federation state anti-entropy synced -41:07.725 [INFO] TestAgent: Synced node info -41:07.725 [DEBUG] TestAgent: Node info in sync -41:07.994 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:52954 latency=1.46488ms -41:07.999 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-3cee5700-9898-82fe-b68a-25ccc2bf66d7 from=127.0.0.1:52956 latency=453.22µs -41:08.003 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:52958 latency=925.14µs -41:08.004 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-f70a9f02-62c0-0174-519b-6afeb4d0ed27 from=127.0.0.1:52956 latency=325.7µs -41:08.009 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:52960 latency=1.391ms -41:08.010 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-258e88f3-b99e-2c96-f512-af61cd159bfb from=127.0.0.1:52956 latency=238.54µs -41:08.011 [INFO] TestAgent: Requesting shutdown -41:08.011 [INFO] TestAgent.server: shutting down server -41:08.011 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -41:08.011 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -41:08.011 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:08.011 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:08.011 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:08.011 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:08.011 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:08.011 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:08.011 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:08.011 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -41:08.011 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -41:08.011 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:08.011 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:08.011 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:08.011 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:08.012Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:08.012Z [INFO] agent.router.manager: shutting down -41:08.012 [INFO] TestAgent: consul server down -41:08.012 [INFO] TestAgent: shutdown complete -41:08.012 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25323 network=tcp -41:08.012 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25323 network=udp -41:08.012 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25324 network=tcp -41:08.512 [INFO] TestAgent: Waiting for endpoints to shut down -41:08.512 [INFO] TestAgent: Endpoints down ---- PASS: TestAuthMethodCreateCommand_JSON (1.06s) - --- PASS: TestAuthMethodCreateCommand_JSON/type_required (0.00s) - --- PASS: TestAuthMethodCreateCommand_JSON/create_testing (0.01s) - --- PASS: TestAuthMethodCreateCommand_JSON/create_testing_with_max_token_ttl (0.00s) - --- PASS: TestAuthMethodCreateCommand_JSON/create_testing_with_token_type_global (0.01s) -2022-09-15T21:41:07.551Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:07.594Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:07.618 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:747c3079-c319-56c5-fcd0-19500b386c76 Address:127.0.0.1:25342}]" -41:07.634 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25342 [Follower]" leader= -41:07.635 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-747c3079-c319-56c5-fcd0-19500b386c76.dc1 127.0.0.1 -41:07.636 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-747c3079-c319-56c5-fcd0-19500b386c76 127.0.0.1 -2022-09-15T21:41:07.636Z [INFO] agent.router: Initializing LAN area manager -41:07.636 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25337 network=udp -41:07.636 [INFO] TestAgent.server: Adding LAN server: server="Node-747c3079-c319-56c5-fcd0-19500b386c76 (Addr: tcp/127.0.0.1:25342) (DC: dc1)" -41:07.637 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-747c3079-c319-56c5-fcd0-19500b386c76.dc1 area=wan -41:07.637 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25337 network=tcp -41:07.642 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25338 network=tcp -41:07.642 [INFO] TestAgent: started state syncer -41:07.642 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25343 network=tcp -41:07.676 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:07.676 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25342 [Candidate]" term=2 -41:07.676 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:07.676 [DEBUG] TestAgent.server.raft: vote granted: from=747c3079-c319-56c5-fcd0-19500b386c76 term=2 tally=1 -41:07.676 [INFO] TestAgent.server.raft: election won: tally=1 -41:07.676 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25342 [Leader]" -41:07.676 [INFO] TestAgent.server: cluster leadership acquired -41:07.676 [INFO] TestAgent.server: initializing acls -41:07.677 [INFO] TestAgent.server: Created ACL 'global-management' policy -41:07.677 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -41:07.678 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -41:07.678 [INFO] TestAgent.server: Created ACL anonymous token from configuration -41:07.678 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -41:07.678 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -41:07.679 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-747c3079-c319-56c5-fcd0-19500b386c76 -41:07.679 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-747c3079-c319-56c5-fcd0-19500b386c76.dc1 -41:07.680 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:07.682 [INFO] TestAgent.server: New leader elected: payload=Node-747c3079-c319-56c5-fcd0-19500b386c76 -41:07.682 [INFO] TestAgent.server: Updating LAN server: server="Node-747c3079-c319-56c5-fcd0-19500b386c76 (Addr: tcp/127.0.0.1:25342) (DC: dc1)" -41:07.682 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-747c3079-c319-56c5-fcd0-19500b386c76.dc1 area=wan -41:07.728 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:07.728 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:07.728 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:07.728 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:07.728 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:07.728 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-747c3079-c319-56c5-fcd0-19500b386c76 -41:07.729 [INFO] TestAgent.server: member joined, marking health alive: member=Node-747c3079-c319-56c5-fcd0-19500b386c76 -41:07.746 [INFO] TestAgent.server: federation state anti-entropy synced -41:07.746 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-747c3079-c319-56c5-fcd0-19500b386c76 -41:07.964 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:07.965 [INFO] TestAgent: Synced node info -41:08.072 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/auth-method from=127.0.0.1:60326 error="Invalid Auth Method: Type should be one of: [jwt kubernetes testing]" -41:08.072 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:60326 latency=425.36µs -41:08.076 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:60328 latency=853.72µs -41:08.077 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-59c0f42e-95d6-fa1a-7e86-e0e53ff57b8c from=127.0.0.1:60330 latency=269.12µs -41:08.080 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:60332 latency=877.9µs -41:08.082 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-c58286a8-85a1-0db7-5f2d-4b197a5f73a9 from=127.0.0.1:60330 latency=389.26µs -41:08.086 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:60334 latency=991.86µs -41:08.087 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-6f0390d9-2bba-ccb8-5ff6-53c68f6ea3b9 from=127.0.0.1:60330 latency=185.48µs -41:08.087 [INFO] TestAgent: Requesting shutdown -41:08.087 [INFO] TestAgent.server: shutting down server -41:08.087 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -41:08.087 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -41:08.087 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:08.087 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:08.087 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:08.087 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:08.087 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:08.087 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:08.087 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:08.088 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -41:08.088 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -41:08.088 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:08.088 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:08.088 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:08.088 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:08.088Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:08.088Z [INFO] agent.router.manager: shutting down -41:08.088 [INFO] TestAgent: consul server down -41:08.088 [INFO] TestAgent: shutdown complete -41:08.088 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25337 network=tcp -41:08.088 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25337 network=udp -41:08.088 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25338 network=tcp -41:08.589 [INFO] TestAgent: Waiting for endpoints to shut down -41:08.589 [INFO] TestAgent: Endpoints down ---- PASS: TestAuthMethodCreateCommand (1.13s) - --- PASS: TestAuthMethodCreateCommand/type_required (0.00s) - --- PASS: TestAuthMethodCreateCommand/name_required (0.00s) - --- PASS: TestAuthMethodCreateCommand/invalid_type (0.00s) - --- PASS: TestAuthMethodCreateCommand/create_testing (0.00s) - --- PASS: TestAuthMethodCreateCommand/create_testing_with_max_token_ttl (0.01s) - --- PASS: TestAuthMethodCreateCommand/create_testing_with_token_type_global (0.00s) + --- PASS: TestAuthMethodCreateCommand_k8s/create_k8s_with_cert_file (0.01s) +2021-08-13T15:33:06.161Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:06.188Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:06.192 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:68dc6738-ed3f-e3ac-2d69-44e3c6923ab9 Address:127.0.0.1:14097}]" +33:06.193 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-68dc6738-ed3f-e3ac-2d69-44e3c6923ab9.dc1 127.0.0.1 +33:06.193 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14097 [Follower]" leader= +33:06.194 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-68dc6738-ed3f-e3ac-2d69-44e3c6923ab9 127.0.0.1 +2021-08-13T15:33:06.194Z [INFO] agent.router: Initializing LAN area manager +33:06.208 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14092 network=tcp +33:06.208 [INFO] TestAgent.server: Adding LAN server: server="Node-68dc6738-ed3f-e3ac-2d69-44e3c6923ab9 (Addr: tcp/127.0.0.1:14097) (DC: dc1)" +33:06.208 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14092 network=udp +33:06.228 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14093 network=tcp +33:06.228 [INFO] TestAgent: started state syncer +33:06.228 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14098 network=tcp +33:06.229 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-68dc6738-ed3f-e3ac-2d69-44e3c6923ab9.dc1 area=wan +33:06.244 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:06.244 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14097 [Candidate]" term=2 +33:06.244 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:06.244 [DEBUG] TestAgent.server.raft: vote granted: from=68dc6738-ed3f-e3ac-2d69-44e3c6923ab9 term=2 tally=1 +33:06.244 [INFO] TestAgent.server.raft: election won: tally=1 +33:06.244 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14097 [Leader]" +33:06.244 [INFO] TestAgent.server: cluster leadership acquired +33:06.245 [INFO] TestAgent.server: New leader elected: payload=Node-68dc6738-ed3f-e3ac-2d69-44e3c6923ab9 +33:06.245 [INFO] TestAgent.server: initializing acls +33:06.245 [INFO] TestAgent.server: Created ACL 'global-management' policy +33:06.245 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +33:06.246 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +33:06.246 [INFO] TestAgent.server: Created ACL anonymous token from configuration +33:06.246 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +33:06.246 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +33:06.246 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-68dc6738-ed3f-e3ac-2d69-44e3c6923ab9 +33:06.247 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-68dc6738-ed3f-e3ac-2d69-44e3c6923ab9.dc1 +33:06.248 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:06.248 [INFO] TestAgent.server: Updating LAN server: server="Node-68dc6738-ed3f-e3ac-2d69-44e3c6923ab9 (Addr: tcp/127.0.0.1:14097) (DC: dc1)" +33:06.248 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-68dc6738-ed3f-e3ac-2d69-44e3c6923ab9.dc1 area=wan +33:06.265 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:06.265 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:06.265 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:06.265 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:06.265 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:06.265 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-68dc6738-ed3f-e3ac-2d69-44e3c6923ab9 +33:06.265 [INFO] TestAgent.server: member joined, marking health alive: member=Node-68dc6738-ed3f-e3ac-2d69-44e3c6923ab9 +33:06.267 [INFO] TestAgent.server: federation state anti-entropy synced +33:06.270 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-68dc6738-ed3f-e3ac-2d69-44e3c6923ab9 +33:06.358 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:06.359 [INFO] TestAgent: Synced node info +33:06.359 [DEBUG] TestAgent: Node info in sync +33:06.644 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:36340 latency=868.742µs +33:06.645 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-a8af2ec1-fcc3-b952-e1ea-0b4a4f05b037 from=127.0.0.1:36342 latency=250.9µs +33:06.648 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:36344 latency=678.681µs +33:06.649 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-588b4cda-6b65-2886-47a5-730a4b0a55c8 from=127.0.0.1:36342 latency=174.66µs +33:06.652 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:36346 latency=988.782µs +33:06.652 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-ae222bb4-edf4-3204-94e3-08e588c5d434 from=127.0.0.1:36342 latency=181.46µs +33:06.653 [INFO] TestAgent: Requesting shutdown +33:06.653 [INFO] TestAgent.server: shutting down server +33:06.653 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +33:06.653 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +33:06.653 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:06.653 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:06.653 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:06.653 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:06.653 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +33:06.653 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:06.653 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:06.653 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:06.653 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:06.653 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:06.653 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +33:06.653 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:06.653 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:06.653Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:06.653Z [INFO] agent.router.manager: shutting down +33:06.653 [INFO] TestAgent: consul server down +33:06.653 [INFO] TestAgent: shutdown complete +33:06.653 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14092 network=tcp +33:06.653 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14092 network=udp +33:06.653 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14093 network=tcp +33:07.154 [INFO] TestAgent: Waiting for endpoints to shut down +33:07.154 [INFO] TestAgent: Endpoints down +--- PASS: TestAuthMethodCreateCommand_config (1.06s) + --- PASS: TestAuthMethodCreateCommand_config/config_file (0.00s) + --- PASS: TestAuthMethodCreateCommand_config/config_std-in (0.00s) + --- PASS: TestAuthMethodCreateCommand_config/config_string (0.00s) PASS -ok github.com/hashicorp/consul/command/acl/authmethod/create 1.256s +ok github.com/hashicorp/consul/command/acl/authmethod/create 1.128s === RUN TestAuthMethodDeleteCommand_noTabs === PAUSE TestAuthMethodDeleteCommand_noTabs === RUN TestAuthMethodDeleteCommand === PAUSE TestAuthMethodDeleteCommand === CONT TestAuthMethodDeleteCommand_noTabs ---- PASS: TestAuthMethodDeleteCommand_noTabs (0.00s) === CONT TestAuthMethodDeleteCommand [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestAuthMethodDeleteCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range === RUN TestAuthMethodDeleteCommand/name_required === RUN TestAuthMethodDeleteCommand/delete_notfound === RUN TestAuthMethodDeleteCommand/delete_works -2022-09-15T21:41:16.237Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:16.259Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:16.265 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:424d53f3-c0dd-4433-509a-aab637e93d72 Address:127.0.0.1:27363}]" -41:16.265 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:27363 [Follower]" leader= -41:16.268 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-424d53f3-c0dd-4433-509a-aab637e93d72.dc1 127.0.0.1 -41:16.272 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-424d53f3-c0dd-4433-509a-aab637e93d72 127.0.0.1 -2022-09-15T21:41:16.272Z [INFO] agent.router: Initializing LAN area manager -41:16.273 [INFO] TestAgent.server: Adding LAN server: server="Node-424d53f3-c0dd-4433-509a-aab637e93d72 (Addr: tcp/127.0.0.1:27363) (DC: dc1)" -41:16.274 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27358 network=udp -41:16.274 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-424d53f3-c0dd-4433-509a-aab637e93d72.dc1 area=wan -41:16.274 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27358 network=tcp -41:16.276 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:27359 network=tcp -41:16.276 [INFO] TestAgent: started state syncer -41:16.276 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:27364 network=tcp -41:16.324 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -41:16.324 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:16.324 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:27363 [Candidate]" term=2 -41:16.324 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:16.324 [DEBUG] TestAgent.server.raft: vote granted: from=424d53f3-c0dd-4433-509a-aab637e93d72 term=2 tally=1 -41:16.324 [INFO] TestAgent.server.raft: election won: tally=1 -41:16.324 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:27363 [Leader]" -41:16.324 [INFO] TestAgent.server: cluster leadership acquired -41:16.325 [INFO] TestAgent.server: New leader elected: payload=Node-424d53f3-c0dd-4433-509a-aab637e93d72 -41:16.325 [INFO] TestAgent.server: initializing acls -41:16.325 [INFO] TestAgent.server: Created ACL 'global-management' policy -41:16.325 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -41:16.326 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -41:16.327 [INFO] TestAgent.server: Created ACL anonymous token from configuration -41:16.327 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -41:16.327 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -41:16.327 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-424d53f3-c0dd-4433-509a-aab637e93d72 -41:16.327 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-424d53f3-c0dd-4433-509a-aab637e93d72.dc1 -41:16.328 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:16.330 [INFO] TestAgent.server: Updating LAN server: server="Node-424d53f3-c0dd-4433-509a-aab637e93d72 (Addr: tcp/127.0.0.1:27363) (DC: dc1)" -41:16.330 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-424d53f3-c0dd-4433-509a-aab637e93d72.dc1 area=wan -41:16.353 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:16.353 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:16.353 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:16.353 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:16.353 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:16.353 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-424d53f3-c0dd-4433-509a-aab637e93d72 -41:16.353 [INFO] TestAgent.server: member joined, marking health alive: member=Node-424d53f3-c0dd-4433-509a-aab637e93d72 -41:16.353 [INFO] TestAgent: Synced node info -41:16.354 [INFO] TestAgent.server: federation state anti-entropy synced -41:16.354 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-424d53f3-c0dd-4433-509a-aab637e93d72 -41:16.371 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/auth-method/notfound from=127.0.0.1:55038 latency=1.10038ms -41:16.374 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:55040 latency=1.28712ms -41:16.382 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/auth-method/test-a8e7ef3f-b0ce-76e5-14e6-33988be02fdc from=127.0.0.1:55042 latency=1.26658ms -41:16.394 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-a8e7ef3f-b0ce-76e5-14e6-33988be02fdc from=127.0.0.1:55040 latency=307.22µs -41:16.394 [INFO] TestAgent: Requesting shutdown -41:16.395 [INFO] TestAgent.server: shutting down server -41:16.395 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -41:16.395 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -41:16.395 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:16.395 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:16.395 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:16.395 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:16.395 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:16.395 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:16.395 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -41:16.395 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:16.395 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:16.395 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -41:16.395 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:16.395 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:16.395 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:16.395Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:16.395Z [INFO] agent.router.manager: shutting down -41:16.395 [INFO] TestAgent: consul server down -41:16.395 [INFO] TestAgent: shutdown complete -41:16.395 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27358 network=tcp -41:16.395 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27358 network=udp -41:16.395 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:27359 network=tcp -41:16.896 [INFO] TestAgent: Waiting for endpoints to shut down -41:16.896 [INFO] TestAgent: Endpoints down ---- PASS: TestAuthMethodDeleteCommand (0.78s) +2021-08-13T15:33:13.705Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:13.727Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:13.730 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:44a21221-2b76-7aa5-6f36-f52e43a74f3a Address:127.0.0.1:18174}]" +33:13.730 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:18174 [Follower]" leader= +33:13.731 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-44a21221-2b76-7aa5-6f36-f52e43a74f3a.dc1 127.0.0.1 +33:13.732 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-44a21221-2b76-7aa5-6f36-f52e43a74f3a 127.0.0.1 +2021-08-13T15:33:13.732Z [INFO] agent.router: Initializing LAN area manager +33:13.732 [INFO] TestAgent.server: Adding LAN server: server="Node-44a21221-2b76-7aa5-6f36-f52e43a74f3a (Addr: tcp/127.0.0.1:18174) (DC: dc1)" +33:13.732 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-44a21221-2b76-7aa5-6f36-f52e43a74f3a.dc1 area=wan +33:13.733 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18169 network=tcp +33:13.733 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18169 network=udp +33:13.734 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:18170 network=tcp +33:13.734 [INFO] TestAgent: started state syncer +33:13.734 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:18175 network=tcp +33:13.781 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:13.781 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:18174 [Candidate]" term=2 +33:13.781 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:13.781 [DEBUG] TestAgent.server.raft: vote granted: from=44a21221-2b76-7aa5-6f36-f52e43a74f3a term=2 tally=1 +33:13.781 [INFO] TestAgent.server.raft: election won: tally=1 +33:13.781 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:18174 [Leader]" +33:13.781 [INFO] TestAgent.server: cluster leadership acquired +33:13.781 [INFO] TestAgent.server: New leader elected: payload=Node-44a21221-2b76-7aa5-6f36-f52e43a74f3a +33:13.781 [INFO] TestAgent.server: initializing acls +33:13.782 [INFO] TestAgent.server: Created ACL 'global-management' policy +33:13.782 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +33:13.782 [INFO] TestAgent.server: initializing acls +33:13.782 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +33:13.783 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +33:13.783 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +33:13.783 [INFO] TestAgent.server: Created ACL anonymous token from configuration +33:13.784 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +33:13.784 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +33:13.784 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode +33:13.784 [INFO] TestAgent.server: Created ACL anonymous token from configuration +33:13.784 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-44a21221-2b76-7aa5-6f36-f52e43a74f3a +33:13.784 [INFO] TestAgent.server: Updating LAN server: server="Node-44a21221-2b76-7aa5-6f36-f52e43a74f3a (Addr: tcp/127.0.0.1:18174) (DC: dc1)" +33:13.784 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-44a21221-2b76-7aa5-6f36-f52e43a74f3a.dc1 +33:13.784 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-44a21221-2b76-7aa5-6f36-f52e43a74f3a.dc1 area=wan +33:13.784 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-44a21221-2b76-7aa5-6f36-f52e43a74f3a +33:13.784 [INFO] TestAgent.server: Updating LAN server: server="Node-44a21221-2b76-7aa5-6f36-f52e43a74f3a (Addr: tcp/127.0.0.1:18174) (DC: dc1)" +33:13.784 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-44a21221-2b76-7aa5-6f36-f52e43a74f3a.dc1 +33:13.784 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-44a21221-2b76-7aa5-6f36-f52e43a74f3a.dc1 area=wan +33:13.787 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:13.811 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:13.811 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:13.811 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:13.811 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:13.811 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:13.812 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-44a21221-2b76-7aa5-6f36-f52e43a74f3a +33:13.812 [INFO] TestAgent.server: member joined, marking health alive: member=Node-44a21221-2b76-7aa5-6f36-f52e43a74f3a +33:13.812 [INFO] TestAgent.server: federation state anti-entropy synced +33:13.813 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-44a21221-2b76-7aa5-6f36-f52e43a74f3a +33:13.813 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-44a21221-2b76-7aa5-6f36-f52e43a74f3a +33:13.899 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:13.900 [INFO] TestAgent: Synced node info +33:14.005 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/auth-method/notfound from=127.0.0.1:35492 latency=1.068043ms +33:14.008 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:35494 latency=1.147763ms +33:14.010 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/auth-method/test-3e5a5523-2941-dd4b-17f5-5cd25c5830f6 from=127.0.0.1:35496 latency=647.642µs +33:14.011 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-3e5a5523-2941-dd4b-17f5-5cd25c5830f6 from=127.0.0.1:35494 latency=126.04µs +33:14.011 [INFO] TestAgent: Requesting shutdown +33:14.011 [INFO] TestAgent.server: shutting down server +33:14.011 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:14.011 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:14.011 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:14.011 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +33:14.011 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +33:14.011 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:14.011 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:14.011 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:14.011 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:14.011 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +33:14.011 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:14.011 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +33:14.011 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:14.011 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:14.011 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:14.011Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:14.011Z [INFO] agent.router.manager: shutting down +33:14.012 [INFO] TestAgent: consul server down +33:14.012 [INFO] TestAgent: shutdown complete +33:14.012 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18169 network=tcp +33:14.012 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18169 network=udp +33:14.012 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:18170 network=tcp +33:14.512 [INFO] TestAgent: Waiting for endpoints to shut down +33:14.512 [INFO] TestAgent: Endpoints down +--- PASS: TestAuthMethodDeleteCommand (0.86s) --- PASS: TestAuthMethodDeleteCommand/name_required (0.00s) --- PASS: TestAuthMethodDeleteCommand/delete_notfound (0.00s) - --- PASS: TestAuthMethodDeleteCommand/delete_works (0.02s) + --- PASS: TestAuthMethodDeleteCommand/delete_works (0.01s) PASS -ok github.com/hashicorp/consul/command/acl/authmethod/delete 0.869s +ok github.com/hashicorp/consul/command/acl/authmethod/delete 0.953s === RUN TestAuthMethodListCommand_noTabs === PAUSE TestAuthMethodListCommand_noTabs === RUN TestAuthMethodListCommand @@ -73185,179 +73213,176 @@ === RUN TestAuthMethodListCommand_JSON === PAUSE TestAuthMethodListCommand_JSON === CONT TestAuthMethodListCommand_noTabs ---- PASS: TestAuthMethodListCommand_noTabs (0.00s) === CONT TestAuthMethodListCommand_JSON -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestAuthMethodListCommand +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestAuthMethodListCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range === RUN TestAuthMethodListCommand_JSON/found_some === RUN TestAuthMethodListCommand/found_none === RUN TestAuthMethodListCommand/found_some -2022-09-15T21:41:19.592Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:19.630Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:19.632 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:73de35b8-0844-810c-d0c7-21df4f4082cb Address:127.0.0.1:12048}]" -41:19.633 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-73de35b8-0844-810c-d0c7-21df4f4082cb.dc1 127.0.0.1 -41:19.633 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:12048 [Follower]" leader= -41:19.634 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-73de35b8-0844-810c-d0c7-21df4f4082cb 127.0.0.1 -2022-09-15T21:41:19.634Z [INFO] agent.router: Initializing LAN area manager -41:19.634 [INFO] TestAgent.server: Adding LAN server: server="Node-73de35b8-0844-810c-d0c7-21df4f4082cb (Addr: tcp/127.0.0.1:12048) (DC: dc1)" -41:19.634 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12043 network=udp -41:19.634 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-73de35b8-0844-810c-d0c7-21df4f4082cb.dc1 area=wan -41:19.635 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12043 network=tcp -41:19.635 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:12044 network=tcp -41:19.635 [INFO] TestAgent: started state syncer -41:19.636 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:12049 network=tcp -41:19.685 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -41:19.696 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:19.696 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:12048 [Candidate]" term=2 -41:19.697 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:19.697 [DEBUG] TestAgent.server.raft: vote granted: from=73de35b8-0844-810c-d0c7-21df4f4082cb term=2 tally=1 -41:19.697 [INFO] TestAgent.server.raft: election won: tally=1 -41:19.697 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:12048 [Leader]" -41:19.697 [INFO] TestAgent.server: cluster leadership acquired -41:19.697 [INFO] TestAgent.server: New leader elected: payload=Node-73de35b8-0844-810c-d0c7-21df4f4082cb -41:19.697 [INFO] TestAgent.server: initializing acls -41:19.697 [INFO] TestAgent.server: Created ACL 'global-management' policy -41:19.697 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -41:19.698 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -41:19.708 [INFO] TestAgent.server: Created ACL anonymous token from configuration -41:19.708 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -41:19.708 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -41:19.708 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-73de35b8-0844-810c-d0c7-21df4f4082cb -41:19.708 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-73de35b8-0844-810c-d0c7-21df4f4082cb.dc1 -41:19.709 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:19.710 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-73de35b8-0844-810c-d0c7-21df4f4082cb.dc1 area=wan -41:19.710 [INFO] TestAgent.server: Updating LAN server: server="Node-73de35b8-0844-810c-d0c7-21df4f4082cb (Addr: tcp/127.0.0.1:12048) (DC: dc1)" -41:19.735 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:19.735 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:19.735 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:19.735 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:19.735 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:19.735 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-73de35b8-0844-810c-d0c7-21df4f4082cb -41:19.735 [INFO] TestAgent.server: member joined, marking health alive: member=Node-73de35b8-0844-810c-d0c7-21df4f4082cb -41:19.736 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-73de35b8-0844-810c-d0c7-21df4f4082cb -41:19.736 [INFO] TestAgent.server: federation state anti-entropy synced -41:19.972 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:19.974 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:42286 latency=2.0827ms -41:19.974 [INFO] TestAgent: Synced node info -41:19.976 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:42286 latency=985µs -41:19.978 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:42286 latency=1.37804ms -41:19.980 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:42286 latency=681.94µs -41:19.981 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:42286 latency=528.1µs -41:19.984 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-methods from=127.0.0.1:42288 latency=641.36µs -41:19.986 [INFO] TestAgent: Requesting shutdown -41:19.986 [INFO] TestAgent.server: shutting down server -41:19.986 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:19.986 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:19.986 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:19.986 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:19.986 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -41:19.986 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -41:19.986 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:19.986 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:19.986 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:19.986 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:19.986 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:19.987 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:19.987 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -41:19.987 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -41:19.987 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:19.987Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:19.987Z [INFO] agent.router.manager: shutting down -41:19.987 [INFO] TestAgent: consul server down -41:19.987 [INFO] TestAgent: shutdown complete -41:19.987 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12043 network=tcp -41:19.987 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12043 network=udp -41:19.987 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:12044 network=tcp -41:20.487 [INFO] TestAgent: Waiting for endpoints to shut down -41:20.487 [INFO] TestAgent: Endpoints down ---- PASS: TestAuthMethodListCommand_JSON (0.99s) +2021-08-13T15:33:21.111Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:21.133Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:21.137 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e7693a90-10fe-61dd-124f-c879de234703 Address:127.0.0.1:14090}]" +33:21.138 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14090 [Follower]" leader= +33:21.139 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e7693a90-10fe-61dd-124f-c879de234703.dc1 127.0.0.1 +33:21.141 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e7693a90-10fe-61dd-124f-c879de234703 127.0.0.1 +2021-08-13T15:33:21.142Z [INFO] agent.router: Initializing LAN area manager +33:21.142 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14085 network=udp +33:21.142 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14085 network=tcp +33:21.143 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e7693a90-10fe-61dd-124f-c879de234703.dc1 area=wan +33:21.143 [INFO] TestAgent.server: Adding LAN server: server="Node-e7693a90-10fe-61dd-124f-c879de234703 (Addr: tcp/127.0.0.1:14090) (DC: dc1)" +33:21.144 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14086 network=tcp +33:21.144 [INFO] TestAgent: started state syncer +33:21.144 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14091 network=tcp +33:21.187 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:21.187 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14090 [Candidate]" term=2 +33:21.187 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:21.187 [DEBUG] TestAgent.server.raft: vote granted: from=e7693a90-10fe-61dd-124f-c879de234703 term=2 tally=1 +33:21.187 [INFO] TestAgent.server.raft: election won: tally=1 +33:21.187 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14090 [Leader]" +33:21.187 [INFO] TestAgent.server: cluster leadership acquired +33:21.188 [INFO] TestAgent.server: New leader elected: payload=Node-e7693a90-10fe-61dd-124f-c879de234703 +33:21.188 [INFO] TestAgent.server: initializing acls +33:21.188 [INFO] TestAgent.server: Created ACL 'global-management' policy +33:21.188 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +33:21.189 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +33:21.189 [INFO] TestAgent.server: Created ACL anonymous token from configuration +33:21.189 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +33:21.189 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +33:21.189 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-e7693a90-10fe-61dd-124f-c879de234703 +33:21.190 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-e7693a90-10fe-61dd-124f-c879de234703.dc1 +33:21.190 [INFO] TestAgent.server: Updating LAN server: server="Node-e7693a90-10fe-61dd-124f-c879de234703 (Addr: tcp/127.0.0.1:14090) (DC: dc1)" +33:21.190 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-e7693a90-10fe-61dd-124f-c879de234703.dc1 area=wan +33:21.190 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:21.212 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:21.212 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:21.212 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:21.212 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:21.212 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:21.212 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e7693a90-10fe-61dd-124f-c879de234703 +33:21.212 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e7693a90-10fe-61dd-124f-c879de234703 +33:21.213 [INFO] TestAgent.server: federation state anti-entropy synced +33:21.215 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e7693a90-10fe-61dd-124f-c879de234703 +33:21.463 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:47666 latency=2.099165ms +33:21.468 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:47666 latency=911.483µs +33:21.469 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:47666 latency=558.682µs +33:21.471 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:47666 latency=709.202µs +33:21.472 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:47666 latency=686.442µs +33:21.474 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-methods from=127.0.0.1:47668 latency=498.141µs +33:21.475 [INFO] TestAgent: Requesting shutdown +33:21.476 [INFO] TestAgent.server: shutting down server +33:21.476 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +33:21.476 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:21.476 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:21.476 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:21.476 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:21.476 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +33:21.476 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:21.476 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +33:21.476 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:21.476 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:21.476 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:21.476 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +33:21.476 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:21.476 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:21.476 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:21.476 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:21.476Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:21.476Z [INFO] agent.router.manager: shutting down +33:21.476 [INFO] TestAgent: consul server down +33:21.476 [INFO] TestAgent: shutdown complete +33:21.476 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14085 network=tcp +33:21.476 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14085 network=udp +33:21.476 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14086 network=tcp +33:21.976 [INFO] TestAgent: Waiting for endpoints to shut down +33:21.976 [INFO] TestAgent: Endpoints down +--- PASS: TestAuthMethodListCommand_JSON (0.92s) --- PASS: TestAuthMethodListCommand_JSON/found_some (0.00s) -2022-09-15T21:41:19.593Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:19.613Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:19.627 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c3c3b60c-d916-b162-3583-dc71c4a23ba8 Address:127.0.0.1:12055}]" -41:19.628 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c3c3b60c-d916-b162-3583-dc71c4a23ba8.dc1 127.0.0.1 -41:19.629 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c3c3b60c-d916-b162-3583-dc71c4a23ba8 127.0.0.1 -2022-09-15T21:41:19.629Z [INFO] agent.router: Initializing LAN area manager -41:19.629 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12050 network=udp -41:19.629 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:12055 [Follower]" leader= -41:19.630 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c3c3b60c-d916-b162-3583-dc71c4a23ba8.dc1 area=wan -41:19.630 [INFO] TestAgent.server: Adding LAN server: server="Node-c3c3b60c-d916-b162-3583-dc71c4a23ba8 (Addr: tcp/127.0.0.1:12055) (DC: dc1)" -41:19.630 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12050 network=tcp -41:19.631 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:12051 network=tcp -41:19.631 [INFO] TestAgent: started state syncer -41:19.632 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:12056 network=tcp -41:19.680 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -41:19.693 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:19.693 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:12055 [Candidate]" term=2 -41:19.694 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:19.694 [DEBUG] TestAgent.server.raft: vote granted: from=c3c3b60c-d916-b162-3583-dc71c4a23ba8 term=2 tally=1 -41:19.694 [INFO] TestAgent.server.raft: election won: tally=1 -41:19.694 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:12055 [Leader]" -41:19.694 [INFO] TestAgent.server: cluster leadership acquired -41:19.694 [INFO] TestAgent.server: New leader elected: payload=Node-c3c3b60c-d916-b162-3583-dc71c4a23ba8 -41:19.694 [INFO] TestAgent.server: initializing acls -41:19.695 [INFO] TestAgent.server: Created ACL 'global-management' policy -41:19.695 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -41:19.695 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -41:19.696 [INFO] TestAgent.server: Created ACL anonymous token from configuration -41:19.696 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -41:19.696 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -41:19.696 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-c3c3b60c-d916-b162-3583-dc71c4a23ba8 -41:19.696 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-c3c3b60c-d916-b162-3583-dc71c4a23ba8.dc1 -41:19.696 [INFO] TestAgent.server: Updating LAN server: server="Node-c3c3b60c-d916-b162-3583-dc71c4a23ba8 (Addr: tcp/127.0.0.1:12055) (DC: dc1)" -41:19.696 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-c3c3b60c-d916-b162-3583-dc71c4a23ba8.dc1 area=wan -41:19.697 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:19.731 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:19.731 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:19.731 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:19.731 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:19.731 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:19.731 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c3c3b60c-d916-b162-3583-dc71c4a23ba8 -41:19.731 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c3c3b60c-d916-b162-3583-dc71c4a23ba8 -41:19.732 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c3c3b60c-d916-b162-3583-dc71c4a23ba8 -41:19.733 [INFO] TestAgent.server: federation state anti-entropy synced -41:19.798 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:19.799 [INFO] TestAgent: Synced node info -41:20.032 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-methods from=127.0.0.1:52780 latency=3.98544ms -41:20.038 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:52782 latency=1.3111ms -41:20.040 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:52782 latency=906.2µs -41:20.042 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:52782 latency=1.06482ms -41:20.045 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:52782 latency=886.66µs -41:20.047 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:52782 latency=850.26µs -41:20.049 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-methods from=127.0.0.1:52784 latency=336.08µs -41:20.050 [INFO] TestAgent: Requesting shutdown -41:20.050 [INFO] TestAgent.server: shutting down server -41:20.050 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -41:20.050 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -41:20.050 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:20.050 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:20.050 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:20.050 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:20.050 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:20.050 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:20.050 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:20.051 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -41:20.051 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -41:20.051 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:20.051 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:20.051 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:20.051 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:20.051Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:20.051Z [INFO] agent.router.manager: shutting down -41:20.051 [INFO] TestAgent: consul server down -41:20.051 [INFO] TestAgent: shutdown complete -41:20.051 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12050 network=tcp -41:20.051 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12050 network=udp -41:20.051 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:12051 network=tcp -41:20.551 [INFO] TestAgent: Waiting for endpoints to shut down -41:20.551 [INFO] TestAgent: Endpoints down ---- PASS: TestAuthMethodListCommand (1.05s) - --- PASS: TestAuthMethodListCommand/found_none (0.01s) +2021-08-13T15:33:21.111Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:21.133Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:21.138 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e25bbc97-79f0-54fe-cb31-a7c2109a793c Address:127.0.0.1:14097}]" +33:21.139 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14097 [Follower]" leader= +33:21.140 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e25bbc97-79f0-54fe-cb31-a7c2109a793c.dc1 127.0.0.1 +33:21.142 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e25bbc97-79f0-54fe-cb31-a7c2109a793c 127.0.0.1 +2021-08-13T15:33:21.142Z [INFO] agent.router: Initializing LAN area manager +33:21.144 [INFO] TestAgent.server: Adding LAN server: server="Node-e25bbc97-79f0-54fe-cb31-a7c2109a793c (Addr: tcp/127.0.0.1:14097) (DC: dc1)" +33:21.144 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e25bbc97-79f0-54fe-cb31-a7c2109a793c.dc1 area=wan +33:21.144 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14092 network=udp +33:21.144 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14092 network=tcp +33:21.145 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14093 network=tcp +33:21.145 [INFO] TestAgent: started state syncer +33:21.145 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14098 network=tcp +33:21.184 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:21.184 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14097 [Candidate]" term=2 +33:21.185 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:21.185 [DEBUG] TestAgent.server.raft: vote granted: from=e25bbc97-79f0-54fe-cb31-a7c2109a793c term=2 tally=1 +33:21.185 [INFO] TestAgent.server.raft: election won: tally=1 +33:21.185 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14097 [Leader]" +33:21.185 [INFO] TestAgent.server: cluster leadership acquired +33:21.185 [INFO] TestAgent.server: New leader elected: payload=Node-e25bbc97-79f0-54fe-cb31-a7c2109a793c +33:21.185 [INFO] TestAgent.server: initializing acls +33:21.186 [INFO] TestAgent.server: Created ACL 'global-management' policy +33:21.186 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +33:21.187 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +33:21.187 [INFO] TestAgent.server: Created ACL anonymous token from configuration +33:21.187 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +33:21.187 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +33:21.188 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-e25bbc97-79f0-54fe-cb31-a7c2109a793c +33:21.188 [INFO] TestAgent.server: Updating LAN server: server="Node-e25bbc97-79f0-54fe-cb31-a7c2109a793c (Addr: tcp/127.0.0.1:14097) (DC: dc1)" +33:21.188 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-e25bbc97-79f0-54fe-cb31-a7c2109a793c.dc1 +33:21.188 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-e25bbc97-79f0-54fe-cb31-a7c2109a793c.dc1 area=wan +33:21.189 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:21.212 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:21.212 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:21.212 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:21.212 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:21.212 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:21.212 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e25bbc97-79f0-54fe-cb31-a7c2109a793c +33:21.212 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e25bbc97-79f0-54fe-cb31-a7c2109a793c +33:21.213 [INFO] TestAgent.server: federation state anti-entropy synced +33:21.214 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e25bbc97-79f0-54fe-cb31-a7c2109a793c +33:21.467 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:21.468 [INFO] TestAgent: Synced node info +33:21.526 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-methods from=127.0.0.1:36366 latency=890.563µs +33:21.528 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:36368 latency=852.642µs +33:21.529 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:36368 latency=856.102µs +33:21.530 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:36368 latency=624.981µs +33:21.532 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:36368 latency=601.041µs +33:21.533 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:36368 latency=614.721µs +33:21.535 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-methods from=127.0.0.1:36370 latency=324.241µs +33:21.536 [INFO] TestAgent: Requesting shutdown +33:21.536 [INFO] TestAgent.server: shutting down server +33:21.536 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +33:21.536 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +33:21.536 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:21.536 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:21.536 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:21.536 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:21.536 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:21.536 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +33:21.536 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:21.536 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +33:21.536 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:21.536 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:21.536 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:21.536 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:21.536 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:21.536Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:21.536Z [INFO] agent.router.manager: shutting down +33:21.536 [INFO] TestAgent: consul server down +33:21.536 [INFO] TestAgent: shutdown complete +33:21.536 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14092 network=tcp +33:21.536 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14092 network=udp +33:21.536 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14093 network=tcp +33:22.037 [INFO] TestAgent: Waiting for endpoints to shut down +33:22.037 [INFO] TestAgent: Endpoints down +--- PASS: TestAuthMethodListCommand (0.98s) + --- PASS: TestAuthMethodListCommand/found_none (0.00s) --- PASS: TestAuthMethodListCommand/found_some (0.00s) PASS -ok github.com/hashicorp/consul/command/acl/authmethod/list 1.141s +ok github.com/hashicorp/consul/command/acl/authmethod/list 1.040s === RUN TestAuthMethodReadCommand_noTabs === PAUSE TestAuthMethodReadCommand_noTabs === RUN TestAuthMethodReadCommand @@ -73365,171 +73390,174 @@ === RUN TestAuthMethodReadCommand_JSON === PAUSE TestAuthMethodReadCommand_JSON === CONT TestAuthMethodReadCommand_noTabs ---- PASS: TestAuthMethodReadCommand_noTabs (0.00s) -=== CONT TestAuthMethodReadCommand === CONT TestAuthMethodReadCommand_JSON +=== CONT TestAuthMethodReadCommand [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestAuthMethodReadCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range === RUN TestAuthMethodReadCommand_JSON/read_by_name === RUN TestAuthMethodReadCommand/name_required === RUN TestAuthMethodReadCommand/not_found === RUN TestAuthMethodReadCommand/read_by_name -2022-09-15T21:41:20.020Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:20.041Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:20.044 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:539960ab-0a83-b59e-76ed-94201ae4dc9c Address:127.0.0.1:31447}]" -41:20.045 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31447 [Follower]" leader= -41:20.045 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-539960ab-0a83-b59e-76ed-94201ae4dc9c.dc1 127.0.0.1 -41:20.046 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-539960ab-0a83-b59e-76ed-94201ae4dc9c 127.0.0.1 -2022-09-15T21:41:20.046Z [INFO] agent.router: Initializing LAN area manager -41:20.047 [INFO] TestAgent.server: Adding LAN server: server="Node-539960ab-0a83-b59e-76ed-94201ae4dc9c (Addr: tcp/127.0.0.1:31447) (DC: dc1)" -41:20.047 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31442 network=udp -41:20.047 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31442 network=tcp -41:20.047 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-539960ab-0a83-b59e-76ed-94201ae4dc9c.dc1 area=wan -41:20.048 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31443 network=tcp -41:20.049 [INFO] TestAgent: started state syncer -41:20.049 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31448 network=tcp -41:20.097 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:20.097 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31447 [Candidate]" term=2 -41:20.097 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -41:20.097 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:20.097 [DEBUG] TestAgent.server.raft: vote granted: from=539960ab-0a83-b59e-76ed-94201ae4dc9c term=2 tally=1 -41:20.097 [INFO] TestAgent.server.raft: election won: tally=1 -41:20.097 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31447 [Leader]" -41:20.097 [INFO] TestAgent.server: cluster leadership acquired -41:20.098 [INFO] TestAgent.server: New leader elected: payload=Node-539960ab-0a83-b59e-76ed-94201ae4dc9c -41:20.098 [INFO] TestAgent.server: initializing acls -41:20.098 [INFO] TestAgent.server: Created ACL 'global-management' policy -41:20.098 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -41:20.098 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="ACL not found" -41:20.099 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -41:20.099 [INFO] TestAgent.server: Created ACL anonymous token from configuration -41:20.099 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -41:20.099 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -41:20.100 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-539960ab-0a83-b59e-76ed-94201ae4dc9c -41:20.100 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-539960ab-0a83-b59e-76ed-94201ae4dc9c.dc1 -41:20.101 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:20.101 [INFO] TestAgent.server: Updating LAN server: server="Node-539960ab-0a83-b59e-76ed-94201ae4dc9c (Addr: tcp/127.0.0.1:31447) (DC: dc1)" -41:20.101 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-539960ab-0a83-b59e-76ed-94201ae4dc9c.dc1 area=wan -41:20.120 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:20.120 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:20.120 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:20.120 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:20.120 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:20.120 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-539960ab-0a83-b59e-76ed-94201ae4dc9c -41:20.120 [INFO] TestAgent.server: member joined, marking health alive: member=Node-539960ab-0a83-b59e-76ed-94201ae4dc9c -41:20.122 [INFO] TestAgent.server: federation state anti-entropy synced -41:20.122 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-539960ab-0a83-b59e-76ed-94201ae4dc9c -41:20.262 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:57820 latency=2.01748ms -41:20.265 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-80dacf80-31d5-e7ff-302b-adc72448481b from=127.0.0.1:57822 latency=464.9µs -41:20.266 [INFO] TestAgent: Requesting shutdown -41:20.266 [INFO] TestAgent.server: shutting down server -41:20.266 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -41:20.266 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -41:20.266 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:20.266 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:20.266 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:20.266 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:20.266 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:20.266 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:20.266 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:20.266 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -41:20.266 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -41:20.266 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:20.266 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:20.266 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:20.266 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:20.266Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:20.266Z [INFO] agent.router.manager: shutting down -41:20.267 [INFO] TestAgent: consul server down -41:20.267 [INFO] TestAgent: shutdown complete -41:20.267 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31442 network=tcp -41:20.267 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31442 network=udp -41:20.267 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31443 network=tcp -41:20.767 [INFO] TestAgent: Waiting for endpoints to shut down -41:20.767 [INFO] TestAgent: Endpoints down ---- PASS: TestAuthMethodReadCommand_JSON (0.82s) +2021-08-13T15:33:26.654Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:26.677Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:26.680 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:85b10d8d-5536-e201-c2ba-86913cac4218 Address:127.0.0.1:10006}]" +33:26.680 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:10006 [Follower]" leader= +33:26.681 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-85b10d8d-5536-e201-c2ba-86913cac4218.dc1 127.0.0.1 +33:26.684 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-85b10d8d-5536-e201-c2ba-86913cac4218 127.0.0.1 +2021-08-13T15:33:26.684Z [INFO] agent.router: Initializing LAN area manager +33:26.684 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10001 network=udp +33:26.685 [INFO] TestAgent.server: Adding LAN server: server="Node-85b10d8d-5536-e201-c2ba-86913cac4218 (Addr: tcp/127.0.0.1:10006) (DC: dc1)" +33:26.685 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-85b10d8d-5536-e201-c2ba-86913cac4218.dc1 area=wan +33:26.686 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10001 network=tcp +33:26.687 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:10002 network=tcp +33:26.687 [INFO] TestAgent: started state syncer +33:26.687 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:10007 network=tcp +33:26.735 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +33:26.746 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:26.746 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:10006 [Candidate]" term=2 +33:26.746 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:26.746 [DEBUG] TestAgent.server.raft: vote granted: from=85b10d8d-5536-e201-c2ba-86913cac4218 term=2 tally=1 +33:26.746 [INFO] TestAgent.server.raft: election won: tally=1 +33:26.746 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:10006 [Leader]" +33:26.746 [INFO] TestAgent.server: cluster leadership acquired +33:26.746 [INFO] TestAgent.server: initializing acls +33:26.747 [INFO] TestAgent.server: Created ACL 'global-management' policy +33:26.747 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +33:26.747 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +33:26.748 [INFO] TestAgent.server: Created ACL anonymous token from configuration +33:26.748 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +33:26.748 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +33:26.748 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-85b10d8d-5536-e201-c2ba-86913cac4218 +33:26.748 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-85b10d8d-5536-e201-c2ba-86913cac4218.dc1 +33:26.749 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:26.749 [INFO] TestAgent.server: New leader elected: payload=Node-85b10d8d-5536-e201-c2ba-86913cac4218 +33:26.749 [INFO] TestAgent.server: Updating LAN server: server="Node-85b10d8d-5536-e201-c2ba-86913cac4218 (Addr: tcp/127.0.0.1:10006) (DC: dc1)" +33:26.749 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-85b10d8d-5536-e201-c2ba-86913cac4218.dc1 area=wan +33:26.770 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:26.770 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:26.770 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:26.770 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:26.770 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:26.770 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-85b10d8d-5536-e201-c2ba-86913cac4218 +33:26.770 [INFO] TestAgent.server: member joined, marking health alive: member=Node-85b10d8d-5536-e201-c2ba-86913cac4218 +33:26.772 [INFO] TestAgent.server: federation state anti-entropy synced +33:26.772 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-85b10d8d-5536-e201-c2ba-86913cac4218 +33:26.917 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:26.917 [INFO] TestAgent: Synced node info +33:26.917 [DEBUG] TestAgent: Node info in sync +33:27.052 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:58406 latency=1.871344ms +33:27.055 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-c5df836f-a5b2-6085-230a-cc86dd4eb34b from=127.0.0.1:58408 latency=1.197983ms +33:27.059 [INFO] TestAgent: Requesting shutdown +33:27.059 [INFO] TestAgent.server: shutting down server +33:27.059 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:27.059 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:27.059 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:27.059 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:27.059 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +33:27.059 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +33:27.059 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:27.059 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:27.060 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:27.060 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:27.060 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:27.060 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:27.060 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +33:27.060 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +33:27.060 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:27.060Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:27.060Z [INFO] agent.router.manager: shutting down +33:27.060 [INFO] TestAgent: consul server down +33:27.060 [INFO] TestAgent: shutdown complete +33:27.060 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10001 network=tcp +33:27.060 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10001 network=udp +33:27.060 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:10002 network=tcp +33:27.563 [INFO] TestAgent: Waiting for endpoints to shut down +33:27.563 [INFO] TestAgent: Endpoints down +--- PASS: TestAuthMethodReadCommand_JSON (0.96s) --- PASS: TestAuthMethodReadCommand_JSON/read_by_name (0.01s) -2022-09-15T21:41:20.019Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:20.039Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:20.044 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:24d0932f-e0e4-a713-381a-e3bb31108a6a Address:127.0.0.1:31454}]" -41:20.045 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31454 [Follower]" leader= -41:20.046 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-24d0932f-e0e4-a713-381a-e3bb31108a6a.dc1 127.0.0.1 -41:20.046 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-24d0932f-e0e4-a713-381a-e3bb31108a6a 127.0.0.1 -2022-09-15T21:41:20.047Z [INFO] agent.router: Initializing LAN area manager -41:20.047 [INFO] TestAgent.server: Adding LAN server: server="Node-24d0932f-e0e4-a713-381a-e3bb31108a6a (Addr: tcp/127.0.0.1:31454) (DC: dc1)" -41:20.047 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-24d0932f-e0e4-a713-381a-e3bb31108a6a.dc1 area=wan -41:20.047 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31449 network=udp -41:20.047 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31449 network=tcp -41:20.048 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31450 network=tcp -41:20.048 [INFO] TestAgent: started state syncer -41:20.048 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31455 network=tcp -41:20.083 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:20.083 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31454 [Candidate]" term=2 -41:20.083 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:20.083 [DEBUG] TestAgent.server.raft: vote granted: from=24d0932f-e0e4-a713-381a-e3bb31108a6a term=2 tally=1 -41:20.083 [INFO] TestAgent.server.raft: election won: tally=1 -41:20.083 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31454 [Leader]" -41:20.083 [INFO] TestAgent.server: cluster leadership acquired -41:20.083 [INFO] TestAgent.server: initializing acls -41:20.084 [INFO] TestAgent.server: Created ACL 'global-management' policy -41:20.084 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -41:20.084 [INFO] TestAgent.server: New leader elected: payload=Node-24d0932f-e0e4-a713-381a-e3bb31108a6a -41:20.085 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -41:20.085 [INFO] TestAgent.server: Created ACL anonymous token from configuration -41:20.085 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -41:20.085 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -41:20.085 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-24d0932f-e0e4-a713-381a-e3bb31108a6a -41:20.086 [INFO] TestAgent.server: Updating LAN server: server="Node-24d0932f-e0e4-a713-381a-e3bb31108a6a (Addr: tcp/127.0.0.1:31454) (DC: dc1)" -41:20.086 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-24d0932f-e0e4-a713-381a-e3bb31108a6a.dc1 -41:20.086 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-24d0932f-e0e4-a713-381a-e3bb31108a6a.dc1 area=wan -41:20.087 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:20.117 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:20.117 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:20.117 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:20.118 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:20.122 [INFO] TestAgent: Synced node info -41:20.122 [DEBUG] TestAgent: Node info in sync -41:20.123 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:20.123 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-24d0932f-e0e4-a713-381a-e3bb31108a6a -41:20.123 [INFO] TestAgent.server: member joined, marking health alive: member=Node-24d0932f-e0e4-a713-381a-e3bb31108a6a -41:20.124 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-24d0932f-e0e4-a713-381a-e3bb31108a6a -41:20.126 [INFO] TestAgent.server: federation state anti-entropy synced -41:20.426 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/notfound from=127.0.0.1:49522 latency=1.0834ms -41:20.429 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:49524 latency=1.94624ms -41:20.440 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-bfcd1eab-3b90-c82e-3b2e-1a90a6a6a639 from=127.0.0.1:49526 latency=625.7µs -41:20.442 [INFO] TestAgent: Requesting shutdown -41:20.442 [INFO] TestAgent.server: shutting down server -41:20.442 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:20.442 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -41:20.442 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -41:20.442 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:20.442 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:20.442 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:20.442 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:20.442 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:20.442 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:20.442 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:20.442 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -41:20.442 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -41:20.442 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:20.442 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:20.442 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:20.442Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:20.442Z [INFO] agent.router.manager: shutting down -41:20.442 [INFO] TestAgent: consul server down -41:20.442 [INFO] TestAgent: shutdown complete -41:20.442 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31449 network=tcp -41:20.443 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31449 network=udp -41:20.443 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31450 network=tcp -41:20.943 [INFO] TestAgent: Waiting for endpoints to shut down -41:20.943 [INFO] TestAgent: Endpoints down ---- PASS: TestAuthMethodReadCommand (0.99s) +2021-08-13T15:33:26.654Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:26.678Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:26.680 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:af1ea489-0faa-252e-d818-f5ffb446f6cb Address:127.0.0.1:10013}]" +33:26.680 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:10013 [Follower]" leader= +33:26.681 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-af1ea489-0faa-252e-d818-f5ffb446f6cb.dc1 127.0.0.1 +33:26.683 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-af1ea489-0faa-252e-d818-f5ffb446f6cb 127.0.0.1 +2021-08-13T15:33:26.683Z [INFO] agent.router: Initializing LAN area manager +33:26.685 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10008 network=udp +33:26.685 [INFO] TestAgent.server: Adding LAN server: server="Node-af1ea489-0faa-252e-d818-f5ffb446f6cb (Addr: tcp/127.0.0.1:10013) (DC: dc1)" +33:26.685 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-af1ea489-0faa-252e-d818-f5ffb446f6cb.dc1 area=wan +33:26.686 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10008 network=tcp +33:26.687 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:10009 network=tcp +33:26.687 [INFO] TestAgent: started state syncer +33:26.687 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:10014 network=tcp +33:26.736 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +33:26.743 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:26.743 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:10013 [Candidate]" term=2 +33:26.743 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:26.743 [DEBUG] TestAgent.server.raft: vote granted: from=af1ea489-0faa-252e-d818-f5ffb446f6cb term=2 tally=1 +33:26.743 [INFO] TestAgent.server.raft: election won: tally=1 +33:26.743 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:10013 [Leader]" +33:26.743 [INFO] TestAgent.server: cluster leadership acquired +33:26.743 [INFO] TestAgent.server: New leader elected: payload=Node-af1ea489-0faa-252e-d818-f5ffb446f6cb +33:26.744 [INFO] TestAgent.server: initializing acls +33:26.744 [INFO] TestAgent.server: Created ACL 'global-management' policy +33:26.744 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +33:26.745 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +33:26.745 [INFO] TestAgent.server: Created ACL anonymous token from configuration +33:26.745 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +33:26.745 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +33:26.746 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-af1ea489-0faa-252e-d818-f5ffb446f6cb +33:26.746 [INFO] TestAgent.server: Updating LAN server: server="Node-af1ea489-0faa-252e-d818-f5ffb446f6cb (Addr: tcp/127.0.0.1:10013) (DC: dc1)" +33:26.746 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-af1ea489-0faa-252e-d818-f5ffb446f6cb.dc1 +33:26.746 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-af1ea489-0faa-252e-d818-f5ffb446f6cb.dc1 area=wan +33:26.747 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:26.770 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:26.770 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:26.770 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:26.770 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:26.770 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:26.770 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-af1ea489-0faa-252e-d818-f5ffb446f6cb +33:26.770 [INFO] TestAgent.server: member joined, marking health alive: member=Node-af1ea489-0faa-252e-d818-f5ffb446f6cb +33:26.772 [INFO] TestAgent.server: federation state anti-entropy synced +33:26.772 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-af1ea489-0faa-252e-d818-f5ffb446f6cb +33:27.072 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:27.073 [INFO] TestAgent: Synced node info +33:27.101 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/notfound from=127.0.0.1:38412 latency=665.041µs +33:27.103 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:38414 latency=967.542µs +33:27.105 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-725304b4-cd59-c05b-4971-3b61add2e55e from=127.0.0.1:38416 latency=319.86µs +33:27.106 [INFO] TestAgent: Requesting shutdown +33:27.106 [INFO] TestAgent.server: shutting down server +33:27.106 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:27.106 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:27.106 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:27.106 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +33:27.106 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +33:27.106 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:27.106 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:27.106 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:27.106 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:27.106 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:27.106 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +33:27.106 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:27.106 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:27.106 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +33:27.106 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:27.107Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:27.107Z [INFO] agent.router.manager: shutting down +33:27.107 [INFO] TestAgent: consul server down +33:27.107 [INFO] TestAgent: shutdown complete +33:27.107 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10008 network=tcp +33:27.107 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10008 network=udp +33:27.107 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:10009 network=tcp +33:27.607 [INFO] TestAgent: Waiting for endpoints to shut down +33:27.607 [INFO] TestAgent: Endpoints down +--- PASS: TestAuthMethodReadCommand (1.01s) --- PASS: TestAuthMethodReadCommand/name_required (0.00s) --- PASS: TestAuthMethodReadCommand/not_found (0.00s) - --- PASS: TestAuthMethodReadCommand/read_by_name (0.02s) + --- PASS: TestAuthMethodReadCommand/read_by_name (0.00s) PASS -ok github.com/hashicorp/consul/command/acl/authmethod/read 1.077s +ok github.com/hashicorp/consul/command/acl/authmethod/read 1.070s === RUN TestAuthMethodUpdateCommand_noTabs === PAUSE TestAuthMethodUpdateCommand_noTabs === RUN TestAuthMethodUpdateCommand @@ -73545,582 +73573,568 @@ === RUN TestAuthMethodUpdateCommand_config === PAUSE TestAuthMethodUpdateCommand_config === CONT TestAuthMethodUpdateCommand_noTabs +=== CONT TestAuthMethodUpdateCommand_k8s +=== CONT TestAuthMethodUpdateCommand_config +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestAuthMethodUpdateCommand_k8s_noMerge -=== CONT TestAuthMethodUpdateCommand_noMerge === CONT TestAuthMethodUpdateCommand_JSON -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... -=== CONT TestAuthMethodUpdateCommand_config +=== CONT TestAuthMethodUpdateCommand_noMerge === CONT TestAuthMethodUpdateCommand -=== CONT TestAuthMethodUpdateCommand_k8s --- PASS: TestAuthMethodUpdateCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -=== RUN TestAuthMethodUpdateCommand/update_without_name -=== RUN TestAuthMethodUpdateCommand/update_nonexistent_method -=== RUN TestAuthMethodUpdateCommand/update_all_fields -=== RUN TestAuthMethodUpdateCommand/update_config_field_and_prove_no_merging_happens +=== RUN TestAuthMethodUpdateCommand_JSON/update_without_name +=== RUN TestAuthMethodUpdateCommand_JSON/update_all_fields === RUN TestAuthMethodUpdateCommand_k8s_noMerge/update_missing_k8s_host === RUN TestAuthMethodUpdateCommand_k8s_noMerge/update_missing_k8s_ca_cert === RUN TestAuthMethodUpdateCommand_k8s_noMerge/update_missing_k8s_jwt === RUN TestAuthMethodUpdateCommand_k8s_noMerge/update_all_fields === RUN TestAuthMethodUpdateCommand_k8s_noMerge/update_all_fields_with_cert_file -=== RUN TestAuthMethodUpdateCommand_JSON/update_without_name -=== RUN TestAuthMethodUpdateCommand_JSON/update_all_fields -=== RUN TestAuthMethodUpdateCommand_noMerge/update_without_name -=== RUN TestAuthMethodUpdateCommand_noMerge/update_nonexistent_method -=== RUN TestAuthMethodUpdateCommand_noMerge/update_all_fields === RUN TestAuthMethodUpdateCommand_k8s/update_all_fields -=== RUN TestAuthMethodUpdateCommand_config/config_file -=== RUN TestAuthMethodUpdateCommand_config/config_stdin -=== RUN TestAuthMethodUpdateCommand_config/config_string === RUN TestAuthMethodUpdateCommand_k8s/update_all_fields_with_cert_file -=== RUN TestAuthMethodUpdateCommand_config/config_with_no_merge === RUN TestAuthMethodUpdateCommand_k8s/update_all_fields_but_k8s_host === RUN TestAuthMethodUpdateCommand_k8s/update_all_fields_but_k8s_ca_cert === RUN TestAuthMethodUpdateCommand_k8s/update_all_fields_but_k8s_jwt -2022-09-15T21:41:30.054Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:30.107Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:30.110 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:69f64baf-1399-248d-64a8-e6e06e919cc9 Address:127.0.0.1:20244}]" -41:30.110 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:20244 [Follower]" leader= -41:30.111 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-69f64baf-1399-248d-64a8-e6e06e919cc9.dc1 127.0.0.1 -41:30.147 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-69f64baf-1399-248d-64a8-e6e06e919cc9 127.0.0.1 -2022-09-15T21:41:30.147Z [INFO] agent.router: Initializing LAN area manager -41:30.148 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20239 network=udp -41:30.150 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-69f64baf-1399-248d-64a8-e6e06e919cc9.dc1 area=wan -41:30.152 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20239 network=tcp -41:30.153 [INFO] TestAgent.server: Adding LAN server: server="Node-69f64baf-1399-248d-64a8-e6e06e919cc9 (Addr: tcp/127.0.0.1:20244) (DC: dc1)" -41:30.160 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:20240 network=tcp -41:30.160 [INFO] TestAgent: started state syncer -41:30.160 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:20245 network=tcp -41:30.161 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:30.161 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:20244 [Candidate]" term=2 -41:30.163 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:30.163 [DEBUG] TestAgent.server.raft: vote granted: from=69f64baf-1399-248d-64a8-e6e06e919cc9 term=2 tally=1 -41:30.163 [INFO] TestAgent.server.raft: election won: tally=1 -41:30.163 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:20244 [Leader]" -41:30.163 [INFO] TestAgent.server: cluster leadership acquired -41:30.163 [INFO] TestAgent.server: initializing acls -41:30.164 [INFO] TestAgent.server: Created ACL 'global-management' policy -41:30.164 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -41:30.164 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -41:30.165 [INFO] TestAgent.server: New leader elected: payload=Node-69f64baf-1399-248d-64a8-e6e06e919cc9 -41:30.165 [INFO] TestAgent.server: Created ACL anonymous token from configuration -41:30.165 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -41:30.165 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -41:30.165 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-69f64baf-1399-248d-64a8-e6e06e919cc9 -41:30.165 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-69f64baf-1399-248d-64a8-e6e06e919cc9.dc1 -41:30.165 [INFO] TestAgent.server: Updating LAN server: server="Node-69f64baf-1399-248d-64a8-e6e06e919cc9 (Addr: tcp/127.0.0.1:20244) (DC: dc1)" -41:30.165 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-69f64baf-1399-248d-64a8-e6e06e919cc9.dc1 area=wan -41:30.166 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:30.188 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:30.188 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:30.188 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:30.188 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:30.188 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:30.189 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-69f64baf-1399-248d-64a8-e6e06e919cc9 -41:30.189 [INFO] TestAgent.server: member joined, marking health alive: member=Node-69f64baf-1399-248d-64a8-e6e06e919cc9 -41:30.192 [INFO] TestAgent.server: federation state anti-entropy synced -41:30.193 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-69f64baf-1399-248d-64a8-e6e06e919cc9 -41:30.280 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-5df7c289-955e-f639-956f-da20b908e496 from=127.0.0.1:56628 latency=876.46µs -41:30.283 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:56630 latency=1.28804ms -41:30.298 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-9d169f75-a680-6e96-f9f8-f0d9d61e5d08 from=127.0.0.1:56632 latency=6.84142ms -41:30.304 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/test-9d169f75-a680-6e96-f9f8-f0d9d61e5d08 from=127.0.0.1:56632 latency=1.68404ms -41:30.306 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-9d169f75-a680-6e96-f9f8-f0d9d61e5d08 from=127.0.0.1:56630 latency=431.06µs -41:30.308 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-9d169f75-a680-6e96-f9f8-f0d9d61e5d08 from=127.0.0.1:56634 latency=431.08µs -41:30.310 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/test-9d169f75-a680-6e96-f9f8-f0d9d61e5d08 from=127.0.0.1:56634 latency=883.7µs -41:30.311 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-9d169f75-a680-6e96-f9f8-f0d9d61e5d08 from=127.0.0.1:56630 latency=204.28µs -41:30.312 [INFO] TestAgent: Requesting shutdown -41:30.312 [INFO] TestAgent.server: shutting down server -41:30.312 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -41:30.312 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -41:30.312 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:30.312 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:30.312 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:30.312 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:30.312 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:30.312 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:30.312 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:30.312 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:30.312 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -41:30.312 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -41:30.312 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:30.312 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:30.312 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:30.312 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:30.312Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:30.312Z [INFO] agent.router.manager: shutting down -41:30.313 [INFO] TestAgent: consul server down -41:30.313 [INFO] TestAgent: shutdown complete -41:30.313 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20239 network=tcp -41:30.313 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20239 network=udp -41:30.313 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:20240 network=tcp -41:30.813 [INFO] TestAgent: Waiting for endpoints to shut down -41:30.813 [INFO] TestAgent: Endpoints down ---- PASS: TestAuthMethodUpdateCommand (0.90s) - --- PASS: TestAuthMethodUpdateCommand/update_without_name (0.00s) - --- PASS: TestAuthMethodUpdateCommand/update_nonexistent_method (0.00s) - --- PASS: TestAuthMethodUpdateCommand/update_all_fields (0.02s) - --- PASS: TestAuthMethodUpdateCommand/update_config_field_and_prove_no_merging_happens (0.01s) -2022-09-15T21:41:30.020Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:30.057Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:30.066 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:71f4be5e-ba4e-def7-503e-7ea7915daec6 Address:127.0.0.1:20223}]" -41:30.067 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-71f4be5e-ba4e-def7-503e-7ea7915daec6.dc1 127.0.0.1 -41:30.069 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-71f4be5e-ba4e-def7-503e-7ea7915daec6 127.0.0.1 -2022-09-15T21:41:30.069Z [INFO] agent.router: Initializing LAN area manager -41:30.069 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20218 network=udp -41:30.069 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:20223 [Follower]" leader= -41:30.072 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20218 network=tcp -41:30.073 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-71f4be5e-ba4e-def7-503e-7ea7915daec6.dc1 area=wan -41:30.078 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:20219 network=tcp -41:30.078 [INFO] TestAgent: started state syncer -41:30.079 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:20224 network=tcp -41:30.081 [INFO] TestAgent.server: Adding LAN server: server="Node-71f4be5e-ba4e-def7-503e-7ea7915daec6 (Addr: tcp/127.0.0.1:20223) (DC: dc1)" -41:30.123 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -41:30.125 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:30.125 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:20223 [Candidate]" term=2 -41:30.125 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:30.125 [DEBUG] TestAgent.server.raft: vote granted: from=71f4be5e-ba4e-def7-503e-7ea7915daec6 term=2 tally=1 -41:30.125 [INFO] TestAgent.server.raft: election won: tally=1 -41:30.125 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:20223 [Leader]" -41:30.125 [INFO] TestAgent.server: cluster leadership acquired -41:30.125 [INFO] TestAgent.server: New leader elected: payload=Node-71f4be5e-ba4e-def7-503e-7ea7915daec6 -41:30.126 [INFO] TestAgent.server: initializing acls -41:30.126 [INFO] TestAgent.server: Created ACL 'global-management' policy -41:30.126 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -41:30.127 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -41:30.128 [INFO] TestAgent.server: Created ACL anonymous token from configuration -41:30.128 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -41:30.128 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -41:30.128 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-71f4be5e-ba4e-def7-503e-7ea7915daec6 -41:30.128 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-71f4be5e-ba4e-def7-503e-7ea7915daec6.dc1 -41:30.129 [INFO] TestAgent.server: Updating LAN server: server="Node-71f4be5e-ba4e-def7-503e-7ea7915daec6 (Addr: tcp/127.0.0.1:20223) (DC: dc1)" -41:30.129 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-71f4be5e-ba4e-def7-503e-7ea7915daec6.dc1 area=wan -41:30.161 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:30.192 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:30.192 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:30.192 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:30.192 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:30.192 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:30.193 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-71f4be5e-ba4e-def7-503e-7ea7915daec6 -41:30.193 [INFO] TestAgent.server: member joined, marking health alive: member=Node-71f4be5e-ba4e-def7-503e-7ea7915daec6 -41:30.193 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-71f4be5e-ba4e-def7-503e-7ea7915daec6 -41:30.194 [INFO] TestAgent.server: federation state anti-entropy synced -41:30.267 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:30.267 [INFO] TestAgent: Synced node info -41:30.328 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:56516 latency=3.88132ms -41:30.332 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-34ecdf39-4b2d-628c-f270-743680a27f49 from=127.0.0.1:56518 latency=476.8µs -41:30.336 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:56516 latency=2.40262ms -41:30.339 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-8c786a91-782c-6cb6-71b2-6aa13198cd8d from=127.0.0.1:56520 latency=440.66µs -41:30.345 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:56516 latency=2.61328ms -41:30.349 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-e15f702c-df3b-4d82-6421-1141fb3e72d4 from=127.0.0.1:56522 latency=411.32µs -41:30.352 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:56516 latency=2.27906ms -41:30.356 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-8a87f6a2-becf-2119-af0c-41b11773f8d9 from=127.0.0.1:56524 latency=434.38µs -41:30.360 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/k8s-8a87f6a2-becf-2119-af0c-41b11773f8d9 from=127.0.0.1:56524 latency=2.91768ms -41:30.362 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-8a87f6a2-becf-2119-af0c-41b11773f8d9 from=127.0.0.1:56516 latency=301.02µs -41:30.366 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:56516 latency=2.3277ms -41:30.371 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-7a154ab1-d953-d3e5-99eb-6c2be8c97aff from=127.0.0.1:56526 latency=429.5µs -41:30.375 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/k8s-7a154ab1-d953-d3e5-99eb-6c2be8c97aff from=127.0.0.1:56526 latency=2.4789ms -41:30.377 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-7a154ab1-d953-d3e5-99eb-6c2be8c97aff from=127.0.0.1:56516 latency=248.1µs -41:30.378 [INFO] TestAgent: Requesting shutdown -41:30.378 [INFO] TestAgent.server: shutting down server -41:30.378 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:30.378 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:30.378 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -41:30.378 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -41:30.378 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:30.378 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:30.378 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:30.378 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:30.378 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:30.378 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:30.378 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:30.378 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -41:30.378 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -41:30.378 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:30.378 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:30.378Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:30.378Z [INFO] agent.router.manager: shutting down -41:30.379 [INFO] TestAgent: consul server down -41:30.379 [INFO] TestAgent: shutdown complete -41:30.379 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20218 network=tcp -41:30.379 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20218 network=udp -41:30.379 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:20219 network=tcp -41:30.879 [INFO] TestAgent: Waiting for endpoints to shut down -41:30.879 [INFO] TestAgent: Endpoints down +=== RUN TestAuthMethodUpdateCommand_config/config_file +=== RUN TestAuthMethodUpdateCommand_config/config_stdin +=== RUN TestAuthMethodUpdateCommand/update_without_name +=== RUN TestAuthMethodUpdateCommand/update_nonexistent_method +=== RUN TestAuthMethodUpdateCommand_config/config_string +=== RUN TestAuthMethodUpdateCommand/update_all_fields +=== RUN TestAuthMethodUpdateCommand_config/config_with_no_merge +=== RUN TestAuthMethodUpdateCommand/update_config_field_and_prove_no_merging_happens +=== RUN TestAuthMethodUpdateCommand_noMerge/update_without_name +=== RUN TestAuthMethodUpdateCommand_noMerge/update_nonexistent_method +=== RUN TestAuthMethodUpdateCommand_noMerge/update_all_fields +2021-08-13T15:33:27.205Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:27.237Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:27.240 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3b556d53-b89b-cb38-41c1-de1fcb740b37 Address:127.0.0.1:19223}]" +33:27.240 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19223 [Follower]" leader= +33:27.243 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3b556d53-b89b-cb38-41c1-de1fcb740b37.dc1 127.0.0.1 +33:27.243 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3b556d53-b89b-cb38-41c1-de1fcb740b37 127.0.0.1 +2021-08-13T15:33:27.244Z [INFO] agent.router: Initializing LAN area manager +33:27.244 [INFO] TestAgent.server: Adding LAN server: server="Node-3b556d53-b89b-cb38-41c1-de1fcb740b37 (Addr: tcp/127.0.0.1:19223) (DC: dc1)" +33:27.244 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3b556d53-b89b-cb38-41c1-de1fcb740b37.dc1 area=wan +33:27.244 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19218 network=udp +33:27.244 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19218 network=tcp +33:27.245 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19219 network=tcp +33:27.294 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +33:27.300 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:27.300 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19223 [Candidate]" term=2 +33:27.300 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:27.300 [DEBUG] TestAgent.server.raft: vote granted: from=3b556d53-b89b-cb38-41c1-de1fcb740b37 term=2 tally=1 +33:27.300 [INFO] TestAgent.server.raft: election won: tally=1 +33:27.300 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19223 [Leader]" +33:27.300 [INFO] TestAgent.server: cluster leadership acquired +33:27.508 [INFO] TestAgent.server: New leader elected: payload=Node-3b556d53-b89b-cb38-41c1-de1fcb740b37 +33:27.508 [INFO] TestAgent: started state syncer +33:27.508 [INFO] TestAgent.server: initializing acls +33:27.509 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="ACL not found" +33:27.509 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19224 network=tcp +33:27.509 [INFO] TestAgent.server: Created ACL 'global-management' policy +33:27.509 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +33:27.510 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +33:27.511 [INFO] TestAgent.server: Created ACL anonymous token from configuration +33:27.511 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +33:27.511 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +33:27.515 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-3b556d53-b89b-cb38-41c1-de1fcb740b37 +33:27.516 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-3b556d53-b89b-cb38-41c1-de1fcb740b37.dc1 +33:27.516 [INFO] TestAgent.server: Updating LAN server: server="Node-3b556d53-b89b-cb38-41c1-de1fcb740b37 (Addr: tcp/127.0.0.1:19223) (DC: dc1)" +33:27.520 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-3b556d53-b89b-cb38-41c1-de1fcb740b37.dc1 area=wan +33:27.522 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:27.528 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:27.528 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:27.528 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:27.528 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:27.528 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:27.528 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3b556d53-b89b-cb38-41c1-de1fcb740b37 +33:27.528 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3b556d53-b89b-cb38-41c1-de1fcb740b37 +33:27.530 [INFO] TestAgent.server: federation state anti-entropy synced +33:27.530 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3b556d53-b89b-cb38-41c1-de1fcb740b37 +33:27.537 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:40796 latency=1.641523ms +33:27.540 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-8a8fe8d3-9937-921d-4268-7ae77b977195 from=127.0.0.1:40798 latency=381.461µs +33:27.541 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/test-8a8fe8d3-9937-921d-4268-7ae77b977195 from=127.0.0.1:40798 latency=687.901µs +33:27.542 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-8a8fe8d3-9937-921d-4268-7ae77b977195 from=127.0.0.1:40796 latency=272.841µs +33:27.543 [INFO] TestAgent: Requesting shutdown +33:27.543 [INFO] TestAgent.server: shutting down server +33:27.543 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:27.543 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:27.543 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:27.543 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:27.543 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +33:27.543 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:27.543 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +33:27.543 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:27.543 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +33:27.543 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:27.543 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:27.543 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:27.543 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:27.543 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +33:27.543 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:27.543Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:27.543Z [INFO] agent.router.manager: shutting down +33:27.544 [INFO] TestAgent: consul server down +33:27.544 [INFO] TestAgent: shutdown complete +33:27.544 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19218 network=tcp +33:27.544 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19218 network=udp +33:27.544 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19219 network=tcp +33:28.044 [INFO] TestAgent: Waiting for endpoints to shut down +33:28.044 [INFO] TestAgent: Endpoints down +--- PASS: TestAuthMethodUpdateCommand_JSON (0.91s) + --- PASS: TestAuthMethodUpdateCommand_JSON/update_without_name (0.00s) + --- PASS: TestAuthMethodUpdateCommand_JSON/update_all_fields (0.01s) +2021-08-13T15:33:27.204Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:27.238Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:27.241 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:371c3827-5a41-3ded-ff88-eafafef14d2a Address:127.0.0.1:19209}]" +33:27.242 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19209 [Follower]" leader= +33:27.243 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-371c3827-5a41-3ded-ff88-eafafef14d2a.dc1 127.0.0.1 +33:27.244 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-371c3827-5a41-3ded-ff88-eafafef14d2a 127.0.0.1 +2021-08-13T15:33:27.244Z [INFO] agent.router: Initializing LAN area manager +33:27.245 [INFO] TestAgent.server: Adding LAN server: server="Node-371c3827-5a41-3ded-ff88-eafafef14d2a (Addr: tcp/127.0.0.1:19209) (DC: dc1)" +33:27.245 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-371c3827-5a41-3ded-ff88-eafafef14d2a.dc1 area=wan +33:27.245 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19204 network=udp +33:27.245 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19204 network=tcp +33:27.282 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:27.282 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19209 [Candidate]" term=2 +33:27.282 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:27.282 [DEBUG] TestAgent.server.raft: vote granted: from=371c3827-5a41-3ded-ff88-eafafef14d2a term=2 tally=1 +33:27.282 [INFO] TestAgent.server.raft: election won: tally=1 +33:27.282 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19209 [Leader]" +33:27.282 [INFO] TestAgent.server: cluster leadership acquired +33:27.282 [INFO] TestAgent.server: New leader elected: payload=Node-371c3827-5a41-3ded-ff88-eafafef14d2a +33:27.282 [INFO] TestAgent.server: initializing acls +33:27.284 [INFO] TestAgent.server: Created ACL 'global-management' policy +33:27.284 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +33:27.284 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +33:27.285 [INFO] TestAgent.server: Created ACL anonymous token from configuration +33:27.285 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +33:27.285 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +33:27.285 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-371c3827-5a41-3ded-ff88-eafafef14d2a +33:27.285 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-371c3827-5a41-3ded-ff88-eafafef14d2a.dc1 +33:27.286 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:27.286 [INFO] TestAgent.server: Updating LAN server: server="Node-371c3827-5a41-3ded-ff88-eafafef14d2a (Addr: tcp/127.0.0.1:19209) (DC: dc1)" +33:27.286 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-371c3827-5a41-3ded-ff88-eafafef14d2a.dc1 area=wan +33:27.511 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19205 network=tcp +33:27.511 [INFO] TestAgent: started state syncer +33:27.511 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19210 network=tcp +33:27.521 [INFO] TestAgent: Synced node info +33:27.521 [DEBUG] TestAgent: Node info in sync +33:27.527 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:27.527 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:27.527 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:27.527 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:27.527 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:27.527 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-371c3827-5a41-3ded-ff88-eafafef14d2a +33:27.527 [INFO] TestAgent.server: member joined, marking health alive: member=Node-371c3827-5a41-3ded-ff88-eafafef14d2a +33:27.531 [INFO] TestAgent.server: federation state anti-entropy synced +33:27.531 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-371c3827-5a41-3ded-ff88-eafafef14d2a +33:27.549 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:36016 latency=3.690509ms +33:27.569 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-57d69ab0-f810-900b-5e53-ec2b7ee24db0 from=127.0.0.1:36018 latency=513.822µs +33:27.572 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:36016 latency=2.007244ms +33:27.575 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-68eec664-50bc-fa9e-286d-e367e6c779db from=127.0.0.1:36020 latency=403.881µs +33:27.578 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:36016 latency=1.921584ms +33:27.581 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-20415e91-1707-0a21-5e0a-615e3899fa61 from=127.0.0.1:36022 latency=402.421µs +33:27.584 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:36016 latency=1.977084ms +33:27.587 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-0459b8e3-04e4-4b10-7a2d-5bddf2fba317 from=127.0.0.1:36024 latency=348.081µs +33:27.590 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/k8s-0459b8e3-04e4-4b10-7a2d-5bddf2fba317 from=127.0.0.1:36024 latency=2.070285ms +33:27.591 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-0459b8e3-04e4-4b10-7a2d-5bddf2fba317 from=127.0.0.1:36016 latency=217.9µs +33:27.595 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:36016 latency=1.811705ms +33:27.597 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-affce171-ec38-e9dc-f68a-23444e770339 from=127.0.0.1:36026 latency=377.001µs +33:27.600 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/k8s-affce171-ec38-e9dc-f68a-23444e770339 from=127.0.0.1:36026 latency=1.813984ms +33:27.602 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-affce171-ec38-e9dc-f68a-23444e770339 from=127.0.0.1:36016 latency=214.581µs +33:27.602 [INFO] TestAgent: Requesting shutdown +33:27.602 [INFO] TestAgent.server: shutting down server +33:27.602 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +33:27.602 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +33:27.603 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:27.603 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:27.603 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:27.603 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:27.603 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:27.603 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +33:27.603 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:27.603 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +33:27.603 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:27.603 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:27.603 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:27.603 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:27.603 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:27.603Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:27.603Z [INFO] agent.router.manager: shutting down +33:27.603 [INFO] TestAgent: consul server down +33:27.603 [INFO] TestAgent: shutdown complete +33:27.603 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19204 network=tcp +33:27.603 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19204 network=udp +33:27.603 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19205 network=tcp +33:28.103 [INFO] TestAgent: Waiting for endpoints to shut down +33:28.103 [INFO] TestAgent: Endpoints down --- PASS: TestAuthMethodUpdateCommand_k8s_noMerge (0.97s) - --- PASS: TestAuthMethodUpdateCommand_k8s_noMerge/update_missing_k8s_host (0.01s) + --- PASS: TestAuthMethodUpdateCommand_k8s_noMerge/update_missing_k8s_host (0.03s) --- PASS: TestAuthMethodUpdateCommand_k8s_noMerge/update_missing_k8s_ca_cert (0.01s) --- PASS: TestAuthMethodUpdateCommand_k8s_noMerge/update_missing_k8s_jwt (0.01s) --- PASS: TestAuthMethodUpdateCommand_k8s_noMerge/update_all_fields (0.01s) --- PASS: TestAuthMethodUpdateCommand_k8s_noMerge/update_all_fields_with_cert_file (0.01s) -2022-09-15T21:41:30.018Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:30.075Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:30.077 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:92a8643a-501e-9e31-4618-5728e6d31517 Address:127.0.0.1:20216}]" -41:30.078 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:20216 [Follower]" leader= -41:30.081 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-92a8643a-501e-9e31-4618-5728e6d31517.dc1 127.0.0.1 -41:30.087 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-92a8643a-501e-9e31-4618-5728e6d31517 127.0.0.1 -2022-09-15T21:41:30.087Z [INFO] agent.router: Initializing LAN area manager -41:30.101 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-92a8643a-501e-9e31-4618-5728e6d31517.dc1 area=wan -41:30.101 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20211 network=tcp -41:30.107 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20211 network=udp -41:30.107 [INFO] TestAgent.server: Adding LAN server: server="Node-92a8643a-501e-9e31-4618-5728e6d31517 (Addr: tcp/127.0.0.1:20216) (DC: dc1)" -41:30.108 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:20212 network=tcp -41:30.108 [INFO] TestAgent: started state syncer -41:30.108 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:20217 network=tcp -41:30.126 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:30.126 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:20216 [Candidate]" term=2 -41:30.126 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:30.126 [DEBUG] TestAgent.server.raft: vote granted: from=92a8643a-501e-9e31-4618-5728e6d31517 term=2 tally=1 -41:30.126 [INFO] TestAgent.server.raft: election won: tally=1 -41:30.126 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:20216 [Leader]" -41:30.126 [INFO] TestAgent.server: cluster leadership acquired -41:30.126 [INFO] TestAgent.server: initializing acls -41:30.126 [INFO] TestAgent.server: New leader elected: payload=Node-92a8643a-501e-9e31-4618-5728e6d31517 -41:30.128 [INFO] TestAgent.server: Created ACL 'global-management' policy -41:30.128 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -41:30.128 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -41:30.159 [INFO] TestAgent.server: initializing acls -41:30.159 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -41:30.159 [INFO] TestAgent.server: Created ACL anonymous token from configuration -41:30.160 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -41:30.160 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -41:30.160 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-92a8643a-501e-9e31-4618-5728e6d31517 -41:30.160 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-92a8643a-501e-9e31-4618-5728e6d31517.dc1 -41:30.161 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode -41:30.161 [INFO] TestAgent.server: Updating LAN server: server="Node-92a8643a-501e-9e31-4618-5728e6d31517 (Addr: tcp/127.0.0.1:20216) (DC: dc1)" -41:30.161 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-92a8643a-501e-9e31-4618-5728e6d31517 -41:30.161 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-92a8643a-501e-9e31-4618-5728e6d31517.dc1 area=wan -41:30.161 [INFO] TestAgent.server: Updating LAN server: server="Node-92a8643a-501e-9e31-4618-5728e6d31517 (Addr: tcp/127.0.0.1:20216) (DC: dc1)" -41:30.161 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-92a8643a-501e-9e31-4618-5728e6d31517.dc1 -41:30.161 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-92a8643a-501e-9e31-4618-5728e6d31517.dc1 area=wan -41:30.161 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:30.188 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:30.188 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:30.188 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:30.188 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:30.188 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:30.188 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-92a8643a-501e-9e31-4618-5728e6d31517 -41:30.188 [INFO] TestAgent.server: member joined, marking health alive: member=Node-92a8643a-501e-9e31-4618-5728e6d31517 -41:30.190 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-92a8643a-501e-9e31-4618-5728e6d31517 -41:30.191 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-92a8643a-501e-9e31-4618-5728e6d31517 -41:30.193 [INFO] TestAgent.server: federation state anti-entropy synced -41:30.381 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:54666 latency=3.90762ms -41:30.389 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-f4349fbb-3c5d-9d14-d8d5-3a72eef13cd2 from=127.0.0.1:54668 latency=1.55894ms -41:30.392 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/test-f4349fbb-3c5d-9d14-d8d5-3a72eef13cd2 from=127.0.0.1:54668 latency=1.26472ms -41:30.393 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-f4349fbb-3c5d-9d14-d8d5-3a72eef13cd2 from=127.0.0.1:54666 latency=564.16µs -41:30.398 [INFO] TestAgent: Requesting shutdown -41:30.398 [INFO] TestAgent.server: shutting down server -41:30.398 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:30.398 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:30.398 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -41:30.398 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -41:30.398 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:30.398 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:30.398 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:30.398 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:30.398 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:30.398 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:30.398 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:30.398 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:30.398 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -41:30.398 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -41:30.398 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:30.399 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:30.399Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:30.399Z [INFO] agent.router.manager: shutting down -41:30.399 [INFO] TestAgent: consul server down -41:30.399 [INFO] TestAgent: shutdown complete -41:30.399 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20211 network=tcp -41:30.399 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20211 network=udp -41:30.399 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:20212 network=tcp -41:30.901 [INFO] TestAgent: Waiting for endpoints to shut down -41:30.901 [INFO] TestAgent: Endpoints down ---- PASS: TestAuthMethodUpdateCommand_JSON (0.99s) - --- PASS: TestAuthMethodUpdateCommand_JSON/update_without_name (0.00s) - --- PASS: TestAuthMethodUpdateCommand_JSON/update_all_fields (0.02s) -2022-09-15T21:41:30.068Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:30.109Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:30.146 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9bc0b22b-c8c9-2825-ef98-84acc701bf63 Address:127.0.0.1:20230}]" -41:30.147 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9bc0b22b-c8c9-2825-ef98-84acc701bf63.dc1 127.0.0.1 -41:30.149 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9bc0b22b-c8c9-2825-ef98-84acc701bf63 127.0.0.1 -2022-09-15T21:41:30.149Z [INFO] agent.router: Initializing LAN area manager -41:30.150 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20225 network=udp -41:30.150 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:20230 [Follower]" leader= -41:30.152 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20225 network=tcp -41:30.159 [INFO] TestAgent.server: Adding LAN server: server="Node-9bc0b22b-c8c9-2825-ef98-84acc701bf63 (Addr: tcp/127.0.0.1:20230) (DC: dc1)" -41:30.159 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9bc0b22b-c8c9-2825-ef98-84acc701bf63.dc1 area=wan -41:30.161 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:20226 network=tcp -41:30.161 [INFO] TestAgent: started state syncer -41:30.161 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:20231 network=tcp -41:30.201 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -41:30.202 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:30.202 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:20230 [Candidate]" term=2 -41:30.202 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:30.202 [DEBUG] TestAgent.server.raft: vote granted: from=9bc0b22b-c8c9-2825-ef98-84acc701bf63 term=2 tally=1 -41:30.202 [INFO] TestAgent.server.raft: election won: tally=1 -41:30.202 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:20230 [Leader]" -41:30.202 [INFO] TestAgent.server: cluster leadership acquired -41:30.202 [INFO] TestAgent.server: New leader elected: payload=Node-9bc0b22b-c8c9-2825-ef98-84acc701bf63 -41:30.202 [INFO] TestAgent.server: initializing acls -41:30.203 [INFO] TestAgent.server: Created ACL 'global-management' policy -41:30.203 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -41:30.204 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -41:30.204 [INFO] TestAgent.server: Created ACL anonymous token from configuration -41:30.204 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -41:30.204 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -41:30.204 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-9bc0b22b-c8c9-2825-ef98-84acc701bf63 -41:30.205 [INFO] TestAgent.server: Updating LAN server: server="Node-9bc0b22b-c8c9-2825-ef98-84acc701bf63 (Addr: tcp/127.0.0.1:20230) (DC: dc1)" -41:30.205 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-9bc0b22b-c8c9-2825-ef98-84acc701bf63.dc1 -41:30.205 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-9bc0b22b-c8c9-2825-ef98-84acc701bf63.dc1 area=wan -41:30.206 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:30.208 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:30.208 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:30.208 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:30.208 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:30.208 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:30.208 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9bc0b22b-c8c9-2825-ef98-84acc701bf63 -41:30.208 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9bc0b22b-c8c9-2825-ef98-84acc701bf63 -41:30.210 [INFO] TestAgent.server: federation state anti-entropy synced -41:30.210 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9bc0b22b-c8c9-2825-ef98-84acc701bf63 -41:30.446 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-fad2a14d-1015-b89a-3ec6-e03c7c3e90c7 from=127.0.0.1:42686 latency=850.4µs -41:30.449 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:42688 latency=1.183ms -41:30.455 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-3c966049-8274-2441-4d3d-e8298ca43182 from=127.0.0.1:42690 latency=495.42µs -41:30.457 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/test-3c966049-8274-2441-4d3d-e8298ca43182 from=127.0.0.1:42690 latency=770.06µs -41:30.458 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-3c966049-8274-2441-4d3d-e8298ca43182 from=127.0.0.1:42688 latency=192.32µs -41:30.461 [INFO] TestAgent: Requesting shutdown -41:30.461 [INFO] TestAgent.server: shutting down server -41:30.461 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -41:30.461 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:30.461 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:30.461 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:30.461 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:30.461 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -41:30.461 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:30.461 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:30.461 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:30.461 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:30.461 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -41:30.461 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:30.461 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:30.462 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:30.462 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -41:30.462 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:30.462Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:30.462Z [INFO] agent.router.manager: shutting down -41:30.462 [INFO] TestAgent: consul server down -41:30.462 [INFO] TestAgent: shutdown complete -41:30.462 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20225 network=tcp -41:30.462 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20225 network=udp -41:30.462 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:20226 network=tcp -41:30.962 [INFO] TestAgent: Waiting for endpoints to shut down -41:30.962 [INFO] TestAgent: Endpoints down ---- PASS: TestAuthMethodUpdateCommand_noMerge (1.05s) - --- PASS: TestAuthMethodUpdateCommand_noMerge/update_without_name (0.00s) - --- PASS: TestAuthMethodUpdateCommand_noMerge/update_nonexistent_method (0.00s) - --- PASS: TestAuthMethodUpdateCommand_noMerge/update_all_fields (0.01s) -2022-09-15T21:41:30.041Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:30.082Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:30.107 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7ee0cda3-b637-25a2-f3e3-ade7cbbab675 Address:127.0.0.1:20237}]" -41:30.107 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:20237 [Follower]" leader= -41:30.108 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7ee0cda3-b637-25a2-f3e3-ade7cbbab675.dc1 127.0.0.1 -41:30.109 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7ee0cda3-b637-25a2-f3e3-ade7cbbab675 127.0.0.1 -2022-09-15T21:41:30.109Z [INFO] agent.router: Initializing LAN area manager -41:30.109 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7ee0cda3-b637-25a2-f3e3-ade7cbbab675.dc1 area=wan -41:30.109 [INFO] TestAgent.server: Adding LAN server: server="Node-7ee0cda3-b637-25a2-f3e3-ade7cbbab675 (Addr: tcp/127.0.0.1:20237) (DC: dc1)" -41:30.109 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20232 network=udp -41:30.109 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20232 network=tcp -41:30.110 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:20233 network=tcp -41:30.110 [INFO] TestAgent: started state syncer -41:30.110 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:20238 network=tcp -41:30.153 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:30.153 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:20237 [Candidate]" term=2 -41:30.153 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:30.153 [DEBUG] TestAgent.server.raft: vote granted: from=7ee0cda3-b637-25a2-f3e3-ade7cbbab675 term=2 tally=1 -41:30.153 [INFO] TestAgent.server.raft: election won: tally=1 -41:30.153 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:20237 [Leader]" -41:30.161 [INFO] TestAgent.server: cluster leadership acquired -41:30.161 [INFO] TestAgent.server: New leader elected: payload=Node-7ee0cda3-b637-25a2-f3e3-ade7cbbab675 -41:30.161 [INFO] TestAgent.server: initializing acls -41:30.161 [INFO] TestAgent.server: initializing acls -41:30.162 [INFO] TestAgent.server: Created ACL 'global-management' policy -41:30.162 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -41:30.163 [INFO] TestAgent.server: Created ACL 'global-management' policy -41:30.163 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -41:30.163 [INFO] TestAgent.server: Created ACL anonymous token from configuration -41:30.163 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -41:30.163 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -41:30.163 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode -41:30.164 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-7ee0cda3-b637-25a2-f3e3-ade7cbbab675 -41:30.164 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-7ee0cda3-b637-25a2-f3e3-ade7cbbab675.dc1 -41:30.166 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-7ee0cda3-b637-25a2-f3e3-ade7cbbab675.dc1 area=wan -41:30.165 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -41:30.167 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-7ee0cda3-b637-25a2-f3e3-ade7cbbab675 -41:30.167 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-7ee0cda3-b637-25a2-f3e3-ade7cbbab675.dc1 -41:30.165 [INFO] TestAgent.server: Updating LAN server: server="Node-7ee0cda3-b637-25a2-f3e3-ade7cbbab675 (Addr: tcp/127.0.0.1:20237) (DC: dc1)" -41:30.168 [INFO] TestAgent.server: Updating LAN server: server="Node-7ee0cda3-b637-25a2-f3e3-ade7cbbab675 (Addr: tcp/127.0.0.1:20237) (DC: dc1)" -41:30.168 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-7ee0cda3-b637-25a2-f3e3-ade7cbbab675.dc1 area=wan -41:30.168 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:30.190 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:30.190 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:30.190 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:30.190 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:30.190 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:30.190 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7ee0cda3-b637-25a2-f3e3-ade7cbbab675 -41:30.190 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7ee0cda3-b637-25a2-f3e3-ade7cbbab675 -41:30.191 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7ee0cda3-b637-25a2-f3e3-ade7cbbab675 -41:30.191 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7ee0cda3-b637-25a2-f3e3-ade7cbbab675 -41:30.192 [INFO] TestAgent.server: federation state anti-entropy synced -41:30.222 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:30.223 [INFO] TestAgent: Synced node info -41:30.530 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:57822 latency=1.61318ms -41:30.533 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/teste1e97f34-bc38-681a-d67f-581a94b9fa35 from=127.0.0.1:57824 latency=343.48µs -41:30.535 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/teste1e97f34-bc38-681a-d67f-581a94b9fa35 from=127.0.0.1:57824 latency=706.78µs -41:30.535 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/teste1e97f34-bc38-681a-d67f-581a94b9fa35 from=127.0.0.1:57822 latency=196.8µs -41:30.537 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:57822 latency=654.12µs -41:30.539 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test2291429f-60c9-e6bb-e97a-f680945daa6d from=127.0.0.1:57828 latency=298.52µs -41:30.541 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/test2291429f-60c9-e6bb-e97a-f680945daa6d from=127.0.0.1:57828 latency=739.44µs -41:30.542 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test2291429f-60c9-e6bb-e97a-f680945daa6d from=127.0.0.1:57822 latency=214.98µs -41:30.544 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:57822 latency=858.7µs -41:30.550 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test61b12d79-5343-4d16-f2b0-531d8520a346 from=127.0.0.1:57830 latency=379.38µs -41:30.552 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/test61b12d79-5343-4d16-f2b0-531d8520a346 from=127.0.0.1:57830 latency=771.78µs -41:30.553 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test61b12d79-5343-4d16-f2b0-531d8520a346 from=127.0.0.1:57822 latency=189.48µs -41:30.554 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:57822 latency=651.28µs -41:30.557 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test26a8181c-2e96-6d86-3787-a752f2d20c63 from=127.0.0.1:57834 latency=263.3µs -41:30.563 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/test26a8181c-2e96-6d86-3787-a752f2d20c63 from=127.0.0.1:57834 latency=889.18µs -41:30.566 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test26a8181c-2e96-6d86-3787-a752f2d20c63 from=127.0.0.1:57822 latency=210.32µs -41:30.566 [INFO] TestAgent: Requesting shutdown -41:30.566 [INFO] TestAgent.server: shutting down server -41:30.566 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:30.566 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:30.566 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:30.566 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -41:30.566 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -41:30.566 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:30.566 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:30.571 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:30.572Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:30.572Z [INFO] agent.router.manager: shutting down -41:30.572 [INFO] TestAgent: consul server down -41:30.572 [INFO] TestAgent: shutdown complete -41:30.572 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20232 network=tcp -41:30.572 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20232 network=udp -41:30.572 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:20233 network=tcp -41:30.573 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:30.573 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:30.573 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:30.573 [DEBUG] TestAgent.server: shutting down leader loop -41:30.573 [INFO] TestAgent.server: cluster leadership lost -41:30.573 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:30.573 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -41:30.573 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -41:30.573 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:31.074 [INFO] TestAgent: Waiting for endpoints to shut down -41:31.074 [INFO] TestAgent: Endpoints down ---- PASS: TestAuthMethodUpdateCommand_config (1.16s) +2021-08-13T15:33:27.195Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:27.222Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:27.230 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:398e04ff-9e29-8fa4-b9c7-ca65474ab7a6 Address:127.0.0.1:19202}]" +33:27.231 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-398e04ff-9e29-8fa4-b9c7-ca65474ab7a6.dc1 127.0.0.1 +33:27.232 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-398e04ff-9e29-8fa4-b9c7-ca65474ab7a6 127.0.0.1 +2021-08-13T15:33:27.232Z [INFO] agent.router: Initializing LAN area manager +33:27.232 [INFO] TestAgent.server: Adding LAN server: server="Node-398e04ff-9e29-8fa4-b9c7-ca65474ab7a6 (Addr: tcp/127.0.0.1:19202) (DC: dc1)" +33:27.232 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19197 network=udp +33:27.232 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-398e04ff-9e29-8fa4-b9c7-ca65474ab7a6.dc1 area=wan +33:27.233 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19197 network=tcp +33:27.234 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19198 network=tcp +33:27.234 [INFO] TestAgent: started state syncer +33:27.234 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19203 network=tcp +33:27.234 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19202 [Follower]" leader= +33:27.286 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +33:27.511 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +33:27.515 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:27.515 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19202 [Candidate]" term=2 +33:27.515 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:27.515 [DEBUG] TestAgent.server.raft: vote granted: from=398e04ff-9e29-8fa4-b9c7-ca65474ab7a6 term=2 tally=1 +33:27.515 [INFO] TestAgent.server.raft: election won: tally=1 +33:27.515 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19202 [Leader]" +33:27.515 [INFO] TestAgent.server: cluster leadership acquired +33:27.516 [INFO] TestAgent.server: initializing acls +33:27.516 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="ACL not found" +33:27.516 [INFO] TestAgent.server: New leader elected: payload=Node-398e04ff-9e29-8fa4-b9c7-ca65474ab7a6 +33:27.516 [INFO] TestAgent.server: Created ACL 'global-management' policy +33:27.516 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +33:27.521 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +33:27.522 [INFO] TestAgent.server: Created ACL anonymous token from configuration +33:27.522 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +33:27.522 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +33:27.523 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-398e04ff-9e29-8fa4-b9c7-ca65474ab7a6 +33:27.523 [INFO] TestAgent.server: Updating LAN server: server="Node-398e04ff-9e29-8fa4-b9c7-ca65474ab7a6 (Addr: tcp/127.0.0.1:19202) (DC: dc1)" +33:27.523 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-398e04ff-9e29-8fa4-b9c7-ca65474ab7a6.dc1 +33:27.523 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-398e04ff-9e29-8fa4-b9c7-ca65474ab7a6.dc1 area=wan +33:27.525 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:27.529 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:27.529 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:27.529 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:27.529 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:27.529 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:27.529 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-398e04ff-9e29-8fa4-b9c7-ca65474ab7a6 +33:27.529 [INFO] TestAgent.server: member joined, marking health alive: member=Node-398e04ff-9e29-8fa4-b9c7-ca65474ab7a6 +33:27.530 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-398e04ff-9e29-8fa4-b9c7-ca65474ab7a6 +33:27.531 [INFO] TestAgent.server: federation state anti-entropy synced +33:27.600 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:47268 latency=2.711487ms +33:27.602 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-f9ddfe74-5db6-cf32-d81b-6af64b247f43 from=127.0.0.1:47270 latency=274.64µs +33:27.606 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/k8s-f9ddfe74-5db6-cf32-d81b-6af64b247f43 from=127.0.0.1:47270 latency=2.153645ms +33:27.607 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-f9ddfe74-5db6-cf32-d81b-6af64b247f43 from=127.0.0.1:47268 latency=232.26µs +33:27.610 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:47268 latency=1.974164ms +33:27.628 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-496ee7d4-29ac-10c2-dfc9-8b9719aa9ba8 from=127.0.0.1:47272 latency=775.622µs +33:27.633 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/k8s-496ee7d4-29ac-10c2-dfc9-8b9719aa9ba8 from=127.0.0.1:47272 latency=3.053027ms +33:27.635 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-496ee7d4-29ac-10c2-dfc9-8b9719aa9ba8 from=127.0.0.1:47268 latency=492.541µs +33:27.639 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:47268 latency=1.910984ms +33:27.641 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-3b8ed9da-aa1a-47b9-d48d-fac2ec7d620d from=127.0.0.1:47274 latency=411.881µs +33:27.645 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/k8s-3b8ed9da-aa1a-47b9-d48d-fac2ec7d620d from=127.0.0.1:47274 latency=2.062205ms +33:27.647 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-3b8ed9da-aa1a-47b9-d48d-fac2ec7d620d from=127.0.0.1:47268 latency=406.281µs +33:27.651 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:47268 latency=2.017165ms +33:27.656 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-acd02d81-2d3b-f887-5ff0-cdd9e6066d2c from=127.0.0.1:47276 latency=355.941µs +33:27.659 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/k8s-acd02d81-2d3b-f887-5ff0-cdd9e6066d2c from=127.0.0.1:47276 latency=1.901264ms +33:27.660 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-acd02d81-2d3b-f887-5ff0-cdd9e6066d2c from=127.0.0.1:47268 latency=215.681µs +33:27.663 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:47268 latency=1.953304ms +33:27.666 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-b4cbf80f-446b-2a2a-f53f-03223560cf63 from=127.0.0.1:47278 latency=332.961µs +33:27.669 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/k8s-b4cbf80f-446b-2a2a-f53f-03223560cf63 from=127.0.0.1:47278 latency=2.045685ms +33:27.670 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-b4cbf80f-446b-2a2a-f53f-03223560cf63 from=127.0.0.1:47268 latency=207.12µs +33:27.671 [INFO] TestAgent: Requesting shutdown +33:27.671 [INFO] TestAgent.server: shutting down server +33:27.671 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:27.671 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:27.671 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:27.671 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:27.671 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +33:27.671 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +33:27.671 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:27.671 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:27.671 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:27.671 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:27.671 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +33:27.671 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +33:27.671 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:27.671 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:27.671 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:27.672Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:27.672Z [INFO] agent.router.manager: shutting down +33:27.672 [INFO] TestAgent: consul server down +33:27.672 [INFO] TestAgent: shutdown complete +33:27.672 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19197 network=tcp +33:27.672 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19197 network=udp +33:27.672 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19198 network=tcp +33:28.172 [INFO] TestAgent: Waiting for endpoints to shut down +33:28.172 [INFO] TestAgent: Endpoints down +--- PASS: TestAuthMethodUpdateCommand_k8s (1.04s) + --- PASS: TestAuthMethodUpdateCommand_k8s/update_all_fields (0.01s) + --- PASS: TestAuthMethodUpdateCommand_k8s/update_all_fields_with_cert_file (0.03s) + --- PASS: TestAuthMethodUpdateCommand_k8s/update_all_fields_but_k8s_host (0.01s) + --- PASS: TestAuthMethodUpdateCommand_k8s/update_all_fields_but_k8s_ca_cert (0.01s) + --- PASS: TestAuthMethodUpdateCommand_k8s/update_all_fields_but_k8s_jwt (0.01s) +2021-08-13T15:33:27.200Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:27.236Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:27.240 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:99cde13d-45a4-a42e-da38-63c00f5195ed Address:127.0.0.1:19230}]" +33:27.241 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-99cde13d-45a4-a42e-da38-63c00f5195ed.dc1 127.0.0.1 +33:27.241 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19230 [Follower]" leader= +33:27.243 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-99cde13d-45a4-a42e-da38-63c00f5195ed 127.0.0.1 +2021-08-13T15:33:27.243Z [INFO] agent.router: Initializing LAN area manager +33:27.244 [INFO] TestAgent.server: Adding LAN server: server="Node-99cde13d-45a4-a42e-da38-63c00f5195ed (Addr: tcp/127.0.0.1:19230) (DC: dc1)" +33:27.244 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-99cde13d-45a4-a42e-da38-63c00f5195ed.dc1 area=wan +33:27.244 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19225 network=udp +33:27.244 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19225 network=tcp +33:27.245 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19226 network=tcp +33:27.245 [INFO] TestAgent: started state syncer +33:27.245 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19231 network=tcp +33:27.294 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +33:27.511 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +33:27.512 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:27.512 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19230 [Candidate]" term=2 +33:27.512 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:27.512 [DEBUG] TestAgent.server.raft: vote granted: from=99cde13d-45a4-a42e-da38-63c00f5195ed term=2 tally=1 +33:27.512 [INFO] TestAgent.server.raft: election won: tally=1 +33:27.512 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19230 [Leader]" +33:27.513 [INFO] TestAgent.server: cluster leadership acquired +33:27.513 [INFO] TestAgent.server: initializing acls +33:27.513 [INFO] TestAgent.server: Created ACL 'global-management' policy +33:27.513 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +33:27.514 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +33:27.515 [INFO] TestAgent.server: Created ACL anonymous token from configuration +33:27.515 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +33:27.515 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +33:27.515 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-99cde13d-45a4-a42e-da38-63c00f5195ed +33:27.516 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-99cde13d-45a4-a42e-da38-63c00f5195ed.dc1 +33:27.516 [INFO] TestAgent.server: New leader elected: payload=Node-99cde13d-45a4-a42e-da38-63c00f5195ed +33:27.518 [INFO] TestAgent.server: Updating LAN server: server="Node-99cde13d-45a4-a42e-da38-63c00f5195ed (Addr: tcp/127.0.0.1:19230) (DC: dc1)" +33:27.518 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-99cde13d-45a4-a42e-da38-63c00f5195ed.dc1 area=wan +33:27.521 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:27.530 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:27.530 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:27.530 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:27.530 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:27.530 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:27.530 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-99cde13d-45a4-a42e-da38-63c00f5195ed +33:27.530 [INFO] TestAgent.server: member joined, marking health alive: member=Node-99cde13d-45a4-a42e-da38-63c00f5195ed +33:27.531 [INFO] TestAgent.server: federation state anti-entropy synced +33:27.531 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-99cde13d-45a4-a42e-da38-63c00f5195ed +33:27.682 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-e6a67815-baef-b686-7879-4d13af695819 from=127.0.0.1:36924 latency=650.121µs +33:27.685 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:36926 latency=2.556466ms +33:27.693 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-36cc077a-cab7-5781-9707-6bc932b06ea9 from=127.0.0.1:36930 latency=248.441µs +33:27.695 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/test-36cc077a-cab7-5781-9707-6bc932b06ea9 from=127.0.0.1:36930 latency=732.582µs +33:27.696 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-36cc077a-cab7-5781-9707-6bc932b06ea9 from=127.0.0.1:36926 latency=161.341µs +33:27.698 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:27.698 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-36cc077a-cab7-5781-9707-6bc932b06ea9 from=127.0.0.1:36932 latency=356.101µs +33:27.698 [INFO] TestAgent: Synced node info +33:27.698 [DEBUG] TestAgent: Node info in sync +33:27.699 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/test-36cc077a-cab7-5781-9707-6bc932b06ea9 from=127.0.0.1:36932 latency=693.242µs +33:27.700 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-36cc077a-cab7-5781-9707-6bc932b06ea9 from=127.0.0.1:36926 latency=164.881µs +33:27.700 [INFO] TestAgent: Requesting shutdown +33:27.700 [INFO] TestAgent.server: shutting down server +33:27.700 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +33:27.700 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +33:27.700 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:27.700 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:27.700 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:27.700 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:27.700 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:27.701 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:27.701 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:27.701 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +33:27.701 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +33:27.701 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +33:27.701 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:27.701 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:27.701 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2021-08-13T15:33:27.701Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:27.701Z [INFO] agent.router.manager: shutting down +33:27.701 [INFO] TestAgent: consul server down +33:27.701 [INFO] TestAgent: shutdown complete +33:27.701 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19225 network=tcp +33:27.701 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19225 network=udp +33:27.701 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19226 network=tcp +33:28.201 [INFO] TestAgent: Waiting for endpoints to shut down +33:28.201 [INFO] TestAgent: Endpoints down +2021-08-13T15:33:27.196Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:27.223Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:27.227 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4ff5b903-83e1-fcd8-36e1-e76a9bb403ef Address:127.0.0.1:19195}]" +33:27.228 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19195 [Follower]" leader= +33:27.231 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4ff5b903-83e1-fcd8-36e1-e76a9bb403ef.dc1 127.0.0.1 +33:27.232 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4ff5b903-83e1-fcd8-36e1-e76a9bb403ef 127.0.0.1 +2021-08-13T15:33:27.232Z [INFO] agent.router: Initializing LAN area manager +33:27.233 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=udp +33:27.233 [INFO] TestAgent.server: Adding LAN server: server="Node-4ff5b903-83e1-fcd8-36e1-e76a9bb403ef (Addr: tcp/127.0.0.1:19195) (DC: dc1)" +33:27.233 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4ff5b903-83e1-fcd8-36e1-e76a9bb403ef.dc1 area=wan +33:27.233 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=tcp +33:27.234 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19191 network=tcp +33:27.234 [INFO] TestAgent: started state syncer +33:27.234 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19196 network=tcp +33:27.279 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:27.279 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19195 [Candidate]" term=2 +33:27.279 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:27.279 [DEBUG] TestAgent.server.raft: vote granted: from=4ff5b903-83e1-fcd8-36e1-e76a9bb403ef term=2 tally=1 +33:27.280 [INFO] TestAgent.server.raft: election won: tally=1 +33:27.280 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19195 [Leader]" +33:27.280 [INFO] TestAgent.server: cluster leadership acquired +33:27.280 [INFO] TestAgent.server: New leader elected: payload=Node-4ff5b903-83e1-fcd8-36e1-e76a9bb403ef +33:27.280 [INFO] TestAgent.server: initializing acls +33:27.280 [INFO] TestAgent.server: Created ACL 'global-management' policy +33:27.280 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +33:27.281 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +33:27.282 [INFO] TestAgent.server: Created ACL anonymous token from configuration +33:27.282 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +33:27.282 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +33:27.282 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-4ff5b903-83e1-fcd8-36e1-e76a9bb403ef +33:27.282 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-4ff5b903-83e1-fcd8-36e1-e76a9bb403ef.dc1 +33:27.282 [INFO] TestAgent.server: Updating LAN server: server="Node-4ff5b903-83e1-fcd8-36e1-e76a9bb403ef (Addr: tcp/127.0.0.1:19195) (DC: dc1)" +33:27.282 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-4ff5b903-83e1-fcd8-36e1-e76a9bb403ef.dc1 area=wan +33:27.283 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:27.526 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:27.526 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:27.526 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:27.526 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:27.526 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:27.526 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4ff5b903-83e1-fcd8-36e1-e76a9bb403ef +33:27.526 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4ff5b903-83e1-fcd8-36e1-e76a9bb403ef +33:27.529 [INFO] TestAgent.server: federation state anti-entropy synced +33:27.529 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4ff5b903-83e1-fcd8-36e1-e76a9bb403ef +33:27.559 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:27.560 [INFO] TestAgent: Synced node info +33:27.560 [DEBUG] TestAgent: Node info in sync +33:27.667 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:33962 latency=1.222243ms +33:27.669 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test14695180-e63c-bb41-5421-7315fc45e1f1 from=127.0.0.1:33964 latency=251µs +33:27.670 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/test14695180-e63c-bb41-5421-7315fc45e1f1 from=127.0.0.1:33964 latency=579.401µs +33:27.671 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test14695180-e63c-bb41-5421-7315fc45e1f1 from=127.0.0.1:33962 latency=159.72µs +33:27.676 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:33962 latency=851.322µs +33:27.679 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test21a263b6-8359-b3d0-af59-3cb5c0eb8247 from=127.0.0.1:33966 latency=333.961µs +33:27.680 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/test21a263b6-8359-b3d0-af59-3cb5c0eb8247 from=127.0.0.1:33966 latency=661.942µs +33:27.681 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test21a263b6-8359-b3d0-af59-3cb5c0eb8247 from=127.0.0.1:33962 latency=178.181µs +33:27.682 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:33962 latency=573.441µs +33:27.691 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/testd85127eb-ae4b-4481-ff0a-b10373478d9a from=127.0.0.1:33972 latency=436.321µs +33:27.694 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/testd85127eb-ae4b-4481-ff0a-b10373478d9a from=127.0.0.1:33972 latency=1.736384ms +33:27.695 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/testd85127eb-ae4b-4481-ff0a-b10373478d9a from=127.0.0.1:33962 latency=280.1µs +33:27.696 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:33962 latency=580.061µs +33:27.698 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test67971309-bcf9-d985-b5f6-88e5e38b7055 from=127.0.0.1:33978 latency=232.02µs +33:27.700 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/test67971309-bcf9-d985-b5f6-88e5e38b7055 from=127.0.0.1:33978 latency=717.661µs +33:27.700 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test67971309-bcf9-d985-b5f6-88e5e38b7055 from=127.0.0.1:33962 latency=154.78µs +33:27.701 [INFO] TestAgent: Requesting shutdown +33:27.701 [INFO] TestAgent.server: shutting down server +33:27.701 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +33:27.701 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +33:27.701 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:27.701 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:27.701 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:27.701 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:27.701 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:27.701 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +33:27.701 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:27.701 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +33:27.701 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:27.701 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:27.701 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:27.701 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:27.701 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:27.701Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:27.701Z [INFO] agent.router.manager: shutting down +33:27.701 [INFO] TestAgent: consul server down +33:27.701 [INFO] TestAgent: shutdown complete +33:27.701 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=tcp +33:27.701 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=udp +33:27.701 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19191 network=tcp +33:28.202 [INFO] TestAgent: Waiting for endpoints to shut down +33:28.202 [INFO] TestAgent: Endpoints down +--- PASS: TestAuthMethodUpdateCommand (1.07s) + --- PASS: TestAuthMethodUpdateCommand/update_without_name (0.00s) + --- PASS: TestAuthMethodUpdateCommand/update_nonexistent_method (0.00s) + --- PASS: TestAuthMethodUpdateCommand/update_all_fields (0.01s) + --- PASS: TestAuthMethodUpdateCommand/update_config_field_and_prove_no_merging_happens (0.00s) +--- PASS: TestAuthMethodUpdateCommand_config (1.07s) --- PASS: TestAuthMethodUpdateCommand_config/config_file (0.01s) --- PASS: TestAuthMethodUpdateCommand_config/config_stdin (0.01s) --- PASS: TestAuthMethodUpdateCommand_config/config_string (0.01s) --- PASS: TestAuthMethodUpdateCommand_config/config_with_no_merge (0.01s) -2022-09-15T21:41:30.024Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:30.062Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:30.082 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5a5aacc3-c73d-76bc-afda-dd87e0f69ac4 Address:127.0.0.1:20251}]" -41:30.084 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5a5aacc3-c73d-76bc-afda-dd87e0f69ac4.dc1 127.0.0.1 -41:30.087 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:20251 [Follower]" leader= -41:30.089 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5a5aacc3-c73d-76bc-afda-dd87e0f69ac4 127.0.0.1 -2022-09-15T21:41:30.090Z [INFO] agent.router: Initializing LAN area manager -41:30.098 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20246 network=udp -41:30.098 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20246 network=tcp -41:30.099 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:20247 network=tcp -41:30.099 [INFO] TestAgent: started state syncer -41:30.100 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:20252 network=tcp -41:30.101 [INFO] TestAgent.server: Adding LAN server: server="Node-5a5aacc3-c73d-76bc-afda-dd87e0f69ac4 (Addr: tcp/127.0.0.1:20251) (DC: dc1)" -41:30.101 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5a5aacc3-c73d-76bc-afda-dd87e0f69ac4.dc1 area=wan -41:30.150 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:30.150 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:20251 [Candidate]" term=2 -41:30.155 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -41:30.160 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:30.160 [DEBUG] TestAgent.server.raft: vote granted: from=5a5aacc3-c73d-76bc-afda-dd87e0f69ac4 term=2 tally=1 -41:30.160 [INFO] TestAgent.server.raft: election won: tally=1 -41:30.160 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:20251 [Leader]" -41:30.160 [INFO] TestAgent.server: cluster leadership acquired -41:30.161 [INFO] TestAgent.server: initializing acls -41:30.161 [INFO] TestAgent.server: New leader elected: payload=Node-5a5aacc3-c73d-76bc-afda-dd87e0f69ac4 -41:30.162 [INFO] TestAgent.server: Created ACL 'global-management' policy -41:30.162 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -41:30.162 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -41:30.163 [INFO] TestAgent.server: Created ACL anonymous token from configuration -41:30.163 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -41:30.163 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -41:30.163 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-5a5aacc3-c73d-76bc-afda-dd87e0f69ac4 -41:30.163 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-5a5aacc3-c73d-76bc-afda-dd87e0f69ac4.dc1 -41:30.164 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:30.164 [INFO] TestAgent.server: Updating LAN server: server="Node-5a5aacc3-c73d-76bc-afda-dd87e0f69ac4 (Addr: tcp/127.0.0.1:20251) (DC: dc1)" -41:30.164 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-5a5aacc3-c73d-76bc-afda-dd87e0f69ac4.dc1 area=wan -41:30.188 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:30.188 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:30.188 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:30.188 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:30.188 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:30.189 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5a5aacc3-c73d-76bc-afda-dd87e0f69ac4 -41:30.189 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5a5aacc3-c73d-76bc-afda-dd87e0f69ac4 -41:30.194 [INFO] TestAgent.server: federation state anti-entropy synced -41:30.194 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5a5aacc3-c73d-76bc-afda-dd87e0f69ac4 -41:30.495 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:30.496 [INFO] TestAgent: Synced node info -41:30.531 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:38984 latency=2.8849ms -41:30.534 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-822f2c88-4a7a-ed3b-3946-e199ac1fb7a3 from=127.0.0.1:38990 latency=470.36µs -41:30.538 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/k8s-822f2c88-4a7a-ed3b-3946-e199ac1fb7a3 from=127.0.0.1:38990 latency=2.58966ms -41:30.541 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-822f2c88-4a7a-ed3b-3946-e199ac1fb7a3 from=127.0.0.1:38984 latency=411.84µs -41:30.546 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:38984 latency=2.9091ms -41:30.551 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-1a2aca95-664a-8ade-b4b2-1d4e589d3dc6 from=127.0.0.1:38996 latency=434.54µs -41:30.556 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/k8s-1a2aca95-664a-8ade-b4b2-1d4e589d3dc6 from=127.0.0.1:38996 latency=2.47394ms -41:30.557 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-1a2aca95-664a-8ade-b4b2-1d4e589d3dc6 from=127.0.0.1:38984 latency=238.66µs -41:30.565 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:38984 latency=2.23268ms -41:30.573 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-9b15c793-b354-e92d-1f4c-7259169ebebd from=127.0.0.1:39000 latency=623.08µs -41:30.581 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/k8s-9b15c793-b354-e92d-1f4c-7259169ebebd from=127.0.0.1:39000 latency=6.47406ms -41:30.587 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-9b15c793-b354-e92d-1f4c-7259169ebebd from=127.0.0.1:38984 latency=507.98µs -41:30.592 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:38984 latency=2.42554ms -41:30.596 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-f6a83d63-506b-37a3-ec0d-d2b9cc4a22b8 from=127.0.0.1:39002 latency=415.26µs -41:30.600 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/k8s-f6a83d63-506b-37a3-ec0d-d2b9cc4a22b8 from=127.0.0.1:39002 latency=2.328ms -41:30.602 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-f6a83d63-506b-37a3-ec0d-d2b9cc4a22b8 from=127.0.0.1:38984 latency=338.24µs -41:30.606 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:38984 latency=2.4517ms -41:30.609 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-dc9f7e8a-a992-aec3-4d6e-c17ae52f64e3 from=127.0.0.1:39004 latency=575.94µs -41:30.613 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/k8s-dc9f7e8a-a992-aec3-4d6e-c17ae52f64e3 from=127.0.0.1:39004 latency=2.16328ms -41:30.617 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-dc9f7e8a-a992-aec3-4d6e-c17ae52f64e3 from=127.0.0.1:38984 latency=343.1µs -41:30.619 [INFO] TestAgent: Requesting shutdown -41:30.619 [INFO] TestAgent.server: shutting down server -41:30.619 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -41:30.619 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -41:30.619 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:30.619 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:30.619 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:30.619 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:30.619 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:30.619 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:30.619 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:30.619 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:30.619 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -41:30.619 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -41:30.619 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:30.619 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:30.619 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:30.620Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:30.620Z [INFO] agent.router.manager: shutting down -41:30.620 [INFO] TestAgent: consul server down -41:30.620 [INFO] TestAgent: shutdown complete -41:30.620 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20246 network=tcp -41:30.620 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20246 network=udp -41:30.620 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:20247 network=tcp -41:31.120 [INFO] TestAgent: Waiting for endpoints to shut down -41:31.120 [INFO] TestAgent: Endpoints down ---- PASS: TestAuthMethodUpdateCommand_k8s (1.21s) - --- PASS: TestAuthMethodUpdateCommand_k8s/update_all_fields (0.01s) - --- PASS: TestAuthMethodUpdateCommand_k8s/update_all_fields_with_cert_file (0.02s) - --- PASS: TestAuthMethodUpdateCommand_k8s/update_all_fields_but_k8s_host (0.03s) - --- PASS: TestAuthMethodUpdateCommand_k8s/update_all_fields_but_k8s_ca_cert (0.02s) - --- PASS: TestAuthMethodUpdateCommand_k8s/update_all_fields_but_k8s_jwt (0.02s) +2021-08-13T15:33:27.206Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:27.513Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:27.523 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3f5c30ff-3da8-bb3e-c892-d56c008c0b15 Address:127.0.0.1:19216}]" +33:27.524 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19216 [Follower]" leader= +33:27.525 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3f5c30ff-3da8-bb3e-c892-d56c008c0b15.dc1 127.0.0.1 +33:27.527 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3f5c30ff-3da8-bb3e-c892-d56c008c0b15 127.0.0.1 +2021-08-13T15:33:27.527Z [INFO] agent.router: Initializing LAN area manager +33:27.528 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19211 network=udp +33:27.528 [INFO] TestAgent.server: Adding LAN server: server="Node-3f5c30ff-3da8-bb3e-c892-d56c008c0b15 (Addr: tcp/127.0.0.1:19216) (DC: dc1)" +33:27.528 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3f5c30ff-3da8-bb3e-c892-d56c008c0b15.dc1 area=wan +33:27.528 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19211 network=tcp +33:27.529 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19212 network=tcp +33:27.529 [INFO] TestAgent: started state syncer +33:27.529 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19217 network=tcp +33:27.562 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:27.562 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19216 [Candidate]" term=2 +33:27.563 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:27.563 [DEBUG] TestAgent.server.raft: vote granted: from=3f5c30ff-3da8-bb3e-c892-d56c008c0b15 term=2 tally=1 +33:27.563 [INFO] TestAgent.server.raft: election won: tally=1 +33:27.563 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19216 [Leader]" +33:27.563 [INFO] TestAgent.server: cluster leadership acquired +33:27.564 [INFO] TestAgent.server: initializing acls +33:27.564 [INFO] TestAgent.server: Created ACL 'global-management' policy +33:27.564 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +33:27.564 [INFO] TestAgent.server: New leader elected: payload=Node-3f5c30ff-3da8-bb3e-c892-d56c008c0b15 +33:27.567 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +33:27.568 [INFO] TestAgent.server: Created ACL anonymous token from configuration +33:27.568 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +33:27.568 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +33:27.568 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-3f5c30ff-3da8-bb3e-c892-d56c008c0b15 +33:27.568 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-3f5c30ff-3da8-bb3e-c892-d56c008c0b15.dc1 +33:27.568 [INFO] TestAgent.server: Updating LAN server: server="Node-3f5c30ff-3da8-bb3e-c892-d56c008c0b15 (Addr: tcp/127.0.0.1:19216) (DC: dc1)" +33:27.568 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-3f5c30ff-3da8-bb3e-c892-d56c008c0b15.dc1 area=wan +33:27.569 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:27.571 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:27.571 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:27.571 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:27.571 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:27.571 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:27.571 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3f5c30ff-3da8-bb3e-c892-d56c008c0b15 +33:27.571 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3f5c30ff-3da8-bb3e-c892-d56c008c0b15 +33:27.572 [INFO] TestAgent.server: federation state anti-entropy synced +33:27.572 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3f5c30ff-3da8-bb3e-c892-d56c008c0b15 +33:27.899 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:27.899 [INFO] TestAgent: Synced node info +33:27.944 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-ff44268e-0866-9ce8-6401-8eff4f867490 from=127.0.0.1:40588 latency=688.442µs +33:27.945 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:40590 latency=793.582µs +33:27.948 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-96969e21-6ee4-0efc-4078-d464780d1040 from=127.0.0.1:40592 latency=361.18µs +33:27.949 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/test-96969e21-6ee4-0efc-4078-d464780d1040 from=127.0.0.1:40592 latency=615.541µs +33:27.950 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-96969e21-6ee4-0efc-4078-d464780d1040 from=127.0.0.1:40590 latency=187.22µs +33:27.950 [INFO] TestAgent: Requesting shutdown +33:27.950 [INFO] TestAgent.server: shutting down server +33:27.950 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:27.950 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:27.950 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:27.950 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +33:27.950 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +33:27.950 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:27.950 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:27.950 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:27.950 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:27.950 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:27.950 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:27.950 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:27.951 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +33:27.951 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +33:27.951 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:27.951Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:27.951Z [INFO] agent.router.manager: shutting down +33:27.951 [INFO] TestAgent: consul server down +33:27.951 [INFO] TestAgent: shutdown complete +33:27.951 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19211 network=tcp +33:27.951 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19211 network=udp +33:27.951 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19212 network=tcp +33:28.452 [INFO] TestAgent: Waiting for endpoints to shut down +33:28.452 [INFO] TestAgent: Endpoints down +--- PASS: TestAuthMethodUpdateCommand_noMerge (1.32s) + --- PASS: TestAuthMethodUpdateCommand_noMerge/update_without_name (0.00s) + --- PASS: TestAuthMethodUpdateCommand_noMerge/update_nonexistent_method (0.00s) + --- PASS: TestAuthMethodUpdateCommand_noMerge/update_all_fields (0.01s) PASS -ok github.com/hashicorp/consul/command/acl/authmethod/update 1.411s +ok github.com/hashicorp/consul/command/acl/authmethod/update 1.382s ? github.com/hashicorp/consul/command/acl/bindingrule [no test files] === RUN TestBindingRuleCreateCommand_noTabs === PAUSE TestBindingRuleCreateCommand_noTabs @@ -74143,280 +74157,284 @@ === RUN TestBindingRuleCreateCommand/create_it_with_a_match_selector === RUN TestBindingRuleCreateCommand/create_it_with_type_role === RUN TestBindingRuleCreateCommand_JSON/create_it_with_no_selector -2022-09-15T21:41:33.683Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:33.713Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:33.716 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1fae4010-3720-82f7-ede9-240979f22600 Address:127.0.0.1:25328}]" -41:33.717 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25328 [Follower]" leader= -41:33.717 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1fae4010-3720-82f7-ede9-240979f22600.dc1 127.0.0.1 -41:33.718 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1fae4010-3720-82f7-ede9-240979f22600 127.0.0.1 -2022-09-15T21:41:33.718Z [INFO] agent.router: Initializing LAN area manager -41:33.719 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25323 network=udp -41:33.719 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1fae4010-3720-82f7-ede9-240979f22600.dc1 area=wan -41:33.719 [INFO] TestAgent.server: Adding LAN server: server="Node-1fae4010-3720-82f7-ede9-240979f22600 (Addr: tcp/127.0.0.1:25328) (DC: dc1)" -41:33.719 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25323 network=tcp -41:33.720 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25324 network=tcp -41:33.720 [INFO] TestAgent: started state syncer -41:33.721 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25329 network=tcp -41:33.769 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -41:33.781 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:33.781 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25328 [Candidate]" term=2 -41:33.782 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:33.782 [DEBUG] TestAgent.server.raft: vote granted: from=1fae4010-3720-82f7-ede9-240979f22600 term=2 tally=1 -41:33.782 [INFO] TestAgent.server.raft: election won: tally=1 -41:33.782 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25328 [Leader]" -41:33.782 [INFO] TestAgent.server: cluster leadership acquired -41:33.782 [INFO] TestAgent.server: initializing acls -41:33.783 [INFO] TestAgent.server: Created ACL 'global-management' policy -41:33.783 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -41:33.783 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -41:33.784 [INFO] TestAgent.server: Created ACL anonymous token from configuration -41:33.784 [INFO] TestAgent.server: New leader elected: payload=Node-1fae4010-3720-82f7-ede9-240979f22600 -41:33.784 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -41:33.784 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -41:33.784 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-1fae4010-3720-82f7-ede9-240979f22600 -41:33.784 [INFO] TestAgent.server: Updating LAN server: server="Node-1fae4010-3720-82f7-ede9-240979f22600 (Addr: tcp/127.0.0.1:25328) (DC: dc1)" -41:33.784 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-1fae4010-3720-82f7-ede9-240979f22600.dc1 -41:33.784 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-1fae4010-3720-82f7-ede9-240979f22600.dc1 area=wan -41:33.785 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:33.797 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:33.797 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:33.797 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:33.797 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:33.797 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:33.797 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1fae4010-3720-82f7-ede9-240979f22600 -41:33.797 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1fae4010-3720-82f7-ede9-240979f22600 -41:33.799 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1fae4010-3720-82f7-ede9-240979f22600 -41:33.800 [INFO] TestAgent.server: federation state anti-entropy synced -41:33.820 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:53064 latency=1.97182ms -41:33.831 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:53066 error="invalid Binding Rule: Selector is invalid: 1:4 (3): no match found, expected: "!=", ".", "==", "[", [ \t\r\n] or [a-zA-Z0-9_]" -41:33.831 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:53066 latency=1.29946ms -41:33.838 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:53068 latency=1.34906ms -41:33.846 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:53070 latency=4.76558ms -41:33.850 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:53072 latency=1.52808ms -41:33.851 [INFO] TestAgent: Requesting shutdown -41:33.851 [INFO] TestAgent.server: shutting down server -41:33.851 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:33.851 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:33.851 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -41:33.851 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -41:33.851 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:33.851 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:33.851 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:33.851 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:33.851 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:33.851 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:33.851 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:33.851 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:33.851 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -41:33.851 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -41:33.851 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:33.851 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:33.851Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:33.851Z [INFO] agent.router.manager: shutting down -41:33.852 [INFO] TestAgent: consul server down -41:33.852 [INFO] TestAgent: shutdown complete -41:33.852 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25323 network=tcp -41:33.852 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25323 network=udp -41:33.852 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25324 network=tcp -41:34.352 [INFO] TestAgent: Waiting for endpoints to shut down -41:34.352 [INFO] TestAgent: Endpoints down ---- PASS: TestBindingRuleCreateCommand (0.73s) +2021-08-13T15:33:48.393Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:48.427Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:48.431 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:77a08bf2-79b0-f4ab-3357-42c3e4f7efba Address:127.0.0.1:12048}]" +33:48.432 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:12048 [Follower]" leader= +33:48.433 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-77a08bf2-79b0-f4ab-3357-42c3e4f7efba.dc1 127.0.0.1 +33:48.434 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-77a08bf2-79b0-f4ab-3357-42c3e4f7efba 127.0.0.1 +2021-08-13T15:33:48.434Z [INFO] agent.router: Initializing LAN area manager +33:48.434 [INFO] TestAgent.server: Adding LAN server: server="Node-77a08bf2-79b0-f4ab-3357-42c3e4f7efba (Addr: tcp/127.0.0.1:12048) (DC: dc1)" +33:48.434 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-77a08bf2-79b0-f4ab-3357-42c3e4f7efba.dc1 area=wan +33:48.434 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12043 network=udp +33:48.434 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12043 network=tcp +33:48.435 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:12044 network=tcp +33:48.436 [INFO] TestAgent: started state syncer +33:48.436 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:12049 network=tcp +33:48.484 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +33:48.490 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:48.491 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:12048 [Candidate]" term=2 +33:48.491 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:48.491 [DEBUG] TestAgent.server.raft: vote granted: from=77a08bf2-79b0-f4ab-3357-42c3e4f7efba term=2 tally=1 +33:48.491 [INFO] TestAgent.server.raft: election won: tally=1 +33:48.491 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:12048 [Leader]" +33:48.491 [INFO] TestAgent.server: cluster leadership acquired +33:48.491 [INFO] TestAgent.server: initializing acls +33:48.492 [INFO] TestAgent.server: Created ACL 'global-management' policy +33:48.492 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +33:48.492 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +33:48.493 [INFO] TestAgent.server: Created ACL anonymous token from configuration +33:48.493 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +33:48.493 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +33:48.493 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-77a08bf2-79b0-f4ab-3357-42c3e4f7efba +33:48.493 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-77a08bf2-79b0-f4ab-3357-42c3e4f7efba.dc1 +33:48.494 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:48.494 [INFO] TestAgent.server: New leader elected: payload=Node-77a08bf2-79b0-f4ab-3357-42c3e4f7efba +33:48.495 [INFO] TestAgent.server: Updating LAN server: server="Node-77a08bf2-79b0-f4ab-3357-42c3e4f7efba (Addr: tcp/127.0.0.1:12048) (DC: dc1)" +33:48.495 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-77a08bf2-79b0-f4ab-3357-42c3e4f7efba.dc1 area=wan +33:48.503 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:48.504 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:48.504 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:48.504 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:48.504 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:48.504 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-77a08bf2-79b0-f4ab-3357-42c3e4f7efba +33:48.504 [INFO] TestAgent.server: member joined, marking health alive: member=Node-77a08bf2-79b0-f4ab-3357-42c3e4f7efba +33:48.505 [INFO] TestAgent.server: federation state anti-entropy synced +33:48.505 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-77a08bf2-79b0-f4ab-3357-42c3e4f7efba +33:48.576 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:47394 latency=2.075445ms +33:48.585 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:47396 error="invalid Binding Rule: Selector is invalid: 1:4 (3): no match found, expected: "!=", ".", "==", "[", [ \t\r\n] or [a-zA-Z0-9_]" +33:48.585 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:47396 latency=1.543284ms +33:48.602 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:47398 latency=1.812524ms +33:48.609 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:47400 latency=2.991667ms +33:48.613 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:47402 latency=1.540284ms +33:48.614 [INFO] TestAgent: Requesting shutdown +33:48.614 [INFO] TestAgent.server: shutting down server +33:48.614 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:48.614 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:48.614 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:48.614 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +33:48.614 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +33:48.614 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:48.614 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:48.614 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:48.614 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:48.614 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +33:48.614 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:48.614 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:48.614 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:48.614 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +33:48.614 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +33:48.614 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:48.614Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:48.614Z [INFO] agent.router.manager: shutting down +33:48.614 [INFO] TestAgent: consul server down +33:48.614 [INFO] TestAgent: shutdown complete +33:48.614 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12043 network=tcp +33:48.614 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12043 network=udp +33:48.615 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:12044 network=tcp +33:49.115 [INFO] TestAgent: Waiting for endpoints to shut down +33:49.115 [INFO] TestAgent: Endpoints down +--- PASS: TestBindingRuleCreateCommand (0.79s) --- PASS: TestBindingRuleCreateCommand/method_is_required (0.00s) --- PASS: TestBindingRuleCreateCommand/bind_type_required (0.00s) --- PASS: TestBindingRuleCreateCommand/bind_name_required (0.00s) - --- PASS: TestBindingRuleCreateCommand/must_use_roughly_valid_selector (0.01s) - --- PASS: TestBindingRuleCreateCommand/create_it_with_no_selector (0.00s) + --- PASS: TestBindingRuleCreateCommand/must_use_roughly_valid_selector (0.02s) + --- PASS: TestBindingRuleCreateCommand/create_it_with_no_selector (0.01s) --- PASS: TestBindingRuleCreateCommand/create_it_with_a_match_selector (0.01s) --- PASS: TestBindingRuleCreateCommand/create_it_with_type_role (0.00s) -2022-09-15T21:41:33.689Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:33.724Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:33.727 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a51920a8-eea3-ab5e-623b-ef9ed8d6fde2 Address:127.0.0.1:25321}]" -41:33.727 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25321 [Follower]" leader= -41:33.728 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a51920a8-eea3-ab5e-623b-ef9ed8d6fde2.dc1 127.0.0.1 -41:33.732 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a51920a8-eea3-ab5e-623b-ef9ed8d6fde2 127.0.0.1 -2022-09-15T21:41:33.733Z [INFO] agent.router: Initializing LAN area manager -41:33.733 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25316 network=udp -41:33.733 [INFO] TestAgent.server: Adding LAN server: server="Node-a51920a8-eea3-ab5e-623b-ef9ed8d6fde2 (Addr: tcp/127.0.0.1:25321) (DC: dc1)" -41:33.734 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a51920a8-eea3-ab5e-623b-ef9ed8d6fde2.dc1 area=wan -41:33.734 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25316 network=tcp -41:33.738 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25317 network=tcp -41:33.738 [INFO] TestAgent: started state syncer -41:33.738 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25322 network=tcp -41:33.769 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:33.769 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25321 [Candidate]" term=2 -41:33.769 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:33.769 [DEBUG] TestAgent.server.raft: vote granted: from=a51920a8-eea3-ab5e-623b-ef9ed8d6fde2 term=2 tally=1 -41:33.769 [INFO] TestAgent.server.raft: election won: tally=1 -41:33.769 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25321 [Leader]" -41:33.769 [INFO] TestAgent.server: cluster leadership acquired -41:33.770 [INFO] TestAgent.server: initializing acls -41:33.770 [INFO] TestAgent.server: Created ACL 'global-management' policy -41:33.770 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -41:33.770 [INFO] TestAgent.server: New leader elected: payload=Node-a51920a8-eea3-ab5e-623b-ef9ed8d6fde2 -41:33.771 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -41:33.772 [INFO] TestAgent.server: Created ACL anonymous token from configuration -41:33.772 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -41:33.772 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -41:33.772 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-a51920a8-eea3-ab5e-623b-ef9ed8d6fde2 -41:33.772 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-a51920a8-eea3-ab5e-623b-ef9ed8d6fde2.dc1 -41:33.772 [INFO] TestAgent.server: Updating LAN server: server="Node-a51920a8-eea3-ab5e-623b-ef9ed8d6fde2 (Addr: tcp/127.0.0.1:25321) (DC: dc1)" -41:33.772 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-a51920a8-eea3-ab5e-623b-ef9ed8d6fde2.dc1 area=wan -41:33.773 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:33.797 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:33.797 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:33.797 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:33.797 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:33.797 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:33.797 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a51920a8-eea3-ab5e-623b-ef9ed8d6fde2 -41:33.797 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a51920a8-eea3-ab5e-623b-ef9ed8d6fde2 -41:33.800 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a51920a8-eea3-ab5e-623b-ef9ed8d6fde2 -41:33.800 [INFO] TestAgent.server: federation state anti-entropy synced -41:33.880 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:33.881 [INFO] TestAgent: Synced node info -41:33.881 [DEBUG] TestAgent: Node info in sync -41:33.915 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:34924 latency=1.56534ms -41:33.919 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:34926 latency=936.18µs -41:33.919 [INFO] TestAgent: Requesting shutdown -41:33.919 [INFO] TestAgent.server: shutting down server -41:33.919 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -41:33.919 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -41:33.919 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:33.919 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:33.919 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:33.919 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:33.919 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:33.920 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:33.920 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:33.920 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -41:33.920 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -41:33.920 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:33.920 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:33.920 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:33.920 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:33.920Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:33.920Z [INFO] agent.router.manager: shutting down -41:33.920 [INFO] TestAgent: consul server down -41:33.920 [INFO] TestAgent: shutdown complete -41:33.920 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25316 network=tcp -41:33.920 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25316 network=udp -41:33.920 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25317 network=tcp -41:34.421 [INFO] TestAgent: Waiting for endpoints to shut down -41:34.421 [INFO] TestAgent: Endpoints down ---- PASS: TestBindingRuleCreateCommand_JSON (0.80s) - --- PASS: TestBindingRuleCreateCommand_JSON/create_it_with_no_selector (0.00s) +2021-08-13T15:33:48.392Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:48.434Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:48.437 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:65238e3b-3a73-c71a-44da-d85b4ec77679 Address:127.0.0.1:12055}]" +33:48.437 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:12055 [Follower]" leader= +33:48.438 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-65238e3b-3a73-c71a-44da-d85b4ec77679.dc1 127.0.0.1 +33:48.444 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-65238e3b-3a73-c71a-44da-d85b4ec77679 127.0.0.1 +2021-08-13T15:33:48.444Z [INFO] agent.router: Initializing LAN area manager +33:48.445 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-65238e3b-3a73-c71a-44da-d85b4ec77679.dc1 area=wan +33:48.445 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12050 network=udp +33:48.445 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12050 network=tcp +33:48.446 [INFO] TestAgent.server: Adding LAN server: server="Node-65238e3b-3a73-c71a-44da-d85b4ec77679 (Addr: tcp/127.0.0.1:12055) (DC: dc1)" +33:48.447 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:12051 network=tcp +33:48.448 [INFO] TestAgent: started state syncer +33:48.448 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:12056 network=tcp +33:48.475 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:48.475 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:12055 [Candidate]" term=2 +33:48.475 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:48.475 [DEBUG] TestAgent.server.raft: vote granted: from=65238e3b-3a73-c71a-44da-d85b4ec77679 term=2 tally=1 +33:48.475 [INFO] TestAgent.server.raft: election won: tally=1 +33:48.475 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:12055 [Leader]" +33:48.475 [INFO] TestAgent.server: cluster leadership acquired +33:48.476 [INFO] TestAgent.server: New leader elected: payload=Node-65238e3b-3a73-c71a-44da-d85b4ec77679 +33:48.476 [INFO] TestAgent.server: initializing acls +33:48.476 [INFO] TestAgent.server: Created ACL 'global-management' policy +33:48.476 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +33:48.477 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +33:48.478 [INFO] TestAgent.server: Created ACL anonymous token from configuration +33:48.478 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +33:48.478 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +33:48.478 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-65238e3b-3a73-c71a-44da-d85b4ec77679 +33:48.478 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-65238e3b-3a73-c71a-44da-d85b4ec77679.dc1 +33:48.478 [INFO] TestAgent.server: Updating LAN server: server="Node-65238e3b-3a73-c71a-44da-d85b4ec77679 (Addr: tcp/127.0.0.1:12055) (DC: dc1)" +33:48.478 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-65238e3b-3a73-c71a-44da-d85b4ec77679.dc1 area=wan +33:48.480 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:48.504 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:48.504 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:48.504 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:48.504 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:48.504 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:48.504 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-65238e3b-3a73-c71a-44da-d85b4ec77679 +33:48.504 [INFO] TestAgent.server: member joined, marking health alive: member=Node-65238e3b-3a73-c71a-44da-d85b4ec77679 +33:48.506 [INFO] TestAgent.server: federation state anti-entropy synced +33:48.506 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-65238e3b-3a73-c71a-44da-d85b4ec77679 +33:48.563 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:48.564 [INFO] TestAgent: Synced node info +33:48.564 [DEBUG] TestAgent: Node info in sync +33:48.825 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:58346 latency=1.615764ms +33:48.833 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:58348 latency=1.150703ms +33:48.834 [INFO] TestAgent: Requesting shutdown +33:48.834 [INFO] TestAgent.server: shutting down server +33:48.834 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:48.834 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:48.834 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:48.834 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:48.834 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +33:48.834 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +33:48.834 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:48.834 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:48.834 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:48.834 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:48.834 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:48.834 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:48.834 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +33:48.834 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +33:48.834 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:48.834Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:48.834Z [INFO] agent.router.manager: shutting down +33:48.835 [INFO] TestAgent: consul server down +33:48.835 [INFO] TestAgent: shutdown complete +33:48.835 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12050 network=tcp +33:48.835 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12050 network=udp +33:48.835 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:12051 network=tcp +33:49.335 [INFO] TestAgent: Waiting for endpoints to shut down +33:49.335 [INFO] TestAgent: Endpoints down +--- PASS: TestBindingRuleCreateCommand_JSON (1.01s) + --- PASS: TestBindingRuleCreateCommand_JSON/create_it_with_no_selector (0.01s) PASS -ok github.com/hashicorp/consul/command/acl/bindingrule/create 0.885s +ok github.com/hashicorp/consul/command/acl/bindingrule/create 1.074s === RUN TestBindingRuleDeleteCommand_noTabs === PAUSE TestBindingRuleDeleteCommand_noTabs === RUN TestBindingRuleDeleteCommand === PAUSE TestBindingRuleDeleteCommand === CONT TestBindingRuleDeleteCommand_noTabs ---- PASS: TestBindingRuleDeleteCommand_noTabs (0.00s) === CONT TestBindingRuleDeleteCommand [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestBindingRuleDeleteCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range === RUN TestBindingRuleDeleteCommand/id_required === RUN TestBindingRuleDeleteCommand/delete_works === RUN TestBindingRuleDeleteCommand/delete_works_via_prefixes === RUN TestBindingRuleDeleteCommand/delete_fails_when_prefix_matches_more_than_one_rule -2022-09-15T21:41:33.713Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:33.738Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:33.746 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0235888a-f176-81a8-084e-bf59d431d7d1 Address:127.0.0.1:11027}]" -41:33.747 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:11027 [Follower]" leader= -41:33.748 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0235888a-f176-81a8-084e-bf59d431d7d1.dc1 127.0.0.1 -41:33.748 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0235888a-f176-81a8-084e-bf59d431d7d1 127.0.0.1 -2022-09-15T21:41:33.749Z [INFO] agent.router: Initializing LAN area manager -41:33.749 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0235888a-f176-81a8-084e-bf59d431d7d1.dc1 area=wan -41:33.749 [INFO] TestAgent.server: Adding LAN server: server="Node-0235888a-f176-81a8-084e-bf59d431d7d1 (Addr: tcp/127.0.0.1:11027) (DC: dc1)" -41:33.749 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11022 network=tcp -41:33.749 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11022 network=udp -41:33.750 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:11023 network=tcp -41:33.750 [INFO] TestAgent: started state syncer -41:33.750 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:11028 network=tcp -41:33.799 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -41:33.811 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:33.811 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:11027 [Candidate]" term=2 -41:33.811 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:33.811 [DEBUG] TestAgent.server.raft: vote granted: from=0235888a-f176-81a8-084e-bf59d431d7d1 term=2 tally=1 -41:33.811 [INFO] TestAgent.server.raft: election won: tally=1 -41:33.811 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:11027 [Leader]" -41:33.812 [INFO] TestAgent.server: cluster leadership acquired -41:33.812 [INFO] TestAgent.server: initializing acls -41:33.812 [INFO] TestAgent.server: Created ACL 'global-management' policy -41:33.813 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -41:33.813 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -41:33.814 [INFO] TestAgent.server: Created ACL anonymous token from configuration -41:33.814 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -41:33.814 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -41:33.814 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-0235888a-f176-81a8-084e-bf59d431d7d1 -41:33.814 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-0235888a-f176-81a8-084e-bf59d431d7d1.dc1 -41:33.816 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:33.818 [INFO] TestAgent.server: New leader elected: payload=Node-0235888a-f176-81a8-084e-bf59d431d7d1 -41:33.818 [INFO] TestAgent.server: Updating LAN server: server="Node-0235888a-f176-81a8-084e-bf59d431d7d1 (Addr: tcp/127.0.0.1:11027) (DC: dc1)" -41:33.818 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-0235888a-f176-81a8-084e-bf59d431d7d1.dc1 area=wan -41:33.849 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:33.849 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:33.849 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:33.849 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:33.849 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:33.849 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0235888a-f176-81a8-084e-bf59d431d7d1 -41:33.849 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0235888a-f176-81a8-084e-bf59d431d7d1 -41:33.850 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0235888a-f176-81a8-084e-bf59d431d7d1 -41:33.850 [INFO] TestAgent.server: federation state anti-entropy synced -41:33.992 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:33.993 [INFO] TestAgent: Synced node info -41:34.191 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:46550 latency=3.7563ms -41:34.199 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:46550 latency=2.81586ms -41:34.202 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/2492eb0f-8be7-76e9-7d8a-5dda0cbe5fb0 from=127.0.0.1:46552 latency=909.04µs -41:34.202 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/2492eb0f-8be7-76e9-7d8a-5dda0cbe5fb0 from=127.0.0.1:46550 latency=123.08µs -41:34.204 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:46550 latency=957.28µs -41:34.206 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:46554 latency=442.18µs -41:34.207 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/bee3c5a9-bfd3-94cb-47cb-e40353b50dd3 from=127.0.0.1:46554 latency=471.3µs -41:34.208 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/bee3c5a9-bfd3-94cb-47cb-e40353b50dd3 from=127.0.0.1:46550 latency=95.32µs -41:34.208 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:46550 latency=149.24µs -41:34.210 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:46550 latency=1.0233ms -41:34.211 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:46550 latency=180.84µs -41:34.212 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:46550 latency=835.94µs -41:34.213 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:46550 latency=171.44µs -41:34.215 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:46550 latency=1.09172ms -41:34.217 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:46550 latency=214.22µs -41:34.219 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:46550 latency=1.04364ms -41:34.220 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:46550 latency=317.6µs -41:34.222 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:46556 latency=281.22µs -41:34.224 [INFO] TestAgent: Requesting shutdown -41:34.224 [INFO] TestAgent.server: shutting down server -41:34.224 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -41:34.224 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -41:34.224 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:34.224 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:34.224 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:34.224 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:34.224 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:34.224 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:34.224 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -41:34.224 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:34.224 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:34.224 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -41:34.224 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:34.224 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:34.224 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:34.224Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:34.224Z [INFO] agent.router.manager: shutting down -41:34.225 [INFO] TestAgent: consul server down -41:34.225 [INFO] TestAgent: shutdown complete -41:34.225 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11022 network=tcp -41:34.225 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11022 network=udp -41:34.225 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:11023 network=tcp -41:34.725 [INFO] TestAgent: Waiting for endpoints to shut down -41:34.725 [INFO] TestAgent: Endpoints down ---- PASS: TestBindingRuleDeleteCommand (1.08s) +2021-08-13T15:33:52.489Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:52.507Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:52.511 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c94f5c9a-5cc3-872b-e222-d886078ebf35 Address:127.0.0.1:30426}]" +33:52.511 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30426 [Follower]" leader= +33:52.513 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c94f5c9a-5cc3-872b-e222-d886078ebf35.dc1 127.0.0.1 +33:52.514 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c94f5c9a-5cc3-872b-e222-d886078ebf35 127.0.0.1 +2021-08-13T15:33:52.515Z [INFO] agent.router: Initializing LAN area manager +33:52.515 [INFO] TestAgent.server: Adding LAN server: server="Node-c94f5c9a-5cc3-872b-e222-d886078ebf35 (Addr: tcp/127.0.0.1:30426) (DC: dc1)" +33:52.515 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c94f5c9a-5cc3-872b-e222-d886078ebf35.dc1 area=wan +33:52.515 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=tcp +33:52.515 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=udp +33:52.516 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30422 network=tcp +33:52.516 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30427 network=tcp +33:52.516 [INFO] TestAgent: started state syncer +33:52.552 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:52.552 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30426 [Candidate]" term=2 +33:52.552 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:52.552 [DEBUG] TestAgent.server.raft: vote granted: from=c94f5c9a-5cc3-872b-e222-d886078ebf35 term=2 tally=1 +33:52.552 [INFO] TestAgent.server.raft: election won: tally=1 +33:52.552 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30426 [Leader]" +33:52.552 [INFO] TestAgent.server: cluster leadership acquired +33:52.553 [INFO] TestAgent.server: New leader elected: payload=Node-c94f5c9a-5cc3-872b-e222-d886078ebf35 +33:52.553 [INFO] TestAgent.server: initializing acls +33:52.553 [INFO] TestAgent.server: Created ACL 'global-management' policy +33:52.553 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +33:52.554 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +33:52.555 [INFO] TestAgent.server: Created ACL anonymous token from configuration +33:52.555 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +33:52.555 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +33:52.555 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-c94f5c9a-5cc3-872b-e222-d886078ebf35 +33:52.555 [INFO] TestAgent.server: Updating LAN server: server="Node-c94f5c9a-5cc3-872b-e222-d886078ebf35 (Addr: tcp/127.0.0.1:30426) (DC: dc1)" +33:52.555 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-c94f5c9a-5cc3-872b-e222-d886078ebf35.dc1 +33:52.555 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-c94f5c9a-5cc3-872b-e222-d886078ebf35.dc1 area=wan +33:52.556 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:52.580 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:52.580 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:52.580 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:52.580 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:52.580 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:52.580 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c94f5c9a-5cc3-872b-e222-d886078ebf35 +33:52.580 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c94f5c9a-5cc3-872b-e222-d886078ebf35 +33:52.581 [INFO] TestAgent.server: federation state anti-entropy synced +33:52.581 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c94f5c9a-5cc3-872b-e222-d886078ebf35 +33:52.890 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:52.891 [INFO] TestAgent: Synced node info +33:52.891 [DEBUG] TestAgent: Node info in sync +33:52.953 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:48320 latency=1.724844ms +33:52.958 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:48320 latency=2.920687ms +33:52.962 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/4a9e9001-17a7-4dd0-8744-e42cb4dcc96c from=127.0.0.1:48322 latency=1.262643ms +33:52.963 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/4a9e9001-17a7-4dd0-8744-e42cb4dcc96c from=127.0.0.1:48320 latency=190.421µs +33:52.965 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:48320 latency=1.857824ms +33:52.968 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:48324 latency=497.921µs +33:52.970 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/928bf433-56cf-ca6f-a71e-e8a273d7172f from=127.0.0.1:48324 latency=530.661µs +33:52.971 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/928bf433-56cf-ca6f-a71e-e8a273d7172f from=127.0.0.1:48320 latency=110.1µs +33:52.972 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:48320 latency=261.881µs +33:52.973 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:48320 latency=1.054742ms +33:52.974 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:48320 latency=438.681µs +33:52.976 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:48320 latency=990.082µs +33:52.977 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:48320 latency=419.401µs +33:52.979 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:48320 latency=1.029823ms +33:52.982 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:48320 latency=340.841µs +33:52.984 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:48320 latency=1.099483ms +33:52.985 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:48320 latency=230.82µs +33:52.987 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:48320 latency=1.050523ms +33:52.988 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:48320 latency=212.701µs +33:52.989 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:48320 latency=959.823µs +33:52.990 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:48320 latency=212µs +33:52.993 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:48326 latency=328.24µs +33:52.993 [INFO] TestAgent: Requesting shutdown +33:52.993 [INFO] TestAgent.server: shutting down server +33:52.993 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:52.993 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:52.993 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:52.993 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +33:52.993 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +33:52.993 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:52.994 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:52.994 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:52.994 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:52.994 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:52.994 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:52.994 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:52.994 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +33:52.994 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +33:52.994 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:52.994Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:52.994Z [INFO] agent.router.manager: shutting down +33:52.994 [INFO] TestAgent: consul server down +33:52.994 [INFO] TestAgent: shutdown complete +33:52.994 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=tcp +33:52.994 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=udp +33:52.994 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30422 network=tcp +33:53.494 [INFO] TestAgent: Waiting for endpoints to shut down +33:53.494 [INFO] TestAgent: Endpoints down +--- PASS: TestBindingRuleDeleteCommand (1.06s) --- PASS: TestBindingRuleDeleteCommand/id_required (0.00s) --- PASS: TestBindingRuleDeleteCommand/delete_works (0.01s) --- PASS: TestBindingRuleDeleteCommand/delete_works_via_prefixes (0.01s) --- PASS: TestBindingRuleDeleteCommand/delete_fails_when_prefix_matches_more_than_one_rule (0.02s) PASS -ok github.com/hashicorp/consul/command/acl/bindingrule/delete 1.174s +ok github.com/hashicorp/consul/command/acl/bindingrule/delete 1.125s === RUN TestBindingRuleListCommand_noTabs === PAUSE TestBindingRuleListCommand_noTabs === RUN TestBindingRuleListCommand @@ -74431,108 +74449,109 @@ === RUN TestBindingRuleListCommand/filter_by_method_1 === RUN TestBindingRuleListCommand/filter_by_method_2 === RUN TestBindingRuleListCommand/normal_json_formatted -2022-09-15T21:41:45.093Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:45.111Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:45.116 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f2fb2e66-b435-1600-c731-33749057f04d Address:127.0.0.1:30426}]" -41:45.117 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f2fb2e66-b435-1600-c731-33749057f04d.dc1 127.0.0.1 -41:45.118 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30426 [Follower]" leader= -41:45.120 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f2fb2e66-b435-1600-c731-33749057f04d 127.0.0.1 -2022-09-15T21:41:45.120Z [INFO] agent.router: Initializing LAN area manager -41:45.120 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f2fb2e66-b435-1600-c731-33749057f04d.dc1 area=wan -41:45.120 [INFO] TestAgent.server: Adding LAN server: server="Node-f2fb2e66-b435-1600-c731-33749057f04d (Addr: tcp/127.0.0.1:30426) (DC: dc1)" -41:45.121 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=udp -41:45.122 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=tcp -41:45.123 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30422 network=tcp -41:45.123 [INFO] TestAgent: started state syncer -41:45.123 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30427 network=tcp -41:45.171 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -41:45.187 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:45.187 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30426 [Candidate]" term=2 -41:45.187 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:45.187 [DEBUG] TestAgent.server.raft: vote granted: from=f2fb2e66-b435-1600-c731-33749057f04d term=2 tally=1 -41:45.187 [INFO] TestAgent.server.raft: election won: tally=1 -41:45.187 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30426 [Leader]" -41:45.188 [INFO] TestAgent.server: cluster leadership acquired -41:45.188 [INFO] TestAgent.server: initializing acls -41:45.189 [INFO] TestAgent.server: Created ACL 'global-management' policy -41:45.189 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -41:45.190 [INFO] TestAgent.server: New leader elected: payload=Node-f2fb2e66-b435-1600-c731-33749057f04d -41:45.190 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -41:45.190 [INFO] TestAgent.server: Created ACL anonymous token from configuration -41:45.190 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -41:45.190 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -41:45.191 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-f2fb2e66-b435-1600-c731-33749057f04d -41:45.191 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-f2fb2e66-b435-1600-c731-33749057f04d.dc1 -41:45.192 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:45.194 [INFO] TestAgent.server: Updating LAN server: server="Node-f2fb2e66-b435-1600-c731-33749057f04d (Addr: tcp/127.0.0.1:30426) (DC: dc1)" -41:45.194 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-f2fb2e66-b435-1600-c731-33749057f04d.dc1 area=wan -41:45.216 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:45.216 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:45.216 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:45.216 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:45.216 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:45.216 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f2fb2e66-b435-1600-c731-33749057f04d -41:45.216 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f2fb2e66-b435-1600-c731-33749057f04d -41:45.218 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f2fb2e66-b435-1600-c731-33749057f04d -41:45.218 [INFO] TestAgent.server: federation state anti-entropy synced -41:45.227 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:34580 latency=1.97604ms -41:45.228 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:34580 latency=830.74µs -41:45.231 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:34580 latency=1.64408ms -41:45.233 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:34580 latency=1.25192ms -41:45.238 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:34580 latency=1.40342ms -41:45.246 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:34580 latency=1.99692ms -41:45.254 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:34580 latency=5.46434ms -41:45.259 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:34580 latency=1.37398ms -41:45.261 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:34580 latency=1.10092ms -41:45.264 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:34580 latency=1.31814ms -41:45.266 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:34580 latency=1.19592ms -41:45.268 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:34580 latency=1.12116ms -41:45.271 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:34582 latency=539.48µs -41:45.275 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test-1 from=127.0.0.1:34584 latency=362.36µs -41:45.277 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test-2 from=127.0.0.1:34586 latency=499.68µs -41:45.281 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:34588 latency=371.02µs -41:45.282 [INFO] TestAgent: Requesting shutdown -41:45.283 [INFO] TestAgent.server: shutting down server -41:45.283 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -41:45.283 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -41:45.283 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:45.283 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:45.283 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:45.283 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:45.283 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:45.283 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:45.283 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:45.283 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:45.283 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -41:45.283 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -41:45.283 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -41:45.283 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:45.283 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:45.283 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2022-09-15T21:41:45.283Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:45.283Z [INFO] agent.router.manager: shutting down -41:45.283 [INFO] TestAgent: consul server down -41:45.283 [INFO] TestAgent: shutdown complete -41:45.283 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=tcp -41:45.283 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=udp -41:45.283 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30422 network=tcp -41:45.783 [INFO] TestAgent: Waiting for endpoints to shut down -41:45.783 [INFO] TestAgent: Endpoints down ---- PASS: TestBindingRuleListCommand (0.77s) +2021-08-13T15:33:52.002Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:33:52.020Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +33:52.024 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b241cdc4-4182-a13e-f53c-5bae6158da04 Address:127.0.0.1:27363}]" +33:52.024 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:27363 [Follower]" leader= +33:52.025 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b241cdc4-4182-a13e-f53c-5bae6158da04.dc1 127.0.0.1 +33:52.026 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b241cdc4-4182-a13e-f53c-5bae6158da04 127.0.0.1 +2021-08-13T15:33:52.026Z [INFO] agent.router: Initializing LAN area manager +33:52.026 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b241cdc4-4182-a13e-f53c-5bae6158da04.dc1 area=wan +33:52.026 [INFO] TestAgent.server: Adding LAN server: server="Node-b241cdc4-4182-a13e-f53c-5bae6158da04 (Addr: tcp/127.0.0.1:27363) (DC: dc1)" +33:52.026 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27358 network=tcp +33:52.026 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27358 network=udp +33:52.027 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:27359 network=tcp +33:52.027 [INFO] TestAgent: started state syncer +33:52.027 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:27364 network=tcp +33:52.059 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +33:52.059 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:27363 [Candidate]" term=2 +33:52.060 [DEBUG] TestAgent.server.raft: votes: needed=1 +33:52.060 [DEBUG] TestAgent.server.raft: vote granted: from=b241cdc4-4182-a13e-f53c-5bae6158da04 term=2 tally=1 +33:52.060 [INFO] TestAgent.server.raft: election won: tally=1 +33:52.060 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:27363 [Leader]" +33:52.060 [INFO] TestAgent.server: cluster leadership acquired +33:52.060 [INFO] TestAgent.server: New leader elected: payload=Node-b241cdc4-4182-a13e-f53c-5bae6158da04 +33:52.060 [INFO] TestAgent.server: initializing acls +33:52.061 [INFO] TestAgent.server: Created ACL 'global-management' policy +33:52.061 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +33:52.061 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +33:52.062 [INFO] TestAgent.server: Created ACL anonymous token from configuration +33:52.062 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +33:52.062 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +33:52.062 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-b241cdc4-4182-a13e-f53c-5bae6158da04 +33:52.062 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-b241cdc4-4182-a13e-f53c-5bae6158da04.dc1 +33:52.062 [INFO] TestAgent.server: Updating LAN server: server="Node-b241cdc4-4182-a13e-f53c-5bae6158da04 (Addr: tcp/127.0.0.1:27363) (DC: dc1)" +33:52.062 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-b241cdc4-4182-a13e-f53c-5bae6158da04.dc1 area=wan +33:52.063 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +33:52.086 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +33:52.086 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +33:52.086 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +33:52.086 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +33:52.086 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +33:52.087 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b241cdc4-4182-a13e-f53c-5bae6158da04 +33:52.087 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b241cdc4-4182-a13e-f53c-5bae6158da04 +33:52.087 [INFO] TestAgent.server: federation state anti-entropy synced +33:52.088 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b241cdc4-4182-a13e-f53c-5bae6158da04 +33:52.386 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +33:52.386 [INFO] TestAgent: Synced node info +33:52.386 [DEBUG] TestAgent: Node info in sync +33:52.408 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:33566 latency=2.940287ms +33:52.414 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:33566 latency=866.922µs +33:52.417 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:33566 latency=1.449883ms +33:52.419 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:33566 latency=1.022702ms +33:52.421 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:33566 latency=1.158023ms +33:52.422 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:33566 latency=939.582µs +33:52.423 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:33566 latency=885.642µs +33:52.425 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:33566 latency=854.822µs +33:52.428 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:33566 latency=891.962µs +33:52.429 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:33566 latency=827.402µs +33:52.431 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:33566 latency=833.421µs +33:52.432 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:33566 latency=955.062µs +33:52.435 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:33568 latency=395.001µs +33:52.437 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test-1 from=127.0.0.1:33570 latency=260.081µs +33:52.440 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test-2 from=127.0.0.1:33572 latency=276.54µs +33:52.442 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:33574 latency=303.96µs +33:52.443 [INFO] TestAgent: Requesting shutdown +33:52.443 [INFO] TestAgent.server: shutting down server +33:52.443 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +33:52.443 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +33:52.443 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +33:52.443 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +33:52.443 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +33:52.443 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +33:52.443 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +33:52.443 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +33:52.443 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +33:52.443 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +33:52.443 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +33:52.443 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +33:52.443 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +33:52.444 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +33:52.444 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:33:52.444Z [INFO] agent.router.manager: shutting down +2021-08-13T15:33:52.444Z [INFO] agent.router.manager: shutting down +33:52.445 [INFO] TestAgent: consul server down +33:52.445 [INFO] TestAgent: shutdown complete +33:52.445 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27358 network=tcp +33:52.445 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27358 network=udp +33:52.445 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:27359 network=tcp +33:52.945 [INFO] TestAgent: Waiting for endpoints to shut down +33:52.945 [INFO] TestAgent: Endpoints down +--- PASS: TestBindingRuleListCommand (1.00s) --- PASS: TestBindingRuleListCommand/normal (0.00s) --- PASS: TestBindingRuleListCommand/filter_by_method_1 (0.00s) --- PASS: TestBindingRuleListCommand/filter_by_method_2 (0.00s) --- PASS: TestBindingRuleListCommand/normal_json_formatted (0.00s) PASS -ok github.com/hashicorp/consul/command/acl/bindingrule/list 0.849s +ok github.com/hashicorp/consul/command/acl/bindingrule/list 1.057s === RUN TestBindingRuleReadCommand_noTabs === PAUSE TestBindingRuleReadCommand_noTabs === RUN TestBindingRuleReadCommand === PAUSE TestBindingRuleReadCommand === CONT TestBindingRuleReadCommand_noTabs +--- PASS: TestBindingRuleReadCommand_noTabs (0.01s) === CONT TestBindingRuleReadCommand [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... ---- PASS: TestBindingRuleReadCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range === RUN TestBindingRuleReadCommand/id_required @@ -74540,103 +74559,104 @@ === RUN TestBindingRuleReadCommand/read_by_id === RUN TestBindingRuleReadCommand/read_by_id_prefix === RUN TestBindingRuleReadCommand/read_by_id_json_formatted -2022-09-15T21:41:47.069Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:47.090Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:47.094 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:047b8721-b3ec-3309-b563-f6dee022189b Address:127.0.0.1:24300}]" -41:47.094 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24300 [Follower]" leader= -41:47.095 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-047b8721-b3ec-3309-b563-f6dee022189b.dc1 127.0.0.1 -41:47.096 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-047b8721-b3ec-3309-b563-f6dee022189b 127.0.0.1 -2022-09-15T21:41:47.096Z [INFO] agent.router: Initializing LAN area manager -41:47.097 [INFO] TestAgent.server: Adding LAN server: server="Node-047b8721-b3ec-3309-b563-f6dee022189b (Addr: tcp/127.0.0.1:24300) (DC: dc1)" -41:47.097 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-047b8721-b3ec-3309-b563-f6dee022189b.dc1 area=wan -41:47.097 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24295 network=udp -41:47.097 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24295 network=tcp -41:47.098 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24296 network=tcp -41:47.098 [INFO] TestAgent: started state syncer -41:47.098 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24301 network=tcp -41:47.144 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:47.144 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24300 [Candidate]" term=2 -41:47.145 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:47.145 [DEBUG] TestAgent.server.raft: vote granted: from=047b8721-b3ec-3309-b563-f6dee022189b term=2 tally=1 -41:47.145 [INFO] TestAgent.server.raft: election won: tally=1 -41:47.145 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24300 [Leader]" -41:47.145 [INFO] TestAgent.server: cluster leadership acquired -41:47.145 [INFO] TestAgent.server: initializing acls -41:47.146 [INFO] TestAgent.server: Created ACL 'global-management' policy -41:47.146 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -41:47.147 [INFO] TestAgent.server: initializing acls -41:47.147 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -41:47.147 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -41:47.147 [INFO] TestAgent.server: New leader elected: payload=Node-047b8721-b3ec-3309-b563-f6dee022189b -41:47.148 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -41:47.148 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -41:47.148 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -41:47.148 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode -41:47.148 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-047b8721-b3ec-3309-b563-f6dee022189b -41:47.148 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-047b8721-b3ec-3309-b563-f6dee022189b.dc1 -41:47.148 [INFO] TestAgent.server: Created ACL anonymous token from configuration -41:47.148 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-047b8721-b3ec-3309-b563-f6dee022189b.dc1 area=wan -41:47.148 [INFO] TestAgent.server: Updating LAN server: server="Node-047b8721-b3ec-3309-b563-f6dee022189b (Addr: tcp/127.0.0.1:24300) (DC: dc1)" -41:47.148 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-047b8721-b3ec-3309-b563-f6dee022189b -41:47.149 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-047b8721-b3ec-3309-b563-f6dee022189b.dc1 -41:47.149 [INFO] TestAgent.server: Updating LAN server: server="Node-047b8721-b3ec-3309-b563-f6dee022189b (Addr: tcp/127.0.0.1:24300) (DC: dc1)" -41:47.149 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-047b8721-b3ec-3309-b563-f6dee022189b.dc1 area=wan -41:47.151 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:47.176 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:47.176 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:47.176 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:47.176 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:47.176 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:47.177 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-047b8721-b3ec-3309-b563-f6dee022189b -41:47.177 [INFO] TestAgent.server: member joined, marking health alive: member=Node-047b8721-b3ec-3309-b563-f6dee022189b -41:47.178 [INFO] TestAgent.server: federation state anti-entropy synced -41:47.178 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-047b8721-b3ec-3309-b563-f6dee022189b -41:47.179 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-047b8721-b3ec-3309-b563-f6dee022189b -41:47.332 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:47.333 [INFO] TestAgent: Synced node info -41:47.426 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:53696 latency=1.8857ms -41:47.430 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/9b005f1b-fe1f-102d-59de-31508d4c6026 from=127.0.0.1:53700 latency=328.26µs -41:47.432 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:53696 latency=1.44814ms -41:47.435 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/11e482ab-df8e-d649-66d0-585c5fe5675f from=127.0.0.1:53702 latency=1.19244ms -41:47.447 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:53696 latency=1.53228ms -41:47.454 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:53704 latency=373.44µs -41:47.455 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/ee23ac30-a72e-726e-9ee5-1208cc72fb2c from=127.0.0.1:53704 latency=162.84µs -41:47.457 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:53696 latency=1.12294ms -41:47.460 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/23186235-6007-4ac1-1893-1fa19a6f656e from=127.0.0.1:53706 latency=418.02µs -41:47.460 [INFO] TestAgent: Requesting shutdown -41:47.461 [INFO] TestAgent.server: shutting down server -41:47.461 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -41:47.461 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -41:47.461 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:47.461 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:47.461 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:47.461 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:47.461 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:47.461 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:47.461 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:47.461 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -41:47.461 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -41:47.461 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:47.461 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:47.461 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:47.461 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:47.461Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:47.461Z [INFO] agent.router.manager: shutting down -41:47.461 [INFO] TestAgent: consul server down -41:47.461 [INFO] TestAgent: shutdown complete -41:47.461 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24295 network=tcp -41:47.461 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24295 network=udp -41:47.461 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24296 network=tcp -41:47.962 [INFO] TestAgent: Waiting for endpoints to shut down -41:47.962 [INFO] TestAgent: Endpoints down +2021-08-13T15:34:06.619Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:06.639Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:06.643 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1340bcf7-360f-fbd0-96b8-81aacf165574 Address:127.0.0.1:15111}]" +34:06.643 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:15111 [Follower]" leader= +34:06.644 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1340bcf7-360f-fbd0-96b8-81aacf165574.dc1 127.0.0.1 +34:06.645 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1340bcf7-360f-fbd0-96b8-81aacf165574 127.0.0.1 +2021-08-13T15:34:06.645Z [INFO] agent.router: Initializing LAN area manager +34:06.646 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1340bcf7-360f-fbd0-96b8-81aacf165574.dc1 area=wan +34:06.646 [INFO] TestAgent.server: Adding LAN server: server="Node-1340bcf7-360f-fbd0-96b8-81aacf165574 (Addr: tcp/127.0.0.1:15111) (DC: dc1)" +34:06.646 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15106 network=tcp +34:06.646 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15106 network=udp +34:06.647 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:15107 network=tcp +34:06.647 [INFO] TestAgent: started state syncer +34:06.648 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:15112 network=tcp +34:06.683 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:06.683 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:15111 [Candidate]" term=2 +34:06.691 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:06.691 [DEBUG] TestAgent.server.raft: vote granted: from=1340bcf7-360f-fbd0-96b8-81aacf165574 term=2 tally=1 +34:06.691 [INFO] TestAgent.server.raft: election won: tally=1 +34:06.691 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:15111 [Leader]" +34:06.691 [INFO] TestAgent.server: cluster leadership acquired +34:06.692 [INFO] TestAgent.server: initializing acls +34:06.692 [INFO] TestAgent.server: New leader elected: payload=Node-1340bcf7-360f-fbd0-96b8-81aacf165574 +34:06.696 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:06.696 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:06.696 [INFO] TestAgent.server: initializing acls +34:06.696 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:06.697 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:06.698 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:06.698 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:06.698 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:06.698 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode +34:06.698 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-1340bcf7-360f-fbd0-96b8-81aacf165574 +34:06.699 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-1340bcf7-360f-fbd0-96b8-81aacf165574.dc1 +34:06.699 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:06.699 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-1340bcf7-360f-fbd0-96b8-81aacf165574.dc1 area=wan +34:06.699 [INFO] TestAgent.server: Updating LAN server: server="Node-1340bcf7-360f-fbd0-96b8-81aacf165574 (Addr: tcp/127.0.0.1:15111) (DC: dc1)" +34:06.700 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-1340bcf7-360f-fbd0-96b8-81aacf165574 +34:06.700 [INFO] TestAgent.server: Updating LAN server: server="Node-1340bcf7-360f-fbd0-96b8-81aacf165574 (Addr: tcp/127.0.0.1:15111) (DC: dc1)" +34:06.700 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-1340bcf7-360f-fbd0-96b8-81aacf165574.dc1 +34:06.700 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-1340bcf7-360f-fbd0-96b8-81aacf165574.dc1 area=wan +34:06.703 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:06.732 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:06.732 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:06.732 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:06.732 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:06.732 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:06.732 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1340bcf7-360f-fbd0-96b8-81aacf165574 +34:06.732 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1340bcf7-360f-fbd0-96b8-81aacf165574 +34:06.734 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1340bcf7-360f-fbd0-96b8-81aacf165574 +34:06.734 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1340bcf7-360f-fbd0-96b8-81aacf165574 +34:06.734 [INFO] TestAgent.server: federation state anti-entropy synced +34:06.876 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:06.876 [INFO] TestAgent: Synced node info +34:06.876 [DEBUG] TestAgent: Node info in sync +34:07.004 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:37518 latency=1.670564ms +34:07.007 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/6183aa95-55e6-a801-28b1-94da62c989c8 from=127.0.0.1:37520 latency=231.621µs +34:07.009 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:37518 latency=1.424103ms +34:07.012 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/f9f209b4-b0e5-4dd8-4d37-61cbf7ee7979 from=127.0.0.1:37524 latency=316.68µs +34:07.014 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:37518 latency=1.801384ms +34:07.017 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:37526 latency=457.081µs +34:07.024 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/b8217463-9fca-da9e-db5a-756baa1ecd1a from=127.0.0.1:37526 latency=5.076371ms +34:07.026 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:37518 latency=1.402743ms +34:07.028 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/58f21c52-f91c-de10-f2f8-bb336903e7ee from=127.0.0.1:37528 latency=366.521µs +34:07.029 [INFO] TestAgent: Requesting shutdown +34:07.029 [INFO] TestAgent.server: shutting down server +34:07.029 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:07.029 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:07.029 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:07.029 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:07.029 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:07.029 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:07.029 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:07.029 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:07.029 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:07.029 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:07.029 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:07.029 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:07.029 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:07.029 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:07.030 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:07.030Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:07.030Z [INFO] agent.router.manager: shutting down +34:07.030 [INFO] TestAgent: consul server down +34:07.030 [INFO] TestAgent: shutdown complete +34:07.030 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15106 network=tcp +34:07.030 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15106 network=udp +34:07.030 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:15107 network=tcp +34:07.530 [INFO] TestAgent: Waiting for endpoints to shut down +34:07.530 [INFO] TestAgent: Endpoints down --- PASS: TestBindingRuleReadCommand (0.97s) --- PASS: TestBindingRuleReadCommand/id_required (0.00s) --- PASS: TestBindingRuleReadCommand/read_by_id_not_found (0.00s) - --- PASS: TestBindingRuleReadCommand/read_by_id (0.01s) + --- PASS: TestBindingRuleReadCommand/read_by_id (0.00s) --- PASS: TestBindingRuleReadCommand/read_by_id_prefix (0.01s) --- PASS: TestBindingRuleReadCommand/read_by_id_json_formatted (0.00s) PASS -ok github.com/hashicorp/consul/command/acl/bindingrule/read 1.043s +ok github.com/hashicorp/consul/command/acl/bindingrule/read 1.076s === RUN TestBindingRuleUpdateCommand_noTabs === PAUSE TestBindingRuleUpdateCommand_noTabs === RUN TestBindingRuleUpdateCommand @@ -74645,307 +74665,310 @@ === PAUSE TestBindingRuleUpdateCommand_noMerge === CONT TestBindingRuleUpdateCommand_noTabs === CONT TestBindingRuleUpdateCommand_noMerge -=== CONT TestBindingRuleUpdateCommand [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... --- PASS: TestBindingRuleUpdateCommand_noTabs (0.00s) +=== CONT TestBindingRuleUpdateCommand [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range === RUN TestBindingRuleUpdateCommand_noMerge/rule_id_required === RUN TestBindingRuleUpdateCommand_noMerge/rule_id_partial_matches_nothing === RUN TestBindingRuleUpdateCommand_noMerge/rule_id_exact_match_doesn't_exist === RUN TestBindingRuleUpdateCommand_noMerge/rule_id_partial_matches_multiple -=== RUN TestBindingRuleUpdateCommand/rule_id_required === RUN TestBindingRuleUpdateCommand_noMerge/must_use_roughly_valid_selector +=== RUN TestBindingRuleUpdateCommand_noMerge/update_all_fields +=== RUN TestBindingRuleUpdateCommand_noMerge/update_all_fields_-_partial +=== RUN TestBindingRuleUpdateCommand_noMerge/update_all_fields_but_description +=== RUN TestBindingRuleUpdateCommand/rule_id_required === RUN TestBindingRuleUpdateCommand/rule_id_partial_matches_nothing +=== RUN TestBindingRuleUpdateCommand_noMerge/missing_bind_name === RUN TestBindingRuleUpdateCommand/rule_id_exact_match_doesn't_exist === RUN TestBindingRuleUpdateCommand/rule_id_partial_matches_multiple -=== RUN TestBindingRuleUpdateCommand_noMerge/update_all_fields -=== RUN TestBindingRuleUpdateCommand_noMerge/update_all_fields_-_partial +=== RUN TestBindingRuleUpdateCommand_noMerge/update_all_fields_but_selector === RUN TestBindingRuleUpdateCommand/must_use_roughly_valid_selector === RUN TestBindingRuleUpdateCommand/update_all_fields === RUN TestBindingRuleUpdateCommand/update_all_fields_-_partial -=== RUN TestBindingRuleUpdateCommand_noMerge/update_all_fields_but_description -=== RUN TestBindingRuleUpdateCommand_noMerge/missing_bind_name -=== RUN TestBindingRuleUpdateCommand_noMerge/update_all_fields_but_selector === RUN TestBindingRuleUpdateCommand/update_all_fields_but_description === RUN TestBindingRuleUpdateCommand/update_all_fields_but_bind_name === RUN TestBindingRuleUpdateCommand/update_all_fields_but_must_exist === RUN TestBindingRuleUpdateCommand/update_all_fields_but_selector === RUN TestBindingRuleUpdateCommand/update_all_fields_clear_selector === RUN TestBindingRuleUpdateCommand/update_all_fields_json_formatted -2022-09-15T21:41:47.118Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:47.139Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:47.146 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0cb68d5a-11ec-9273-c42d-5c132b7faba1 Address:127.0.0.1:10006}]" -41:47.146 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:10006 [Follower]" leader= -41:47.147 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0cb68d5a-11ec-9273-c42d-5c132b7faba1.dc1 127.0.0.1 -41:47.147 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0cb68d5a-11ec-9273-c42d-5c132b7faba1 127.0.0.1 -2022-09-15T21:41:47.148Z [INFO] agent.router: Initializing LAN area manager -41:47.148 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10001 network=udp -41:47.148 [INFO] TestAgent.server: Adding LAN server: server="Node-0cb68d5a-11ec-9273-c42d-5c132b7faba1 (Addr: tcp/127.0.0.1:10006) (DC: dc1)" -41:47.148 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0cb68d5a-11ec-9273-c42d-5c132b7faba1.dc1 area=wan -41:47.148 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10001 network=tcp -41:47.149 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:10002 network=tcp -41:47.149 [INFO] TestAgent: started state syncer -41:47.149 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:10007 network=tcp -41:47.198 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -41:47.210 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:47.211 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:10006 [Candidate]" term=2 -41:47.211 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:47.211 [DEBUG] TestAgent.server.raft: vote granted: from=0cb68d5a-11ec-9273-c42d-5c132b7faba1 term=2 tally=1 -41:47.211 [INFO] TestAgent.server.raft: election won: tally=1 -41:47.211 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:10006 [Leader]" -41:47.211 [INFO] TestAgent.server: cluster leadership acquired -41:47.211 [INFO] TestAgent.server: initializing acls -41:47.212 [INFO] TestAgent.server: Created ACL 'global-management' policy -41:47.212 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -41:47.212 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -41:47.213 [INFO] TestAgent.server: Created ACL anonymous token from configuration -41:47.213 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -41:47.213 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -41:47.213 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-0cb68d5a-11ec-9273-c42d-5c132b7faba1 -41:47.213 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-0cb68d5a-11ec-9273-c42d-5c132b7faba1.dc1 -41:47.214 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-0cb68d5a-11ec-9273-c42d-5c132b7faba1.dc1 area=wan -41:47.214 [INFO] TestAgent.server: New leader elected: payload=Node-0cb68d5a-11ec-9273-c42d-5c132b7faba1 -41:47.214 [INFO] TestAgent.server: Updating LAN server: server="Node-0cb68d5a-11ec-9273-c42d-5c132b7faba1 (Addr: tcp/127.0.0.1:10006) (DC: dc1)" -41:47.214 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:47.217 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:47.217 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:47.217 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:47.217 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:47.217 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:47.217 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0cb68d5a-11ec-9273-c42d-5c132b7faba1 -41:47.217 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0cb68d5a-11ec-9273-c42d-5c132b7faba1 -41:47.218 [INFO] TestAgent.server: federation state anti-entropy synced -41:47.218 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0cb68d5a-11ec-9273-c42d-5c132b7faba1 -41:47.221 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:47.222 [INFO] TestAgent: Synced node info -41:47.529 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:32864 latency=1.88682ms -41:47.534 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:32866 latency=569.08µs -41:47.537 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/835f55ff-3dce-17b9-c273-498b82233426 from=127.0.0.1:32868 latency=213.4µs -41:47.544 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:32864 latency=4.1465ms -41:47.548 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:32864 latency=2.73118ms -41:47.549 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:32864 latency=263.84µs -41:47.551 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:32864 latency=1.22158ms -41:47.553 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:32864 latency=207.06µs -41:47.555 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:32864 latency=984.7µs -41:47.556 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:32864 latency=324.82µs -41:47.558 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:32864 latency=1.0014ms -41:47.559 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:32864 latency=205.26µs -41:47.561 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:32864 latency=953.4µs -41:47.562 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:32864 latency=213.16µs -41:47.564 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:32872 latency=267.98µs -41:47.569 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:32864 latency=3.6801ms -41:47.577 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/5e81e843-d4d9-db51-99f6-294463ebf809 from=127.0.0.1:32878 latency=274.14µs -41:47.579 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/binding-rule/5e81e843-d4d9-db51-99f6-294463ebf809 from=127.0.0.1:32878 error="invalid Binding Rule: Selector is invalid: 1:4 (3): no match found, expected: "!=", ".", "==", "[", [ \t\r\n] or [a-zA-Z0-9_]" -41:47.579 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/5e81e843-d4d9-db51-99f6-294463ebf809 from=127.0.0.1:32878 latency=882.72µs -41:47.583 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:32864 latency=1.22054ms -41:47.586 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/c5240e3d-9d94-ee2b-dfdd-1f01e841a3e3 from=127.0.0.1:32880 latency=295.68µs -41:47.588 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/c5240e3d-9d94-ee2b-dfdd-1f01e841a3e3 from=127.0.0.1:32880 latency=1.23908ms -41:47.588 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/c5240e3d-9d94-ee2b-dfdd-1f01e841a3e3 from=127.0.0.1:32864 latency=151.26µs -41:47.589 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:32864 latency=216.54µs -41:47.594 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/5e81e843-d4d9-db51-99f6-294463ebf809 from=127.0.0.1:32864 latency=3.9089ms -41:47.597 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/6a03d546-35fd-72c3-ba03-e66d664fd733 from=127.0.0.1:32884 latency=742.28µs -41:47.598 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/7ad20dd4-f8d2-c4c7-137e-5aa30789f179 from=127.0.0.1:32886 latency=691.64µs -41:47.603 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/7c86fcd2-63a3-f7c7-a867-9e135253e37d from=127.0.0.1:32888 latency=2.15812ms -41:47.604 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/c5240e3d-9d94-ee2b-dfdd-1f01e841a3e3 from=127.0.0.1:32892 latency=593.84µs -41:47.605 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/ca5434b6-e73d-b994-bafc-5212117dc3f4 from=127.0.0.1:32894 latency=697.18µs -41:47.607 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/e45bb089-9bec-f9de-146d-bc4a9d67ffea from=127.0.0.1:32896 latency=655.26µs -41:47.609 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:32898 latency=1.1281ms -41:47.611 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:32902 latency=239.74µs -41:47.612 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/d39f8ccc-d9f2-0091-041c-eed9c70ba132 from=127.0.0.1:32902 latency=167.26µs -41:47.614 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/d39f8ccc-d9f2-0091-041c-eed9c70ba132 from=127.0.0.1:32902 latency=1.37106ms -41:47.615 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/d39f8ccc-d9f2-0091-041c-eed9c70ba132 from=127.0.0.1:32898 latency=164.04µs -41:47.617 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:32898 latency=992.54µs -41:47.619 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/bddeb3d8-7aa7-f63e-ba52-9994468257a5 from=127.0.0.1:32910 latency=240.68µs -41:47.621 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/bddeb3d8-7aa7-f63e-ba52-9994468257a5 from=127.0.0.1:32910 latency=1.47296ms -41:47.622 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/bddeb3d8-7aa7-f63e-ba52-9994468257a5 from=127.0.0.1:32898 latency=175.14µs -41:47.624 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:32898 latency=1.09876ms -41:47.627 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/6a2b2645-c7b2-187c-c3f6-8bf9bc49db01 from=127.0.0.1:32918 latency=236.76µs -41:47.628 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:32898 latency=930.96µs -41:47.671 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/49f08f94-fbc0-4ccb-f499-4483b62fe6f0 from=127.0.0.1:32922 latency=370.18µs -41:47.672 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/49f08f94-fbc0-4ccb-f499-4483b62fe6f0 from=127.0.0.1:32922 latency=730.98µs -41:47.673 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/49f08f94-fbc0-4ccb-f499-4483b62fe6f0 from=127.0.0.1:32898 latency=163.82µs -41:47.674 [INFO] TestAgent: Requesting shutdown -41:47.674 [INFO] TestAgent.server: shutting down server -41:47.674 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:47.674 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:47.674 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:47.674 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:47.674 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -41:47.674 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -41:47.674 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:47.674 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:47.674 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:47.674 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:47.674 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:47.674 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:47.674 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -41:47.674 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -41:47.674 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:47.674Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:47.674Z [INFO] agent.router.manager: shutting down -41:47.674 [INFO] TestAgent: consul server down -41:47.674 [INFO] TestAgent: shutdown complete -41:47.674 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10001 network=tcp -41:47.675 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10001 network=udp -41:47.675 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:10002 network=tcp -41:48.175 [INFO] TestAgent: Waiting for endpoints to shut down -41:48.175 [INFO] TestAgent: Endpoints down ---- PASS: TestBindingRuleUpdateCommand_noMerge (1.15s) +2021-08-13T15:34:05.416Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:05.437Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:05.441 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:13ecfaea-a642-5ef1-3fef-fd3e5fd2e2f6 Address:127.0.0.1:14090}]" +34:05.441 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14090 [Follower]" leader= +34:05.442 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-13ecfaea-a642-5ef1-3fef-fd3e5fd2e2f6.dc1 127.0.0.1 +34:05.443 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-13ecfaea-a642-5ef1-3fef-fd3e5fd2e2f6 127.0.0.1 +2021-08-13T15:34:05.443Z [INFO] agent.router: Initializing LAN area manager +34:05.443 [INFO] TestAgent.server: Adding LAN server: server="Node-13ecfaea-a642-5ef1-3fef-fd3e5fd2e2f6 (Addr: tcp/127.0.0.1:14090) (DC: dc1)" +34:05.443 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-13ecfaea-a642-5ef1-3fef-fd3e5fd2e2f6.dc1 area=wan +34:05.443 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14085 network=udp +34:05.443 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14085 network=tcp +34:05.446 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14086 network=tcp +34:05.447 [INFO] TestAgent: started state syncer +34:05.447 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14091 network=tcp +34:05.486 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:05.486 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14090 [Candidate]" term=2 +34:05.486 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:05.486 [DEBUG] TestAgent.server.raft: vote granted: from=13ecfaea-a642-5ef1-3fef-fd3e5fd2e2f6 term=2 tally=1 +34:05.486 [INFO] TestAgent.server.raft: election won: tally=1 +34:05.486 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14090 [Leader]" +34:05.486 [INFO] TestAgent.server: cluster leadership acquired +34:05.487 [INFO] TestAgent.server: New leader elected: payload=Node-13ecfaea-a642-5ef1-3fef-fd3e5fd2e2f6 +34:05.487 [INFO] TestAgent.server: initializing acls +34:05.487 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:05.487 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:05.488 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:05.489 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:05.489 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:05.489 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:05.489 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-13ecfaea-a642-5ef1-3fef-fd3e5fd2e2f6 +34:05.489 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-13ecfaea-a642-5ef1-3fef-fd3e5fd2e2f6.dc1 +34:05.489 [INFO] TestAgent.server: Updating LAN server: server="Node-13ecfaea-a642-5ef1-3fef-fd3e5fd2e2f6 (Addr: tcp/127.0.0.1:14090) (DC: dc1)" +34:05.489 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-13ecfaea-a642-5ef1-3fef-fd3e5fd2e2f6.dc1 area=wan +34:05.490 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:05.514 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:05.514 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:05.514 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:05.514 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:05.514 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:05.514 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-13ecfaea-a642-5ef1-3fef-fd3e5fd2e2f6 +34:05.514 [INFO] TestAgent.server: member joined, marking health alive: member=Node-13ecfaea-a642-5ef1-3fef-fd3e5fd2e2f6 +34:05.516 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-13ecfaea-a642-5ef1-3fef-fd3e5fd2e2f6 +34:05.516 [INFO] TestAgent.server: federation state anti-entropy synced +34:05.543 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:05.544 [INFO] TestAgent: Synced node info +34:05.544 [DEBUG] TestAgent: Node info in sync +34:05.653 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:48752 latency=1.636624ms +34:05.656 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:48754 latency=517.341µs +34:05.663 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/d854213a-a9b9-6264-c4d3-d195514b3f6f from=127.0.0.1:48756 latency=245.26µs +34:05.664 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:48752 latency=383.181µs +34:05.667 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:48752 latency=1.536883ms +34:05.668 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:48752 latency=324.661µs +34:05.670 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:48752 latency=1.281643ms +34:05.671 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:48752 latency=214.26µs +34:05.673 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:48752 latency=1.041343ms +34:05.673 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:48752 latency=296.821µs +34:05.675 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:48752 latency=1.001803ms +34:05.676 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:48752 latency=296.261µs +34:05.679 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:48760 latency=370.941µs +34:05.681 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:48752 latency=941.862µs +34:05.683 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/a80831aa-07b2-8096-3c6a-60624297fbea from=127.0.0.1:48762 latency=290.141µs +34:05.684 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/binding-rule/a80831aa-07b2-8096-3c6a-60624297fbea from=127.0.0.1:48762 error="invalid Binding Rule: Selector is invalid: 1:4 (3): no match found, expected: "!=", ".", "==", "[", [ \t\r\n] or [a-zA-Z0-9_]" +34:05.684 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/a80831aa-07b2-8096-3c6a-60624297fbea from=127.0.0.1:48762 latency=820.722µs +34:05.686 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:48752 latency=1.003343ms +34:05.688 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/c7a26e21-4f19-bc40-2858-b0c9eee55f46 from=127.0.0.1:48764 latency=245.221µs +34:05.689 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/c7a26e21-4f19-bc40-2858-b0c9eee55f46 from=127.0.0.1:48764 latency=1.153263ms +34:05.690 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/c7a26e21-4f19-bc40-2858-b0c9eee55f46 from=127.0.0.1:48752 latency=154.681µs +34:05.691 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:48752 latency=205.801µs +34:05.692 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/264c7d05-908d-f6ac-6673-884cf92c8f4f from=127.0.0.1:48752 latency=577.641µs +34:05.694 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/60dc904b-5e72-2d09-2f29-1a2a3202ec0f from=127.0.0.1:48766 latency=540.361µs +34:05.695 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/a80831aa-07b2-8096-3c6a-60624297fbea from=127.0.0.1:48768 latency=552.922µs +34:05.705 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/c7a26e21-4f19-bc40-2858-b0c9eee55f46 from=127.0.0.1:48770 latency=9.457742ms +34:05.708 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/f89929fb-4ff7-15e9-1398-9626624b6282 from=127.0.0.1:48772 latency=1.207783ms +34:05.710 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/fbdc5315-6c04-26ef-b17f-d1a78dcaff42 from=127.0.0.1:48774 latency=948.482µs +34:05.712 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:48776 latency=1.611144ms +34:05.715 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:48778 latency=648.462µs +34:05.716 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/331020a3-d529-4702-9b32-268c3d3d87f8 from=127.0.0.1:48778 latency=425.441µs +34:05.718 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/331020a3-d529-4702-9b32-268c3d3d87f8 from=127.0.0.1:48778 latency=1.157283ms +34:05.719 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/331020a3-d529-4702-9b32-268c3d3d87f8 from=127.0.0.1:48776 latency=175.54µs +34:05.721 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:48776 latency=1.030262ms +34:05.723 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/9efed3c9-1efa-2276-b2a1-4bab9f084d84 from=127.0.0.1:48782 latency=223.021µs +34:05.724 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/9efed3c9-1efa-2276-b2a1-4bab9f084d84 from=127.0.0.1:48782 latency=1.148603ms +34:05.725 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/9efed3c9-1efa-2276-b2a1-4bab9f084d84 from=127.0.0.1:48776 latency=186.921µs +34:05.727 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:48776 latency=1.145402ms +34:05.729 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/bd1b54be-5cd7-4be1-a1f4-0e50d7af78fd from=127.0.0.1:48788 latency=212.741µs +34:05.731 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:48776 latency=921.523µs +34:05.733 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/50409e22-86ba-bda9-d01a-52501780a50a from=127.0.0.1:48790 latency=213.72µs +34:05.734 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/50409e22-86ba-bda9-d01a-52501780a50a from=127.0.0.1:48790 latency=794.142µs +34:05.735 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/50409e22-86ba-bda9-d01a-52501780a50a from=127.0.0.1:48776 latency=280.06µs +34:05.736 [INFO] TestAgent: Requesting shutdown +34:05.736 [INFO] TestAgent.server: shutting down server +34:05.736 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:05.736 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:05.736 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:05.736 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:05.736 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:05.736 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:05.736 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:05.736 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:05.736 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:05.736 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:05.736 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:05.736 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:05.736 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:05.736 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:05.736 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:05.736Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:05.736Z [INFO] agent.router.manager: shutting down +34:05.737 [INFO] TestAgent: consul server down +34:05.737 [INFO] TestAgent: shutdown complete +34:05.737 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14085 network=tcp +34:05.737 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14085 network=udp +34:05.737 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14086 network=tcp +34:06.237 [INFO] TestAgent: Waiting for endpoints to shut down +34:06.237 [INFO] TestAgent: Endpoints down +--- PASS: TestBindingRuleUpdateCommand_noMerge (0.88s) --- PASS: TestBindingRuleUpdateCommand_noMerge/rule_id_required (0.00s) --- PASS: TestBindingRuleUpdateCommand_noMerge/rule_id_partial_matches_nothing (0.00s) - --- PASS: TestBindingRuleUpdateCommand_noMerge/rule_id_exact_match_doesn't_exist (0.00s) - --- PASS: TestBindingRuleUpdateCommand_noMerge/rule_id_partial_matches_multiple (0.03s) - --- PASS: TestBindingRuleUpdateCommand_noMerge/must_use_roughly_valid_selector (0.02s) + --- PASS: TestBindingRuleUpdateCommand_noMerge/rule_id_exact_match_doesn't_exist (0.01s) + --- PASS: TestBindingRuleUpdateCommand_noMerge/rule_id_partial_matches_multiple (0.02s) + --- PASS: TestBindingRuleUpdateCommand_noMerge/must_use_roughly_valid_selector (0.00s) --- PASS: TestBindingRuleUpdateCommand_noMerge/update_all_fields (0.01s) --- PASS: TestBindingRuleUpdateCommand_noMerge/update_all_fields_-_partial (0.03s) --- PASS: TestBindingRuleUpdateCommand_noMerge/update_all_fields_but_description (0.01s) --- PASS: TestBindingRuleUpdateCommand_noMerge/missing_bind_name (0.00s) - --- PASS: TestBindingRuleUpdateCommand_noMerge/update_all_fields_but_selector (0.05s) -2022-09-15T21:41:47.120Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:47.140Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:47.144 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:87270102-24cb-dad9-4d6d-d44153920c2c Address:127.0.0.1:10013}]" -41:47.144 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:10013 [Follower]" leader= -41:47.145 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-87270102-24cb-dad9-4d6d-d44153920c2c.dc1 127.0.0.1 -41:47.146 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-87270102-24cb-dad9-4d6d-d44153920c2c 127.0.0.1 -2022-09-15T21:41:47.146Z [INFO] agent.router: Initializing LAN area manager -41:47.146 [INFO] TestAgent.server: Adding LAN server: server="Node-87270102-24cb-dad9-4d6d-d44153920c2c (Addr: tcp/127.0.0.1:10013) (DC: dc1)" -41:47.146 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-87270102-24cb-dad9-4d6d-d44153920c2c.dc1 area=wan -41:47.146 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10008 network=udp -41:47.146 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10008 network=tcp -41:47.147 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:10009 network=tcp -41:47.147 [INFO] TestAgent: started state syncer -41:47.148 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:10014 network=tcp -41:47.183 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:47.183 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:10013 [Candidate]" term=2 -41:47.183 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:47.184 [DEBUG] TestAgent.server.raft: vote granted: from=87270102-24cb-dad9-4d6d-d44153920c2c term=2 tally=1 -41:47.184 [INFO] TestAgent.server.raft: election won: tally=1 -41:47.184 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:10013 [Leader]" -41:47.184 [INFO] TestAgent.server: cluster leadership acquired -41:47.184 [INFO] TestAgent.server: New leader elected: payload=Node-87270102-24cb-dad9-4d6d-d44153920c2c -41:47.184 [INFO] TestAgent.server: initializing acls -41:47.185 [INFO] TestAgent.server: Created ACL 'global-management' policy -41:47.185 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -41:47.185 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -41:47.186 [INFO] TestAgent.server: Created ACL anonymous token from configuration -41:47.186 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -41:47.186 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -41:47.186 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-87270102-24cb-dad9-4d6d-d44153920c2c -41:47.187 [INFO] TestAgent.server: Updating LAN server: server="Node-87270102-24cb-dad9-4d6d-d44153920c2c (Addr: tcp/127.0.0.1:10013) (DC: dc1)" -41:47.187 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-87270102-24cb-dad9-4d6d-d44153920c2c.dc1 -41:47.187 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-87270102-24cb-dad9-4d6d-d44153920c2c.dc1 area=wan -41:47.190 [INFO] TestAgent: Synced node info -41:47.190 [DEBUG] TestAgent: Node info in sync -41:47.191 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:47.215 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:47.215 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:47.215 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:47.215 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:47.215 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:47.215 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-87270102-24cb-dad9-4d6d-d44153920c2c -41:47.215 [INFO] TestAgent.server: member joined, marking health alive: member=Node-87270102-24cb-dad9-4d6d-d44153920c2c -41:47.216 [INFO] TestAgent.server: federation state anti-entropy synced -41:47.216 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-87270102-24cb-dad9-4d6d-d44153920c2c -41:47.564 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:52788 latency=1.57782ms -41:47.567 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:52792 latency=356.98µs -41:47.574 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/a40544af-ac0f-e11f-53e9-36080a9c0836 from=127.0.0.1:52794 latency=278.46µs -41:47.577 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:52788 latency=233.82µs -41:47.579 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:52788 latency=1.39196ms -41:47.582 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:52788 latency=223.96µs -41:47.584 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:52788 latency=949.04µs -41:47.585 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:52788 latency=196.5µs -41:47.587 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:52788 latency=1.05904ms -41:47.587 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:52788 latency=197.08µs -41:47.589 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:52788 latency=907.3µs -41:47.590 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:52788 latency=196µs -41:47.595 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:52800 latency=351.92µs -41:47.599 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:52788 latency=1.77476ms -41:47.603 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/df367f74-a6df-5f8a-6802-43bbbdc09c9f from=127.0.0.1:52808 latency=260.64µs -41:47.605 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/binding-rule/df367f74-a6df-5f8a-6802-43bbbdc09c9f from=127.0.0.1:52808 error="invalid Binding Rule: Selector is invalid: 1:4 (3): no match found, expected: "!=", ".", "==", "[", [ \t\r\n] or [a-zA-Z0-9_]" -41:47.605 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/df367f74-a6df-5f8a-6802-43bbbdc09c9f from=127.0.0.1:52808 latency=836.22µs -41:47.607 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:52788 latency=1.08174ms -41:47.609 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/26ae48b2-274f-399a-b09d-3a6d9da1d106 from=127.0.0.1:52818 latency=232.84µs -41:47.611 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/26ae48b2-274f-399a-b09d-3a6d9da1d106 from=127.0.0.1:52818 latency=1.15038ms -41:47.612 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/26ae48b2-274f-399a-b09d-3a6d9da1d106 from=127.0.0.1:52788 latency=168.4µs -41:47.613 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:52788 latency=218.46µs -41:47.615 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/26ae48b2-274f-399a-b09d-3a6d9da1d106 from=127.0.0.1:52788 latency=452.48µs -41:47.616 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/2e31e41d-a99e-63ce-4035-bda3316ee279 from=127.0.0.1:52822 latency=536.32µs -41:47.618 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/311b852d-7150-5bab-68c6-85cec3205c90 from=127.0.0.1:52824 latency=759.1µs -41:47.619 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/39f892b6-53be-32e6-7f98-63e433e9e4e1 from=127.0.0.1:52826 latency=785.64µs -41:47.621 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/69a92ec5-ad02-648e-4691-4a771dbe4047 from=127.0.0.1:52830 latency=570.22µs -41:47.623 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/df367f74-a6df-5f8a-6802-43bbbdc09c9f from=127.0.0.1:52832 latency=504.1µs -41:47.625 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:52834 latency=1.05736ms -41:47.627 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:52838 latency=215.34µs -41:47.628 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/d7806cc6-5b06-6837-b9a2-17139150c157 from=127.0.0.1:52838 latency=139.44µs -41:47.629 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/d7806cc6-5b06-6837-b9a2-17139150c157 from=127.0.0.1:52838 latency=1.02588ms -41:47.630 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/d7806cc6-5b06-6837-b9a2-17139150c157 from=127.0.0.1:52834 latency=165.98µs -41:47.632 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:52834 latency=1.02662ms -41:47.672 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/a912cede-5eb9-a41f-6a95-5034a78ff528 from=127.0.0.1:52842 latency=243.58µs -41:47.674 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/a912cede-5eb9-a41f-6a95-5034a78ff528 from=127.0.0.1:52842 latency=1.48468ms -41:47.683 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/a912cede-5eb9-a41f-6a95-5034a78ff528 from=127.0.0.1:52834 latency=331.84µs -41:47.689 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:52834 latency=2.80604ms -41:47.692 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/2bc08f1e-306a-37b8-c3e8-dfcefcf60de4 from=127.0.0.1:52844 latency=276.98µs -41:47.694 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/2bc08f1e-306a-37b8-c3e8-dfcefcf60de4 from=127.0.0.1:52844 latency=1.27772ms -41:47.706 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/2bc08f1e-306a-37b8-c3e8-dfcefcf60de4 from=127.0.0.1:52834 latency=247.56µs -41:47.712 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:52834 latency=1.1683ms -41:47.715 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/09e6111d-bd00-f929-15be-6dc379eadbe5 from=127.0.0.1:52846 latency=338.72µs -41:47.717 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/09e6111d-bd00-f929-15be-6dc379eadbe5 from=127.0.0.1:52846 latency=1.27692ms -41:47.718 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/09e6111d-bd00-f929-15be-6dc379eadbe5 from=127.0.0.1:52834 latency=156.22µs -41:47.719 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:52834 latency=900.16µs -41:47.722 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/fe11ab04-648f-3736-2746-cbfab31500ad from=127.0.0.1:52848 latency=286.78µs -41:47.723 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/fe11ab04-648f-3736-2746-cbfab31500ad from=127.0.0.1:52848 latency=1.11676ms -41:47.724 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/fe11ab04-648f-3736-2746-cbfab31500ad from=127.0.0.1:52834 latency=152.28µs -41:47.728 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:52834 latency=1.04126ms -41:47.733 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/b51894d1-ee11-84e9-587d-2a142fc6318a from=127.0.0.1:52850 latency=423.92µs -41:47.734 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/b51894d1-ee11-84e9-587d-2a142fc6318a from=127.0.0.1:52850 latency=643.22µs -41:47.735 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/b51894d1-ee11-84e9-587d-2a142fc6318a from=127.0.0.1:52834 latency=152.96µs -41:47.737 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:52834 latency=942.04µs -41:47.739 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/6766bdc0-4bef-3320-9290-f2a912d75d0e from=127.0.0.1:52852 latency=247.46µs -41:47.741 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/6766bdc0-4bef-3320-9290-f2a912d75d0e from=127.0.0.1:52852 latency=1.08814ms -41:47.742 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/6766bdc0-4bef-3320-9290-f2a912d75d0e from=127.0.0.1:52834 latency=239.06µs -41:47.742 [INFO] TestAgent: Requesting shutdown -41:47.742 [INFO] TestAgent.server: shutting down server -41:47.742 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:47.742 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:47.742 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -41:47.742 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -41:47.742 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:47.742 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:47.742 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:47.743 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:47.743 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:47.743 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:47.743 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:47.743 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -41:47.743 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -41:47.743 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:47.743 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:47.743Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:47.743Z [INFO] agent.router.manager: shutting down -41:47.743 [INFO] TestAgent: consul server down -41:47.743 [INFO] TestAgent: shutdown complete -41:47.743 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10008 network=tcp -41:47.743 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10008 network=udp -41:47.743 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:10009 network=tcp -41:48.244 [INFO] TestAgent: Waiting for endpoints to shut down -41:48.244 [INFO] TestAgent: Endpoints down ---- PASS: TestBindingRuleUpdateCommand (1.22s) + --- PASS: TestBindingRuleUpdateCommand_noMerge/update_all_fields_but_selector (0.01s) +2021-08-13T15:34:05.415Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:05.437Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:05.441 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ae2224ee-840c-f1fd-9623-34e73c11b659 Address:127.0.0.1:14097}]" +34:05.441 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14097 [Follower]" leader= +34:05.442 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ae2224ee-840c-f1fd-9623-34e73c11b659.dc1 127.0.0.1 +34:05.443 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ae2224ee-840c-f1fd-9623-34e73c11b659 127.0.0.1 +2021-08-13T15:34:05.443Z [INFO] agent.router: Initializing LAN area manager +34:05.443 [INFO] TestAgent.server: Adding LAN server: server="Node-ae2224ee-840c-f1fd-9623-34e73c11b659 (Addr: tcp/127.0.0.1:14097) (DC: dc1)" +34:05.443 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ae2224ee-840c-f1fd-9623-34e73c11b659.dc1 area=wan +34:05.443 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14092 network=tcp +34:05.443 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14092 network=udp +34:05.445 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14093 network=tcp +34:05.445 [INFO] TestAgent: started state syncer +34:05.446 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14098 network=tcp +34:05.493 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:05.501 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:05.501 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14097 [Candidate]" term=2 +34:05.501 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:05.501 [DEBUG] TestAgent.server.raft: vote granted: from=ae2224ee-840c-f1fd-9623-34e73c11b659 term=2 tally=1 +34:05.501 [INFO] TestAgent.server.raft: election won: tally=1 +34:05.501 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14097 [Leader]" +34:05.501 [INFO] TestAgent.server: cluster leadership acquired +34:05.501 [INFO] TestAgent.server: New leader elected: payload=Node-ae2224ee-840c-f1fd-9623-34e73c11b659 +34:05.501 [INFO] TestAgent.server: initializing acls +34:05.502 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:05.502 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:05.502 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:05.503 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:05.503 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:05.503 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:05.503 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-ae2224ee-840c-f1fd-9623-34e73c11b659 +34:05.503 [INFO] TestAgent.server: Updating LAN server: server="Node-ae2224ee-840c-f1fd-9623-34e73c11b659 (Addr: tcp/127.0.0.1:14097) (DC: dc1)" +34:05.503 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-ae2224ee-840c-f1fd-9623-34e73c11b659.dc1 +34:05.504 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-ae2224ee-840c-f1fd-9623-34e73c11b659.dc1 area=wan +34:05.504 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:05.514 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:05.514 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:05.514 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:05.514 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:05.514 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:05.514 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ae2224ee-840c-f1fd-9623-34e73c11b659 +34:05.514 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ae2224ee-840c-f1fd-9623-34e73c11b659 +34:05.515 [INFO] TestAgent.server: federation state anti-entropy synced +34:05.517 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ae2224ee-840c-f1fd-9623-34e73c11b659 +34:05.723 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:37476 latency=1.430643ms +34:05.726 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:37480 latency=230.601µs +34:05.727 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/dbbc4209-b6f1-1071-e860-d813a7d0cf79 from=127.0.0.1:37482 latency=164.601µs +34:05.728 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:37476 latency=161.54µs +34:05.730 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:37476 latency=1.204022ms +34:05.730 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:37476 latency=161.38µs +34:05.732 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:37476 latency=873.982µs +34:05.733 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:37476 latency=212.82µs +34:05.734 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:37476 latency=841.542µs +34:05.735 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:37476 latency=203.88µs +34:05.736 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:37476 latency=918.602µs +34:05.737 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:37476 latency=211.961µs +34:05.739 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:37476 latency=886.182µs +34:05.740 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:37476 latency=203.68µs +34:05.741 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:37476 latency=909.002µs +34:05.742 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:37476 latency=184.34µs +34:05.744 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:37488 latency=251.26µs +34:05.746 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:37476 latency=966.942µs +34:05.748 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/50adbe8f-8a87-acea-d74d-1881547fdc8b from=127.0.0.1:37490 latency=226.141µs +34:05.749 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/binding-rule/50adbe8f-8a87-acea-d74d-1881547fdc8b from=127.0.0.1:37490 error="invalid Binding Rule: Selector is invalid: 1:4 (3): no match found, expected: "!=", ".", "==", "[", [ \t\r\n] or [a-zA-Z0-9_]" +34:05.751 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/50adbe8f-8a87-acea-d74d-1881547fdc8b from=127.0.0.1:37490 latency=2.248845ms +34:05.757 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:37476 latency=2.086345ms +34:05.759 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/df747be3-7d40-9b38-a358-a875f10524fa from=127.0.0.1:37492 latency=279.481µs +34:05.761 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/df747be3-7d40-9b38-a358-a875f10524fa from=127.0.0.1:37492 latency=1.118723ms +34:05.761 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/df747be3-7d40-9b38-a358-a875f10524fa from=127.0.0.1:37476 latency=169.18µs +34:05.762 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:37476 latency=219.321µs +34:05.764 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/335bc1ed-564a-81e2-d54b-548ce230abf4 from=127.0.0.1:37476 latency=448.081µs +34:05.765 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/50adbe8f-8a87-acea-d74d-1881547fdc8b from=127.0.0.1:37494 latency=514.481µs +34:05.770 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/85883d84-5b45-8b61-6b4f-f2cfff98dcc4 from=127.0.0.1:37496 latency=624.141µs +34:05.771 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/8b7b7a1a-3799-9cbe-bec5-45d14deec0a6 from=127.0.0.1:37498 latency=521.921µs +34:05.772 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/db9375d6-6625-219d-7798-4435f1b8a5a1 from=127.0.0.1:37500 latency=540.761µs +34:05.774 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/df747be3-7d40-9b38-a358-a875f10524fa from=127.0.0.1:37502 latency=506.661µs +34:05.775 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/e949b99a-fdf0-730d-17b6-bff129dece24 from=127.0.0.1:37504 latency=492.381µs +34:05.776 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/f1c4234d-549c-b7cd-92f7-a92cccd15430 from=127.0.0.1:37506 latency=439.441µs +34:05.778 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:37508 latency=1.113242ms +34:05.780 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:37510 latency=240.401µs +34:05.780 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/b78a4f62-4768-81d0-3fe4-9cf62a9bb423 from=127.0.0.1:37510 latency=143.3µs +34:05.782 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/b78a4f62-4768-81d0-3fe4-9cf62a9bb423 from=127.0.0.1:37510 latency=1.113462ms +34:05.783 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/b78a4f62-4768-81d0-3fe4-9cf62a9bb423 from=127.0.0.1:37508 latency=171.26µs +34:05.785 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:37508 latency=1.069002ms +34:05.787 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/ee103017-ae46-7745-4fc1-66b0ac5cc36a from=127.0.0.1:37512 latency=232.641µs +34:05.788 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/ee103017-ae46-7745-4fc1-66b0ac5cc36a from=127.0.0.1:37512 latency=1.129023ms +34:05.789 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/ee103017-ae46-7745-4fc1-66b0ac5cc36a from=127.0.0.1:37508 latency=157.14µs +34:05.791 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:37508 latency=956.422µs +34:05.793 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/270e5e93-7d82-a79f-38d0-c07937bfdbeb from=127.0.0.1:37514 latency=239.561µs +34:05.794 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/270e5e93-7d82-a79f-38d0-c07937bfdbeb from=127.0.0.1:37514 latency=1.158602ms +34:05.795 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/270e5e93-7d82-a79f-38d0-c07937bfdbeb from=127.0.0.1:37508 latency=162.64µs +34:05.797 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:37508 latency=966.143µs +34:05.799 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/395e6826-0933-0f42-18aa-4990164fbe18 from=127.0.0.1:37516 latency=238.941µs +34:05.801 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/395e6826-0933-0f42-18aa-4990164fbe18 from=127.0.0.1:37516 latency=1.122183ms +34:05.801 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/395e6826-0933-0f42-18aa-4990164fbe18 from=127.0.0.1:37508 latency=146.901µs +34:05.803 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:37508 latency=924.803µs +34:05.805 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/2efaa629-bcc3-3d3f-d2d2-0e70e3ce1e62 from=127.0.0.1:37518 latency=231.021µs +34:05.807 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/2efaa629-bcc3-3d3f-d2d2-0e70e3ce1e62 from=127.0.0.1:37518 latency=1.144303ms +34:05.807 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/2efaa629-bcc3-3d3f-d2d2-0e70e3ce1e62 from=127.0.0.1:37508 latency=152.66µs +34:05.809 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:37508 latency=874.962µs +34:05.811 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/3f1634d2-01eb-1732-277f-8bc3c72c1c08 from=127.0.0.1:37520 latency=243.741µs +34:05.812 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/3f1634d2-01eb-1732-277f-8bc3c72c1c08 from=127.0.0.1:37520 latency=576.602µs +34:05.813 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/3f1634d2-01eb-1732-277f-8bc3c72c1c08 from=127.0.0.1:37508 latency=155.3µs +34:05.814 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:37508 latency=886.822µs +34:05.816 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/e25f4000-fbf3-3057-d777-59d8a76da79b from=127.0.0.1:37522 latency=213.501µs +34:05.818 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/e25f4000-fbf3-3057-d777-59d8a76da79b from=127.0.0.1:37522 latency=1.187083ms +34:05.819 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/e25f4000-fbf3-3057-d777-59d8a76da79b from=127.0.0.1:37508 latency=143.74µs +34:05.819 [INFO] TestAgent: Requesting shutdown +34:05.819 [INFO] TestAgent.server: shutting down server +34:05.819 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:05.819 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:05.819 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:05.819 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:05.819 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:05.819 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:05.819 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:05.819 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:05.819 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:05.819 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:05.819 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:05.819 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:05.819 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:05.819 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:05.819 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:05.820 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:05.820Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:05.820Z [INFO] agent.router.manager: shutting down +34:05.820 [INFO] TestAgent: consul server down +34:05.820 [INFO] TestAgent: shutdown complete +34:05.820 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14092 network=tcp +34:05.820 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14092 network=udp +34:05.820 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14093 network=tcp +34:06.320 [INFO] TestAgent: Waiting for endpoints to shut down +34:06.320 [INFO] TestAgent: Endpoints down +--- PASS: TestBindingRuleUpdateCommand (0.97s) --- PASS: TestBindingRuleUpdateCommand/rule_id_required (0.00s) --- PASS: TestBindingRuleUpdateCommand/rule_id_partial_matches_nothing (0.00s) - --- PASS: TestBindingRuleUpdateCommand/rule_id_exact_match_doesn't_exist (0.01s) + --- PASS: TestBindingRuleUpdateCommand/rule_id_exact_match_doesn't_exist (0.00s) --- PASS: TestBindingRuleUpdateCommand/rule_id_partial_matches_multiple (0.02s) --- PASS: TestBindingRuleUpdateCommand/must_use_roughly_valid_selector (0.01s) --- PASS: TestBindingRuleUpdateCommand/update_all_fields (0.01s) --- PASS: TestBindingRuleUpdateCommand/update_all_fields_-_partial (0.02s) - --- PASS: TestBindingRuleUpdateCommand/update_all_fields_but_description (0.05s) - --- PASS: TestBindingRuleUpdateCommand/update_all_fields_but_bind_name (0.02s) + --- PASS: TestBindingRuleUpdateCommand/update_all_fields_but_description (0.01s) + --- PASS: TestBindingRuleUpdateCommand/update_all_fields_but_bind_name (0.01s) --- PASS: TestBindingRuleUpdateCommand/update_all_fields_but_must_exist (0.01s) --- PASS: TestBindingRuleUpdateCommand/update_all_fields_but_selector (0.01s) --- PASS: TestBindingRuleUpdateCommand/update_all_fields_clear_selector (0.01s) --- PASS: TestBindingRuleUpdateCommand/update_all_fields_json_formatted (0.01s) PASS -ok github.com/hashicorp/consul/command/acl/bindingrule/update 1.303s +ok github.com/hashicorp/consul/command/acl/bindingrule/update 1.070s === RUN TestBootstrapCommand_noTabs === PAUSE TestBootstrapCommand_noTabs === RUN TestBootstrapCommand_Pretty @@ -74953,168 +74976,162 @@ === RUN TestBootstrapCommand_JSON === PAUSE TestBootstrapCommand_JSON === CONT TestBootstrapCommand_noTabs ---- PASS: TestBootstrapCommand_noTabs (0.00s) === CONT TestBootstrapCommand_JSON -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestBootstrapCommand_Pretty +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestBootstrapCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:41:58.990Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:59.071Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:59.083 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0d50fff5-76dc-7694-8ca5-e6c30dd9af66 Address:127.0.0.1:11034}]" -41:59.084 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:11034 [Follower]" leader= -41:59.084 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0d50fff5-76dc-7694-8ca5-e6c30dd9af66.dc1 127.0.0.1 -41:59.085 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0d50fff5-76dc-7694-8ca5-e6c30dd9af66 127.0.0.1 -2022-09-15T21:41:59.098Z [INFO] agent.router: Initializing LAN area manager -41:59.098 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11029 network=udp -41:59.099 [INFO] TestAgent.server: Adding LAN server: server="Node-0d50fff5-76dc-7694-8ca5-e6c30dd9af66 (Addr: tcp/127.0.0.1:11034) (DC: dc1)" -41:59.099 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0d50fff5-76dc-7694-8ca5-e6c30dd9af66.dc1 area=wan -41:59.099 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11029 network=tcp -41:59.100 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:11030 network=tcp -41:59.100 [INFO] TestAgent: started state syncer -41:59.100 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:11035 network=tcp -41:59.148 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:59.148 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:11034 [Candidate]" term=2 -41:59.148 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:59.148 [DEBUG] TestAgent.server.raft: vote granted: from=0d50fff5-76dc-7694-8ca5-e6c30dd9af66 term=2 tally=1 -41:59.148 [INFO] TestAgent.server.raft: election won: tally=1 -41:59.148 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:11034 [Leader]" -41:59.148 [INFO] TestAgent.server: cluster leadership acquired -41:59.149 [INFO] TestAgent.server: initializing acls -41:59.149 [INFO] TestAgent.server: New leader elected: payload=Node-0d50fff5-76dc-7694-8ca5-e6c30dd9af66 -41:59.149 [INFO] TestAgent.server: Created ACL 'global-management' policy -41:59.149 [INFO] TestAgent.server: initializing acls -41:59.150 [INFO] TestAgent.server: Created ACL anonymous token from configuration -41:59.150 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -41:59.150 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -41:59.151 [INFO] TestAgent.server: Created ACL anonymous token from configuration -41:59.151 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode -41:59.151 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-0d50fff5-76dc-7694-8ca5-e6c30dd9af66 -41:59.151 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-0d50fff5-76dc-7694-8ca5-e6c30dd9af66.dc1 -41:59.151 [INFO] TestAgent.server: Updating LAN server: server="Node-0d50fff5-76dc-7694-8ca5-e6c30dd9af66 (Addr: tcp/127.0.0.1:11034) (DC: dc1)" -41:59.151 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-0d50fff5-76dc-7694-8ca5-e6c30dd9af66.dc1 area=wan -41:59.152 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-0d50fff5-76dc-7694-8ca5-e6c30dd9af66 -41:59.152 [INFO] TestAgent.server: Updating LAN server: server="Node-0d50fff5-76dc-7694-8ca5-e6c30dd9af66 (Addr: tcp/127.0.0.1:11034) (DC: dc1)" -41:59.154 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-0d50fff5-76dc-7694-8ca5-e6c30dd9af66.dc1 -41:59.154 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-0d50fff5-76dc-7694-8ca5-e6c30dd9af66.dc1 area=wan -41:59.154 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:59.172 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:59.172 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:59.172 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:59.172 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:59.172 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:59.172 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0d50fff5-76dc-7694-8ca5-e6c30dd9af66 -41:59.172 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0d50fff5-76dc-7694-8ca5-e6c30dd9af66 -41:59.174 [INFO] TestAgent.server: federation state anti-entropy synced -41:59.174 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0d50fff5-76dc-7694-8ca5-e6c30dd9af66 -41:59.174 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0d50fff5-76dc-7694-8ca5-e6c30dd9af66 -41:59.380 [WARN] TestAgent.server.acl: failed to remove bootstrap file: error="remove /tmp/consul-test/TestBootstrapCommand_Pretty-TestAgent-agent152935036/acl-bootstrap-reset: no such file or directory" -41:59.381 [INFO] TestAgent.server.acl: ACL bootstrap completed -41:59.382 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/bootstrap from=127.0.0.1:33754 latency=2.4575ms -41:59.384 [INFO] TestAgent: Requesting shutdown -41:59.384 [INFO] TestAgent.server: shutting down server -41:59.384 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:59.384 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -41:59.384 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -41:59.384 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:59.384 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:59.384 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:59.384 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:59.384 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:59.384 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:59.385 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -41:59.385 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:59.385 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -41:59.385 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -41:59.385 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:59.385 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:59.385 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:59.385Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:59.385Z [INFO] agent.router.manager: shutting down -41:59.385 [INFO] TestAgent: consul server down -41:59.385 [INFO] TestAgent: shutdown complete -41:59.385 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11029 network=tcp -41:59.385 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11029 network=udp -41:59.385 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:11030 network=tcp -41:59.886 [INFO] TestAgent: Waiting for endpoints to shut down -41:59.886 [INFO] TestAgent: Endpoints down +2021-08-13T15:34:06.715Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:06.742Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:06.745 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d010922f-77dd-7f8f-1e4c-8c55e5a9346f Address:127.0.0.1:18181}]" +34:06.745 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:18181 [Follower]" leader= +34:06.746 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d010922f-77dd-7f8f-1e4c-8c55e5a9346f.dc1 127.0.0.1 +34:06.747 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d010922f-77dd-7f8f-1e4c-8c55e5a9346f 127.0.0.1 +2021-08-13T15:34:06.747Z [INFO] agent.router: Initializing LAN area manager +34:06.747 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18176 network=udp +34:06.748 [INFO] TestAgent.server: Adding LAN server: server="Node-d010922f-77dd-7f8f-1e4c-8c55e5a9346f (Addr: tcp/127.0.0.1:18181) (DC: dc1)" +34:06.748 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d010922f-77dd-7f8f-1e4c-8c55e5a9346f.dc1 area=wan +34:06.748 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18176 network=tcp +34:06.750 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:18177 network=tcp +34:06.754 [INFO] TestAgent: started state syncer +34:06.755 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:18182 network=tcp +34:06.798 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:06.806 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:06.807 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:18181 [Candidate]" term=2 +34:06.807 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:06.807 [DEBUG] TestAgent.server.raft: vote granted: from=d010922f-77dd-7f8f-1e4c-8c55e5a9346f term=2 tally=1 +34:06.807 [INFO] TestAgent.server.raft: election won: tally=1 +34:06.807 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:18181 [Leader]" +34:06.807 [INFO] TestAgent.server: cluster leadership acquired +34:06.807 [INFO] TestAgent.server: New leader elected: payload=Node-d010922f-77dd-7f8f-1e4c-8c55e5a9346f +34:06.807 [INFO] TestAgent.server: initializing acls +34:06.808 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:06.809 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:06.809 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:06.809 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:06.809 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-d010922f-77dd-7f8f-1e4c-8c55e5a9346f +34:06.809 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-d010922f-77dd-7f8f-1e4c-8c55e5a9346f.dc1 +34:06.810 [INFO] TestAgent.server: Updating LAN server: server="Node-d010922f-77dd-7f8f-1e4c-8c55e5a9346f (Addr: tcp/127.0.0.1:18181) (DC: dc1)" +34:06.810 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-d010922f-77dd-7f8f-1e4c-8c55e5a9346f.dc1 area=wan +34:06.811 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:06.834 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:06.834 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:06.834 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:06.834 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:06.834 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:06.834 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d010922f-77dd-7f8f-1e4c-8c55e5a9346f +34:06.834 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d010922f-77dd-7f8f-1e4c-8c55e5a9346f +34:06.835 [INFO] TestAgent.server: federation state anti-entropy synced +34:06.836 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d010922f-77dd-7f8f-1e4c-8c55e5a9346f +34:06.861 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:06.861 [INFO] TestAgent: Synced node info +34:06.947 [WARN] TestAgent.server.acl: failed to remove bootstrap file: error="remove /tmp/consul-test/TestBootstrapCommand_JSON-TestAgent-agent952337278/acl-bootstrap-reset: no such file or directory" +34:06.948 [INFO] TestAgent.server.acl: ACL bootstrap completed +34:06.949 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/bootstrap from=127.0.0.1:36064 latency=1.701284ms +34:06.950 [INFO] TestAgent: Requesting shutdown +34:06.950 [INFO] TestAgent.server: shutting down server +34:06.950 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:06.950 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:06.950 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:06.950 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:06.950 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:06.950 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:06.950 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:06.950 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:06.950 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:06.950 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:06.950 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:06.950 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:06.950 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:06.950 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:06.950 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:06.950Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:06.951Z [INFO] agent.router.manager: shutting down +34:06.951 [INFO] TestAgent: consul server down +34:06.951 [INFO] TestAgent: shutdown complete +34:06.951 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18176 network=tcp +34:06.951 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18176 network=udp +34:06.951 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:18177 network=tcp +34:07.451 [INFO] TestAgent: Waiting for endpoints to shut down +34:07.451 [INFO] TestAgent: Endpoints down +--- PASS: TestBootstrapCommand_JSON (0.82s) +2021-08-13T15:34:06.711Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:06.735Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:06.739 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:83c08794-64db-6bb6-9950-467d42c30ba5 Address:127.0.0.1:18174}]" +34:06.739 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:18174 [Follower]" leader= +34:06.740 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-83c08794-64db-6bb6-9950-467d42c30ba5.dc1 127.0.0.1 +34:06.741 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-83c08794-64db-6bb6-9950-467d42c30ba5 127.0.0.1 +2021-08-13T15:34:06.741Z [INFO] agent.router: Initializing LAN area manager +34:06.741 [INFO] TestAgent.server: Adding LAN server: server="Node-83c08794-64db-6bb6-9950-467d42c30ba5 (Addr: tcp/127.0.0.1:18174) (DC: dc1)" +34:06.741 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-83c08794-64db-6bb6-9950-467d42c30ba5.dc1 area=wan +34:06.741 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18169 network=udp +34:06.741 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18169 network=tcp +34:06.742 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:18170 network=tcp +34:06.742 [INFO] TestAgent: started state syncer +34:06.743 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:18175 network=tcp +34:06.792 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:06.807 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:06.807 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:18174 [Candidate]" term=2 +34:06.807 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:06.807 [DEBUG] TestAgent.server.raft: vote granted: from=83c08794-64db-6bb6-9950-467d42c30ba5 term=2 tally=1 +34:06.807 [INFO] TestAgent.server.raft: election won: tally=1 +34:06.807 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:18174 [Leader]" +34:06.807 [INFO] TestAgent.server: cluster leadership acquired +34:06.808 [INFO] TestAgent.server: initializing acls +34:06.808 [INFO] TestAgent.server: New leader elected: payload=Node-83c08794-64db-6bb6-9950-467d42c30ba5 +34:06.809 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:06.809 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:06.809 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:06.809 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:06.809 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-83c08794-64db-6bb6-9950-467d42c30ba5 +34:06.809 [INFO] TestAgent.server: Updating LAN server: server="Node-83c08794-64db-6bb6-9950-467d42c30ba5 (Addr: tcp/127.0.0.1:18174) (DC: dc1)" +34:06.810 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-83c08794-64db-6bb6-9950-467d42c30ba5.dc1 +34:06.810 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-83c08794-64db-6bb6-9950-467d42c30ba5.dc1 area=wan +34:06.811 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:06.834 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:06.834 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:06.834 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:06.834 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:06.834 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:06.834 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-83c08794-64db-6bb6-9950-467d42c30ba5 +34:06.834 [INFO] TestAgent.server: member joined, marking health alive: member=Node-83c08794-64db-6bb6-9950-467d42c30ba5 +34:06.835 [INFO] TestAgent.server: federation state anti-entropy synced +34:06.836 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-83c08794-64db-6bb6-9950-467d42c30ba5 +34:06.998 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:06.999 [INFO] TestAgent: Synced node info +34:07.157 [WARN] TestAgent.server.acl: failed to remove bootstrap file: error="remove /tmp/consul-test/TestBootstrapCommand_Pretty-TestAgent-agent523260379/acl-bootstrap-reset: no such file or directory" +34:07.158 [INFO] TestAgent.server.acl: ACL bootstrap completed +34:07.159 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/bootstrap from=127.0.0.1:36684 latency=2.319885ms +34:07.161 [INFO] TestAgent: Requesting shutdown +34:07.161 [INFO] TestAgent.server: shutting down server +34:07.161 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:07.161 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:07.161 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:07.161 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:07.161 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:07.161 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:07.161 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:07.161 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:07.161 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:07.161 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:07.161 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:07.161 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:07.161 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:07.161 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:07.161 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:07.162Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:07.162Z [INFO] agent.router.manager: shutting down +34:07.162 [INFO] TestAgent: consul server down +34:07.162 [INFO] TestAgent: shutdown complete +34:07.162 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18169 network=tcp +34:07.162 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18169 network=udp +34:07.162 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:18170 network=tcp +34:07.662 [INFO] TestAgent: Waiting for endpoints to shut down +34:07.662 [INFO] TestAgent: Endpoints down --- PASS: TestBootstrapCommand_Pretty (1.03s) -2022-09-15T21:41:58.990Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:41:59.069Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -41:59.080 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d603bf5d-35fa-f50e-2de3-736357f7fb54 Address:127.0.0.1:11027}]" -41:59.081 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:11027 [Follower]" leader= -41:59.081 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d603bf5d-35fa-f50e-2de3-736357f7fb54.dc1 127.0.0.1 -41:59.085 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d603bf5d-35fa-f50e-2de3-736357f7fb54 127.0.0.1 -2022-09-15T21:41:59.085Z [INFO] agent.router: Initializing LAN area manager -41:59.086 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11022 network=udp -41:59.086 [INFO] TestAgent.server: Adding LAN server: server="Node-d603bf5d-35fa-f50e-2de3-736357f7fb54 (Addr: tcp/127.0.0.1:11027) (DC: dc1)" -41:59.086 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d603bf5d-35fa-f50e-2de3-736357f7fb54.dc1 area=wan -41:59.087 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11022 network=tcp -41:59.088 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:11023 network=tcp -41:59.088 [INFO] TestAgent: started state syncer -41:59.088 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:11028 network=tcp -41:59.119 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -41:59.119 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:11027 [Candidate]" term=2 -41:59.119 [DEBUG] TestAgent.server.raft: votes: needed=1 -41:59.119 [DEBUG] TestAgent.server.raft: vote granted: from=d603bf5d-35fa-f50e-2de3-736357f7fb54 term=2 tally=1 -41:59.119 [INFO] TestAgent.server.raft: election won: tally=1 -41:59.119 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:11027 [Leader]" -41:59.119 [INFO] TestAgent.server: cluster leadership acquired -41:59.120 [INFO] TestAgent.server: initializing acls -41:59.120 [INFO] TestAgent.server: Created ACL 'global-management' policy -41:59.121 [INFO] TestAgent.server: Created ACL anonymous token from configuration -41:59.121 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -41:59.121 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -41:59.121 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-d603bf5d-35fa-f50e-2de3-736357f7fb54 -41:59.122 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-d603bf5d-35fa-f50e-2de3-736357f7fb54.dc1 -41:59.122 [INFO] TestAgent.server: New leader elected: payload=Node-d603bf5d-35fa-f50e-2de3-736357f7fb54 -41:59.122 [INFO] TestAgent.server: Updating LAN server: server="Node-d603bf5d-35fa-f50e-2de3-736357f7fb54 (Addr: tcp/127.0.0.1:11027) (DC: dc1)" -41:59.122 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-d603bf5d-35fa-f50e-2de3-736357f7fb54.dc1 area=wan -41:59.124 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -41:59.151 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -41:59.151 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -41:59.151 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -41:59.151 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -41:59.151 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -41:59.151 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d603bf5d-35fa-f50e-2de3-736357f7fb54 -41:59.151 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d603bf5d-35fa-f50e-2de3-736357f7fb54 -41:59.152 [INFO] TestAgent.server: federation state anti-entropy synced -41:59.162 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d603bf5d-35fa-f50e-2de3-736357f7fb54 -41:59.326 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -41:59.326 [INFO] TestAgent: Synced node info -41:59.327 [DEBUG] TestAgent: Node info in sync -41:59.425 [WARN] TestAgent.server.acl: failed to remove bootstrap file: error="remove /tmp/consul-test/TestBootstrapCommand_JSON-TestAgent-agent060223802/acl-bootstrap-reset: no such file or directory" -41:59.426 [INFO] TestAgent.server.acl: ACL bootstrap completed -41:59.427 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/bootstrap from=127.0.0.1:47586 latency=1.25534ms -41:59.428 [INFO] TestAgent: Requesting shutdown -41:59.428 [INFO] TestAgent.server: shutting down server -41:59.428 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -41:59.428 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -41:59.428 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -41:59.428 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -41:59.428 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -41:59.428 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -41:59.428 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -41:59.428 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -41:59.428 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -41:59.428 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -41:59.428 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -41:59.428 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -41:59.428 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -41:59.428 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -41:59.428 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:41:59.428Z [INFO] agent.router.manager: shutting down -2022-09-15T21:41:59.428Z [INFO] agent.router.manager: shutting down -41:59.428 [INFO] TestAgent: consul server down -41:59.428 [INFO] TestAgent: shutdown complete -41:59.428 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11022 network=tcp -41:59.429 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11022 network=udp -41:59.429 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:11023 network=tcp -41:59.929 [INFO] TestAgent: Waiting for endpoints to shut down -41:59.929 [INFO] TestAgent: Endpoints down ---- PASS: TestBootstrapCommand_JSON (1.07s) PASS -ok github.com/hashicorp/consul/command/acl/bootstrap 1.174s +ok github.com/hashicorp/consul/command/acl/bootstrap 1.110s ? github.com/hashicorp/consul/command/acl/policy [no test files] === RUN TestPolicyCreateCommand_noTabs === PAUSE TestPolicyCreateCommand_noTabs @@ -75124,248 +75141,252 @@ === PAUSE TestPolicyCreateCommand_JSON === CONT TestPolicyCreateCommand_noTabs === CONT TestPolicyCreateCommand_JSON +=== CONT TestPolicyCreateCommand [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... --- PASS: TestPolicyCreateCommand_noTabs (0.00s) -=== CONT TestPolicyCreateCommand [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:42:01.350Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:01.378Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:01.380 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:75f5f8bc-fc95-d676-f1f8-c0b6f46fb9db Address:127.0.0.1:24307}]" -42:01.380 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24307 [Follower]" leader= -42:01.381 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-75f5f8bc-fc95-d676-f1f8-c0b6f46fb9db.dc1 127.0.0.1 -42:01.382 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-75f5f8bc-fc95-d676-f1f8-c0b6f46fb9db 127.0.0.1 -2022-09-15T21:42:01.382Z [INFO] agent.router: Initializing LAN area manager -42:01.382 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24302 network=udp -42:01.382 [INFO] TestAgent.server: Adding LAN server: server="Node-75f5f8bc-fc95-d676-f1f8-c0b6f46fb9db (Addr: tcp/127.0.0.1:24307) (DC: dc1)" -42:01.382 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-75f5f8bc-fc95-d676-f1f8-c0b6f46fb9db.dc1 area=wan -42:01.382 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24302 network=tcp -42:01.383 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24303 network=tcp -42:01.383 [INFO] TestAgent: started state syncer -42:01.383 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24308 network=tcp -42:01.434 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -42:01.448 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:01.448 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24307 [Candidate]" term=2 -42:01.448 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:01.448 [DEBUG] TestAgent.server.raft: vote granted: from=75f5f8bc-fc95-d676-f1f8-c0b6f46fb9db term=2 tally=1 -42:01.448 [INFO] TestAgent.server.raft: election won: tally=1 -42:01.448 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24307 [Leader]" -42:01.449 [INFO] TestAgent.server: cluster leadership acquired -42:01.449 [INFO] TestAgent.server: initializing acls -42:01.449 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:01.449 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:01.450 [INFO] TestAgent.server: New leader elected: payload=Node-75f5f8bc-fc95-d676-f1f8-c0b6f46fb9db -42:01.450 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:01.450 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:01.450 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:01.450 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:01.451 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-75f5f8bc-fc95-d676-f1f8-c0b6f46fb9db -42:01.451 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-75f5f8bc-fc95-d676-f1f8-c0b6f46fb9db.dc1 -42:01.452 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:01.452 [INFO] TestAgent.server: Updating LAN server: server="Node-75f5f8bc-fc95-d676-f1f8-c0b6f46fb9db (Addr: tcp/127.0.0.1:24307) (DC: dc1)" -42:01.452 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-75f5f8bc-fc95-d676-f1f8-c0b6f46fb9db.dc1 area=wan -42:01.472 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:01.473 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:01.473 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:01.473 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:01.473 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:01.473 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-75f5f8bc-fc95-d676-f1f8-c0b6f46fb9db -42:01.473 [INFO] TestAgent.server: member joined, marking health alive: member=Node-75f5f8bc-fc95-d676-f1f8-c0b6f46fb9db -42:01.473 [INFO] TestAgent.server: federation state anti-entropy synced -42:01.475 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-75f5f8bc-fc95-d676-f1f8-c0b6f46fb9db -42:01.493 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:01.494 [INFO] TestAgent: Synced node info -42:01.637 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:53244 latency=2.1005ms -42:01.638 [INFO] TestAgent: Requesting shutdown -42:01.638 [INFO] TestAgent.server: shutting down server -42:01.638 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:01.638 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:01.638 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:01.638 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:01.638 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:01.638 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:01.638 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:01.638 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:01.638 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:01.638 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:01.638 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:01.638 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:01.638 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:01.640 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:01.641 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:01.641Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:01.641Z [INFO] agent.router.manager: shutting down -42:01.641 [INFO] TestAgent: consul server down -42:01.641 [INFO] TestAgent: shutdown complete -42:01.641 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24302 network=tcp -42:01.641 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24302 network=udp -42:01.641 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24303 network=tcp -42:02.141 [INFO] TestAgent: Waiting for endpoints to shut down -42:02.141 [INFO] TestAgent: Endpoints down ---- PASS: TestPolicyCreateCommand (0.86s) -2022-09-15T21:42:01.346Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:01.372Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:01.376 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e008bde6-ac9a-4e4f-146d-ff8985c230d1 Address:127.0.0.1:24300}]" -42:01.376 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24300 [Follower]" leader= -42:01.377 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e008bde6-ac9a-4e4f-146d-ff8985c230d1.dc1 127.0.0.1 -42:01.378 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e008bde6-ac9a-4e4f-146d-ff8985c230d1 127.0.0.1 -2022-09-15T21:42:01.378Z [INFO] agent.router: Initializing LAN area manager -42:01.379 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24295 network=udp -42:01.379 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e008bde6-ac9a-4e4f-146d-ff8985c230d1.dc1 area=wan -42:01.379 [INFO] TestAgent.server: Adding LAN server: server="Node-e008bde6-ac9a-4e4f-146d-ff8985c230d1 (Addr: tcp/127.0.0.1:24300) (DC: dc1)" -42:01.379 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24295 network=tcp -42:01.380 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24296 network=tcp -42:01.380 [INFO] TestAgent: started state syncer -42:01.380 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24301 network=tcp -42:01.430 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -42:01.434 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:01.434 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24300 [Candidate]" term=2 -42:01.434 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:01.434 [DEBUG] TestAgent.server.raft: vote granted: from=e008bde6-ac9a-4e4f-146d-ff8985c230d1 term=2 tally=1 -42:01.434 [INFO] TestAgent.server.raft: election won: tally=1 -42:01.434 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24300 [Leader]" -42:01.434 [INFO] TestAgent.server: cluster leadership acquired -42:01.435 [INFO] TestAgent.server: initializing acls -42:01.435 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:01.435 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:01.436 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:01.437 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:01.437 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:01.437 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:01.437 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-e008bde6-ac9a-4e4f-146d-ff8985c230d1 -42:01.438 [INFO] TestAgent.server: New leader elected: payload=Node-e008bde6-ac9a-4e4f-146d-ff8985c230d1 -42:01.438 [INFO] TestAgent.server: Updating LAN server: server="Node-e008bde6-ac9a-4e4f-146d-ff8985c230d1 (Addr: tcp/127.0.0.1:24300) (DC: dc1)" -42:01.441 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-e008bde6-ac9a-4e4f-146d-ff8985c230d1.dc1 -42:01.442 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-e008bde6-ac9a-4e4f-146d-ff8985c230d1.dc1 area=wan -42:01.444 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:01.475 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:01.475 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:01.475 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:01.475 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:01.475 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:01.475 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e008bde6-ac9a-4e4f-146d-ff8985c230d1 -42:01.475 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e008bde6-ac9a-4e4f-146d-ff8985c230d1 -42:01.476 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e008bde6-ac9a-4e4f-146d-ff8985c230d1 -42:01.476 [INFO] TestAgent.server: federation state anti-entropy synced -42:01.729 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:01.730 [INFO] TestAgent: Synced node info -42:01.824 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:54712 latency=1.93052ms -42:01.825 [INFO] TestAgent: Requesting shutdown -42:01.825 [INFO] TestAgent.server: shutting down server -42:01.825 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:01.825 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:01.825 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:01.825 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:01.825 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:01.825 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:01.825 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:01.825 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:01.825 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:01.825 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:01.826 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:01.826 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:01.826 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:01.826 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:01.826 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:01.826Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:01.826Z [INFO] agent.router.manager: shutting down -42:01.826 [INFO] TestAgent: consul server down -42:01.826 [INFO] TestAgent: shutdown complete -42:01.826 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24295 network=tcp -42:01.826 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24295 network=udp -42:01.826 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24296 network=tcp -42:02.326 [INFO] TestAgent: Waiting for endpoints to shut down -42:02.327 [INFO] TestAgent: Endpoints down ---- PASS: TestPolicyCreateCommand_JSON (1.05s) +2021-08-13T15:34:18.848Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:18.873Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:18.876 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c09e5b08-edb7-2ad5-8fe2-6d2bd4d34096 Address:127.0.0.1:14090}]" +34:18.876 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14090 [Follower]" leader= +34:18.878 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c09e5b08-edb7-2ad5-8fe2-6d2bd4d34096.dc1 127.0.0.1 +34:18.880 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c09e5b08-edb7-2ad5-8fe2-6d2bd4d34096 127.0.0.1 +2021-08-13T15:34:18.880Z [INFO] agent.router: Initializing LAN area manager +34:18.881 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14085 network=udp +34:18.882 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c09e5b08-edb7-2ad5-8fe2-6d2bd4d34096.dc1 area=wan +34:18.882 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14085 network=tcp +34:18.882 [INFO] TestAgent.server: Adding LAN server: server="Node-c09e5b08-edb7-2ad5-8fe2-6d2bd4d34096 (Addr: tcp/127.0.0.1:14090) (DC: dc1)" +34:18.883 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14086 network=tcp +34:18.883 [INFO] TestAgent: started state syncer +34:18.883 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14091 network=tcp +34:18.936 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:18.945 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:18.945 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14090 [Candidate]" term=2 +34:18.945 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:18.945 [DEBUG] TestAgent.server.raft: vote granted: from=c09e5b08-edb7-2ad5-8fe2-6d2bd4d34096 term=2 tally=1 +34:18.945 [INFO] TestAgent.server.raft: election won: tally=1 +34:18.945 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14090 [Leader]" +34:18.945 [INFO] TestAgent.server: cluster leadership acquired +34:18.946 [INFO] TestAgent.server: initializing acls +34:18.946 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:18.946 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:18.947 [INFO] TestAgent.server: New leader elected: payload=Node-c09e5b08-edb7-2ad5-8fe2-6d2bd4d34096 +34:18.947 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:18.947 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:18.948 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:18.948 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:18.948 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-c09e5b08-edb7-2ad5-8fe2-6d2bd4d34096 +34:18.948 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-c09e5b08-edb7-2ad5-8fe2-6d2bd4d34096.dc1 +34:18.948 [INFO] TestAgent.server: Updating LAN server: server="Node-c09e5b08-edb7-2ad5-8fe2-6d2bd4d34096 (Addr: tcp/127.0.0.1:14090) (DC: dc1)" +34:18.948 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-c09e5b08-edb7-2ad5-8fe2-6d2bd4d34096.dc1 area=wan +34:18.949 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:18.995 [INFO] TestAgent: Synced node info +34:18.997 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:18.997 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:18.997 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:18.997 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:18.997 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:18.997 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c09e5b08-edb7-2ad5-8fe2-6d2bd4d34096 +34:18.997 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c09e5b08-edb7-2ad5-8fe2-6d2bd4d34096 +34:18.998 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c09e5b08-edb7-2ad5-8fe2-6d2bd4d34096 +34:18.998 [INFO] TestAgent.server: federation state anti-entropy synced +34:19.125 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:48864 latency=1.918164ms +34:19.129 [INFO] TestAgent: Requesting shutdown +34:19.129 [INFO] TestAgent.server: shutting down server +34:19.129 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:19.129 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:19.129 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:19.129 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:19.129 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:19.129 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:19.130 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:19.130 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:19.130 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:19.130 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:19.130 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:19.130 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:19.130 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:19.130 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:19.130 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:19.130Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:19.130Z [INFO] agent.router.manager: shutting down +34:19.130 [INFO] TestAgent: consul server down +34:19.130 [INFO] TestAgent: shutdown complete +34:19.130 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14085 network=tcp +34:19.131 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14085 network=udp +34:19.131 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14086 network=tcp +34:19.631 [INFO] TestAgent: Waiting for endpoints to shut down +34:19.631 [INFO] TestAgent: Endpoints down +--- PASS: TestPolicyCreateCommand_JSON (0.84s) +2021-08-13T15:34:18.848Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:18.872Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:18.876 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b4da1a79-e878-a3a8-07f9-e8a3454bb692 Address:127.0.0.1:14097}]" +34:18.876 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14097 [Follower]" leader= +34:18.878 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b4da1a79-e878-a3a8-07f9-e8a3454bb692.dc1 127.0.0.1 +34:18.880 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b4da1a79-e878-a3a8-07f9-e8a3454bb692 127.0.0.1 +2021-08-13T15:34:18.880Z [INFO] agent.router: Initializing LAN area manager +34:18.880 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14092 network=udp +34:18.882 [INFO] TestAgent.server: Adding LAN server: server="Node-b4da1a79-e878-a3a8-07f9-e8a3454bb692 (Addr: tcp/127.0.0.1:14097) (DC: dc1)" +34:18.882 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b4da1a79-e878-a3a8-07f9-e8a3454bb692.dc1 area=wan +34:18.882 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14092 network=tcp +34:18.883 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14093 network=tcp +34:18.883 [INFO] TestAgent: started state syncer +34:18.883 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14098 network=tcp +34:18.935 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:18.947 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:18.947 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14097 [Candidate]" term=2 +34:18.947 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:18.947 [DEBUG] TestAgent.server.raft: vote granted: from=b4da1a79-e878-a3a8-07f9-e8a3454bb692 term=2 tally=1 +34:18.947 [INFO] TestAgent.server.raft: election won: tally=1 +34:18.947 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14097 [Leader]" +34:18.947 [INFO] TestAgent.server: cluster leadership acquired +34:18.948 [INFO] TestAgent.server: initializing acls +34:18.948 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:18.948 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:18.948 [INFO] TestAgent.server: New leader elected: payload=Node-b4da1a79-e878-a3a8-07f9-e8a3454bb692 +34:18.949 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:18.950 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:18.950 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:18.950 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:18.950 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-b4da1a79-e878-a3a8-07f9-e8a3454bb692 +34:18.950 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-b4da1a79-e878-a3a8-07f9-e8a3454bb692.dc1 +34:18.951 [INFO] TestAgent.server: Updating LAN server: server="Node-b4da1a79-e878-a3a8-07f9-e8a3454bb692 (Addr: tcp/127.0.0.1:14097) (DC: dc1)" +34:18.951 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:18.951 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-b4da1a79-e878-a3a8-07f9-e8a3454bb692.dc1 area=wan +34:18.990 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:18.990 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:18.990 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:18.990 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:18.990 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:18.990 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b4da1a79-e878-a3a8-07f9-e8a3454bb692 +34:18.990 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b4da1a79-e878-a3a8-07f9-e8a3454bb692 +34:18.991 [INFO] TestAgent.server: federation state anti-entropy synced +34:18.992 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b4da1a79-e878-a3a8-07f9-e8a3454bb692 +34:19.046 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:19.048 [INFO] TestAgent: Synced node info +34:19.048 [DEBUG] TestAgent: Node info in sync +34:19.204 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:37562 latency=1.614964ms +34:19.205 [INFO] TestAgent: Requesting shutdown +34:19.205 [INFO] TestAgent.server: shutting down server +34:19.205 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:19.205 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:19.205 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:19.205 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:19.205 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:19.205 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:19.205 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:19.205 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:19.205 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:19.206 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:19.206 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:19.206 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:19.206 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:19.206 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:19.206 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:19.206Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:19.206Z [INFO] agent.router.manager: shutting down +34:19.206 [INFO] TestAgent: consul server down +34:19.206 [INFO] TestAgent: shutdown complete +34:19.206 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14092 network=tcp +34:19.206 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14092 network=udp +34:19.206 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14093 network=tcp +34:19.706 [INFO] TestAgent: Waiting for endpoints to shut down +34:19.706 [INFO] TestAgent: Endpoints down +--- PASS: TestPolicyCreateCommand (0.91s) PASS -ok github.com/hashicorp/consul/command/acl/policy/create 1.135s +ok github.com/hashicorp/consul/command/acl/policy/create 0.973s === RUN TestPolicyDeleteCommand_noTabs === PAUSE TestPolicyDeleteCommand_noTabs === RUN TestPolicyDeleteCommand === PAUSE TestPolicyDeleteCommand === CONT TestPolicyDeleteCommand_noTabs ---- PASS: TestPolicyDeleteCommand_noTabs (0.00s) === CONT TestPolicyDeleteCommand [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestPolicyDeleteCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:42:01.846Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:01.866Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:01.870 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9a8627b3-0ed5-b26c-9c97-09a6ecd30cf0 Address:127.0.0.1:30426}]" -42:01.870 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30426 [Follower]" leader= -42:01.871 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9a8627b3-0ed5-b26c-9c97-09a6ecd30cf0.dc1 127.0.0.1 -42:01.872 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9a8627b3-0ed5-b26c-9c97-09a6ecd30cf0 127.0.0.1 -2022-09-15T21:42:01.872Z [INFO] agent.router: Initializing LAN area manager -42:01.873 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=udp -42:01.874 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=tcp -42:01.874 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9a8627b3-0ed5-b26c-9c97-09a6ecd30cf0.dc1 area=wan -42:01.875 [INFO] TestAgent.server: Adding LAN server: server="Node-9a8627b3-0ed5-b26c-9c97-09a6ecd30cf0 (Addr: tcp/127.0.0.1:30426) (DC: dc1)" -42:01.876 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30422 network=tcp -42:01.876 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30427 network=tcp -42:01.876 [INFO] TestAgent: started state syncer -42:01.910 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:01.910 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30426 [Candidate]" term=2 -42:01.911 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:01.911 [DEBUG] TestAgent.server.raft: vote granted: from=9a8627b3-0ed5-b26c-9c97-09a6ecd30cf0 term=2 tally=1 -42:01.911 [INFO] TestAgent.server.raft: election won: tally=1 -42:01.911 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30426 [Leader]" -42:01.911 [INFO] TestAgent.server: cluster leadership acquired -42:01.911 [INFO] TestAgent.server: initializing acls -42:01.912 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:01.912 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:01.913 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:01.913 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:01.913 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:01.913 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:01.914 [INFO] TestAgent.server: New leader elected: payload=Node-9a8627b3-0ed5-b26c-9c97-09a6ecd30cf0 -42:01.914 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-9a8627b3-0ed5-b26c-9c97-09a6ecd30cf0 -42:01.914 [INFO] TestAgent.server: Updating LAN server: server="Node-9a8627b3-0ed5-b26c-9c97-09a6ecd30cf0 (Addr: tcp/127.0.0.1:30426) (DC: dc1)" -42:01.914 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-9a8627b3-0ed5-b26c-9c97-09a6ecd30cf0.dc1 -42:01.914 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-9a8627b3-0ed5-b26c-9c97-09a6ecd30cf0.dc1 area=wan -42:01.915 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:01.939 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:01.939 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:01.939 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:01.939 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:01.939 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:01.939 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9a8627b3-0ed5-b26c-9c97-09a6ecd30cf0 -42:01.939 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9a8627b3-0ed5-b26c-9c97-09a6ecd30cf0 -42:01.940 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9a8627b3-0ed5-b26c-9c97-09a6ecd30cf0 -42:01.941 [INFO] TestAgent.server: federation state anti-entropy synced -42:01.964 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:35610 latency=2.1992ms -42:01.966 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/policy/3d21ad21-c722-f7a2-e5e0-e3e4ce9ca4bd from=127.0.0.1:35612 latency=808.42µs -42:01.967 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/policy/3d21ad21-c722-f7a2-e5e0-e3e4ce9ca4bd from=127.0.0.1:35610 error="ACL not found" -42:01.967 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policy/3d21ad21-c722-f7a2-e5e0-e3e4ce9ca4bd from=127.0.0.1:35610 latency=451.3µs -42:01.967 [INFO] TestAgent: Requesting shutdown -42:01.967 [INFO] TestAgent.server: shutting down server -42:01.967 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:01.967 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:01.967 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:01.967 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:01.968 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:01.968 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:01.968 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:01.968 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:01.968 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:01.968 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -42:01.968 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:01.968 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:01.968 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:01.968 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:01.968 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:01.968 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -2022-09-15T21:42:01.968Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:01.968Z [INFO] agent.router.manager: shutting down -42:01.968 [INFO] TestAgent: consul server down -42:01.968 [INFO] TestAgent: shutdown complete -42:01.968 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=tcp -42:01.968 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=udp -42:01.968 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30422 network=tcp -42:02.468 [INFO] TestAgent: Waiting for endpoints to shut down -42:02.469 [INFO] TestAgent: Endpoints down ---- PASS: TestPolicyDeleteCommand (0.69s) +2021-08-13T15:34:20.680Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:20.698Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:20.704 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c6a6e7b0-9162-c0d5-09a0-3fe7a211413a Address:127.0.0.1:30426}]" +34:20.704 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30426 [Follower]" leader= +34:20.706 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c6a6e7b0-9162-c0d5-09a0-3fe7a211413a.dc1 127.0.0.1 +34:20.707 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c6a6e7b0-9162-c0d5-09a0-3fe7a211413a 127.0.0.1 +2021-08-13T15:34:20.707Z [INFO] agent.router: Initializing LAN area manager +34:20.707 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=udp +34:20.707 [INFO] TestAgent.server: Adding LAN server: server="Node-c6a6e7b0-9162-c0d5-09a0-3fe7a211413a (Addr: tcp/127.0.0.1:30426) (DC: dc1)" +34:20.707 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c6a6e7b0-9162-c0d5-09a0-3fe7a211413a.dc1 area=wan +34:20.708 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=tcp +34:20.708 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30422 network=tcp +34:20.709 [INFO] TestAgent: started state syncer +34:20.709 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30427 network=tcp +34:20.746 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:20.746 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30426 [Candidate]" term=2 +34:20.747 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:20.747 [DEBUG] TestAgent.server.raft: vote granted: from=c6a6e7b0-9162-c0d5-09a0-3fe7a211413a term=2 tally=1 +34:20.747 [INFO] TestAgent.server.raft: election won: tally=1 +34:20.747 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30426 [Leader]" +34:20.747 [INFO] TestAgent.server: cluster leadership acquired +34:20.747 [INFO] TestAgent.server: initializing acls +34:20.748 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:20.748 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:20.749 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:20.749 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:20.749 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:20.749 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:20.749 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-c6a6e7b0-9162-c0d5-09a0-3fe7a211413a +34:20.750 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-c6a6e7b0-9162-c0d5-09a0-3fe7a211413a.dc1 +34:20.750 [INFO] TestAgent.server: New leader elected: payload=Node-c6a6e7b0-9162-c0d5-09a0-3fe7a211413a +34:20.750 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-c6a6e7b0-9162-c0d5-09a0-3fe7a211413a.dc1 area=wan +34:20.750 [INFO] TestAgent.server: Updating LAN server: server="Node-c6a6e7b0-9162-c0d5-09a0-3fe7a211413a (Addr: tcp/127.0.0.1:30426) (DC: dc1)" +34:20.751 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:20.774 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:20.774 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:20.774 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:20.774 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:20.774 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:20.774 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c6a6e7b0-9162-c0d5-09a0-3fe7a211413a +34:20.774 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c6a6e7b0-9162-c0d5-09a0-3fe7a211413a +34:20.775 [INFO] TestAgent.server: federation state anti-entropy synced +34:20.776 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c6a6e7b0-9162-c0d5-09a0-3fe7a211413a +34:20.805 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:20.806 [INFO] TestAgent: Synced node info +34:20.831 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:20.831 [DEBUG] TestAgent: Node info in sync +34:20.831 [DEBUG] TestAgent: Node info in sync +34:20.842 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:48488 latency=2.408426ms +34:20.858 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/policy/763c8531-5638-f457-6380-9d7e491b577c from=127.0.0.1:48490 latency=1.424883ms +34:20.859 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/policy/763c8531-5638-f457-6380-9d7e491b577c from=127.0.0.1:48488 error="ACL not found" +34:20.860 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policy/763c8531-5638-f457-6380-9d7e491b577c from=127.0.0.1:48488 latency=1.133703ms +34:20.860 [INFO] TestAgent: Requesting shutdown +34:20.860 [INFO] TestAgent.server: shutting down server +34:20.860 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:20.860 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:20.860 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:20.860 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:20.860 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:20.860 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:20.860 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:20.860 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:20.860 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:20.860 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:20.860 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:20.860 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:20.860 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:20.860 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:20.860 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:20.861Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:20.861Z [INFO] agent.router.manager: shutting down +34:20.861 [INFO] TestAgent: consul server down +34:20.861 [INFO] TestAgent: shutdown complete +34:20.861 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=tcp +34:20.861 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=udp +34:20.861 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30422 network=tcp +34:21.361 [INFO] TestAgent: Waiting for endpoints to shut down +34:21.361 [INFO] TestAgent: Endpoints down +--- PASS: TestPolicyDeleteCommand (0.73s) PASS -ok github.com/hashicorp/consul/command/acl/policy/delete 0.772s +ok github.com/hashicorp/consul/command/acl/policy/delete 0.808s === RUN TestPolicyListCommand_noTabs === PAUSE TestPolicyListCommand_noTabs === RUN TestPolicyListCommand @@ -75373,172 +75394,172 @@ === RUN TestPolicyListCommand_JSON === PAUSE TestPolicyListCommand_JSON === CONT TestPolicyListCommand_noTabs -=== CONT TestPolicyListCommand_JSON --- PASS: TestPolicyListCommand_noTabs (0.00s) -=== CONT TestPolicyListCommand +=== CONT TestPolicyListCommand_JSON [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +=== CONT TestPolicyListCommand [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:42:12.167Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:12.211Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:12.216 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0272f569-89ab-4a11-a8a0-314197088f1c Address:127.0.0.1:22265}]" -42:12.216 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22265 [Follower]" leader= -42:12.217 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0272f569-89ab-4a11-a8a0-314197088f1c.dc1 127.0.0.1 -42:12.218 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0272f569-89ab-4a11-a8a0-314197088f1c 127.0.0.1 -2022-09-15T21:42:12.218Z [INFO] agent.router: Initializing LAN area manager -42:12.218 [INFO] TestAgent.server: Adding LAN server: server="Node-0272f569-89ab-4a11-a8a0-314197088f1c (Addr: tcp/127.0.0.1:22265) (DC: dc1)" -42:12.218 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0272f569-89ab-4a11-a8a0-314197088f1c.dc1 area=wan -42:12.218 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22260 network=udp -42:12.218 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22260 network=tcp -42:12.220 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22261 network=tcp -42:12.221 [INFO] TestAgent: started state syncer -42:12.221 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22266 network=tcp -42:12.268 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -42:12.278 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:12.278 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22265 [Candidate]" term=2 -42:12.278 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:12.278 [DEBUG] TestAgent.server.raft: vote granted: from=0272f569-89ab-4a11-a8a0-314197088f1c term=2 tally=1 -42:12.278 [INFO] TestAgent.server.raft: election won: tally=1 -42:12.278 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22265 [Leader]" -42:12.279 [INFO] TestAgent.server: cluster leadership acquired -42:12.279 [INFO] TestAgent.server: initializing acls -42:12.279 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:12.279 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:12.280 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:12.281 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:12.281 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:12.281 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:12.281 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-0272f569-89ab-4a11-a8a0-314197088f1c -42:12.281 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-0272f569-89ab-4a11-a8a0-314197088f1c.dc1 -42:12.282 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:12.282 [INFO] TestAgent.server: New leader elected: payload=Node-0272f569-89ab-4a11-a8a0-314197088f1c -42:12.282 [INFO] TestAgent.server: Updating LAN server: server="Node-0272f569-89ab-4a11-a8a0-314197088f1c (Addr: tcp/127.0.0.1:22265) (DC: dc1)" -42:12.282 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-0272f569-89ab-4a11-a8a0-314197088f1c.dc1 area=wan -42:12.294 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:12.294 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:12.294 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:12.294 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:12.294 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:12.294 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0272f569-89ab-4a11-a8a0-314197088f1c -42:12.294 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0272f569-89ab-4a11-a8a0-314197088f1c -42:12.295 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0272f569-89ab-4a11-a8a0-314197088f1c -42:12.297 [INFO] TestAgent.server: federation state anti-entropy synced -42:12.336 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:33700 latency=2.0308ms -42:12.339 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:33700 latency=981.18µs -42:12.340 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:33700 latency=658.22µs -42:12.342 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:33700 latency=809.84µs -42:12.344 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:33700 latency=745.82µs -42:12.345 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policies from=127.0.0.1:33702 latency=470.84µs -42:12.346 [INFO] TestAgent: Requesting shutdown -42:12.347 [INFO] TestAgent.server: shutting down server -42:12.347 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:12.347 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:12.347 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:12.347 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:12.347 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:12.347 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:12.347 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:12.347 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:12.347 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:12.347 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -42:12.347 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:12.347 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:12.347 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:12.347 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:12.347 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:12.347 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2022-09-15T21:42:12.347Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:12.347Z [INFO] agent.router.manager: shutting down -42:12.347 [INFO] TestAgent: consul server down -42:12.347 [INFO] TestAgent: shutdown complete -42:12.347 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22260 network=tcp -42:12.347 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22260 network=udp -42:12.347 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22261 network=tcp -42:12.848 [INFO] TestAgent: Waiting for endpoints to shut down -42:12.848 [INFO] TestAgent: Endpoints down ---- PASS: TestPolicyListCommand_JSON (0.76s) -2022-09-15T21:42:12.165Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:12.212Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:12.216 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9b2ff978-a828-f2ec-f503-1cbe27f5efac Address:127.0.0.1:22258}]" -42:12.216 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22258 [Follower]" leader= -42:12.217 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9b2ff978-a828-f2ec-f503-1cbe27f5efac.dc1 127.0.0.1 -42:12.218 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9b2ff978-a828-f2ec-f503-1cbe27f5efac 127.0.0.1 -2022-09-15T21:42:12.218Z [INFO] agent.router: Initializing LAN area manager -42:12.218 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9b2ff978-a828-f2ec-f503-1cbe27f5efac.dc1 area=wan -42:12.218 [INFO] TestAgent.server: Adding LAN server: server="Node-9b2ff978-a828-f2ec-f503-1cbe27f5efac (Addr: tcp/127.0.0.1:22258) (DC: dc1)" -42:12.218 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22253 network=udp -42:12.218 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22253 network=tcp -42:12.222 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22254 network=tcp -42:12.223 [INFO] TestAgent: started state syncer -42:12.223 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22259 network=tcp -42:12.265 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:12.265 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22258 [Candidate]" term=2 -42:12.266 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:12.266 [DEBUG] TestAgent.server.raft: vote granted: from=9b2ff978-a828-f2ec-f503-1cbe27f5efac term=2 tally=1 -42:12.266 [INFO] TestAgent.server.raft: election won: tally=1 -42:12.266 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22258 [Leader]" -42:12.266 [INFO] TestAgent.server: cluster leadership acquired -42:12.266 [INFO] TestAgent.server: New leader elected: payload=Node-9b2ff978-a828-f2ec-f503-1cbe27f5efac -42:12.266 [INFO] TestAgent.server: initializing acls -42:12.267 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:12.267 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:12.268 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:12.268 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:12.268 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:12.268 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:12.269 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-9b2ff978-a828-f2ec-f503-1cbe27f5efac -42:12.269 [INFO] TestAgent.server: Updating LAN server: server="Node-9b2ff978-a828-f2ec-f503-1cbe27f5efac (Addr: tcp/127.0.0.1:22258) (DC: dc1)" -42:12.269 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-9b2ff978-a828-f2ec-f503-1cbe27f5efac.dc1 -42:12.269 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-9b2ff978-a828-f2ec-f503-1cbe27f5efac.dc1 area=wan -42:12.270 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:12.288 [INFO] TestAgent: Synced node info -42:12.294 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:12.294 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:12.294 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:12.294 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:12.294 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:12.294 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9b2ff978-a828-f2ec-f503-1cbe27f5efac -42:12.294 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9b2ff978-a828-f2ec-f503-1cbe27f5efac -42:12.296 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9b2ff978-a828-f2ec-f503-1cbe27f5efac -42:12.297 [INFO] TestAgent.server: federation state anti-entropy synced -42:12.448 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:12.449 [DEBUG] TestAgent: Node info in sync -42:12.449 [DEBUG] TestAgent: Node info in sync -42:12.467 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:48912 latency=1.695ms -42:12.468 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:48912 latency=704.08µs -42:12.470 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:48912 latency=904.8µs -42:12.472 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:48912 latency=705.1µs -42:12.473 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:48912 latency=686.24µs -42:12.475 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policies from=127.0.0.1:48914 latency=318.84µs -42:12.476 [INFO] TestAgent: Requesting shutdown -42:12.476 [INFO] TestAgent.server: shutting down server -42:12.476 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:12.476 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:12.476 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:12.476 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:12.476 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:12.476 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:12.476 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:12.476 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:12.476 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:12.476 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:12.476 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:12.476 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:12.476 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:12.476 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:12.476 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:12.476Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:12.476Z [INFO] agent.router.manager: shutting down -42:12.476 [INFO] TestAgent: consul server down -42:12.476 [INFO] TestAgent: shutdown complete -42:12.476 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22253 network=tcp -42:12.476 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22253 network=udp -42:12.476 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22254 network=tcp -42:12.977 [INFO] TestAgent: Waiting for endpoints to shut down -42:12.977 [INFO] TestAgent: Endpoints down ---- PASS: TestPolicyListCommand (0.88s) +2021-08-13T15:34:24.966Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:24.985Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:24.995 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f92de718-2e6a-0151-3e53-09b3c50b568b Address:127.0.0.1:19202}]" +34:24.995 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19202 [Follower]" leader= +34:24.997 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f92de718-2e6a-0151-3e53-09b3c50b568b.dc1 127.0.0.1 +34:24.998 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f92de718-2e6a-0151-3e53-09b3c50b568b 127.0.0.1 +2021-08-13T15:34:24.998Z [INFO] agent.router: Initializing LAN area manager +34:24.998 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19197 network=udp +34:24.998 [INFO] TestAgent.server: Adding LAN server: server="Node-f92de718-2e6a-0151-3e53-09b3c50b568b (Addr: tcp/127.0.0.1:19202) (DC: dc1)" +34:24.998 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f92de718-2e6a-0151-3e53-09b3c50b568b.dc1 area=wan +34:24.998 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19197 network=tcp +34:24.999 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19198 network=tcp +34:24.999 [INFO] TestAgent: started state syncer +34:24.999 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19203 network=tcp +34:25.048 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:25.059 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:25.059 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19202 [Candidate]" term=2 +34:25.059 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:25.059 [DEBUG] TestAgent.server.raft: vote granted: from=f92de718-2e6a-0151-3e53-09b3c50b568b term=2 tally=1 +34:25.059 [INFO] TestAgent.server.raft: election won: tally=1 +34:25.059 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19202 [Leader]" +34:25.059 [INFO] TestAgent.server: cluster leadership acquired +34:25.060 [INFO] TestAgent.server: New leader elected: payload=Node-f92de718-2e6a-0151-3e53-09b3c50b568b +34:25.060 [INFO] TestAgent.server: initializing acls +34:25.060 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:25.060 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:25.061 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:25.063 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:25.063 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:25.063 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:25.063 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-f92de718-2e6a-0151-3e53-09b3c50b568b +34:25.063 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-f92de718-2e6a-0151-3e53-09b3c50b568b.dc1 +34:25.064 [INFO] TestAgent.server: Updating LAN server: server="Node-f92de718-2e6a-0151-3e53-09b3c50b568b (Addr: tcp/127.0.0.1:19202) (DC: dc1)" +34:25.064 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:25.065 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-f92de718-2e6a-0151-3e53-09b3c50b568b.dc1 area=wan +34:25.076 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:25.076 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:25.076 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:25.076 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:25.076 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:25.076 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f92de718-2e6a-0151-3e53-09b3c50b568b +34:25.076 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f92de718-2e6a-0151-3e53-09b3c50b568b +34:25.078 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f92de718-2e6a-0151-3e53-09b3c50b568b +34:25.079 [INFO] TestAgent.server: federation state anti-entropy synced +34:25.236 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:25.237 [INFO] TestAgent: Synced node info +34:25.237 [DEBUG] TestAgent: Node info in sync +34:25.291 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:48446 latency=1.859345ms +34:25.293 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:48446 latency=789.982µs +34:25.294 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:48446 latency=664.462µs +34:25.295 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:48446 latency=644.322µs +34:25.296 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:48446 latency=619.642µs +34:25.298 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policies from=127.0.0.1:48448 latency=452.681µs +34:25.298 [INFO] TestAgent: Requesting shutdown +34:25.299 [INFO] TestAgent.server: shutting down server +34:25.299 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:25.299 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:25.299 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:25.299 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:25.299 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:25.299 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:25.299 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:25.299 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:25.299 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:25.299 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:25.299 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:25.299 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:25.299 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:25.299 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:25.299 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:25.299Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:25.299Z [INFO] agent.router.manager: shutting down +34:25.299 [INFO] TestAgent: consul server down +34:25.299 [INFO] TestAgent: shutdown complete +34:25.299 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19197 network=tcp +34:25.299 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19197 network=udp +34:25.300 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19198 network=tcp +34:25.800 [INFO] TestAgent: Waiting for endpoints to shut down +34:25.800 [INFO] TestAgent: Endpoints down +--- PASS: TestPolicyListCommand (0.89s) +2021-08-13T15:34:24.967Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:24.986Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:24.995 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ff6258a6-c44f-4147-0bb7-647c8ffbbd34 Address:127.0.0.1:19195}]" +34:24.995 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19195 [Follower]" leader= +34:24.996 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ff6258a6-c44f-4147-0bb7-647c8ffbbd34.dc1 127.0.0.1 +34:24.997 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ff6258a6-c44f-4147-0bb7-647c8ffbbd34 127.0.0.1 +2021-08-13T15:34:24.997Z [INFO] agent.router: Initializing LAN area manager +34:24.998 [INFO] TestAgent.server: Adding LAN server: server="Node-ff6258a6-c44f-4147-0bb7-647c8ffbbd34 (Addr: tcp/127.0.0.1:19195) (DC: dc1)" +34:24.998 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ff6258a6-c44f-4147-0bb7-647c8ffbbd34.dc1 area=wan +34:24.998 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=udp +34:24.998 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=tcp +34:24.999 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19191 network=tcp +34:24.999 [INFO] TestAgent: started state syncer +34:24.999 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19196 network=tcp +34:25.044 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:25.044 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19195 [Candidate]" term=2 +34:25.044 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:25.044 [DEBUG] TestAgent.server.raft: vote granted: from=ff6258a6-c44f-4147-0bb7-647c8ffbbd34 term=2 tally=1 +34:25.044 [INFO] TestAgent.server.raft: election won: tally=1 +34:25.044 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19195 [Leader]" +34:25.044 [INFO] TestAgent.server: cluster leadership acquired +34:25.045 [INFO] TestAgent.server: New leader elected: payload=Node-ff6258a6-c44f-4147-0bb7-647c8ffbbd34 +34:25.045 [INFO] TestAgent.server: initializing acls +34:25.046 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:25.046 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:25.046 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:25.047 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:25.047 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:25.047 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:25.047 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-ff6258a6-c44f-4147-0bb7-647c8ffbbd34 +34:25.047 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-ff6258a6-c44f-4147-0bb7-647c8ffbbd34.dc1 +34:25.047 [INFO] TestAgent.server: Updating LAN server: server="Node-ff6258a6-c44f-4147-0bb7-647c8ffbbd34 (Addr: tcp/127.0.0.1:19195) (DC: dc1)" +34:25.048 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-ff6258a6-c44f-4147-0bb7-647c8ffbbd34.dc1 area=wan +34:25.048 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:25.078 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:25.078 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:25.078 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:25.078 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:25.078 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:25.078 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ff6258a6-c44f-4147-0bb7-647c8ffbbd34 +34:25.078 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ff6258a6-c44f-4147-0bb7-647c8ffbbd34 +34:25.079 [INFO] TestAgent.server: federation state anti-entropy synced +34:25.079 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ff6258a6-c44f-4147-0bb7-647c8ffbbd34 +34:25.381 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:25.382 [INFO] TestAgent: Synced node info +34:25.425 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:35132 latency=1.477164ms +34:25.427 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:35132 latency=713.022µs +34:25.429 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:35132 latency=1.546584ms +34:25.439 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:35132 latency=925.502µs +34:25.440 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:35132 latency=528.461µs +34:25.441 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policies from=127.0.0.1:35134 latency=251.341µs +34:25.442 [INFO] TestAgent: Requesting shutdown +34:25.442 [INFO] TestAgent.server: shutting down server +34:25.442 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:25.442 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:25.442 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:25.442 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:25.442 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:25.442 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:25.442 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:25.442 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:25.442 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:25.442 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:25.442 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:25.442 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:25.442 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:25.442 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:25.442 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:25.442Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:25.443Z [INFO] agent.router.manager: shutting down +34:25.443 [INFO] TestAgent: consul server down +34:25.443 [INFO] TestAgent: shutdown complete +34:25.443 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=tcp +34:25.443 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=udp +34:25.443 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19191 network=tcp +34:25.943 [INFO] TestAgent: Waiting for endpoints to shut down +34:25.943 [INFO] TestAgent: Endpoints down +--- PASS: TestPolicyListCommand_JSON (1.03s) PASS -ok github.com/hashicorp/consul/command/acl/policy/list 0.988s +ok github.com/hashicorp/consul/command/acl/policy/list 1.103s === RUN TestPolicyReadCommand_noTabs === PAUSE TestPolicyReadCommand_noTabs === RUN TestPolicyReadCommand @@ -75546,165 +75567,164 @@ === RUN TestPolicyReadCommand_JSON === PAUSE TestPolicyReadCommand_JSON === CONT TestPolicyReadCommand_noTabs ---- PASS: TestPolicyReadCommand_noTabs (0.02s) === CONT TestPolicyReadCommand_JSON -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestPolicyReadCommand +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestPolicyReadCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:42:14.881Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:14.959Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:14.969 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f7c453e3-03c6-4c4c-806a-8112c2ce3e0a Address:127.0.0.1:14090}]" -42:14.970 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f7c453e3-03c6-4c4c-806a-8112c2ce3e0a.dc1 127.0.0.1 -42:14.971 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f7c453e3-03c6-4c4c-806a-8112c2ce3e0a 127.0.0.1 -2022-09-15T21:42:14.971Z [INFO] agent.router: Initializing LAN area manager -42:14.972 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14085 network=udp -42:14.972 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14090 [Follower]" leader= -42:14.972 [INFO] TestAgent.server: Adding LAN server: server="Node-f7c453e3-03c6-4c4c-806a-8112c2ce3e0a (Addr: tcp/127.0.0.1:14090) (DC: dc1)" -42:14.972 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f7c453e3-03c6-4c4c-806a-8112c2ce3e0a.dc1 area=wan -42:14.973 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14085 network=tcp -42:14.973 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14086 network=tcp -42:14.982 [INFO] TestAgent: started state syncer -42:14.982 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14091 network=tcp -42:15.013 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:15.013 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14090 [Candidate]" term=2 -42:15.026 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -42:15.044 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:15.044 [DEBUG] TestAgent.server.raft: vote granted: from=f7c453e3-03c6-4c4c-806a-8112c2ce3e0a term=2 tally=1 -42:15.044 [INFO] TestAgent.server.raft: election won: tally=1 -42:15.044 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14090 [Leader]" -42:15.045 [INFO] TestAgent.server: cluster leadership acquired -42:15.045 [INFO] TestAgent.server: initializing acls -42:15.046 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:15.046 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:15.047 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:15.048 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:15.048 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:15.048 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:15.048 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-f7c453e3-03c6-4c4c-806a-8112c2ce3e0a -42:15.052 [INFO] TestAgent.server: New leader elected: payload=Node-f7c453e3-03c6-4c4c-806a-8112c2ce3e0a -42:15.052 [INFO] TestAgent.server: Updating LAN server: server="Node-f7c453e3-03c6-4c4c-806a-8112c2ce3e0a (Addr: tcp/127.0.0.1:14090) (DC: dc1)" -42:15.054 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-f7c453e3-03c6-4c4c-806a-8112c2ce3e0a.dc1 -42:15.056 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:15.056 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-f7c453e3-03c6-4c4c-806a-8112c2ce3e0a.dc1 area=wan -42:15.129 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:15.129 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:15.129 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:15.129 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:15.129 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:15.129 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f7c453e3-03c6-4c4c-806a-8112c2ce3e0a -42:15.129 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f7c453e3-03c6-4c4c-806a-8112c2ce3e0a -42:15.138 [INFO] TestAgent.server: federation state anti-entropy synced -42:15.138 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f7c453e3-03c6-4c4c-806a-8112c2ce3e0a -42:15.222 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:15.223 [INFO] TestAgent: Synced node info -42:15.239 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:56948 latency=1.2604ms -42:15.241 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policy/1a865833-242b-7f5e-c67f-12643512151a from=127.0.0.1:56950 latency=459.34µs -42:15.254 [INFO] TestAgent: Requesting shutdown -42:15.254 [INFO] TestAgent.server: shutting down server -42:15.254 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:15.254 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:15.254 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:15.254 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:15.254 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:15.254 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:15.254 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:15.254 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:15.255 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:15.255 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:15.255 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:15.255 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:15.255 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:15.255 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:15.255 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:15.255Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:15.255Z [INFO] agent.router.manager: shutting down -42:15.255 [INFO] TestAgent: consul server down -42:15.255 [INFO] TestAgent: shutdown complete -42:15.255 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14085 network=tcp -42:15.255 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14085 network=udp -42:15.255 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14086 network=tcp -42:15.758 [INFO] TestAgent: Waiting for endpoints to shut down -42:15.758 [INFO] TestAgent: Endpoints down ---- PASS: TestPolicyReadCommand_JSON (1.09s) -2022-09-15T21:42:14.871Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:14.946Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:14.963 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:961c20af-a576-b4e1-2827-74b0dc1bcc32 Address:127.0.0.1:14097}]" -42:14.964 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-961c20af-a576-b4e1-2827-74b0dc1bcc32.dc1 127.0.0.1 -42:14.965 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-961c20af-a576-b4e1-2827-74b0dc1bcc32 127.0.0.1 -2022-09-15T21:42:14.965Z [INFO] agent.router: Initializing LAN area manager -42:14.966 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14097 [Follower]" leader= -42:14.966 [INFO] TestAgent.server: Adding LAN server: server="Node-961c20af-a576-b4e1-2827-74b0dc1bcc32 (Addr: tcp/127.0.0.1:14097) (DC: dc1)" -42:14.966 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-961c20af-a576-b4e1-2827-74b0dc1bcc32.dc1 area=wan -42:14.967 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14092 network=tcp -42:14.974 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14092 network=udp -42:14.975 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14093 network=tcp -42:14.975 [INFO] TestAgent: started state syncer -42:14.975 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14098 network=tcp -42:15.010 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:15.010 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14097 [Candidate]" term=2 -42:15.010 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:15.010 [DEBUG] TestAgent.server.raft: vote granted: from=961c20af-a576-b4e1-2827-74b0dc1bcc32 term=2 tally=1 -42:15.010 [INFO] TestAgent.server.raft: election won: tally=1 -42:15.010 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14097 [Leader]" -42:15.010 [INFO] TestAgent.server: cluster leadership acquired -42:15.011 [INFO] TestAgent.server: initializing acls -42:15.011 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:15.011 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:15.012 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:15.013 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:15.013 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:15.013 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:15.013 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-961c20af-a576-b4e1-2827-74b0dc1bcc32 -42:15.013 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-961c20af-a576-b4e1-2827-74b0dc1bcc32.dc1 -42:15.014 [INFO] TestAgent.server: New leader elected: payload=Node-961c20af-a576-b4e1-2827-74b0dc1bcc32 -42:15.014 [INFO] TestAgent.server: Updating LAN server: server="Node-961c20af-a576-b4e1-2827-74b0dc1bcc32 (Addr: tcp/127.0.0.1:14097) (DC: dc1)" -42:15.014 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-961c20af-a576-b4e1-2827-74b0dc1bcc32.dc1 area=wan -42:15.023 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:15.103 [INFO] TestAgent: Synced node info -42:15.103 [DEBUG] TestAgent: Node info in sync -42:15.135 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:15.135 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:15.135 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:15.135 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:15.135 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:15.135 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-961c20af-a576-b4e1-2827-74b0dc1bcc32 -42:15.135 [INFO] TestAgent.server: member joined, marking health alive: member=Node-961c20af-a576-b4e1-2827-74b0dc1bcc32 -42:15.136 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-961c20af-a576-b4e1-2827-74b0dc1bcc32 -42:15.137 [INFO] TestAgent.server: federation state anti-entropy synced -42:15.282 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:15.282 [DEBUG] TestAgent: Node info in sync -42:15.334 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:55702 latency=1.14174ms -42:15.336 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policy/55c8f92c-1bed-43ec-20f3-934197c8d0e0 from=127.0.0.1:55704 latency=475.06µs -42:15.337 [INFO] TestAgent: Requesting shutdown -42:15.337 [INFO] TestAgent.server: shutting down server -42:15.337 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:15.337 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:15.337 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:15.337 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:15.337 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:15.337 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:15.337 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:15.337 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:15.337 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:15.338 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:15.338 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:15.338 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:15.338 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:15.338 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:15.338 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:15.338Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:15.338Z [INFO] agent.router.manager: shutting down -42:15.338 [INFO] TestAgent: consul server down -42:15.338 [INFO] TestAgent: shutdown complete -42:15.338 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14092 network=tcp -42:15.338 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14092 network=udp -42:15.338 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14093 network=tcp -42:15.839 [INFO] TestAgent: Waiting for endpoints to shut down -42:15.839 [INFO] TestAgent: Endpoints down ---- PASS: TestPolicyReadCommand (1.17s) +2021-08-13T15:34:33.371Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:33.390Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:33.395 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f1b29631-6436-51cd-02db-0ce7b52813ab Address:127.0.0.1:12048}]" +34:33.396 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:12048 [Follower]" leader= +34:33.396 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f1b29631-6436-51cd-02db-0ce7b52813ab.dc1 127.0.0.1 +34:33.397 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f1b29631-6436-51cd-02db-0ce7b52813ab 127.0.0.1 +2021-08-13T15:34:33.397Z [INFO] agent.router: Initializing LAN area manager +34:33.398 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12043 network=udp +34:33.398 [INFO] TestAgent.server: Adding LAN server: server="Node-f1b29631-6436-51cd-02db-0ce7b52813ab (Addr: tcp/127.0.0.1:12048) (DC: dc1)" +34:33.398 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12043 network=tcp +34:33.398 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f1b29631-6436-51cd-02db-0ce7b52813ab.dc1 area=wan +34:33.399 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:12044 network=tcp +34:33.399 [INFO] TestAgent: started state syncer +34:33.399 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:12049 network=tcp +34:33.448 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:33.454 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:33.454 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:12048 [Candidate]" term=2 +34:33.455 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:33.455 [DEBUG] TestAgent.server.raft: vote granted: from=f1b29631-6436-51cd-02db-0ce7b52813ab term=2 tally=1 +34:33.455 [INFO] TestAgent.server.raft: election won: tally=1 +34:33.455 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:12048 [Leader]" +34:33.455 [INFO] TestAgent.server: cluster leadership acquired +34:33.455 [INFO] TestAgent.server: initializing acls +34:33.455 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:33.455 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:33.455 [INFO] TestAgent.server: New leader elected: payload=Node-f1b29631-6436-51cd-02db-0ce7b52813ab +34:33.456 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:33.457 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:33.457 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:33.457 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:33.458 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-f1b29631-6436-51cd-02db-0ce7b52813ab +34:33.458 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-f1b29631-6436-51cd-02db-0ce7b52813ab.dc1 +34:33.459 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:33.459 [INFO] TestAgent.server: Updating LAN server: server="Node-f1b29631-6436-51cd-02db-0ce7b52813ab (Addr: tcp/127.0.0.1:12048) (DC: dc1)" +34:33.460 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-f1b29631-6436-51cd-02db-0ce7b52813ab.dc1 area=wan +34:33.488 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:33.488 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:33.488 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:33.488 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:33.488 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:33.488 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f1b29631-6436-51cd-02db-0ce7b52813ab +34:33.488 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f1b29631-6436-51cd-02db-0ce7b52813ab +34:33.489 [INFO] TestAgent.server: federation state anti-entropy synced +34:33.489 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f1b29631-6436-51cd-02db-0ce7b52813ab +34:33.621 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:33.622 [INFO] TestAgent: Synced node info +34:33.622 [DEBUG] TestAgent: Node info in sync +34:33.701 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:47646 latency=1.328143ms +34:33.703 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policy/68f5e4a7-addb-f6ee-cfc6-3f282b14768a from=127.0.0.1:47648 latency=489.361µs +34:33.704 [INFO] TestAgent: Requesting shutdown +34:33.704 [INFO] TestAgent.server: shutting down server +34:33.704 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:33.704 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:33.704 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:33.704 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:33.704 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:33.704 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:33.704 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:33.704 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:33.704 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:33.705 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:33.705 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:33.705 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:33.704 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:33.705 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:33.705 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:33.705Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:33.705Z [INFO] agent.router.manager: shutting down +34:33.705 [INFO] TestAgent: consul server down +34:33.705 [INFO] TestAgent: shutdown complete +34:33.705 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12043 network=tcp +34:33.705 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12043 network=udp +34:33.705 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:12044 network=tcp +34:34.205 [INFO] TestAgent: Waiting for endpoints to shut down +34:34.205 [INFO] TestAgent: Endpoints down +--- PASS: TestPolicyReadCommand (0.89s) +2021-08-13T15:34:33.371Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:33.390Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:33.396 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:822564e7-88a4-2887-b7ec-6ec3d4eb9ff9 Address:127.0.0.1:12055}]" +34:33.396 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:12055 [Follower]" leader= +34:33.396 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-822564e7-88a4-2887-b7ec-6ec3d4eb9ff9.dc1 127.0.0.1 +34:33.397 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-822564e7-88a4-2887-b7ec-6ec3d4eb9ff9 127.0.0.1 +2021-08-13T15:34:33.397Z [INFO] agent.router: Initializing LAN area manager +34:33.397 [INFO] TestAgent.server: Adding LAN server: server="Node-822564e7-88a4-2887-b7ec-6ec3d4eb9ff9 (Addr: tcp/127.0.0.1:12055) (DC: dc1)" +34:33.397 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-822564e7-88a4-2887-b7ec-6ec3d4eb9ff9.dc1 area=wan +34:33.398 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12050 network=udp +34:33.398 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12050 network=tcp +34:33.399 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:12051 network=tcp +34:33.399 [INFO] TestAgent: started state syncer +34:33.399 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:12056 network=tcp +34:33.444 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:33.444 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:12055 [Candidate]" term=2 +34:33.445 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:33.445 [DEBUG] TestAgent.server.raft: vote granted: from=822564e7-88a4-2887-b7ec-6ec3d4eb9ff9 term=2 tally=1 +34:33.445 [INFO] TestAgent.server.raft: election won: tally=1 +34:33.445 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:12055 [Leader]" +34:33.445 [INFO] TestAgent.server: cluster leadership acquired +34:33.445 [INFO] TestAgent.server: New leader elected: payload=Node-822564e7-88a4-2887-b7ec-6ec3d4eb9ff9 +34:33.445 [INFO] TestAgent.server: initializing acls +34:33.446 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:33.446 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:33.446 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:33.447 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:33.447 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:33.447 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:33.447 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-822564e7-88a4-2887-b7ec-6ec3d4eb9ff9 +34:33.447 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-822564e7-88a4-2887-b7ec-6ec3d4eb9ff9.dc1 +34:33.447 [INFO] TestAgent.server: Updating LAN server: server="Node-822564e7-88a4-2887-b7ec-6ec3d4eb9ff9 (Addr: tcp/127.0.0.1:12055) (DC: dc1)" +34:33.448 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-822564e7-88a4-2887-b7ec-6ec3d4eb9ff9.dc1 area=wan +34:33.448 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:33.487 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:33.487 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:33.488 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:33.488 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:33.488 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:33.488 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-822564e7-88a4-2887-b7ec-6ec3d4eb9ff9 +34:33.488 [INFO] TestAgent.server: member joined, marking health alive: member=Node-822564e7-88a4-2887-b7ec-6ec3d4eb9ff9 +34:33.489 [INFO] TestAgent.server: federation state anti-entropy synced +34:33.489 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-822564e7-88a4-2887-b7ec-6ec3d4eb9ff9 +34:33.641 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:33.642 [INFO] TestAgent: Synced node info +34:33.789 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:58592 latency=882.403µs +34:33.791 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policy/36b81b37-8f55-3b31-ba4b-f1921de61e47 from=127.0.0.1:58594 latency=1.249583ms +34:33.795 [INFO] TestAgent: Requesting shutdown +34:33.796 [INFO] TestAgent.server: shutting down server +34:33.796 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:33.796 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:33.796 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:33.796 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:33.796 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:33.796 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:33.796 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:33.796 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:33.796 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:33.796 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:33.796 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:33.796 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:33.796 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:33.796 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +34:33.796 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +2021-08-13T15:34:33.796Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:33.796Z [INFO] agent.router.manager: shutting down +34:33.797 [INFO] TestAgent: consul server down +34:33.797 [INFO] TestAgent: shutdown complete +34:33.797 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12050 network=tcp +34:33.797 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12050 network=udp +34:33.797 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:12051 network=tcp +34:34.297 [INFO] TestAgent: Waiting for endpoints to shut down +34:34.297 [INFO] TestAgent: Endpoints down +--- PASS: TestPolicyReadCommand_JSON (0.98s) PASS -ok github.com/hashicorp/consul/command/acl/policy/read 1.366s +ok github.com/hashicorp/consul/command/acl/policy/read 1.038s === RUN TestPolicyUpdateCommand_noTabs === PAUSE TestPolicyUpdateCommand_noTabs === RUN TestPolicyUpdateCommand @@ -75712,204 +75732,203 @@ === RUN TestPolicyUpdateCommand_JSON === PAUSE TestPolicyUpdateCommand_JSON === CONT TestPolicyUpdateCommand_noTabs ---- PASS: TestPolicyUpdateCommand_noTabs (0.00s) === CONT TestPolicyUpdateCommand_JSON [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestPolicyUpdateCommand +--- PASS: TestPolicyUpdateCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:42:17.329Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:17.372Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:17.375 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1c68db81-a726-ea7b-baf5-e51ffdc5d2db Address:127.0.0.1:21244}]" -42:17.375 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:21244 [Follower]" leader= -42:17.379 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1c68db81-a726-ea7b-baf5-e51ffdc5d2db.dc1 127.0.0.1 -42:17.380 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1c68db81-a726-ea7b-baf5-e51ffdc5d2db 127.0.0.1 -2022-09-15T21:42:17.380Z [INFO] agent.router: Initializing LAN area manager -42:17.381 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21239 network=udp -42:17.381 [INFO] TestAgent.server: Adding LAN server: server="Node-1c68db81-a726-ea7b-baf5-e51ffdc5d2db (Addr: tcp/127.0.0.1:21244) (DC: dc1)" -42:17.381 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1c68db81-a726-ea7b-baf5-e51ffdc5d2db.dc1 area=wan -42:17.381 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21239 network=tcp -42:17.382 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:21240 network=tcp -42:17.382 [INFO] TestAgent: started state syncer -42:17.383 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:21245 network=tcp -42:17.432 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -42:17.443 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:17.443 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:21244 [Candidate]" term=2 -42:17.444 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:17.444 [DEBUG] TestAgent.server.raft: vote granted: from=1c68db81-a726-ea7b-baf5-e51ffdc5d2db term=2 tally=1 -42:17.444 [INFO] TestAgent.server.raft: election won: tally=1 -42:17.444 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:21244 [Leader]" -42:17.444 [INFO] TestAgent.server: cluster leadership acquired -42:17.444 [INFO] TestAgent.server: New leader elected: payload=Node-1c68db81-a726-ea7b-baf5-e51ffdc5d2db -42:17.444 [INFO] TestAgent.server: initializing acls -42:17.444 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:17.444 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:17.452 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:17.453 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:17.453 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:17.453 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:17.454 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-1c68db81-a726-ea7b-baf5-e51ffdc5d2db -42:17.454 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-1c68db81-a726-ea7b-baf5-e51ffdc5d2db.dc1 -42:17.456 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:17.456 [INFO] TestAgent.server: Updating LAN server: server="Node-1c68db81-a726-ea7b-baf5-e51ffdc5d2db (Addr: tcp/127.0.0.1:21244) (DC: dc1)" -42:17.456 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-1c68db81-a726-ea7b-baf5-e51ffdc5d2db.dc1 area=wan -42:17.496 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:17.496 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:17.496 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:17.496 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:17.496 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:17.496 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1c68db81-a726-ea7b-baf5-e51ffdc5d2db -42:17.496 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1c68db81-a726-ea7b-baf5-e51ffdc5d2db -42:17.497 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1c68db81-a726-ea7b-baf5-e51ffdc5d2db -42:17.498 [INFO] TestAgent.server: federation state anti-entropy synced -42:17.508 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:17.509 [INFO] TestAgent: Synced node info -42:17.509 [DEBUG] TestAgent: Node info in sync -42:17.532 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:45474 latency=1.78406ms -42:17.534 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policy/c13e7e28-436b-0828-ad2d-1f134a6716f3 from=127.0.0.1:45476 latency=467.4µs -42:17.537 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy/c13e7e28-436b-0828-ad2d-1f134a6716f3 from=127.0.0.1:45476 latency=1.03856ms -42:17.537 [INFO] TestAgent: Requesting shutdown -42:17.537 [INFO] TestAgent.server: shutting down server -42:17.537 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:17.537 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:17.537 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:17.537 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:17.537 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:17.537 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:17.537 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:17.537 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:17.537 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:17.537 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:17.537 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:17.537 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:17.537 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:17.537 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:17.538 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:17.538Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:17.538Z [INFO] agent.router.manager: shutting down -42:17.538 [INFO] TestAgent: consul server down -42:17.538 [INFO] TestAgent: shutdown complete -42:17.538 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21239 network=tcp -42:17.538 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21239 network=udp -42:17.538 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:21240 network=tcp -42:18.038 [INFO] TestAgent: Waiting for endpoints to shut down -42:18.038 [INFO] TestAgent: Endpoints down ---- PASS: TestPolicyUpdateCommand (0.83s) -2022-09-15T21:42:17.304Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:17.367Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:17.373 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5d27ec1d-800e-905d-7177-122689d362d0 Address:127.0.0.1:21237}]" -42:17.373 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:21237 [Follower]" leader= -42:17.374 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5d27ec1d-800e-905d-7177-122689d362d0.dc1 127.0.0.1 -42:17.375 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5d27ec1d-800e-905d-7177-122689d362d0 127.0.0.1 -2022-09-15T21:42:17.375Z [INFO] agent.router: Initializing LAN area manager -42:17.375 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5d27ec1d-800e-905d-7177-122689d362d0.dc1 area=wan -42:17.375 [INFO] TestAgent.server: Adding LAN server: server="Node-5d27ec1d-800e-905d-7177-122689d362d0 (Addr: tcp/127.0.0.1:21237) (DC: dc1)" -42:17.375 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21232 network=udp -42:17.375 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21232 network=tcp -42:17.377 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:21233 network=tcp -42:17.377 [INFO] TestAgent: started state syncer -42:17.377 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:21238 network=tcp -42:17.425 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -42:17.438 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:17.438 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:21237 [Candidate]" term=2 -42:17.438 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:17.438 [DEBUG] TestAgent.server.raft: vote granted: from=5d27ec1d-800e-905d-7177-122689d362d0 term=2 tally=1 -42:17.438 [INFO] TestAgent.server.raft: election won: tally=1 -42:17.438 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:21237 [Leader]" -42:17.438 [INFO] TestAgent.server: cluster leadership acquired -42:17.439 [INFO] TestAgent.server: initializing acls -42:17.439 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:17.439 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:17.440 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:17.441 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:17.441 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:17.441 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:17.441 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-5d27ec1d-800e-905d-7177-122689d362d0 -42:17.441 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-5d27ec1d-800e-905d-7177-122689d362d0.dc1 -42:17.442 [INFO] TestAgent.server: New leader elected: payload=Node-5d27ec1d-800e-905d-7177-122689d362d0 -42:17.442 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-5d27ec1d-800e-905d-7177-122689d362d0.dc1 area=wan -42:17.442 [INFO] TestAgent.server: Updating LAN server: server="Node-5d27ec1d-800e-905d-7177-122689d362d0 (Addr: tcp/127.0.0.1:21237) (DC: dc1)" -42:17.444 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:17.493 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:17.493 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:17.493 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:17.493 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:17.493 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:17.493 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5d27ec1d-800e-905d-7177-122689d362d0 -42:17.493 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5d27ec1d-800e-905d-7177-122689d362d0 -42:17.494 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5d27ec1d-800e-905d-7177-122689d362d0 -42:17.495 [INFO] TestAgent.server: federation state anti-entropy synced -42:17.590 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:56696 latency=1.57532ms -42:17.592 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policy/fba29ffb-1ad8-c694-8151-7bbfd063c157 from=127.0.0.1:56698 latency=411.06µs -42:17.594 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy/fba29ffb-1ad8-c694-8151-7bbfd063c157 from=127.0.0.1:56698 latency=851.72µs -42:17.595 [INFO] TestAgent: Requesting shutdown -42:17.600 [INFO] TestAgent.server: shutting down server -42:17.600 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:17.600 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:17.600 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:17.600 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:17.600 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:17.600 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:17.600 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:17.600 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:17.601 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:17.601 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:17.601 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:17.601 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:17.601 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:17.601 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:17.601 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:17.601 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:17.601Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:17.601Z [INFO] agent.router.manager: shutting down -42:17.601 [INFO] TestAgent: consul server down -42:17.601 [INFO] TestAgent: shutdown complete -42:17.601 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21232 network=tcp -42:17.601 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21232 network=udp -42:17.601 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:21233 network=tcp -42:18.102 [INFO] TestAgent: Waiting for endpoints to shut down -42:18.102 [INFO] TestAgent: Endpoints down ---- PASS: TestPolicyUpdateCommand_JSON (0.90s) +2021-08-13T15:34:40.325Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:40.346Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:40.349 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3ee186d9-4936-03e1-e5c0-5b7be4cb4cf3 Address:127.0.0.1:22258}]" +34:40.349 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22258 [Follower]" leader= +34:40.350 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3ee186d9-4936-03e1-e5c0-5b7be4cb4cf3.dc1 127.0.0.1 +34:40.351 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3ee186d9-4936-03e1-e5c0-5b7be4cb4cf3 127.0.0.1 +2021-08-13T15:34:40.351Z [INFO] agent.router: Initializing LAN area manager +34:40.352 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22253 network=udp +34:40.352 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3ee186d9-4936-03e1-e5c0-5b7be4cb4cf3.dc1 area=wan +34:40.352 [INFO] TestAgent.server: Adding LAN server: server="Node-3ee186d9-4936-03e1-e5c0-5b7be4cb4cf3 (Addr: tcp/127.0.0.1:22258) (DC: dc1)" +34:40.352 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22253 network=tcp +34:40.354 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22254 network=tcp +34:40.354 [INFO] TestAgent: started state syncer +34:40.354 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22259 network=tcp +34:40.402 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:40.405 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:40.405 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22258 [Candidate]" term=2 +34:40.405 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:40.405 [DEBUG] TestAgent.server.raft: vote granted: from=3ee186d9-4936-03e1-e5c0-5b7be4cb4cf3 term=2 tally=1 +34:40.405 [INFO] TestAgent.server.raft: election won: tally=1 +34:40.405 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22258 [Leader]" +34:40.405 [INFO] TestAgent.server: cluster leadership acquired +34:40.405 [INFO] TestAgent.server: New leader elected: payload=Node-3ee186d9-4936-03e1-e5c0-5b7be4cb4cf3 +34:40.405 [INFO] TestAgent.server: initializing acls +34:40.406 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:40.406 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:40.407 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:40.407 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:40.407 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:40.407 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:40.408 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-3ee186d9-4936-03e1-e5c0-5b7be4cb4cf3 +34:40.408 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-3ee186d9-4936-03e1-e5c0-5b7be4cb4cf3.dc1 +34:40.408 [INFO] TestAgent.server: Updating LAN server: server="Node-3ee186d9-4936-03e1-e5c0-5b7be4cb4cf3 (Addr: tcp/127.0.0.1:22258) (DC: dc1)" +34:40.408 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-3ee186d9-4936-03e1-e5c0-5b7be4cb4cf3.dc1 area=wan +34:40.409 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:40.452 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:40.452 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:40.452 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:40.452 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:40.452 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:40.452 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3ee186d9-4936-03e1-e5c0-5b7be4cb4cf3 +34:40.452 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3ee186d9-4936-03e1-e5c0-5b7be4cb4cf3 +34:40.453 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3ee186d9-4936-03e1-e5c0-5b7be4cb4cf3 +34:40.453 [INFO] TestAgent.server: federation state anti-entropy synced +34:40.497 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:45482 latency=1.809924ms +34:40.499 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policy/908d81f8-bb88-0d80-08cc-23a3966267c5 from=127.0.0.1:45484 latency=419.941µs +34:40.500 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy/908d81f8-bb88-0d80-08cc-23a3966267c5 from=127.0.0.1:45484 latency=718.062µs +34:40.501 [INFO] TestAgent: Requesting shutdown +34:40.502 [INFO] TestAgent.server: shutting down server +34:40.502 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:40.502 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:40.502 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:40.502 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:40.502 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:40.502 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:40.502 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:40.502 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:40.502 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:40.502 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:40.502 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:40.502 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:40.502 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:40.502 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:40.502 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:40.502 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:40.503Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:40.503Z [INFO] agent.router.manager: shutting down +34:40.503 [INFO] TestAgent: consul server down +34:40.503 [INFO] TestAgent: shutdown complete +34:40.503 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22253 network=tcp +34:40.503 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22253 network=udp +34:40.503 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22254 network=tcp +34:41.003 [INFO] TestAgent: Waiting for endpoints to shut down +34:41.003 [INFO] TestAgent: Endpoints down +--- PASS: TestPolicyUpdateCommand_JSON (0.74s) +2021-08-13T15:34:40.328Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:40.350Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:40.354 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9bb08c30-2700-2bb0-2990-1732f87be7df Address:127.0.0.1:22265}]" +34:40.355 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9bb08c30-2700-2bb0-2990-1732f87be7df.dc1 127.0.0.1 +34:40.356 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9bb08c30-2700-2bb0-2990-1732f87be7df 127.0.0.1 +2021-08-13T15:34:40.356Z [INFO] agent.router: Initializing LAN area manager +34:40.357 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22260 network=udp +34:40.357 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22265 [Follower]" leader= +34:40.357 [INFO] TestAgent.server: Adding LAN server: server="Node-9bb08c30-2700-2bb0-2990-1732f87be7df (Addr: tcp/127.0.0.1:22265) (DC: dc1)" +34:40.358 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9bb08c30-2700-2bb0-2990-1732f87be7df.dc1 area=wan +34:40.358 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22260 network=tcp +34:40.359 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22261 network=tcp +34:40.359 [INFO] TestAgent: started state syncer +34:40.359 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22266 network=tcp +34:40.408 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:40.427 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:40.427 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22265 [Candidate]" term=2 +34:40.427 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:40.427 [DEBUG] TestAgent.server.raft: vote granted: from=9bb08c30-2700-2bb0-2990-1732f87be7df term=2 tally=1 +34:40.427 [INFO] TestAgent.server.raft: election won: tally=1 +34:40.427 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22265 [Leader]" +34:40.428 [INFO] TestAgent.server: cluster leadership acquired +34:40.428 [INFO] TestAgent.server: initializing acls +34:40.428 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:40.428 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:40.429 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:40.429 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:40.429 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:40.429 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:40.430 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-9bb08c30-2700-2bb0-2990-1732f87be7df +34:40.430 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-9bb08c30-2700-2bb0-2990-1732f87be7df.dc1 +34:40.431 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:40.431 [INFO] TestAgent.server: New leader elected: payload=Node-9bb08c30-2700-2bb0-2990-1732f87be7df +34:40.431 [INFO] TestAgent.server: Updating LAN server: server="Node-9bb08c30-2700-2bb0-2990-1732f87be7df (Addr: tcp/127.0.0.1:22265) (DC: dc1)" +34:40.431 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-9bb08c30-2700-2bb0-2990-1732f87be7df.dc1 area=wan +34:40.448 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:40.448 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:40.448 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:40.448 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:40.448 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:40.448 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9bb08c30-2700-2bb0-2990-1732f87be7df +34:40.448 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9bb08c30-2700-2bb0-2990-1732f87be7df +34:40.449 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9bb08c30-2700-2bb0-2990-1732f87be7df +34:40.450 [INFO] TestAgent.server: federation state anti-entropy synced +34:40.632 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:40.633 [INFO] TestAgent: Synced node info +34:40.704 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:50894 latency=1.572564ms +34:40.706 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policy/bac5171b-8fb9-e006-5445-79b2aae09721 from=127.0.0.1:50896 latency=364.041µs +34:40.707 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy/bac5171b-8fb9-e006-5445-79b2aae09721 from=127.0.0.1:50896 latency=890.642µs +34:40.708 [INFO] TestAgent: Requesting shutdown +34:40.708 [INFO] TestAgent.server: shutting down server +34:40.708 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:40.708 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:40.708 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:40.708 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:40.708 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:40.708 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:40.708 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:40.708 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:40.708 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:40.708 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:40.708 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:40.708 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:40.708 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:40.708 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:40.708 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:40.708Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:40.708Z [INFO] agent.router.manager: shutting down +34:40.709 [INFO] TestAgent: consul server down +34:40.709 [INFO] TestAgent: shutdown complete +34:40.709 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22260 network=tcp +34:40.709 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22260 network=udp +34:40.709 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22261 network=tcp +34:41.209 [INFO] TestAgent: Waiting for endpoints to shut down +34:41.209 [INFO] TestAgent: Endpoints down +--- PASS: TestPolicyUpdateCommand (0.95s) PASS -ok github.com/hashicorp/consul/command/acl/policy/update 1.061s +ok github.com/hashicorp/consul/command/acl/policy/update 1.023s === RUN TestFormatRole === RUN TestFormatRole/basic -=== RUN TestFormatRole/basic/pretty === RUN TestFormatRole/basic/pretty-meta === RUN TestFormatRole/basic/json +=== RUN TestFormatRole/basic/pretty === RUN TestFormatRole/complex +=== RUN TestFormatRole/complex/pretty === RUN TestFormatRole/complex/pretty-meta === RUN TestFormatRole/complex/json -=== RUN TestFormatRole/complex/pretty ---- PASS: TestFormatRole (0.01s) +--- PASS: TestFormatRole (0.00s) --- PASS: TestFormatRole/basic (0.00s) - --- PASS: TestFormatRole/basic/pretty (0.00s) --- PASS: TestFormatRole/basic/pretty-meta (0.00s) --- PASS: TestFormatRole/basic/json (0.00s) - --- PASS: TestFormatRole/complex (0.01s) + --- PASS: TestFormatRole/basic/pretty (0.00s) + --- PASS: TestFormatRole/complex (0.00s) + --- PASS: TestFormatRole/complex/pretty (0.00s) --- PASS: TestFormatRole/complex/pretty-meta (0.00s) --- PASS: TestFormatRole/complex/json (0.00s) - --- PASS: TestFormatRole/complex/pretty (0.01s) === RUN TestFormatTokenList -=== RUN TestFormatTokenList/complex -=== RUN TestFormatTokenList/complex/pretty -=== RUN TestFormatTokenList/complex/pretty-meta -=== RUN TestFormatTokenList/complex/json === RUN TestFormatTokenList/basic === RUN TestFormatTokenList/basic/pretty === RUN TestFormatTokenList/basic/pretty-meta === RUN TestFormatTokenList/basic/json +=== RUN TestFormatTokenList/complex +=== RUN TestFormatTokenList/complex/pretty +=== RUN TestFormatTokenList/complex/pretty-meta +=== RUN TestFormatTokenList/complex/json --- PASS: TestFormatTokenList (0.00s) - --- PASS: TestFormatTokenList/complex (0.00s) - --- PASS: TestFormatTokenList/complex/pretty (0.00s) - --- PASS: TestFormatTokenList/complex/pretty-meta (0.00s) - --- PASS: TestFormatTokenList/complex/json (0.00s) --- PASS: TestFormatTokenList/basic (0.00s) --- PASS: TestFormatTokenList/basic/pretty (0.00s) --- PASS: TestFormatTokenList/basic/pretty-meta (0.00s) --- PASS: TestFormatTokenList/basic/json (0.00s) + --- PASS: TestFormatTokenList/complex (0.00s) + --- PASS: TestFormatTokenList/complex/pretty (0.00s) + --- PASS: TestFormatTokenList/complex/pretty-meta (0.00s) + --- PASS: TestFormatTokenList/complex/json (0.00s) PASS -ok github.com/hashicorp/consul/command/acl/role 0.060s +ok github.com/hashicorp/consul/command/acl/role 0.009s === RUN TestRoleCreateCommand_noTabs === PAUSE TestRoleCreateCommand_noTabs === RUN TestRoleCreateCommand_Pretty @@ -75917,10 +75936,10 @@ === RUN TestRoleCreateCommand_JSON === PAUSE TestRoleCreateCommand_JSON === CONT TestRoleCreateCommand_noTabs ---- PASS: TestRoleCreateCommand_noTabs (0.00s) === CONT TestRoleCreateCommand_JSON -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestRoleCreateCommand_noTabs (0.00s) === CONT TestRoleCreateCommand_Pretty +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range === RUN TestRoleCreateCommand_Pretty/policy-name @@ -75928,175 +75947,167 @@ === RUN TestRoleCreateCommand_Pretty/service-identity === RUN TestRoleCreateCommand_Pretty/dc-scoped-service-identity === RUN TestRoleCreateCommand_Pretty/node-identity -2022-09-15T21:42:28.130Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:28.155Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:28.165 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e2f89b52-92dd-4016-d9d3-ecc969c512ae Address:127.0.0.1:29412}]" -42:28.166 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29412 [Follower]" leader= -42:28.166 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e2f89b52-92dd-4016-d9d3-ecc969c512ae.dc1 127.0.0.1 -42:28.167 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e2f89b52-92dd-4016-d9d3-ecc969c512ae 127.0.0.1 -2022-09-15T21:42:28.167Z [INFO] agent.router: Initializing LAN area manager -42:28.167 [INFO] TestAgent.server: Adding LAN server: server="Node-e2f89b52-92dd-4016-d9d3-ecc969c512ae (Addr: tcp/127.0.0.1:29412) (DC: dc1)" -42:28.167 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e2f89b52-92dd-4016-d9d3-ecc969c512ae.dc1 area=wan -42:28.167 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29407 network=udp -42:28.167 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29407 network=tcp -42:28.168 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29408 network=tcp -42:28.169 [INFO] TestAgent: started state syncer -42:28.169 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29413 network=tcp -42:28.215 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:28.215 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29412 [Candidate]" term=2 -42:28.216 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:28.216 [DEBUG] TestAgent.server.raft: vote granted: from=e2f89b52-92dd-4016-d9d3-ecc969c512ae term=2 tally=1 -42:28.216 [INFO] TestAgent.server.raft: election won: tally=1 -42:28.216 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29412 [Leader]" -42:28.216 [INFO] TestAgent.server: cluster leadership acquired -42:28.216 [INFO] TestAgent.server: initializing acls -42:28.217 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:28.217 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:28.217 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:28.218 [INFO] TestAgent.server: New leader elected: payload=Node-e2f89b52-92dd-4016-d9d3-ecc969c512ae -42:28.218 [INFO] TestAgent.server: initializing acls -42:28.218 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:28.219 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:28.219 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:28.219 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:28.219 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:28.219 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode -42:28.219 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-e2f89b52-92dd-4016-d9d3-ecc969c512ae -42:28.219 [INFO] TestAgent.server: Updating LAN server: server="Node-e2f89b52-92dd-4016-d9d3-ecc969c512ae (Addr: tcp/127.0.0.1:29412) (DC: dc1)" -42:28.219 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-e2f89b52-92dd-4016-d9d3-ecc969c512ae.dc1 -42:28.224 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-e2f89b52-92dd-4016-d9d3-ecc969c512ae -42:28.225 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-e2f89b52-92dd-4016-d9d3-ecc969c512ae.dc1 -42:28.225 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-e2f89b52-92dd-4016-d9d3-ecc969c512ae.dc1 area=wan -42:28.225 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-e2f89b52-92dd-4016-d9d3-ecc969c512ae.dc1 area=wan -42:28.225 [INFO] TestAgent.server: Updating LAN server: server="Node-e2f89b52-92dd-4016-d9d3-ecc969c512ae (Addr: tcp/127.0.0.1:29412) (DC: dc1)" -42:28.230 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:28.254 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:28.254 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:28.254 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:28.254 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:28.254 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:28.254 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e2f89b52-92dd-4016-d9d3-ecc969c512ae -42:28.254 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e2f89b52-92dd-4016-d9d3-ecc969c512ae -42:28.255 [INFO] TestAgent.server: federation state anti-entropy synced -42:28.256 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e2f89b52-92dd-4016-d9d3-ecc969c512ae -42:28.256 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e2f89b52-92dd-4016-d9d3-ecc969c512ae -42:28.279 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:37540 latency=1.82008ms -42:28.284 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:37542 latency=1.34024ms -42:28.287 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:37544 latency=831.04µs -42:28.290 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:37546 latency=1.01738ms -42:28.317 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:37548 latency=1.43368ms -42:28.337 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:37550 latency=1.15108ms -42:28.338 [INFO] TestAgent: Requesting shutdown -42:28.339 [INFO] TestAgent.server: shutting down server -42:28.339 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:28.339 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:28.339 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:28.339 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:28.339 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:28.339 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:28.339 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:28.339 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:28.339 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -42:28.339 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:28.339 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:28.339 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:28.339 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:28.339 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:28.339 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -2022-09-15T21:42:28.339Z [INFO] agent.router.manager: shutting down -42:28.339 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -2022-09-15T21:42:28.339Z [INFO] agent.router.manager: shutting down -42:28.339 [INFO] TestAgent: consul server down -42:28.339 [INFO] TestAgent: shutdown complete -42:28.339 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29407 network=tcp -42:28.339 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29407 network=udp -42:28.339 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29408 network=tcp -42:28.842 [INFO] TestAgent: Waiting for endpoints to shut down -42:28.842 [INFO] TestAgent: Endpoints down ---- PASS: TestRoleCreateCommand_Pretty (0.83s) - --- PASS: TestRoleCreateCommand_Pretty/policy-name (0.00s) - --- PASS: TestRoleCreateCommand_Pretty/policy-id (0.00s) +2021-08-13T15:34:48.277Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:48.328Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:48.332 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3ac936c1-0a41-86fe-047d-150b8bcead31 Address:127.0.0.1:25328}]" +34:48.333 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3ac936c1-0a41-86fe-047d-150b8bcead31.dc1 127.0.0.1 +34:48.334 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3ac936c1-0a41-86fe-047d-150b8bcead31 127.0.0.1 +2021-08-13T15:34:48.334Z [INFO] agent.router: Initializing LAN area manager +34:48.334 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25323 network=udp +34:48.334 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25328 [Follower]" leader= +34:48.335 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3ac936c1-0a41-86fe-047d-150b8bcead31.dc1 area=wan +34:48.335 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25323 network=tcp +34:48.343 [INFO] TestAgent.server: Adding LAN server: server="Node-3ac936c1-0a41-86fe-047d-150b8bcead31 (Addr: tcp/127.0.0.1:25328) (DC: dc1)" +34:48.343 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25324 network=tcp +34:48.343 [INFO] TestAgent: started state syncer +34:48.344 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25329 network=tcp +34:48.385 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:48.399 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:48.399 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25328 [Candidate]" term=2 +34:48.400 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:48.400 [DEBUG] TestAgent.server.raft: vote granted: from=3ac936c1-0a41-86fe-047d-150b8bcead31 term=2 tally=1 +34:48.400 [INFO] TestAgent.server.raft: election won: tally=1 +34:48.400 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25328 [Leader]" +34:48.400 [INFO] TestAgent.server: cluster leadership acquired +34:48.400 [INFO] TestAgent.server: New leader elected: payload=Node-3ac936c1-0a41-86fe-047d-150b8bcead31 +34:48.400 [INFO] TestAgent.server: initializing acls +34:48.401 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:48.401 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:48.401 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:48.402 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:48.402 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:48.402 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:48.402 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-3ac936c1-0a41-86fe-047d-150b8bcead31 +34:48.402 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-3ac936c1-0a41-86fe-047d-150b8bcead31.dc1 +34:48.403 [INFO] TestAgent.server: Updating LAN server: server="Node-3ac936c1-0a41-86fe-047d-150b8bcead31 (Addr: tcp/127.0.0.1:25328) (DC: dc1)" +34:48.403 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-3ac936c1-0a41-86fe-047d-150b8bcead31.dc1 area=wan +34:48.412 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:48.437 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:48.437 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:48.437 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:48.437 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:48.437 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:48.437 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3ac936c1-0a41-86fe-047d-150b8bcead31 +34:48.437 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3ac936c1-0a41-86fe-047d-150b8bcead31 +34:48.438 [INFO] TestAgent.server: federation state anti-entropy synced +34:48.439 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3ac936c1-0a41-86fe-047d-150b8bcead31 +34:48.578 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:34924 latency=1.992164ms +34:48.582 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:34926 latency=1.315683ms +34:48.583 [INFO] TestAgent: Requesting shutdown +34:48.583 [INFO] TestAgent.server: shutting down server +34:48.583 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:48.583 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:48.583 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:48.583 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:48.583 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:48.583 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:48.583 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:48.583 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:48.591 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:48.595 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:48.595 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:48.595 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:48.595 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:48.595 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:48.595 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:48.595 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:48.596Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:48.596Z [INFO] agent.router.manager: shutting down +34:48.596 [INFO] TestAgent: consul server down +34:48.596 [INFO] TestAgent: shutdown complete +34:48.596 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25323 network=tcp +34:48.596 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25323 network=udp +34:48.596 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25324 network=tcp +34:49.096 [INFO] TestAgent: Waiting for endpoints to shut down +34:49.096 [INFO] TestAgent: Endpoints down +--- PASS: TestRoleCreateCommand_JSON (0.90s) +2021-08-13T15:34:48.278Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:48.337Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:48.343 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ac0c378c-8687-eed2-be2d-ca071bf0a38d Address:127.0.0.1:25321}]" +34:48.344 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25321 [Follower]" leader= +34:48.344 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ac0c378c-8687-eed2-be2d-ca071bf0a38d.dc1 127.0.0.1 +34:48.345 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ac0c378c-8687-eed2-be2d-ca071bf0a38d 127.0.0.1 +2021-08-13T15:34:48.345Z [INFO] agent.router: Initializing LAN area manager +34:48.346 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25316 network=udp +34:48.346 [INFO] TestAgent.server: Adding LAN server: server="Node-ac0c378c-8687-eed2-be2d-ca071bf0a38d (Addr: tcp/127.0.0.1:25321) (DC: dc1)" +34:48.346 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ac0c378c-8687-eed2-be2d-ca071bf0a38d.dc1 area=wan +34:48.346 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25316 network=tcp +34:48.351 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25317 network=tcp +34:48.351 [INFO] TestAgent: started state syncer +34:48.351 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25322 network=tcp +34:48.396 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +34:48.407 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:48.407 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25321 [Candidate]" term=2 +34:48.407 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:48.407 [DEBUG] TestAgent.server.raft: vote granted: from=ac0c378c-8687-eed2-be2d-ca071bf0a38d term=2 tally=1 +34:48.407 [INFO] TestAgent.server.raft: election won: tally=1 +34:48.408 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25321 [Leader]" +34:48.408 [INFO] TestAgent.server: cluster leadership acquired +34:48.408 [INFO] TestAgent.server: initializing acls +34:48.409 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:48.409 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:48.409 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:48.410 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:48.410 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:48.410 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:48.415 [INFO] TestAgent.server: New leader elected: payload=Node-ac0c378c-8687-eed2-be2d-ca071bf0a38d +34:48.428 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-ac0c378c-8687-eed2-be2d-ca071bf0a38d +34:48.428 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-ac0c378c-8687-eed2-be2d-ca071bf0a38d.dc1 +34:48.429 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-ac0c378c-8687-eed2-be2d-ca071bf0a38d.dc1 area=wan +34:48.429 [INFO] TestAgent.server: Updating LAN server: server="Node-ac0c378c-8687-eed2-be2d-ca071bf0a38d (Addr: tcp/127.0.0.1:25321) (DC: dc1)" +34:48.429 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:48.437 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:48.437 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:48.437 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:48.437 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:48.437 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:48.437 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ac0c378c-8687-eed2-be2d-ca071bf0a38d +34:48.437 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ac0c378c-8687-eed2-be2d-ca071bf0a38d +34:48.438 [INFO] TestAgent.server: federation state anti-entropy synced +34:48.439 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ac0c378c-8687-eed2-be2d-ca071bf0a38d +34:48.446 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +34:48.447 [INFO] TestAgent: Synced node info +34:48.447 [DEBUG] TestAgent: Node info in sync +34:48.638 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:41112 latency=1.493744ms +34:48.643 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:41114 latency=1.588904ms +34:48.660 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:41116 latency=1.284843ms +34:48.665 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:41118 latency=1.019763ms +34:48.668 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:41120 latency=976.142µs +34:48.673 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:41122 latency=1.102482ms +34:48.674 [INFO] TestAgent: Requesting shutdown +34:48.674 [INFO] TestAgent.server: shutting down server +34:48.674 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:48.674 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:48.674 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:48.674 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:48.674 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:48.674 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:48.674 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:48.674 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:48.674 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:48.674 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:48.674 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:48.674 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:48.674 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:48.674 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:48.674 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:48.674Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:48.674Z [INFO] agent.router.manager: shutting down +34:48.675 [INFO] TestAgent: consul server down +34:48.675 [INFO] TestAgent: shutdown complete +34:48.675 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25316 network=tcp +34:48.675 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25316 network=udp +34:48.675 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25317 network=tcp +34:49.175 [INFO] TestAgent: Waiting for endpoints to shut down +34:49.175 [INFO] TestAgent: Endpoints down +--- PASS: TestRoleCreateCommand_Pretty (0.98s) + --- PASS: TestRoleCreateCommand_Pretty/policy-name (0.01s) + --- PASS: TestRoleCreateCommand_Pretty/policy-id (0.01s) --- PASS: TestRoleCreateCommand_Pretty/service-identity (0.00s) - --- PASS: TestRoleCreateCommand_Pretty/dc-scoped-service-identity (0.04s) - --- PASS: TestRoleCreateCommand_Pretty/node-identity (0.01s) -2022-09-15T21:42:28.120Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:28.164Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:28.169 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:30a5bec5-3a6a-e339-614c-70288ec98e52 Address:127.0.0.1:29405}]" -42:28.169 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29405 [Follower]" leader= -42:28.170 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-30a5bec5-3a6a-e339-614c-70288ec98e52.dc1 127.0.0.1 -42:28.171 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-30a5bec5-3a6a-e339-614c-70288ec98e52 127.0.0.1 -2022-09-15T21:42:28.171Z [INFO] agent.router: Initializing LAN area manager -42:28.171 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29400 network=udp -42:28.171 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-30a5bec5-3a6a-e339-614c-70288ec98e52.dc1 area=wan -42:28.171 [INFO] TestAgent.server: Adding LAN server: server="Node-30a5bec5-3a6a-e339-614c-70288ec98e52 (Addr: tcp/127.0.0.1:29405) (DC: dc1)" -42:28.171 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29400 network=tcp -42:28.172 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29401 network=tcp -42:28.172 [INFO] TestAgent: started state syncer -42:28.172 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29406 network=tcp -42:28.224 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -42:28.237 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:28.237 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29405 [Candidate]" term=2 -42:28.237 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:28.237 [DEBUG] TestAgent.server.raft: vote granted: from=30a5bec5-3a6a-e339-614c-70288ec98e52 term=2 tally=1 -42:28.237 [INFO] TestAgent.server.raft: election won: tally=1 -42:28.237 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29405 [Leader]" -42:28.237 [INFO] TestAgent.server: cluster leadership acquired -42:28.238 [INFO] TestAgent.server: initializing acls -42:28.238 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:28.238 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:28.239 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:28.239 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:28.239 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:28.239 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:28.240 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-30a5bec5-3a6a-e339-614c-70288ec98e52 -42:28.240 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-30a5bec5-3a6a-e339-614c-70288ec98e52.dc1 -42:28.241 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:28.241 [INFO] TestAgent.server: New leader elected: payload=Node-30a5bec5-3a6a-e339-614c-70288ec98e52 -42:28.241 [INFO] TestAgent.server: Updating LAN server: server="Node-30a5bec5-3a6a-e339-614c-70288ec98e52 (Addr: tcp/127.0.0.1:29405) (DC: dc1)" -42:28.241 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-30a5bec5-3a6a-e339-614c-70288ec98e52.dc1 area=wan -42:28.254 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:28.254 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:28.254 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:28.254 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:28.254 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:28.254 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-30a5bec5-3a6a-e339-614c-70288ec98e52 -42:28.254 [INFO] TestAgent.server: member joined, marking health alive: member=Node-30a5bec5-3a6a-e339-614c-70288ec98e52 -42:28.255 [INFO] TestAgent.server: federation state anti-entropy synced -42:28.256 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-30a5bec5-3a6a-e339-614c-70288ec98e52 -42:28.538 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:28.539 [INFO] TestAgent: Synced node info -42:28.539 [DEBUG] TestAgent: Node info in sync -42:28.600 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:34910 latency=1.70648ms -42:28.603 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:34912 latency=1.04928ms -42:28.606 [INFO] TestAgent: Requesting shutdown -42:28.606 [INFO] TestAgent.server: shutting down server -42:28.606 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:28.606 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:28.606 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:28.606 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:28.606 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:28.606 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:28.606 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:28.606 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:28.606 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:28.606 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:28.606 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:28.606 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:28.606 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:28.606 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:28.607 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:28.607Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:28.607Z [INFO] agent.router.manager: shutting down -42:28.607 [INFO] TestAgent: consul server down -42:28.607 [INFO] TestAgent: shutdown complete -42:28.607 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29400 network=tcp -42:28.607 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29400 network=udp -42:28.607 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29401 network=tcp -42:29.107 [INFO] TestAgent: Waiting for endpoints to shut down -42:29.107 [INFO] TestAgent: Endpoints down ---- PASS: TestRoleCreateCommand_JSON (1.10s) + --- PASS: TestRoleCreateCommand_Pretty/dc-scoped-service-identity (0.00s) + --- PASS: TestRoleCreateCommand_Pretty/node-identity (0.00s) PASS -ok github.com/hashicorp/consul/command/acl/role/create 1.195s +ok github.com/hashicorp/consul/command/acl/role/create 1.070s === RUN TestRoleDeleteCommand_noTabs === PAUSE TestRoleDeleteCommand_noTabs === RUN TestRoleDeleteCommand @@ -76110,90 +76121,89 @@ === RUN TestRoleDeleteCommand/id_or_name_required === RUN TestRoleDeleteCommand/delete_works === RUN TestRoleDeleteCommand/delete_works_via_prefixes -2022-09-15T21:42:28.731Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:28.759Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:28.763 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:46cd44b3-ea4b-5245-3b63-3e633c5808c7 Address:127.0.0.1:31447}]" -42:28.763 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31447 [Follower]" leader= -42:28.764 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-46cd44b3-ea4b-5245-3b63-3e633c5808c7.dc1 127.0.0.1 -42:28.765 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-46cd44b3-ea4b-5245-3b63-3e633c5808c7 127.0.0.1 -2022-09-15T21:42:28.765Z [INFO] agent.router: Initializing LAN area manager -42:28.765 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31442 network=udp -42:28.766 [INFO] TestAgent.server: Adding LAN server: server="Node-46cd44b3-ea4b-5245-3b63-3e633c5808c7 (Addr: tcp/127.0.0.1:31447) (DC: dc1)" -42:28.766 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-46cd44b3-ea4b-5245-3b63-3e633c5808c7.dc1 area=wan -42:28.766 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31442 network=tcp -42:28.767 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31443 network=tcp -42:28.767 [INFO] TestAgent: started state syncer -42:28.767 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31448 network=tcp -42:28.802 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:28.802 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31447 [Candidate]" term=2 -42:28.802 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:28.802 [DEBUG] TestAgent.server.raft: vote granted: from=46cd44b3-ea4b-5245-3b63-3e633c5808c7 term=2 tally=1 -42:28.802 [INFO] TestAgent.server.raft: election won: tally=1 -42:28.802 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31447 [Leader]" -42:28.802 [INFO] TestAgent.server: cluster leadership acquired -42:28.803 [INFO] TestAgent.server: initializing acls -42:28.803 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:28.803 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:28.804 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:28.805 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:28.805 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:28.805 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:28.805 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-46cd44b3-ea4b-5245-3b63-3e633c5808c7 -42:28.805 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-46cd44b3-ea4b-5245-3b63-3e633c5808c7.dc1 -42:28.806 [INFO] TestAgent.server: New leader elected: payload=Node-46cd44b3-ea4b-5245-3b63-3e633c5808c7 -42:28.806 [INFO] TestAgent.server: Updating LAN server: server="Node-46cd44b3-ea4b-5245-3b63-3e633c5808c7 (Addr: tcp/127.0.0.1:31447) (DC: dc1)" -42:28.806 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-46cd44b3-ea4b-5245-3b63-3e633c5808c7.dc1 area=wan -42:28.807 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:28.845 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:28.845 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:28.845 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:28.845 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:28.845 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:28.845 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-46cd44b3-ea4b-5245-3b63-3e633c5808c7 -42:28.845 [INFO] TestAgent.server: member joined, marking health alive: member=Node-46cd44b3-ea4b-5245-3b63-3e633c5808c7 -42:28.847 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-46cd44b3-ea4b-5245-3b63-3e633c5808c7 -42:28.847 [INFO] TestAgent.server: federation state anti-entropy synced -42:28.874 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:28.875 [INFO] TestAgent: Synced node info -42:28.887 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:59050 latency=25.9513ms -42:28.891 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/role/8e7bc684-24e5-31cd-5bad-40088ffa5b6c from=127.0.0.1:59052 latency=959.88µs -42:28.892 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/8e7bc684-24e5-31cd-5bad-40088ffa5b6c from=127.0.0.1:59050 latency=152.16µs -42:28.894 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:59050 latency=1.33538ms -42:28.900 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/roles from=127.0.0.1:59054 latency=485.82µs -42:28.901 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/role/2cbaca25-06f4-f1c7-02df-26c479617efc from=127.0.0.1:59054 latency=609.88µs -42:28.902 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/2cbaca25-06f4-f1c7-02df-26c479617efc from=127.0.0.1:59050 latency=129.9µs -42:28.906 [INFO] TestAgent: Requesting shutdown -42:28.906 [INFO] TestAgent.server: shutting down server -42:28.906 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:28.906 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:28.906 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:28.906 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:28.906 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:28.906 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:28.906 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:28.906 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:28.906 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:28.906 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:28.906 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:28.906 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:28.906 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:28.906 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:28.906 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:28.907Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:28.907Z [INFO] agent.router.manager: shutting down -42:28.907 [INFO] TestAgent: consul server down -42:28.907 [INFO] TestAgent: shutdown complete -42:28.907 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31442 network=tcp -42:28.907 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31442 network=udp -42:28.907 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31443 network=tcp -42:29.407 [INFO] TestAgent: Waiting for endpoints to shut down -42:29.407 [INFO] TestAgent: Endpoints down ---- PASS: TestRoleDeleteCommand (0.75s) +2021-08-13T15:34:54.218Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:34:54.249Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +34:54.258 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c756b372-2ce9-82ac-9bc4-131544fca6f2 Address:127.0.0.1:31447}]" +34:54.258 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31447 [Follower]" leader= +34:54.273 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c756b372-2ce9-82ac-9bc4-131544fca6f2.dc1 127.0.0.1 +34:54.274 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c756b372-2ce9-82ac-9bc4-131544fca6f2 127.0.0.1 +2021-08-13T15:34:54.274Z [INFO] agent.router: Initializing LAN area manager +34:54.275 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31442 network=udp +34:54.276 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31442 network=tcp +34:54.276 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31443 network=tcp +34:54.277 [INFO] TestAgent: started state syncer +34:54.275 [INFO] TestAgent.server: Adding LAN server: server="Node-c756b372-2ce9-82ac-9bc4-131544fca6f2 (Addr: tcp/127.0.0.1:31447) (DC: dc1)" +34:54.277 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31448 network=tcp +34:54.275 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c756b372-2ce9-82ac-9bc4-131544fca6f2.dc1 area=wan +34:54.323 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +34:54.323 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31447 [Candidate]" term=2 +34:54.323 [DEBUG] TestAgent.server.raft: votes: needed=1 +34:54.323 [DEBUG] TestAgent.server.raft: vote granted: from=c756b372-2ce9-82ac-9bc4-131544fca6f2 term=2 tally=1 +34:54.323 [INFO] TestAgent.server.raft: election won: tally=1 +34:54.323 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31447 [Leader]" +34:54.324 [INFO] TestAgent.server: cluster leadership acquired +34:54.324 [INFO] TestAgent.server: initializing acls +34:54.325 [INFO] TestAgent.server: Created ACL 'global-management' policy +34:54.325 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +34:54.325 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +34:54.326 [INFO] TestAgent.server: Created ACL anonymous token from configuration +34:54.326 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +34:54.326 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +34:54.326 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-c756b372-2ce9-82ac-9bc4-131544fca6f2 +34:54.327 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-c756b372-2ce9-82ac-9bc4-131544fca6f2.dc1 +34:54.327 [INFO] TestAgent.server: New leader elected: payload=Node-c756b372-2ce9-82ac-9bc4-131544fca6f2 +34:54.327 [INFO] TestAgent.server: Updating LAN server: server="Node-c756b372-2ce9-82ac-9bc4-131544fca6f2 (Addr: tcp/127.0.0.1:31447) (DC: dc1)" +34:54.332 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-c756b372-2ce9-82ac-9bc4-131544fca6f2.dc1 area=wan +34:54.338 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +34:54.369 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +34:54.369 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +34:54.370 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +34:54.370 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +34:54.370 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +34:54.370 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c756b372-2ce9-82ac-9bc4-131544fca6f2 +34:54.370 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c756b372-2ce9-82ac-9bc4-131544fca6f2 +34:54.372 [INFO] TestAgent.server: federation state anti-entropy synced +34:54.372 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c756b372-2ce9-82ac-9bc4-131544fca6f2 +34:54.440 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:60230 latency=2.114004ms +34:54.453 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/role/5af715dd-9d0e-d7d9-6473-040cabb2f26d from=127.0.0.1:60232 latency=2.150405ms +34:54.454 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/5af715dd-9d0e-d7d9-6473-040cabb2f26d from=127.0.0.1:60230 latency=430.421µs +34:54.457 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:60230 latency=1.128963ms +34:54.460 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/roles from=127.0.0.1:60234 latency=326.741µs +34:54.461 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/role/c717f979-0a5e-096c-4417-db051b063f90 from=127.0.0.1:60234 latency=531.641µs +34:54.461 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/c717f979-0a5e-096c-4417-db051b063f90 from=127.0.0.1:60230 latency=114.52µs +34:54.462 [INFO] TestAgent: Requesting shutdown +34:54.462 [INFO] TestAgent.server: shutting down server +34:54.462 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +34:54.462 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +34:54.462 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +34:54.462 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +34:54.462 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +34:54.462 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +34:54.462 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +34:54.462 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +34:54.462 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +34:54.462 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +34:54.462 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +34:54.462 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +34:54.462 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +34:54.462 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +34:54.462 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +34:54.462 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:34:54.462Z [INFO] agent.router.manager: shutting down +2021-08-13T15:34:54.462Z [INFO] agent.router.manager: shutting down +34:54.462 [INFO] TestAgent: consul server down +34:54.462 [INFO] TestAgent: shutdown complete +34:54.463 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31442 network=tcp +34:54.463 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31442 network=udp +34:54.463 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31443 network=tcp +34:54.963 [INFO] TestAgent: Waiting for endpoints to shut down +34:54.963 [INFO] TestAgent: Endpoints down +--- PASS: TestRoleDeleteCommand (0.85s) --- PASS: TestRoleDeleteCommand/id_or_name_required (0.00s) - --- PASS: TestRoleDeleteCommand/delete_works (0.03s) + --- PASS: TestRoleDeleteCommand/delete_works (0.02s) --- PASS: TestRoleDeleteCommand/delete_works_via_prefixes (0.01s) PASS -ok github.com/hashicorp/consul/command/acl/role/delete 0.923s +ok github.com/hashicorp/consul/command/acl/role/delete 0.979s === RUN TestRoleListCommand_noTabs === PAUSE TestRoleListCommand_noTabs === RUN TestRoleListCommand @@ -76201,172 +76211,172 @@ === RUN TestRoleListCommand_JSON === PAUSE TestRoleListCommand_JSON === CONT TestRoleListCommand_noTabs ---- PASS: TestRoleListCommand_noTabs (0.01s) +--- PASS: TestRoleListCommand_noTabs (0.00s) === CONT TestRoleListCommand_JSON [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestRoleListCommand [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:42:32.075Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:32.094Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:32.100 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7f79f6b2-e836-0cdf-96b3-42145a0840fa Address:127.0.0.1:25321}]" -42:32.100 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25321 [Follower]" leader= -42:32.101 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7f79f6b2-e836-0cdf-96b3-42145a0840fa.dc1 127.0.0.1 -42:32.104 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7f79f6b2-e836-0cdf-96b3-42145a0840fa 127.0.0.1 -2022-09-15T21:42:32.104Z [INFO] agent.router: Initializing LAN area manager -42:32.104 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25316 network=udp -42:32.104 [INFO] TestAgent.server: Adding LAN server: server="Node-7f79f6b2-e836-0cdf-96b3-42145a0840fa (Addr: tcp/127.0.0.1:25321) (DC: dc1)" -42:32.105 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7f79f6b2-e836-0cdf-96b3-42145a0840fa.dc1 area=wan -42:32.105 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25316 network=tcp -42:32.106 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25317 network=tcp -42:32.106 [INFO] TestAgent: started state syncer -42:32.106 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25322 network=tcp -42:32.148 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:32.148 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25321 [Candidate]" term=2 -42:32.148 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:32.148 [DEBUG] TestAgent.server.raft: vote granted: from=7f79f6b2-e836-0cdf-96b3-42145a0840fa term=2 tally=1 -42:32.148 [INFO] TestAgent.server.raft: election won: tally=1 -42:32.148 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25321 [Leader]" -42:32.148 [INFO] TestAgent.server: cluster leadership acquired -42:32.149 [INFO] TestAgent.server: New leader elected: payload=Node-7f79f6b2-e836-0cdf-96b3-42145a0840fa -42:32.149 [INFO] TestAgent.server: initializing acls -42:32.149 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:32.149 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:32.150 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:32.151 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:32.151 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:32.151 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:32.151 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-7f79f6b2-e836-0cdf-96b3-42145a0840fa -42:32.151 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-7f79f6b2-e836-0cdf-96b3-42145a0840fa.dc1 -42:32.151 [INFO] TestAgent.server: Updating LAN server: server="Node-7f79f6b2-e836-0cdf-96b3-42145a0840fa (Addr: tcp/127.0.0.1:25321) (DC: dc1)" -42:32.152 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-7f79f6b2-e836-0cdf-96b3-42145a0840fa.dc1 area=wan -42:32.159 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:32.189 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:32.189 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:32.189 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:32.189 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:32.189 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:32.190 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7f79f6b2-e836-0cdf-96b3-42145a0840fa -42:32.190 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7f79f6b2-e836-0cdf-96b3-42145a0840fa -42:32.191 [INFO] TestAgent.server: federation state anti-entropy synced -42:32.193 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7f79f6b2-e836-0cdf-96b3-42145a0840fa -42:32.332 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:32.333 [INFO] TestAgent: Synced node info -42:32.333 [DEBUG] TestAgent: Node info in sync -42:32.455 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:36086 latency=2.26054ms -42:32.457 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:36086 latency=915.56µs -42:32.460 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:36086 latency=893.32µs -42:32.461 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:36086 latency=671µs -42:32.467 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:36086 latency=1.07258ms -42:32.470 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/roles from=127.0.0.1:36088 latency=579.1µs -42:32.471 [INFO] TestAgent: Requesting shutdown -42:32.472 [INFO] TestAgent.server: shutting down server -42:32.472 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:32.472 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:32.472 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:32.472 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:32.472 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:32.472 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:32.472 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:32.472 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:32.472 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:32.472 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:32.472 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:32.472 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:32.472 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:32.472 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:32.472 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:32.472Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:32.472Z [INFO] agent.router.manager: shutting down -42:32.472 [INFO] TestAgent: consul server down -42:32.473 [INFO] TestAgent: shutdown complete -42:32.473 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25316 network=tcp -42:32.473 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25316 network=udp -42:32.473 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25317 network=tcp -42:32.973 [INFO] TestAgent: Waiting for endpoints to shut down -42:32.973 [INFO] TestAgent: Endpoints down ---- PASS: TestRoleListCommand_JSON (1.01s) -2022-09-15T21:42:32.082Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:32.109Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:32.112 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2719841d-7ef5-52f7-041b-21f419286149 Address:127.0.0.1:25328}]" -42:32.115 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25328 [Follower]" leader= -42:32.116 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2719841d-7ef5-52f7-041b-21f419286149.dc1 127.0.0.1 -42:32.117 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2719841d-7ef5-52f7-041b-21f419286149 127.0.0.1 -2022-09-15T21:42:32.117Z [INFO] agent.router: Initializing LAN area manager -42:32.117 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25323 network=udp -42:32.117 [INFO] TestAgent.server: Adding LAN server: server="Node-2719841d-7ef5-52f7-041b-21f419286149 (Addr: tcp/127.0.0.1:25328) (DC: dc1)" -42:32.117 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2719841d-7ef5-52f7-041b-21f419286149.dc1 area=wan -42:32.118 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25323 network=tcp -42:32.120 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25324 network=tcp -42:32.120 [INFO] TestAgent: started state syncer -42:32.120 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25329 network=tcp -42:32.162 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:32.162 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25328 [Candidate]" term=2 -42:32.163 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:32.163 [DEBUG] TestAgent.server.raft: vote granted: from=2719841d-7ef5-52f7-041b-21f419286149 term=2 tally=1 -42:32.163 [INFO] TestAgent.server.raft: election won: tally=1 -42:32.163 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25328 [Leader]" -42:32.163 [INFO] TestAgent.server: cluster leadership acquired -42:32.163 [INFO] TestAgent.server: initializing acls -42:32.164 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:32.164 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:32.164 [INFO] TestAgent.server: New leader elected: payload=Node-2719841d-7ef5-52f7-041b-21f419286149 -42:32.165 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:32.166 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:32.166 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:32.166 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:32.167 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-2719841d-7ef5-52f7-041b-21f419286149 -42:32.167 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-2719841d-7ef5-52f7-041b-21f419286149.dc1 -42:32.168 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-2719841d-7ef5-52f7-041b-21f419286149.dc1 area=wan -42:32.168 [INFO] TestAgent.server: Updating LAN server: server="Node-2719841d-7ef5-52f7-041b-21f419286149 (Addr: tcp/127.0.0.1:25328) (DC: dc1)" -42:32.169 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:32.189 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:32.189 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:32.189 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:32.189 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:32.189 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:32.190 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2719841d-7ef5-52f7-041b-21f419286149 -42:32.190 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2719841d-7ef5-52f7-041b-21f419286149 -42:32.190 [INFO] TestAgent.server: federation state anti-entropy synced -42:32.192 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2719841d-7ef5-52f7-041b-21f419286149 -42:32.554 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:32.555 [INFO] TestAgent: Synced node info -42:32.555 [DEBUG] TestAgent: Node info in sync -42:32.558 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:54242 latency=2.11726ms -42:32.561 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:54242 latency=1.07618ms -42:32.583 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:54242 latency=989.22µs -42:32.585 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:54242 latency=710.64µs -42:32.587 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:54242 latency=961.04µs -42:32.589 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/roles from=127.0.0.1:54244 latency=445.88µs -42:32.590 [INFO] TestAgent: Requesting shutdown -42:32.591 [INFO] TestAgent.server: shutting down server -42:32.591 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:32.591 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:32.591 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:32.591 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:32.591 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:32.591 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:32.591 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:32.591 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:32.591 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:32.591 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:32.591 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:32.591 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:32.591 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:32.591 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:32.591 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:32.591Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:32.591Z [INFO] agent.router.manager: shutting down -42:32.592 [INFO] TestAgent: consul server down -42:32.592 [INFO] TestAgent: shutdown complete -42:32.592 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25323 network=tcp -42:32.592 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25323 network=udp -42:32.592 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25324 network=tcp -42:33.095 [INFO] TestAgent: Waiting for endpoints to shut down -42:33.095 [INFO] TestAgent: Endpoints down ---- PASS: TestRoleListCommand (1.13s) +2021-08-13T15:35:00.021Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:35:00.080Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +35:00.085 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:066707bf-2651-9da8-2380-f9ab5a0e62ec Address:127.0.0.1:17153}]" +35:00.086 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-066707bf-2651-9da8-2380-f9ab5a0e62ec.dc1 127.0.0.1 +35:00.087 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-066707bf-2651-9da8-2380-f9ab5a0e62ec 127.0.0.1 +2021-08-13T15:35:00.087Z [INFO] agent.router: Initializing LAN area manager +35:00.087 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17148 network=udp +35:00.087 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17153 [Follower]" leader= +35:00.088 [INFO] TestAgent.server: Adding LAN server: server="Node-066707bf-2651-9da8-2380-f9ab5a0e62ec (Addr: tcp/127.0.0.1:17153) (DC: dc1)" +35:00.088 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-066707bf-2651-9da8-2380-f9ab5a0e62ec.dc1 area=wan +35:00.088 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17148 network=tcp +35:00.089 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17149 network=tcp +35:00.089 [INFO] TestAgent: started state syncer +35:00.089 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17154 network=tcp +35:00.138 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +35:00.161 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +35:00.161 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17153 [Candidate]" term=2 +35:00.161 [DEBUG] TestAgent.server.raft: votes: needed=1 +35:00.161 [DEBUG] TestAgent.server.raft: vote granted: from=066707bf-2651-9da8-2380-f9ab5a0e62ec term=2 tally=1 +35:00.161 [INFO] TestAgent.server.raft: election won: tally=1 +35:00.162 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17153 [Leader]" +35:00.162 [INFO] TestAgent.server: cluster leadership acquired +35:00.162 [INFO] TestAgent.server: initializing acls +35:00.162 [INFO] TestAgent.server: New leader elected: payload=Node-066707bf-2651-9da8-2380-f9ab5a0e62ec +35:00.166 [INFO] TestAgent.server: Created ACL 'global-management' policy +35:00.166 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +35:00.167 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +35:00.173 [INFO] TestAgent.server: Created ACL anonymous token from configuration +35:00.173 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +35:00.173 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +35:00.174 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-066707bf-2651-9da8-2380-f9ab5a0e62ec +35:00.174 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-066707bf-2651-9da8-2380-f9ab5a0e62ec.dc1 +35:00.174 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-066707bf-2651-9da8-2380-f9ab5a0e62ec.dc1 area=wan +35:00.175 [INFO] TestAgent.server: Updating LAN server: server="Node-066707bf-2651-9da8-2380-f9ab5a0e62ec (Addr: tcp/127.0.0.1:17153) (DC: dc1)" +35:00.176 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +35:00.193 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +35:00.193 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +35:00.193 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +35:00.193 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +35:00.193 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +35:00.193 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-066707bf-2651-9da8-2380-f9ab5a0e62ec +35:00.193 [INFO] TestAgent.server: member joined, marking health alive: member=Node-066707bf-2651-9da8-2380-f9ab5a0e62ec +35:00.194 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-066707bf-2651-9da8-2380-f9ab5a0e62ec +35:00.195 [INFO] TestAgent.server: federation state anti-entropy synced +35:00.221 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:37490 latency=2.318465ms +35:00.224 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:37490 latency=889.362µs +35:00.225 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:37490 latency=639.241µs +35:00.226 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:37490 latency=667.982µs +35:00.232 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:37490 latency=1.120823ms +35:00.244 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/roles from=127.0.0.1:37492 latency=663.042µs +35:00.245 [INFO] TestAgent: Requesting shutdown +35:00.245 [INFO] TestAgent.server: shutting down server +35:00.245 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +35:00.245 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +35:00.245 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +35:00.245 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +35:00.245 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +35:00.245 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +35:00.245 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +35:00.246 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +35:00.246 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +35:00.246 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +35:00.246 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +35:00.246 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +35:00.246 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +35:00.246 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +35:00.246 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +35:00.246 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:35:00.246Z [INFO] agent.router.manager: shutting down +2021-08-13T15:35:00.246Z [INFO] agent.router.manager: shutting down +35:00.247 [INFO] TestAgent: consul server down +35:00.247 [INFO] TestAgent: shutdown complete +35:00.247 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17148 network=tcp +35:00.255 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17148 network=udp +35:00.255 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17149 network=tcp +35:00.758 [INFO] TestAgent: Waiting for endpoints to shut down +35:00.758 [INFO] TestAgent: Endpoints down +--- PASS: TestRoleListCommand_JSON (0.84s) +2021-08-13T15:35:00.020Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:35:00.056Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +35:00.077 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5a6e3b06-7f0b-ca6c-6c51-0b85aafba854 Address:127.0.0.1:17160}]" +35:00.078 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5a6e3b06-7f0b-ca6c-6c51-0b85aafba854.dc1 127.0.0.1 +35:00.079 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17160 [Follower]" leader= +35:00.079 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5a6e3b06-7f0b-ca6c-6c51-0b85aafba854 127.0.0.1 +2021-08-13T15:35:00.079Z [INFO] agent.router: Initializing LAN area manager +35:00.079 [INFO] TestAgent.server: Adding LAN server: server="Node-5a6e3b06-7f0b-ca6c-6c51-0b85aafba854 (Addr: tcp/127.0.0.1:17160) (DC: dc1)" +35:00.079 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5a6e3b06-7f0b-ca6c-6c51-0b85aafba854.dc1 area=wan +35:00.080 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17155 network=tcp +35:00.080 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17155 network=udp +35:00.081 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17156 network=tcp +35:00.081 [INFO] TestAgent: started state syncer +35:00.081 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17161 network=tcp +35:00.131 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +35:00.151 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +35:00.151 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17160 [Candidate]" term=2 +35:00.151 [DEBUG] TestAgent.server.raft: votes: needed=1 +35:00.151 [DEBUG] TestAgent.server.raft: vote granted: from=5a6e3b06-7f0b-ca6c-6c51-0b85aafba854 term=2 tally=1 +35:00.151 [INFO] TestAgent.server.raft: election won: tally=1 +35:00.151 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17160 [Leader]" +35:00.152 [INFO] TestAgent.server: cluster leadership acquired +35:00.152 [INFO] TestAgent.server: initializing acls +35:00.153 [INFO] TestAgent.server: Created ACL 'global-management' policy +35:00.153 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +35:00.153 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +35:00.161 [INFO] TestAgent.server: New leader elected: payload=Node-5a6e3b06-7f0b-ca6c-6c51-0b85aafba854 +35:00.161 [INFO] TestAgent.server: Created ACL anonymous token from configuration +35:00.161 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +35:00.161 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +35:00.162 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-5a6e3b06-7f0b-ca6c-6c51-0b85aafba854 +35:00.162 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-5a6e3b06-7f0b-ca6c-6c51-0b85aafba854.dc1 +35:00.163 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-5a6e3b06-7f0b-ca6c-6c51-0b85aafba854.dc1 area=wan +35:00.163 [INFO] TestAgent.server: Updating LAN server: server="Node-5a6e3b06-7f0b-ca6c-6c51-0b85aafba854 (Addr: tcp/127.0.0.1:17160) (DC: dc1)" +35:00.164 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +35:00.192 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +35:00.192 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +35:00.192 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +35:00.192 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +35:00.192 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +35:00.192 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5a6e3b06-7f0b-ca6c-6c51-0b85aafba854 +35:00.192 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5a6e3b06-7f0b-ca6c-6c51-0b85aafba854 +35:00.194 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5a6e3b06-7f0b-ca6c-6c51-0b85aafba854 +35:00.195 [INFO] TestAgent.server: federation state anti-entropy synced +35:00.327 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +35:00.328 [INFO] TestAgent: Synced node info +35:00.328 [DEBUG] TestAgent: Node info in sync +35:00.415 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:40210 latency=5.891994ms +35:00.418 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:40210 latency=982.122µs +35:00.424 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:40210 latency=5.135971ms +35:00.426 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:40210 latency=745.922µs +35:00.427 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:40210 latency=676.442µs +35:00.430 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/roles from=127.0.0.1:40212 latency=476.161µs +35:00.432 [INFO] TestAgent: Requesting shutdown +35:00.432 [INFO] TestAgent.server: shutting down server +35:00.432 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +35:00.432 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +35:00.432 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +35:00.432 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +35:00.432 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +35:00.432 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +35:00.432 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +35:00.432 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +35:00.432 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +35:00.432 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +35:00.432 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +35:00.432 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +35:00.432 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +35:00.432 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +35:00.432 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:35:00.432Z [INFO] agent.router.manager: shutting down +2021-08-13T15:35:00.432Z [INFO] agent.router.manager: shutting down +35:00.433 [INFO] TestAgent: consul server down +35:00.433 [INFO] TestAgent: shutdown complete +35:00.433 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17155 network=tcp +35:00.433 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17155 network=udp +35:00.433 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17156 network=tcp +35:00.933 [INFO] TestAgent: Waiting for endpoints to shut down +35:00.933 [INFO] TestAgent: Endpoints down +--- PASS: TestRoleListCommand (1.01s) PASS -ok github.com/hashicorp/consul/command/acl/role/list 1.267s +ok github.com/hashicorp/consul/command/acl/role/list 1.101s === RUN TestRoleReadCommand_noTabs === PAUSE TestRoleReadCommand_noTabs === RUN TestRoleReadCommand @@ -76374,12 +76384,12 @@ === RUN TestRoleReadCommand_JSON === PAUSE TestRoleReadCommand_JSON === CONT TestRoleReadCommand_noTabs +--- PASS: TestRoleReadCommand_noTabs (0.00s) +=== CONT TestRoleReadCommand_JSON === CONT TestRoleReadCommand [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... -=== CONT TestRoleReadCommand_JSON [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range ---- PASS: TestRoleReadCommand_noTabs (0.02s) === RUN TestRoleReadCommand/id_or_name_required === RUN TestRoleReadCommand/read_by_id_not_found === RUN TestRoleReadCommand/read_by_name_not_found @@ -76387,178 +76397,170 @@ === RUN TestRoleReadCommand/read_by_id_prefix === RUN TestRoleReadCommand/read_by_name === RUN TestRoleReadCommand_JSON/read_by_id -2022-09-15T21:42:42.809Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:42.851Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:42.868 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2b778f38-03f0-1028-01f2-b81adb0e019c Address:127.0.0.1:26342}]" -42:42.869 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2b778f38-03f0-1028-01f2-b81adb0e019c.dc1 127.0.0.1 -42:42.870 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:26342 [Follower]" leader= -42:42.870 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2b778f38-03f0-1028-01f2-b81adb0e019c 127.0.0.1 -2022-09-15T21:42:42.870Z [INFO] agent.router: Initializing LAN area manager -42:42.870 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2b778f38-03f0-1028-01f2-b81adb0e019c.dc1 area=wan -42:42.870 [INFO] TestAgent.server: Adding LAN server: server="Node-2b778f38-03f0-1028-01f2-b81adb0e019c (Addr: tcp/127.0.0.1:26342) (DC: dc1)" -42:42.870 [INFO] TestAgent: Started DNS server: address=127.0.0.1:26337 network=udp -42:42.871 [INFO] TestAgent: Started DNS server: address=127.0.0.1:26337 network=tcp -42:42.872 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:26338 network=tcp -42:42.872 [INFO] TestAgent: started state syncer -42:42.872 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:26343 network=tcp -42:42.910 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:42.910 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:26342 [Candidate]" term=2 -42:42.910 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:42.910 [DEBUG] TestAgent.server.raft: vote granted: from=2b778f38-03f0-1028-01f2-b81adb0e019c term=2 tally=1 -42:42.910 [INFO] TestAgent.server.raft: election won: tally=1 -42:42.910 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:26342 [Leader]" -42:42.910 [INFO] TestAgent.server: cluster leadership acquired -42:42.911 [INFO] TestAgent.server: initializing acls -42:42.911 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:42.911 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:42.912 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:42.913 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:42.913 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:42.913 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:42.914 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-2b778f38-03f0-1028-01f2-b81adb0e019c -42:42.915 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-2b778f38-03f0-1028-01f2-b81adb0e019c.dc1 -42:42.918 [INFO] TestAgent.server: New leader elected: payload=Node-2b778f38-03f0-1028-01f2-b81adb0e019c -42:42.918 [INFO] TestAgent.server: Updating LAN server: server="Node-2b778f38-03f0-1028-01f2-b81adb0e019c (Addr: tcp/127.0.0.1:26342) (DC: dc1)" -42:42.918 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-2b778f38-03f0-1028-01f2-b81adb0e019c.dc1 area=wan -42:42.925 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:42.977 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:42.977 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:42.977 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:42.977 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:42.977 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:42.977 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2b778f38-03f0-1028-01f2-b81adb0e019c -42:42.977 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2b778f38-03f0-1028-01f2-b81adb0e019c -42:42.978 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2b778f38-03f0-1028-01f2-b81adb0e019c -42:42.978 [INFO] TestAgent.server: federation state anti-entropy synced -42:43.093 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:43.094 [INFO] TestAgent: Synced node info -42:43.153 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/654d9402-f624-9b75-3d2e-04359cfc433a from=127.0.0.1:36854 latency=940.48µs -42:43.156 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/name/blah from=127.0.0.1:36856 latency=216.22µs -42:43.158 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:36858 latency=1.43962ms -42:43.161 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/b805b299-2211-3d21-e75b-e3b412cec812 from=127.0.0.1:36860 latency=406.94µs -42:43.163 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:36858 latency=838.66µs -42:43.166 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/roles from=127.0.0.1:36862 latency=530µs -42:43.173 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/011d2ce2-eeaa-b461-53d4-5aee465805d6 from=127.0.0.1:36862 latency=553.24µs -42:43.175 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:36858 latency=1.22868ms -42:43.185 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/name/test-role-by-name from=127.0.0.1:36866 latency=2.51424ms -42:43.186 [INFO] TestAgent: Requesting shutdown -42:43.186 [INFO] TestAgent.server: shutting down server -42:43.187 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:43.187 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:43.187 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:43.187 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:43.187 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:43.187 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:43.187 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:43.187 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:43.187 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:43.187 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:43.187 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:43.187 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:43.187 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:43.187 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:43.187 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:43.187Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:43.187Z [INFO] agent.router.manager: shutting down -42:43.187 [INFO] TestAgent: consul server down -42:43.187 [INFO] TestAgent: shutdown complete -42:43.187 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:26337 network=tcp -42:43.187 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:26337 network=udp -42:43.187 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:26338 network=tcp -42:43.688 [INFO] TestAgent: Waiting for endpoints to shut down -42:43.688 [INFO] TestAgent: Endpoints down ---- PASS: TestRoleReadCommand (1.00s) +2021-08-13T15:35:01.692Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:35:01.717Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +35:01.720 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:69c71475-53e9-044a-1b83-8c1c19949425 Address:127.0.0.1:23286}]" +35:01.720 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23286 [Follower]" leader= +35:01.721 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-69c71475-53e9-044a-1b83-8c1c19949425.dc1 127.0.0.1 +35:01.722 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-69c71475-53e9-044a-1b83-8c1c19949425 127.0.0.1 +2021-08-13T15:35:01.722Z [INFO] agent.router: Initializing LAN area manager +35:01.722 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-69c71475-53e9-044a-1b83-8c1c19949425.dc1 area=wan +35:01.722 [INFO] TestAgent.server: Adding LAN server: server="Node-69c71475-53e9-044a-1b83-8c1c19949425 (Addr: tcp/127.0.0.1:23286) (DC: dc1)" +35:01.722 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23281 network=udp +35:01.722 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23281 network=tcp +35:01.723 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23282 network=tcp +35:01.723 [INFO] TestAgent: started state syncer +35:01.723 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23287 network=tcp +35:01.772 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +35:01.788 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +35:01.789 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23286 [Candidate]" term=2 +35:01.789 [DEBUG] TestAgent.server.raft: votes: needed=1 +35:01.789 [DEBUG] TestAgent.server.raft: vote granted: from=69c71475-53e9-044a-1b83-8c1c19949425 term=2 tally=1 +35:01.789 [INFO] TestAgent.server.raft: election won: tally=1 +35:01.789 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23286 [Leader]" +35:01.789 [INFO] TestAgent.server: cluster leadership acquired +35:01.789 [INFO] TestAgent.server: initializing acls +35:01.790 [INFO] TestAgent.server: Created ACL 'global-management' policy +35:01.790 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +35:01.790 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +35:01.791 [INFO] TestAgent.server: Created ACL anonymous token from configuration +35:01.791 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +35:01.791 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +35:01.791 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-69c71475-53e9-044a-1b83-8c1c19949425 +35:01.792 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-69c71475-53e9-044a-1b83-8c1c19949425.dc1 +35:01.793 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +35:01.793 [INFO] TestAgent.server: New leader elected: payload=Node-69c71475-53e9-044a-1b83-8c1c19949425 +35:01.793 [INFO] TestAgent.server: Updating LAN server: server="Node-69c71475-53e9-044a-1b83-8c1c19949425 (Addr: tcp/127.0.0.1:23286) (DC: dc1)" +35:01.793 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-69c71475-53e9-044a-1b83-8c1c19949425.dc1 area=wan +35:01.797 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +35:01.797 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +35:01.797 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +35:01.797 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +35:01.797 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +35:01.797 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-69c71475-53e9-044a-1b83-8c1c19949425 +35:01.797 [INFO] TestAgent.server: member joined, marking health alive: member=Node-69c71475-53e9-044a-1b83-8c1c19949425 +35:01.798 [INFO] TestAgent.server: federation state anti-entropy synced +35:01.800 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-69c71475-53e9-044a-1b83-8c1c19949425 +35:01.889 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/857678c7-fd29-1490-5074-38cf0db40aaa from=127.0.0.1:60124 latency=1.225623ms +35:01.892 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/name/blah from=127.0.0.1:60126 latency=390.081µs +35:01.895 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +35:01.896 [INFO] TestAgent: Synced node info +35:01.896 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:60128 latency=1.741884ms +35:01.899 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/c0c5b0f8-8217-a734-3d5a-182886d79565 from=127.0.0.1:60130 latency=530.901µs +35:01.905 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:60128 latency=1.058103ms +35:01.916 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/roles from=127.0.0.1:60132 latency=582.041µs +35:01.932 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/718b0416-cd20-ee6f-6042-5f8d5eb00bca from=127.0.0.1:60132 latency=671.182µs +35:01.938 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:60128 latency=4.109309ms +35:01.942 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/name/test-role-by-name from=127.0.0.1:60134 latency=378.581µs +35:01.942 [INFO] TestAgent: Requesting shutdown +35:01.942 [INFO] TestAgent.server: shutting down server +35:01.943 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +35:01.943 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +35:01.943 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +35:01.943 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +35:01.943 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +35:01.943 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +35:01.943 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +35:01.943 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +35:01.943 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +35:01.943 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +35:01.943 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +35:01.943 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +35:01.943 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +35:01.943 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +35:01.947 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:35:01.947Z [INFO] agent.router.manager: shutting down +2021-08-13T15:35:01.947Z [INFO] agent.router.manager: shutting down +35:01.948 [INFO] TestAgent: consul server down +35:01.948 [INFO] TestAgent: shutdown complete +35:01.948 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23281 network=tcp +35:01.948 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23281 network=udp +35:01.948 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23282 network=tcp +35:02.449 [INFO] TestAgent: Waiting for endpoints to shut down +35:02.449 [INFO] TestAgent: Endpoints down +--- PASS: TestRoleReadCommand (0.92s) --- PASS: TestRoleReadCommand/id_or_name_required (0.00s) --- PASS: TestRoleReadCommand/read_by_id_not_found (0.00s) --- PASS: TestRoleReadCommand/read_by_name_not_found (0.00s) --- PASS: TestRoleReadCommand/read_by_id (0.01s) - --- PASS: TestRoleReadCommand/read_by_id_prefix (0.01s) + --- PASS: TestRoleReadCommand/read_by_id_prefix (0.03s) --- PASS: TestRoleReadCommand/read_by_name (0.01s) -2022-09-15T21:42:42.803Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:42.859Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:42.862 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1c1d05ed-43a5-dfd5-3021-627ae1ea08f3 Address:127.0.0.1:26349}]" -42:42.866 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:26349 [Follower]" leader= -42:42.867 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1c1d05ed-43a5-dfd5-3021-627ae1ea08f3.dc1 127.0.0.1 -42:42.868 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1c1d05ed-43a5-dfd5-3021-627ae1ea08f3 127.0.0.1 -2022-09-15T21:42:42.868Z [INFO] agent.router: Initializing LAN area manager -42:42.869 [INFO] TestAgent: Started DNS server: address=127.0.0.1:26344 network=udp -42:42.869 [INFO] TestAgent.server: Adding LAN server: server="Node-1c1d05ed-43a5-dfd5-3021-627ae1ea08f3 (Addr: tcp/127.0.0.1:26349) (DC: dc1)" -42:42.869 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1c1d05ed-43a5-dfd5-3021-627ae1ea08f3.dc1 area=wan -42:42.869 [INFO] TestAgent: Started DNS server: address=127.0.0.1:26344 network=tcp -42:42.872 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:26345 network=tcp -42:42.872 [INFO] TestAgent: started state syncer -42:42.872 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:26350 network=tcp -42:42.919 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:42.919 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:26349 [Candidate]" term=2 -42:42.919 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:42.919 [DEBUG] TestAgent.server.raft: vote granted: from=1c1d05ed-43a5-dfd5-3021-627ae1ea08f3 term=2 tally=1 -42:42.919 [INFO] TestAgent.server.raft: election won: tally=1 -42:42.919 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:26349 [Leader]" -42:42.919 [INFO] TestAgent.server: cluster leadership acquired -42:42.920 [INFO] TestAgent.server: initializing acls -42:42.920 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:42.920 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:42.921 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:42.922 [INFO] TestAgent.server: New leader elected: payload=Node-1c1d05ed-43a5-dfd5-3021-627ae1ea08f3 -42:42.926 [INFO] TestAgent.server: initializing acls -42:42.926 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:42.926 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:42.926 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:42.926 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:42.926 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode -42:42.926 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-1c1d05ed-43a5-dfd5-3021-627ae1ea08f3 -42:42.927 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-1c1d05ed-43a5-dfd5-3021-627ae1ea08f3.dc1 -42:42.928 [INFO] TestAgent.server: Updating LAN server: server="Node-1c1d05ed-43a5-dfd5-3021-627ae1ea08f3 (Addr: tcp/127.0.0.1:26349) (DC: dc1)" -42:42.928 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-1c1d05ed-43a5-dfd5-3021-627ae1ea08f3.dc1 area=wan -42:42.929 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-1c1d05ed-43a5-dfd5-3021-627ae1ea08f3 -42:42.929 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-1c1d05ed-43a5-dfd5-3021-627ae1ea08f3.dc1 -42:42.929 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-1c1d05ed-43a5-dfd5-3021-627ae1ea08f3.dc1 area=wan -42:42.929 [INFO] TestAgent.server: Updating LAN server: server="Node-1c1d05ed-43a5-dfd5-3021-627ae1ea08f3 (Addr: tcp/127.0.0.1:26349) (DC: dc1)" -42:42.934 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:42.975 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:42.975 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:42.975 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:42.975 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:42.975 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:42.975 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1c1d05ed-43a5-dfd5-3021-627ae1ea08f3 -42:42.975 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1c1d05ed-43a5-dfd5-3021-627ae1ea08f3 -42:42.977 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1c1d05ed-43a5-dfd5-3021-627ae1ea08f3 -42:42.977 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1c1d05ed-43a5-dfd5-3021-627ae1ea08f3 -42:42.978 [INFO] TestAgent.server: federation state anti-entropy synced -42:43.112 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:43.113 [INFO] TestAgent: Synced node info -42:43.256 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:46464 latency=1.87404ms -42:43.259 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/4086e5c9-52fc-b3e4-39a5-27e85bfa4e44 from=127.0.0.1:46466 latency=424.72µs -42:43.259 [INFO] TestAgent: Requesting shutdown -42:43.260 [INFO] TestAgent.server: shutting down server -42:43.260 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:43.260 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:43.260 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:43.260 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:43.260 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:43.260 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:43.260 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:43.260 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:43.260 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:43.260 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:43.260 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:43.260 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:43.260 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:43.260 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:43.260 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:43.260Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:43.260Z [INFO] agent.router.manager: shutting down -42:43.260 [INFO] TestAgent: consul server down -42:43.260 [INFO] TestAgent: shutdown complete -42:43.260 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:26344 network=tcp -42:43.260 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:26344 network=udp -42:43.261 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:26345 network=tcp -42:43.761 [INFO] TestAgent: Waiting for endpoints to shut down -42:43.761 [INFO] TestAgent: Endpoints down ---- PASS: TestRoleReadCommand_JSON (1.07s) - --- PASS: TestRoleReadCommand_JSON/read_by_id (0.02s) +2021-08-13T15:35:01.692Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:35:01.715Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +35:01.720 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8bc89c9c-8ef2-cb37-5d5b-cf3c3f378e3b Address:127.0.0.1:23279}]" +35:01.720 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23279 [Follower]" leader= +35:01.721 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8bc89c9c-8ef2-cb37-5d5b-cf3c3f378e3b.dc1 127.0.0.1 +35:01.722 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8bc89c9c-8ef2-cb37-5d5b-cf3c3f378e3b 127.0.0.1 +2021-08-13T15:35:01.722Z [INFO] agent.router: Initializing LAN area manager +35:01.722 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8bc89c9c-8ef2-cb37-5d5b-cf3c3f378e3b.dc1 area=wan +35:01.722 [INFO] TestAgent.server: Adding LAN server: server="Node-8bc89c9c-8ef2-cb37-5d5b-cf3c3f378e3b (Addr: tcp/127.0.0.1:23279) (DC: dc1)" +35:01.722 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23274 network=udp +35:01.722 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23274 network=tcp +35:01.723 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23275 network=tcp +35:01.723 [INFO] TestAgent: started state syncer +35:01.723 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23280 network=tcp +35:01.763 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +35:01.763 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23279 [Candidate]" term=2 +35:01.763 [DEBUG] TestAgent.server.raft: votes: needed=1 +35:01.763 [DEBUG] TestAgent.server.raft: vote granted: from=8bc89c9c-8ef2-cb37-5d5b-cf3c3f378e3b term=2 tally=1 +35:01.763 [INFO] TestAgent.server.raft: election won: tally=1 +35:01.763 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23279 [Leader]" +35:01.763 [INFO] TestAgent.server: cluster leadership acquired +35:01.764 [INFO] TestAgent.server: New leader elected: payload=Node-8bc89c9c-8ef2-cb37-5d5b-cf3c3f378e3b +35:01.764 [INFO] TestAgent.server: initializing acls +35:01.764 [INFO] TestAgent.server: Created ACL 'global-management' policy +35:01.764 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +35:01.765 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +35:01.766 [INFO] TestAgent.server: Created ACL anonymous token from configuration +35:01.766 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +35:01.766 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +35:01.766 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-8bc89c9c-8ef2-cb37-5d5b-cf3c3f378e3b +35:01.766 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-8bc89c9c-8ef2-cb37-5d5b-cf3c3f378e3b.dc1 +35:01.766 [INFO] TestAgent.server: Updating LAN server: server="Node-8bc89c9c-8ef2-cb37-5d5b-cf3c3f378e3b (Addr: tcp/127.0.0.1:23279) (DC: dc1)" +35:01.766 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-8bc89c9c-8ef2-cb37-5d5b-cf3c3f378e3b.dc1 area=wan +35:01.773 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +35:01.797 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +35:01.797 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +35:01.797 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +35:01.797 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +35:01.797 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +35:01.797 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8bc89c9c-8ef2-cb37-5d5b-cf3c3f378e3b +35:01.797 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8bc89c9c-8ef2-cb37-5d5b-cf3c3f378e3b +35:01.799 [INFO] TestAgent.server: federation state anti-entropy synced +35:01.800 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8bc89c9c-8ef2-cb37-5d5b-cf3c3f378e3b +35:01.975 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:46278 latency=1.698624ms +35:01.978 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/0d3bd006-23d3-7bae-365a-fe9d5fdc28c6 from=127.0.0.1:46280 latency=410.941µs +35:01.978 [INFO] TestAgent: Requesting shutdown +35:01.978 [INFO] TestAgent.server: shutting down server +35:01.978 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +35:01.978 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +35:01.978 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +35:01.978 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +35:01.978 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +35:01.978 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +35:01.978 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +35:01.979 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +35:01.979 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +35:01.979 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +35:01.979 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +35:01.979 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +35:01.979 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +35:01.979 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +35:01.979 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +35:01.979 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:35:01.979Z [INFO] agent.router.manager: shutting down +2021-08-13T15:35:01.979Z [INFO] agent.router.manager: shutting down +35:01.979 [INFO] TestAgent: consul server down +35:01.979 [INFO] TestAgent: shutdown complete +35:01.979 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23274 network=tcp +35:01.979 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23274 network=udp +35:01.979 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23275 network=tcp +35:02.480 [INFO] TestAgent: Waiting for endpoints to shut down +35:02.480 [INFO] TestAgent: Endpoints down +--- PASS: TestRoleReadCommand_JSON (0.95s) + --- PASS: TestRoleReadCommand_JSON/read_by_id (0.01s) PASS -ok github.com/hashicorp/consul/command/acl/role/read 1.223s +ok github.com/hashicorp/consul/command/acl/role/read 1.089s === RUN TestRoleUpdateCommand_noTabs === PAUSE TestRoleUpdateCommand_noTabs === RUN TestRoleUpdateCommand @@ -76588,282 +76590,283 @@ === RUN TestRoleUpdateCommand/update_with_service_identity === RUN TestRoleUpdateCommand/update_with_service_identity_scoped_to_2_DCs === RUN TestRoleUpdateCommand/update_with_node_identity -2022-09-15T21:42:44.195Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:44.237Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:44.249 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:08a55aba-ce0c-185b-bc68-a18234945d70 Address:127.0.0.1:23286}]" -42:44.251 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23286 [Follower]" leader= -42:44.251 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-08a55aba-ce0c-185b-bc68-a18234945d70.dc1 127.0.0.1 -42:44.263 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-08a55aba-ce0c-185b-bc68-a18234945d70 127.0.0.1 -2022-09-15T21:42:44.264Z [INFO] agent.router: Initializing LAN area manager -42:44.264 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23281 network=udp -42:44.264 [INFO] TestAgent.server: Adding LAN server: server="Node-08a55aba-ce0c-185b-bc68-a18234945d70 (Addr: tcp/127.0.0.1:23286) (DC: dc1)" -42:44.264 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-08a55aba-ce0c-185b-bc68-a18234945d70.dc1 area=wan -42:44.265 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23281 network=tcp -42:44.266 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23282 network=tcp -42:44.266 [INFO] TestAgent: started state syncer -42:44.266 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23287 network=tcp -42:44.299 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:44.299 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23286 [Candidate]" term=2 -42:44.300 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:44.300 [DEBUG] TestAgent.server.raft: vote granted: from=08a55aba-ce0c-185b-bc68-a18234945d70 term=2 tally=1 -42:44.300 [INFO] TestAgent.server.raft: election won: tally=1 -42:44.300 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23286 [Leader]" -42:44.300 [INFO] TestAgent.server: cluster leadership acquired -42:44.300 [INFO] TestAgent.server: New leader elected: payload=Node-08a55aba-ce0c-185b-bc68-a18234945d70 -42:44.300 [INFO] TestAgent.server: initializing acls -42:44.301 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:44.301 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:44.302 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:44.303 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:44.303 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:44.303 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:44.303 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-08a55aba-ce0c-185b-bc68-a18234945d70 -42:44.303 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-08a55aba-ce0c-185b-bc68-a18234945d70.dc1 -42:44.304 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-08a55aba-ce0c-185b-bc68-a18234945d70.dc1 area=wan -42:44.304 [INFO] TestAgent.server: Updating LAN server: server="Node-08a55aba-ce0c-185b-bc68-a18234945d70 (Addr: tcp/127.0.0.1:23286) (DC: dc1)" -42:44.306 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:44.333 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:44.333 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:44.333 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:44.333 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:44.333 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:44.334 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-08a55aba-ce0c-185b-bc68-a18234945d70 -42:44.334 [INFO] TestAgent.server: member joined, marking health alive: member=Node-08a55aba-ce0c-185b-bc68-a18234945d70 -42:44.334 [INFO] TestAgent.server: federation state anti-entropy synced -42:44.336 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-08a55aba-ce0c-185b-bc68-a18234945d70 -42:44.405 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:34502 latency=1.89198ms -42:44.416 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:34502 latency=2.21504ms -42:44.428 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/d61efed4-5868-265c-a877-507e1e588309 from=127.0.0.1:34504 latency=347.74µs -42:44.433 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/3daa12f4-ae7f-ddfd-9d08-3ce8a4271367 from=127.0.0.1:34506 latency=966.5µs -42:44.437 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/3daa12f4-ae7f-ddfd-9d08-3ce8a4271367 from=127.0.0.1:34506 latency=1.69508ms -42:44.437 [INFO] TestAgent: Requesting shutdown -42:44.438 [INFO] TestAgent.server: shutting down server -42:44.438 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:44.438 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:44.438 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:44.438 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:44.438 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:44.438 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:44.438 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:44.438 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:44.438 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:44.438 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:44.438 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:44.438 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:44.438 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:44.438 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:44.438Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:44.438Z [INFO] agent.router.manager: shutting down -42:44.439 [INFO] TestAgent: consul server down -42:44.439 [INFO] TestAgent: shutdown complete -42:44.439 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23281 network=tcp -42:44.439 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23281 network=udp -42:44.439 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23282 network=tcp -42:44.439 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:44.439 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:44.942 [INFO] TestAgent: Waiting for endpoints to shut down -42:44.942 [INFO] TestAgent: Endpoints down ---- PASS: TestRoleUpdateCommand_JSON (0.90s) - --- PASS: TestRoleUpdateCommand_JSON/update_a_role_that_does_not_exist (0.01s) - --- PASS: TestRoleUpdateCommand_JSON/update_with_policy_by_name (0.01s) -2022-09-15T21:42:44.206Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:44.230Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:44.250 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:272417b2-45a3-99ef-743c-833058794bd0 Address:127.0.0.1:23279}]" -42:44.251 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-272417b2-45a3-99ef-743c-833058794bd0.dc1 127.0.0.1 -42:44.251 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23279 [Follower]" leader= -42:44.252 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-272417b2-45a3-99ef-743c-833058794bd0 127.0.0.1 -2022-09-15T21:42:44.252Z [INFO] agent.router: Initializing LAN area manager -42:44.252 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23274 network=udp -42:44.253 [INFO] TestAgent.server: Adding LAN server: server="Node-272417b2-45a3-99ef-743c-833058794bd0 (Addr: tcp/127.0.0.1:23279) (DC: dc1)" -42:44.253 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-272417b2-45a3-99ef-743c-833058794bd0.dc1 area=wan -42:44.253 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23274 network=tcp -42:44.254 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23275 network=tcp -42:44.254 [INFO] TestAgent: started state syncer -42:44.254 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23280 network=tcp -42:44.303 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -42:44.307 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:44.307 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23279 [Candidate]" term=2 -42:44.307 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:44.307 [DEBUG] TestAgent.server.raft: vote granted: from=272417b2-45a3-99ef-743c-833058794bd0 term=2 tally=1 -42:44.307 [INFO] TestAgent.server.raft: election won: tally=1 -42:44.307 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23279 [Leader]" -42:44.307 [INFO] TestAgent.server: cluster leadership acquired -42:44.307 [INFO] TestAgent.server: initializing acls -42:44.308 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:44.308 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:44.309 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:44.309 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:44.309 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:44.309 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:44.310 [INFO] TestAgent.server: New leader elected: payload=Node-272417b2-45a3-99ef-743c-833058794bd0 -42:44.310 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-272417b2-45a3-99ef-743c-833058794bd0 -42:44.311 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-272417b2-45a3-99ef-743c-833058794bd0.dc1 -42:44.312 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:44.312 [INFO] TestAgent.server: Updating LAN server: server="Node-272417b2-45a3-99ef-743c-833058794bd0 (Addr: tcp/127.0.0.1:23279) (DC: dc1)" -42:44.312 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-272417b2-45a3-99ef-743c-833058794bd0.dc1 area=wan -42:44.334 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:44.334 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:44.334 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:44.334 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:44.334 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:44.335 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-272417b2-45a3-99ef-743c-833058794bd0 -42:44.335 [INFO] TestAgent.server: member joined, marking health alive: member=Node-272417b2-45a3-99ef-743c-833058794bd0 -42:44.336 [INFO] TestAgent.server: federation state anti-entropy synced -42:44.336 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-272417b2-45a3-99ef-743c-833058794bd0 -42:44.484 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:37386 latency=1.94694ms -42:44.495 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:37386 latency=1.10824ms -42:44.498 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:37386 latency=908.54µs -42:44.504 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/b337e6f0-21e0-8c5e-ca15-7fc2e737d248 from=127.0.0.1:37388 latency=295.72µs -42:44.506 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:37386 latency=981.62µs -42:44.512 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/b652ae87-c4d6-82cc-724b-a5dc6e2df2a1 from=127.0.0.1:37390 latency=475.58µs -42:44.515 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/b652ae87-c4d6-82cc-724b-a5dc6e2df2a1 from=127.0.0.1:37390 latency=904.38µs -42:44.516 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/b652ae87-c4d6-82cc-724b-a5dc6e2df2a1 from=127.0.0.1:37386 latency=197.16µs -42:44.517 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:37386 latency=745.86µs -42:44.520 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/05fbe815-b593-1af3-ecca-5bead0a10ef1 from=127.0.0.1:37392 latency=480.74µs -42:44.522 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/05fbe815-b593-1af3-ecca-5bead0a10ef1 from=127.0.0.1:37392 latency=889.8µs -42:44.524 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/05fbe815-b593-1af3-ecca-5bead0a10ef1 from=127.0.0.1:37386 latency=196.74µs -42:44.525 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:37386 latency=716.62µs -42:44.528 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/3bad30c6-afc8-038e-602c-9ad5ae39c667 from=127.0.0.1:37394 latency=444.52µs -42:44.530 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/3bad30c6-afc8-038e-602c-9ad5ae39c667 from=127.0.0.1:37394 latency=812.86µs -42:44.532 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/3bad30c6-afc8-038e-602c-9ad5ae39c667 from=127.0.0.1:37386 latency=279.58µs -42:44.535 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:37386 latency=1.58252ms -42:44.539 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/db6a6647-b079-40f9-1437-54d625d2f3d4 from=127.0.0.1:37396 latency=501.56µs -42:44.542 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/db6a6647-b079-40f9-1437-54d625d2f3d4 from=127.0.0.1:37396 latency=1.64192ms -42:44.544 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/db6a6647-b079-40f9-1437-54d625d2f3d4 from=127.0.0.1:37386 latency=231.6µs -42:44.544 [INFO] TestAgent: Requesting shutdown -42:44.545 [INFO] TestAgent.server: shutting down server -42:44.545 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:44.545 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:44.545 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:44.545 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:44.545 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:44.545 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:44.545 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:44.545 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:44.545 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:44.545 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:44.545 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:44.545 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:44.545 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:44.545 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:44.545 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:44.545 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:44.545Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:44.545Z [INFO] agent.router.manager: shutting down -42:44.546 [INFO] TestAgent: consul server down -42:44.546 [INFO] TestAgent: shutdown complete -42:44.546 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23274 network=tcp -42:44.546 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23274 network=udp -42:44.546 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23275 network=tcp -42:45.046 [INFO] TestAgent: Waiting for endpoints to shut down -42:45.046 [INFO] TestAgent: Endpoints down ---- PASS: TestRoleUpdateCommand_noMerge (1.00s) +2021-08-13T15:35:07.045Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:35:07.098Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +35:07.107 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d051b6ef-3f99-c414-ebbe-2e6457d3e5e8 Address:127.0.0.1:24307}]" +35:07.107 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24307 [Follower]" leader= +35:07.117 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d051b6ef-3f99-c414-ebbe-2e6457d3e5e8.dc1 127.0.0.1 +35:07.118 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d051b6ef-3f99-c414-ebbe-2e6457d3e5e8 127.0.0.1 +2021-08-13T15:35:07.118Z [INFO] agent.router: Initializing LAN area manager +35:07.119 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24302 network=udp +35:07.119 [INFO] TestAgent.server: Adding LAN server: server="Node-d051b6ef-3f99-c414-ebbe-2e6457d3e5e8 (Addr: tcp/127.0.0.1:24307) (DC: dc1)" +35:07.119 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d051b6ef-3f99-c414-ebbe-2e6457d3e5e8.dc1 area=wan +35:07.119 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24302 network=tcp +35:07.120 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24303 network=tcp +35:07.120 [INFO] TestAgent: started state syncer +35:07.120 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24308 network=tcp +35:07.166 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +35:07.166 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24307 [Candidate]" term=2 +35:07.166 [DEBUG] TestAgent.server.raft: votes: needed=1 +35:07.166 [DEBUG] TestAgent.server.raft: vote granted: from=d051b6ef-3f99-c414-ebbe-2e6457d3e5e8 term=2 tally=1 +35:07.166 [INFO] TestAgent.server.raft: election won: tally=1 +35:07.166 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24307 [Leader]" +35:07.167 [INFO] TestAgent.server: cluster leadership acquired +35:07.167 [INFO] TestAgent.server: New leader elected: payload=Node-d051b6ef-3f99-c414-ebbe-2e6457d3e5e8 +35:07.167 [INFO] TestAgent.server: initializing acls +35:07.168 [INFO] TestAgent.server: Created ACL 'global-management' policy +35:07.168 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +35:07.169 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +35:07.169 [INFO] TestAgent.server: initializing acls +35:07.169 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +35:07.170 [INFO] TestAgent.server: Created ACL anonymous token from configuration +35:07.170 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +35:07.170 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +35:07.170 [INFO] TestAgent.server: Created ACL anonymous token from configuration +35:07.170 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode +35:07.170 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-d051b6ef-3f99-c414-ebbe-2e6457d3e5e8 +35:07.170 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-d051b6ef-3f99-c414-ebbe-2e6457d3e5e8.dc1 +35:07.171 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +35:07.171 [INFO] TestAgent.server: Updating LAN server: server="Node-d051b6ef-3f99-c414-ebbe-2e6457d3e5e8 (Addr: tcp/127.0.0.1:24307) (DC: dc1)" +35:07.171 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-d051b6ef-3f99-c414-ebbe-2e6457d3e5e8 +35:07.172 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-d051b6ef-3f99-c414-ebbe-2e6457d3e5e8.dc1 +35:07.172 [INFO] TestAgent.server: Updating LAN server: server="Node-d051b6ef-3f99-c414-ebbe-2e6457d3e5e8 (Addr: tcp/127.0.0.1:24307) (DC: dc1)" +35:07.172 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-d051b6ef-3f99-c414-ebbe-2e6457d3e5e8.dc1 area=wan +35:07.172 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-d051b6ef-3f99-c414-ebbe-2e6457d3e5e8.dc1 area=wan +35:07.194 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +35:07.194 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +35:07.194 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +35:07.194 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +35:07.194 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +35:07.194 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d051b6ef-3f99-c414-ebbe-2e6457d3e5e8 +35:07.194 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d051b6ef-3f99-c414-ebbe-2e6457d3e5e8 +35:07.195 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d051b6ef-3f99-c414-ebbe-2e6457d3e5e8 +35:07.195 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d051b6ef-3f99-c414-ebbe-2e6457d3e5e8 +35:07.198 [INFO] TestAgent.server: federation state anti-entropy synced +35:07.218 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:41112 latency=5.567033ms +35:07.226 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:41112 latency=1.776224ms +35:07.229 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/f68d85fd-45b0-82fa-97dc-ff037928af3c from=127.0.0.1:41116 latency=195.4µs +35:07.231 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/3f080e36-3a50-5c59-0dc3-ec47597516c7 from=127.0.0.1:41118 latency=351.461µs +35:07.232 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/3f080e36-3a50-5c59-0dc3-ec47597516c7 from=127.0.0.1:41118 latency=688.262µs +35:07.232 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +35:07.232 [INFO] TestAgent: Requesting shutdown +35:07.233 [INFO] TestAgent.server: shutting down server +35:07.233 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +35:07.233 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +35:07.233 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +35:07.233 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +35:07.233 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +35:07.233 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +35:07.233 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +35:07.233 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +35:07.233 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +35:07.233 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +35:07.233 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +35:07.233 [INFO] TestAgent: Synced node info +35:07.233 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +35:07.233 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +35:07.233 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +35:07.233 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:35:07.233Z [INFO] agent.router.manager: shutting down +2021-08-13T15:35:07.233Z [INFO] agent.router.manager: shutting down +35:07.233 [INFO] TestAgent: consul server down +35:07.233 [INFO] TestAgent: shutdown complete +35:07.233 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24302 network=tcp +35:07.233 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24302 network=udp +35:07.233 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24303 network=tcp +35:07.734 [INFO] TestAgent: Waiting for endpoints to shut down +35:07.734 [INFO] TestAgent: Endpoints down +--- PASS: TestRoleUpdateCommand_JSON (0.76s) + --- PASS: TestRoleUpdateCommand_JSON/update_a_role_that_does_not_exist (0.00s) + --- PASS: TestRoleUpdateCommand_JSON/update_with_policy_by_name (0.00s) +2021-08-13T15:35:07.051Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:35:07.119Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +35:07.122 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1929c266-ab70-cc2f-ba36-87579ca425d6 Address:127.0.0.1:24300}]" +35:07.123 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24300 [Follower]" leader= +35:07.124 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1929c266-ab70-cc2f-ba36-87579ca425d6.dc1 127.0.0.1 +35:07.125 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1929c266-ab70-cc2f-ba36-87579ca425d6 127.0.0.1 +2021-08-13T15:35:07.125Z [INFO] agent.router: Initializing LAN area manager +35:07.125 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24295 network=udp +35:07.125 [INFO] TestAgent.server: Adding LAN server: server="Node-1929c266-ab70-cc2f-ba36-87579ca425d6 (Addr: tcp/127.0.0.1:24300) (DC: dc1)" +35:07.126 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1929c266-ab70-cc2f-ba36-87579ca425d6.dc1 area=wan +35:07.126 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24295 network=tcp +35:07.127 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24296 network=tcp +35:07.127 [INFO] TestAgent: started state syncer +35:07.127 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24301 network=tcp +35:07.160 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +35:07.160 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24300 [Candidate]" term=2 +35:07.160 [DEBUG] TestAgent.server.raft: votes: needed=1 +35:07.160 [DEBUG] TestAgent.server.raft: vote granted: from=1929c266-ab70-cc2f-ba36-87579ca425d6 term=2 tally=1 +35:07.160 [INFO] TestAgent.server.raft: election won: tally=1 +35:07.160 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24300 [Leader]" +35:07.160 [INFO] TestAgent.server: cluster leadership acquired +35:07.161 [INFO] TestAgent.server: New leader elected: payload=Node-1929c266-ab70-cc2f-ba36-87579ca425d6 +35:07.161 [INFO] TestAgent.server: initializing acls +35:07.162 [INFO] TestAgent.server: Created ACL 'global-management' policy +35:07.162 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +35:07.163 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +35:07.163 [INFO] TestAgent.server: Created ACL anonymous token from configuration +35:07.163 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +35:07.163 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +35:07.164 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-1929c266-ab70-cc2f-ba36-87579ca425d6 +35:07.164 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-1929c266-ab70-cc2f-ba36-87579ca425d6.dc1 +35:07.164 [INFO] TestAgent.server: Updating LAN server: server="Node-1929c266-ab70-cc2f-ba36-87579ca425d6 (Addr: tcp/127.0.0.1:24300) (DC: dc1)" +35:07.164 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-1929c266-ab70-cc2f-ba36-87579ca425d6.dc1 area=wan +35:07.165 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +35:07.192 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +35:07.192 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +35:07.192 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +35:07.192 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +35:07.192 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +35:07.192 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1929c266-ab70-cc2f-ba36-87579ca425d6 +35:07.192 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1929c266-ab70-cc2f-ba36-87579ca425d6 +35:07.196 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1929c266-ab70-cc2f-ba36-87579ca425d6 +35:07.197 [INFO] TestAgent.server: federation state anti-entropy synced +35:07.225 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:34730 latency=2.118045ms +35:07.227 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:34730 latency=671.101µs +35:07.229 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:34730 latency=691.061µs +35:07.231 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/dc1206f3-051b-782b-7fbd-e81b88ac33b0 from=127.0.0.1:34736 latency=148.12µs +35:07.232 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:34730 latency=704.122µs +35:07.234 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/b2755ca1-324e-9ee7-bad2-57592c02967b from=127.0.0.1:34738 latency=235.98µs +35:07.236 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/b2755ca1-324e-9ee7-bad2-57592c02967b from=127.0.0.1:34738 latency=656.541µs +35:07.236 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/b2755ca1-324e-9ee7-bad2-57592c02967b from=127.0.0.1:34730 latency=250.52µs +35:07.238 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:34730 latency=675.482µs +35:07.240 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/1ed813fb-18a0-b82d-2e89-0337a14038bc from=127.0.0.1:34740 latency=218.8µs +35:07.241 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/1ed813fb-18a0-b82d-2e89-0337a14038bc from=127.0.0.1:34740 latency=681.621µs +35:07.242 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/1ed813fb-18a0-b82d-2e89-0337a14038bc from=127.0.0.1:34730 latency=160.62µs +35:07.244 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:34730 latency=724.142µs +35:07.246 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/2f29bdc6-897b-334e-671d-7c0c506df13c from=127.0.0.1:34742 latency=222.261µs +35:07.247 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/2f29bdc6-897b-334e-671d-7c0c506df13c from=127.0.0.1:34742 latency=617.622µs +35:07.248 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/2f29bdc6-897b-334e-671d-7c0c506df13c from=127.0.0.1:34730 latency=148.341µs +35:07.249 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:34730 latency=669.961µs +35:07.251 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/2579b579-6656-1a65-198b-553d230184a0 from=127.0.0.1:34744 latency=222.04µs +35:07.253 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/2579b579-6656-1a65-198b-553d230184a0 from=127.0.0.1:34744 latency=618.081µs +35:07.253 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/2579b579-6656-1a65-198b-553d230184a0 from=127.0.0.1:34730 latency=144.941µs +35:07.254 [INFO] TestAgent: Requesting shutdown +35:07.254 [INFO] TestAgent.server: shutting down server +35:07.254 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +35:07.254 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +35:07.254 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +35:07.254 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +35:07.254 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +35:07.254 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +35:07.254 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +35:07.254 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +35:07.254 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +35:07.254 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +35:07.254 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +35:07.254 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +35:07.254 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +35:07.254 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +35:07.254 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +35:07.254 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:35:07.254Z [INFO] agent.router.manager: shutting down +2021-08-13T15:35:07.254Z [INFO] agent.router.manager: shutting down +35:07.255 [INFO] TestAgent: consul server down +35:07.255 [INFO] TestAgent: shutdown complete +35:07.255 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24295 network=tcp +35:07.255 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24295 network=udp +35:07.255 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24296 network=tcp +35:07.755 [INFO] TestAgent: Waiting for endpoints to shut down +35:07.755 [INFO] TestAgent: Endpoints down +--- PASS: TestRoleUpdateCommand_noMerge (0.78s) --- PASS: TestRoleUpdateCommand_noMerge/update_a_role_that_does_not_exist (0.00s) --- PASS: TestRoleUpdateCommand_noMerge/update_with_policy_by_name (0.01s) --- PASS: TestRoleUpdateCommand_noMerge/update_with_policy_by_id (0.01s) --- PASS: TestRoleUpdateCommand_noMerge/update_with_service_identity (0.01s) --- PASS: TestRoleUpdateCommand_noMerge/update_with_service_identity_scoped_to_2_DCs (0.01s) -2022-09-15T21:42:44.187Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:44.260Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:44.268 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:26bea60b-8cec-dbdb-b994-ad66b71efec8 Address:127.0.0.1:23293}]" -42:44.272 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-26bea60b-8cec-dbdb-b994-ad66b71efec8.dc1 127.0.0.1 -42:44.279 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-26bea60b-8cec-dbdb-b994-ad66b71efec8 127.0.0.1 -2022-09-15T21:42:44.279Z [INFO] agent.router: Initializing LAN area manager -42:44.279 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23288 network=udp -42:44.280 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23288 network=tcp -42:44.281 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23289 network=tcp -42:44.281 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23293 [Follower]" leader= -42:44.281 [INFO] TestAgent: started state syncer -42:44.282 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23294 network=tcp -42:44.282 [INFO] TestAgent.server: Adding LAN server: server="Node-26bea60b-8cec-dbdb-b994-ad66b71efec8 (Addr: tcp/127.0.0.1:23293) (DC: dc1)" -42:44.282 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-26bea60b-8cec-dbdb-b994-ad66b71efec8.dc1 area=wan -42:44.329 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:44.329 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23293 [Candidate]" term=2 -42:44.329 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:44.329 [DEBUG] TestAgent.server.raft: vote granted: from=26bea60b-8cec-dbdb-b994-ad66b71efec8 term=2 tally=1 -42:44.329 [INFO] TestAgent.server.raft: election won: tally=1 -42:44.329 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23293 [Leader]" -42:44.329 [INFO] TestAgent.server: cluster leadership acquired -42:44.330 [INFO] TestAgent.server: initializing acls -42:44.330 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:44.330 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:44.330 [INFO] TestAgent.server: New leader elected: payload=Node-26bea60b-8cec-dbdb-b994-ad66b71efec8 -42:44.330 [INFO] TestAgent.server: initializing acls -42:44.330 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:44.331 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:44.338 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:44.339 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:44.339 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:44.339 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:44.339 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode -42:44.339 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-26bea60b-8cec-dbdb-b994-ad66b71efec8 -42:44.339 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-26bea60b-8cec-dbdb-b994-ad66b71efec8.dc1 -42:44.339 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-26bea60b-8cec-dbdb-b994-ad66b71efec8.dc1 area=wan -42:44.340 [INFO] TestAgent.server: Updating LAN server: server="Node-26bea60b-8cec-dbdb-b994-ad66b71efec8 (Addr: tcp/127.0.0.1:23293) (DC: dc1)" -42:44.340 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:44.341 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-26bea60b-8cec-dbdb-b994-ad66b71efec8 -42:44.341 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-26bea60b-8cec-dbdb-b994-ad66b71efec8.dc1 -42:44.342 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:44.345 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:44.345 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:44.345 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:44.345 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:44.345 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:44.345 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-26bea60b-8cec-dbdb-b994-ad66b71efec8 -42:44.345 [INFO] TestAgent.server: member joined, marking health alive: member=Node-26bea60b-8cec-dbdb-b994-ad66b71efec8 -42:44.346 [INFO] TestAgent.server: Updating LAN server: server="Node-26bea60b-8cec-dbdb-b994-ad66b71efec8 (Addr: tcp/127.0.0.1:23293) (DC: dc1)" -42:44.346 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-26bea60b-8cec-dbdb-b994-ad66b71efec8.dc1 area=wan -42:44.353 [INFO] TestAgent.server: federation state anti-entropy synced -42:44.354 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-26bea60b-8cec-dbdb-b994-ad66b71efec8 -42:44.354 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-26bea60b-8cec-dbdb-b994-ad66b71efec8 -42:44.456 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:44.457 [INFO] TestAgent: Synced node info -42:44.650 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:44004 latency=1.62564ms -42:44.653 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:44004 latency=1.5348ms -42:44.660 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:44004 latency=849.22µs -42:44.662 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/9e6eba6c-5927-cd3b-eafb-f51f510b1819 from=127.0.0.1:44006 latency=223.64µs -42:44.664 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/c381d5b3-6ddf-502c-df2c-ac6e52662738 from=127.0.0.1:44008 latency=355.46µs -42:44.666 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/c381d5b3-6ddf-502c-df2c-ac6e52662738 from=127.0.0.1:44008 latency=801.86µs -42:44.667 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/c381d5b3-6ddf-502c-df2c-ac6e52662738 from=127.0.0.1:44004 latency=193.72µs -42:44.669 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/c381d5b3-6ddf-502c-df2c-ac6e52662738 from=127.0.0.1:44010 latency=292.68µs -42:44.671 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/c381d5b3-6ddf-502c-df2c-ac6e52662738 from=127.0.0.1:44010 latency=768.38µs -42:44.672 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/c381d5b3-6ddf-502c-df2c-ac6e52662738 from=127.0.0.1:44004 latency=183.84µs -42:44.675 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/c381d5b3-6ddf-502c-df2c-ac6e52662738 from=127.0.0.1:44012 latency=305.26µs -42:44.676 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/c381d5b3-6ddf-502c-df2c-ac6e52662738 from=127.0.0.1:44012 latency=791.4µs -42:44.677 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/c381d5b3-6ddf-502c-df2c-ac6e52662738 from=127.0.0.1:44004 latency=174µs -42:44.680 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/c381d5b3-6ddf-502c-df2c-ac6e52662738 from=127.0.0.1:44014 latency=275.4µs -42:44.681 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/c381d5b3-6ddf-502c-df2c-ac6e52662738 from=127.0.0.1:44014 latency=811.1µs -42:44.683 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/c381d5b3-6ddf-502c-df2c-ac6e52662738 from=127.0.0.1:44004 latency=208.72µs -42:44.685 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/c381d5b3-6ddf-502c-df2c-ac6e52662738 from=127.0.0.1:44016 latency=305.04µs -42:44.687 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/c381d5b3-6ddf-502c-df2c-ac6e52662738 from=127.0.0.1:44016 latency=904.56µs -42:44.688 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/c381d5b3-6ddf-502c-df2c-ac6e52662738 from=127.0.0.1:44004 latency=199.46µs -42:44.689 [INFO] TestAgent: Requesting shutdown -42:44.689 [INFO] TestAgent.server: shutting down server -42:44.689 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:44.689 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:44.689 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:44.689 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:44.689 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:44.689 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:44.689 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:44.689 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:44.689 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:44.689 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:44.689 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:44.689 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:44.689 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:44.689 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:44.689 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:44.689Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:44.689Z [INFO] agent.router.manager: shutting down -42:44.690 [INFO] TestAgent: consul server down -42:44.690 [INFO] TestAgent: shutdown complete -42:44.690 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23288 network=tcp -42:44.690 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23288 network=udp -42:44.690 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23289 network=tcp -42:45.194 [INFO] TestAgent: Waiting for endpoints to shut down -42:45.194 [INFO] TestAgent: Endpoints down ---- PASS: TestRoleUpdateCommand (1.15s) +2021-08-13T15:35:07.096Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:35:07.129Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +35:07.134 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:dcfa9dfa-ffa0-c973-b5bf-6324af7a62fe Address:127.0.0.1:24314}]" +35:07.134 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24314 [Follower]" leader= +35:07.135 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-dcfa9dfa-ffa0-c973-b5bf-6324af7a62fe.dc1 127.0.0.1 +35:07.136 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-dcfa9dfa-ffa0-c973-b5bf-6324af7a62fe 127.0.0.1 +2021-08-13T15:35:07.136Z [INFO] agent.router: Initializing LAN area manager +35:07.136 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-dcfa9dfa-ffa0-c973-b5bf-6324af7a62fe.dc1 area=wan +35:07.136 [INFO] TestAgent.server: Adding LAN server: server="Node-dcfa9dfa-ffa0-c973-b5bf-6324af7a62fe (Addr: tcp/127.0.0.1:24314) (DC: dc1)" +35:07.136 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24309 network=tcp +35:07.136 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24309 network=udp +35:07.137 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24310 network=tcp +35:07.137 [INFO] TestAgent: started state syncer +35:07.137 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24315 network=tcp +35:07.187 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +35:07.187 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24314 [Candidate]" term=2 +35:07.187 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +35:07.187 [DEBUG] TestAgent.server.raft: votes: needed=1 +35:07.187 [DEBUG] TestAgent.server.raft: vote granted: from=dcfa9dfa-ffa0-c973-b5bf-6324af7a62fe term=2 tally=1 +35:07.187 [INFO] TestAgent.server.raft: election won: tally=1 +35:07.187 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24314 [Leader]" +35:07.187 [INFO] TestAgent.server: cluster leadership acquired +35:07.188 [INFO] TestAgent.server: initializing acls +35:07.188 [INFO] TestAgent.server: Created ACL 'global-management' policy +35:07.188 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +35:07.189 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +35:07.189 [INFO] TestAgent.server: New leader elected: payload=Node-dcfa9dfa-ffa0-c973-b5bf-6324af7a62fe +35:07.189 [INFO] TestAgent.server: Created ACL anonymous token from configuration +35:07.189 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +35:07.189 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +35:07.190 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-dcfa9dfa-ffa0-c973-b5bf-6324af7a62fe +35:07.190 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-dcfa9dfa-ffa0-c973-b5bf-6324af7a62fe.dc1 +35:07.191 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +35:07.191 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-dcfa9dfa-ffa0-c973-b5bf-6324af7a62fe.dc1 area=wan +35:07.191 [INFO] TestAgent.server: Updating LAN server: server="Node-dcfa9dfa-ffa0-c973-b5bf-6324af7a62fe (Addr: tcp/127.0.0.1:24314) (DC: dc1)" +35:07.196 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +35:07.196 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +35:07.196 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +35:07.196 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +35:07.196 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +35:07.196 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-dcfa9dfa-ffa0-c973-b5bf-6324af7a62fe +35:07.196 [INFO] TestAgent.server: member joined, marking health alive: member=Node-dcfa9dfa-ffa0-c973-b5bf-6324af7a62fe +35:07.197 [INFO] TestAgent.server: federation state anti-entropy synced +35:07.197 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-dcfa9dfa-ffa0-c973-b5bf-6324af7a62fe +35:07.512 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:38608 latency=1.368043ms +35:07.513 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:38608 latency=578.322µs +35:07.515 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:38608 latency=689.042µs +35:07.517 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/c3f5d725-5bc7-50e6-192c-d18089ef1437 from=127.0.0.1:38610 latency=174.601µs +35:07.519 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/02f09a18-0544-0586-bcd9-89619d9fdde1 from=127.0.0.1:38612 latency=232.001µs +35:07.520 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/02f09a18-0544-0586-bcd9-89619d9fdde1 from=127.0.0.1:38612 latency=705.942µs +35:07.521 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/02f09a18-0544-0586-bcd9-89619d9fdde1 from=127.0.0.1:38608 latency=149.161µs +35:07.523 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/02f09a18-0544-0586-bcd9-89619d9fdde1 from=127.0.0.1:38614 latency=234.261µs +35:07.524 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/02f09a18-0544-0586-bcd9-89619d9fdde1 from=127.0.0.1:38614 latency=671.382µs +35:07.525 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/02f09a18-0544-0586-bcd9-89619d9fdde1 from=127.0.0.1:38608 latency=154.741µs +35:07.528 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/02f09a18-0544-0586-bcd9-89619d9fdde1 from=127.0.0.1:38616 latency=275.64µs +35:07.533 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/02f09a18-0544-0586-bcd9-89619d9fdde1 from=127.0.0.1:38616 latency=1.302303ms +35:07.535 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/02f09a18-0544-0586-bcd9-89619d9fdde1 from=127.0.0.1:38608 latency=321.741µs +35:07.535 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +35:07.536 [INFO] TestAgent: Synced node info +35:07.536 [DEBUG] TestAgent: Node info in sync +35:07.540 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/02f09a18-0544-0586-bcd9-89619d9fdde1 from=127.0.0.1:38618 latency=324.261µs +35:07.542 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/02f09a18-0544-0586-bcd9-89619d9fdde1 from=127.0.0.1:38618 latency=862.082µs +35:07.543 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/02f09a18-0544-0586-bcd9-89619d9fdde1 from=127.0.0.1:38608 latency=216.301µs +35:07.545 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/02f09a18-0544-0586-bcd9-89619d9fdde1 from=127.0.0.1:38620 latency=254.521µs +35:07.547 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/02f09a18-0544-0586-bcd9-89619d9fdde1 from=127.0.0.1:38620 latency=965.442µs +35:07.548 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/02f09a18-0544-0586-bcd9-89619d9fdde1 from=127.0.0.1:38608 latency=185.26µs +35:07.549 [INFO] TestAgent: Requesting shutdown +35:07.549 [INFO] TestAgent.server: shutting down server +35:07.549 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +35:07.549 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +35:07.549 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +35:07.549 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +35:07.549 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +35:07.549 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +35:07.549 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +35:07.549 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +35:07.549 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +35:07.549 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +35:07.549 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +35:07.549 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +35:07.549 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +35:07.549 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +35:07.549 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:35:07.549Z [INFO] agent.router.manager: shutting down +2021-08-13T15:35:07.549Z [INFO] agent.router.manager: shutting down +35:07.549 [INFO] TestAgent: consul server down +35:07.549 [INFO] TestAgent: shutdown complete +35:07.549 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24309 network=tcp +35:07.549 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24309 network=udp +35:07.549 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24310 network=tcp +35:08.050 [INFO] TestAgent: Waiting for endpoints to shut down +35:08.050 [INFO] TestAgent: Endpoints down +--- PASS: TestRoleUpdateCommand (1.08s) --- PASS: TestRoleUpdateCommand/update_a_role_that_does_not_exist (0.00s) --- PASS: TestRoleUpdateCommand/update_with_policy_by_name (0.00s) --- PASS: TestRoleUpdateCommand/update_with_policy_by_id (0.00s) @@ -76871,7 +76874,7 @@ --- PASS: TestRoleUpdateCommand/update_with_service_identity_scoped_to_2_DCs (0.01s) --- PASS: TestRoleUpdateCommand/update_with_node_identity (0.01s) PASS -ok github.com/hashicorp/consul/command/acl/role/update 1.309s +ok github.com/hashicorp/consul/command/acl/role/update 1.146s === RUN TestRulesTranslateCommand_noTabs === PAUSE TestRulesTranslateCommand_noTabs === RUN TestRulesTranslateCommand @@ -76879,121 +76882,117 @@ === CONT TestRulesTranslateCommand_noTabs === CONT TestRulesTranslateCommand [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... ---- PASS: TestRulesTranslateCommand_noTabs (0.01s) +--- PASS: TestRulesTranslateCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range === RUN TestRulesTranslateCommand/file === RUN TestRulesTranslateCommand/stdin === RUN TestRulesTranslateCommand/arg === RUN TestRulesTranslateCommand/exclusive-options -2022-09-15T21:42:46.132Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:46.188Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:46.197 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:14391b19-a48c-ed93-9d34-cfd474dae29e Address:127.0.0.1:10006}]" -42:46.197 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:10006 [Follower]" leader= -42:46.202 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-14391b19-a48c-ed93-9d34-cfd474dae29e.dc1 127.0.0.1 -42:46.204 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-14391b19-a48c-ed93-9d34-cfd474dae29e 127.0.0.1 -2022-09-15T21:42:46.204Z [INFO] agent.router: Initializing LAN area manager -42:46.204 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10001 network=udp -42:46.204 [INFO] TestAgent.server: Adding LAN server: server="Node-14391b19-a48c-ed93-9d34-cfd474dae29e (Addr: tcp/127.0.0.1:10006) (DC: dc1)" -42:46.204 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-14391b19-a48c-ed93-9d34-cfd474dae29e.dc1 area=wan -42:46.205 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10001 network=tcp -42:46.206 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:10002 network=tcp -42:46.206 [INFO] TestAgent: started state syncer -42:46.206 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:10007 network=tcp -42:46.254 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:46.254 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:10006 [Candidate]" term=2 -42:46.254 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:46.254 [DEBUG] TestAgent.server.raft: vote granted: from=14391b19-a48c-ed93-9d34-cfd474dae29e term=2 tally=1 -42:46.254 [INFO] TestAgent.server.raft: election won: tally=1 -42:46.254 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:10006 [Leader]" -42:46.255 [INFO] TestAgent.server: cluster leadership acquired -42:46.255 [INFO] TestAgent.server: initializing acls -42:46.255 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:46.255 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:46.256 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:46.257 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:46.257 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:46.257 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:46.257 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode -42:46.257 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-14391b19-a48c-ed93-9d34-cfd474dae29e -42:46.258 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-14391b19-a48c-ed93-9d34-cfd474dae29e.dc1 -42:46.260 [INFO] TestAgent.server: initializing acls -42:46.260 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:46.261 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:46.266 [INFO] TestAgent.server: Updating LAN server: server="Node-14391b19-a48c-ed93-9d34-cfd474dae29e (Addr: tcp/127.0.0.1:10006) (DC: dc1)" -42:46.266 [INFO] TestAgent.server: New leader elected: payload=Node-14391b19-a48c-ed93-9d34-cfd474dae29e -42:46.266 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-14391b19-a48c-ed93-9d34-cfd474dae29e.dc1 area=wan -42:46.289 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:46.289 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:46.289 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:46.289 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:46.289 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:46.289 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-14391b19-a48c-ed93-9d34-cfd474dae29e -42:46.289 [INFO] TestAgent.server: member joined, marking health alive: member=Node-14391b19-a48c-ed93-9d34-cfd474dae29e -42:46.303 [INFO] TestAgent.server: federation state anti-entropy synced -42:46.304 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-14391b19-a48c-ed93-9d34-cfd474dae29e -42:46.381 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:46.382 [INFO] TestAgent: Synced node info -42:46.382 [DEBUG] TestAgent: Node info in sync -42:46.540 [INFO] TestAgent: Requesting shutdown -42:46.540 [INFO] TestAgent.server: shutting down server -42:46.540 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:46.540 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:46.540 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:46.540 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:46.540 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:46.540 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:46.540 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:46.540 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:46.540 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:46.541 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:46.541 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:46.541 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:46.541 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:46.541 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:46.541 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:46.541Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:46.541Z [INFO] agent.router.manager: shutting down -42:46.541 [INFO] TestAgent: consul server down -42:46.541 [INFO] TestAgent: shutdown complete -42:46.541 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10001 network=tcp -42:46.541 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10001 network=udp -42:46.541 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:10002 network=tcp -42:47.042 [INFO] TestAgent: Waiting for endpoints to shut down -42:47.042 [INFO] TestAgent: Endpoints down ---- PASS: TestRulesTranslateCommand (1.05s) +2021-08-13T15:35:12.831Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:35:12.853Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +35:12.856 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:48af98f6-5ae3-9661-f35c-f83a2f124976 Address:127.0.0.1:29405}]" +35:12.856 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29405 [Follower]" leader= +35:12.857 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-48af98f6-5ae3-9661-f35c-f83a2f124976.dc1 127.0.0.1 +35:12.858 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-48af98f6-5ae3-9661-f35c-f83a2f124976 127.0.0.1 +2021-08-13T15:35:12.858Z [INFO] agent.router: Initializing LAN area manager +35:12.858 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29400 network=udp +35:12.859 [INFO] TestAgent.server: Adding LAN server: server="Node-48af98f6-5ae3-9661-f35c-f83a2f124976 (Addr: tcp/127.0.0.1:29405) (DC: dc1)" +35:12.859 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-48af98f6-5ae3-9661-f35c-f83a2f124976.dc1 area=wan +35:12.859 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29400 network=tcp +35:12.860 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29401 network=tcp +35:12.860 [INFO] TestAgent: started state syncer +35:12.860 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29406 network=tcp +35:12.909 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +35:12.918 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +35:12.918 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29405 [Candidate]" term=2 +35:12.918 [DEBUG] TestAgent.server.raft: votes: needed=1 +35:12.918 [DEBUG] TestAgent.server.raft: vote granted: from=48af98f6-5ae3-9661-f35c-f83a2f124976 term=2 tally=1 +35:12.918 [INFO] TestAgent.server.raft: election won: tally=1 +35:12.918 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29405 [Leader]" +35:12.918 [INFO] TestAgent.server: cluster leadership acquired +35:12.919 [INFO] TestAgent.server: initializing acls +35:12.919 [INFO] TestAgent.server: Created ACL 'global-management' policy +35:12.919 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +35:12.920 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +35:12.920 [INFO] TestAgent.server: Created ACL anonymous token from configuration +35:12.920 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +35:12.920 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +35:12.921 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-48af98f6-5ae3-9661-f35c-f83a2f124976 +35:12.922 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-48af98f6-5ae3-9661-f35c-f83a2f124976.dc1 +35:12.927 [INFO] TestAgent.server: New leader elected: payload=Node-48af98f6-5ae3-9661-f35c-f83a2f124976 +35:12.927 [INFO] TestAgent.server: Updating LAN server: server="Node-48af98f6-5ae3-9661-f35c-f83a2f124976 (Addr: tcp/127.0.0.1:29405) (DC: dc1)" +35:12.927 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-48af98f6-5ae3-9661-f35c-f83a2f124976.dc1 area=wan +35:12.929 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +35:12.953 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +35:12.953 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +35:12.953 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +35:12.953 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +35:12.954 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +35:12.954 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-48af98f6-5ae3-9661-f35c-f83a2f124976 +35:12.954 [INFO] TestAgent.server: member joined, marking health alive: member=Node-48af98f6-5ae3-9661-f35c-f83a2f124976 +35:12.954 [INFO] TestAgent.server: federation state anti-entropy synced +35:12.955 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-48af98f6-5ae3-9661-f35c-f83a2f124976 +35:12.980 [INFO] TestAgent: Requesting shutdown +35:12.980 [INFO] TestAgent.server: shutting down server +35:12.980 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +35:12.980 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +35:12.980 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +35:12.980 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +35:12.980 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +35:12.980 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +35:12.980 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +35:12.980 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +35:12.980 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +35:12.980 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +35:12.980 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +35:12.980 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +35:12.980 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +35:12.980 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +35:12.980 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +35:12.980 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +2021-08-13T15:35:12.980Z [INFO] agent.router.manager: shutting down +2021-08-13T15:35:12.980Z [INFO] agent.router.manager: shutting down +35:12.981 [INFO] TestAgent: consul server down +35:12.981 [INFO] TestAgent: shutdown complete +35:12.981 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29400 network=tcp +35:12.981 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29400 network=udp +35:12.981 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29401 network=tcp +35:13.481 [INFO] TestAgent: Waiting for endpoints to shut down +35:13.481 [INFO] TestAgent: Endpoints down +--- PASS: TestRulesTranslateCommand (0.71s) --- PASS: TestRulesTranslateCommand/file (0.00s) --- PASS: TestRulesTranslateCommand/stdin (0.00s) --- PASS: TestRulesTranslateCommand/arg (0.00s) --- PASS: TestRulesTranslateCommand/exclusive-options (0.00s) PASS -ok github.com/hashicorp/consul/command/acl/rules 1.238s +ok github.com/hashicorp/consul/command/acl/rules 0.771s === RUN TestFormatToken -=== RUN TestFormatToken/basic -=== RUN TestFormatToken/basic/pretty-meta -=== RUN TestFormatToken/basic/json -=== RUN TestFormatToken/basic/pretty === RUN TestFormatToken/complex === RUN TestFormatToken/complex/pretty === RUN TestFormatToken/complex/pretty-meta === RUN TestFormatToken/complex/json === RUN TestFormatToken/legacy +=== RUN TestFormatToken/legacy/pretty-meta === RUN TestFormatToken/legacy/json === RUN TestFormatToken/legacy/pretty -=== RUN TestFormatToken/legacy/pretty-meta ---- PASS: TestFormatToken (0.00s) - --- PASS: TestFormatToken/basic (0.00s) - --- PASS: TestFormatToken/basic/pretty-meta (0.00s) - --- PASS: TestFormatToken/basic/json (0.00s) - --- PASS: TestFormatToken/basic/pretty (0.00s) +=== RUN TestFormatToken/basic +=== RUN TestFormatToken/basic/pretty +=== RUN TestFormatToken/basic/pretty-meta +=== RUN TestFormatToken/basic/json +--- PASS: TestFormatToken (0.01s) --- PASS: TestFormatToken/complex (0.00s) --- PASS: TestFormatToken/complex/pretty (0.00s) --- PASS: TestFormatToken/complex/pretty-meta (0.00s) --- PASS: TestFormatToken/complex/json (0.00s) --- PASS: TestFormatToken/legacy (0.00s) + --- PASS: TestFormatToken/legacy/pretty-meta (0.00s) --- PASS: TestFormatToken/legacy/json (0.00s) --- PASS: TestFormatToken/legacy/pretty (0.00s) - --- PASS: TestFormatToken/legacy/pretty-meta (0.00s) + --- PASS: TestFormatToken/basic (0.00s) + --- PASS: TestFormatToken/basic/pretty (0.00s) + --- PASS: TestFormatToken/basic/pretty-meta (0.00s) + --- PASS: TestFormatToken/basic/json (0.00s) === RUN TestFormatTokenList === RUN TestFormatTokenList/basic === RUN TestFormatTokenList/basic/pretty @@ -77007,7 +77006,7 @@ === RUN TestFormatTokenList/complex/pretty === RUN TestFormatTokenList/complex/pretty-meta === RUN TestFormatTokenList/complex/json ---- PASS: TestFormatTokenList (0.01s) +--- PASS: TestFormatTokenList (0.00s) --- PASS: TestFormatTokenList/basic (0.00s) --- PASS: TestFormatTokenList/basic/pretty (0.00s) --- PASS: TestFormatTokenList/basic/pretty-meta (0.00s) @@ -77021,7 +77020,7 @@ --- PASS: TestFormatTokenList/complex/pretty-meta (0.00s) --- PASS: TestFormatTokenList/complex/json (0.00s) PASS -ok github.com/hashicorp/consul/command/acl/token 0.020s +ok github.com/hashicorp/consul/command/acl/token 0.037s === RUN TestTokenCloneCommand_noTabs === PAUSE TestTokenCloneCommand_noTabs === RUN TestTokenCloneCommand_Pretty @@ -77029,7 +77028,7 @@ === RUN TestTokenCloneCommand_JSON === PAUSE TestTokenCloneCommand_JSON === CONT TestTokenCloneCommand_noTabs ---- PASS: TestTokenCloneCommand_noTabs (0.02s) +--- PASS: TestTokenCloneCommand_noTabs (0.00s) === CONT TestTokenCloneCommand_JSON [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestTokenCloneCommand_Pretty @@ -77039,169 +77038,167 @@ === RUN TestTokenCloneCommand_JSON/Without_Description === RUN TestTokenCloneCommand_Pretty/Description === RUN TestTokenCloneCommand_Pretty/Without_Description -2022-09-15T21:42:58.334Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:58.359Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:58.361 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:78930060-60af-77a6-2080-0b48d01d7516 Address:127.0.0.1:10006}]" -42:58.361 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:10006 [Follower]" leader= -42:58.362 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-78930060-60af-77a6-2080-0b48d01d7516.dc1 127.0.0.1 -42:58.363 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-78930060-60af-77a6-2080-0b48d01d7516 127.0.0.1 -2022-09-15T21:42:58.363Z [INFO] agent.router: Initializing LAN area manager -42:58.363 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10001 network=udp -42:58.363 [INFO] TestAgent.server: Adding LAN server: server="Node-78930060-60af-77a6-2080-0b48d01d7516 (Addr: tcp/127.0.0.1:10006) (DC: dc1)" -42:58.363 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-78930060-60af-77a6-2080-0b48d01d7516.dc1 area=wan -42:58.364 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10001 network=tcp -42:58.365 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:10002 network=tcp -42:58.365 [INFO] TestAgent: started state syncer -42:58.365 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:10007 network=tcp -42:58.414 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -42:58.425 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:58.425 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:10006 [Candidate]" term=2 -42:58.425 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:58.425 [DEBUG] TestAgent.server.raft: vote granted: from=78930060-60af-77a6-2080-0b48d01d7516 term=2 tally=1 -42:58.425 [INFO] TestAgent.server.raft: election won: tally=1 -42:58.425 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:10006 [Leader]" -42:58.426 [INFO] TestAgent.server: cluster leadership acquired -42:58.426 [INFO] TestAgent.server: initializing acls -42:58.426 [INFO] TestAgent.server: New leader elected: payload=Node-78930060-60af-77a6-2080-0b48d01d7516 -42:58.426 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:58.426 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:58.427 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:58.430 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:58.430 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:58.430 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:58.431 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-78930060-60af-77a6-2080-0b48d01d7516 -42:58.431 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-78930060-60af-77a6-2080-0b48d01d7516.dc1 -42:58.433 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:58.433 [INFO] TestAgent.server: Updating LAN server: server="Node-78930060-60af-77a6-2080-0b48d01d7516 (Addr: tcp/127.0.0.1:10006) (DC: dc1)" -42:58.433 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-78930060-60af-77a6-2080-0b48d01d7516.dc1 area=wan -42:58.449 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:58.449 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:58.449 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:58.449 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:58.449 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:58.449 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-78930060-60af-77a6-2080-0b48d01d7516 -42:58.449 [INFO] TestAgent.server: member joined, marking health alive: member=Node-78930060-60af-77a6-2080-0b48d01d7516 -42:58.451 [INFO] TestAgent.server: federation state anti-entropy synced -42:58.451 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-78930060-60af-77a6-2080-0b48d01d7516 -42:58.565 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:58.566 [INFO] TestAgent: Synced node info -42:58.666 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:34188 latency=1.9752ms -42:58.670 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:34188 latency=1.54168ms -42:58.673 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/c566c0ca-87f4-8586-909c-f3d941118792/clone from=127.0.0.1:34190 latency=1.16624ms -42:58.677 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/c566c0ca-87f4-8586-909c-f3d941118792/clone from=127.0.0.1:34192 latency=1.10488ms -42:58.678 [INFO] TestAgent: Requesting shutdown -42:58.678 [INFO] TestAgent.server: shutting down server -42:58.678 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:58.678 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:58.678 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:58.678 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:58.678 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:58.678 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:58.678 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:58.678 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:58.678 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:58.678 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:58.678 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:58.678 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:58.678 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:58.678 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:58.679 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:58.679Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:58.679Z [INFO] agent.router.manager: shutting down -42:58.679 [INFO] TestAgent: consul server down -42:58.679 [INFO] TestAgent: shutdown complete -42:58.679 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10001 network=tcp -42:58.679 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10001 network=udp -42:58.679 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:10002 network=tcp -42:59.180 [INFO] TestAgent: Waiting for endpoints to shut down -42:59.180 [INFO] TestAgent: Endpoints down ---- PASS: TestTokenCloneCommand_JSON (0.93s) +2021-08-13T15:35:18.812Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:35:18.833Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +35:18.838 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8e2a5c66-b401-e747-fb91-1de55d2273a9 Address:127.0.0.1:15111}]" +35:18.839 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:15111 [Follower]" leader= +35:18.840 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8e2a5c66-b401-e747-fb91-1de55d2273a9.dc1 127.0.0.1 +35:18.841 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8e2a5c66-b401-e747-fb91-1de55d2273a9 127.0.0.1 +2021-08-13T15:35:18.841Z [INFO] agent.router: Initializing LAN area manager +35:18.841 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15106 network=udp +35:18.841 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8e2a5c66-b401-e747-fb91-1de55d2273a9.dc1 area=wan +35:18.842 [INFO] TestAgent.server: Adding LAN server: server="Node-8e2a5c66-b401-e747-fb91-1de55d2273a9 (Addr: tcp/127.0.0.1:15111) (DC: dc1)" +35:18.842 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15106 network=tcp +35:18.843 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:15107 network=tcp +35:18.843 [INFO] TestAgent: started state syncer +35:18.843 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:15112 network=tcp +35:18.885 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +35:18.885 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:15111 [Candidate]" term=2 +35:18.886 [DEBUG] TestAgent.server.raft: votes: needed=1 +35:18.886 [DEBUG] TestAgent.server.raft: vote granted: from=8e2a5c66-b401-e747-fb91-1de55d2273a9 term=2 tally=1 +35:18.886 [INFO] TestAgent.server.raft: election won: tally=1 +35:18.886 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:15111 [Leader]" +35:18.886 [INFO] TestAgent.server: cluster leadership acquired +35:18.886 [INFO] TestAgent.server: initializing acls +35:18.887 [INFO] TestAgent.server: Created ACL 'global-management' policy +35:18.887 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +35:18.887 [INFO] TestAgent.server: New leader elected: payload=Node-8e2a5c66-b401-e747-fb91-1de55d2273a9 +35:18.888 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +35:18.888 [INFO] TestAgent.server: Created ACL anonymous token from configuration +35:18.888 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +35:18.888 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +35:18.889 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-8e2a5c66-b401-e747-fb91-1de55d2273a9 +35:18.889 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-8e2a5c66-b401-e747-fb91-1de55d2273a9.dc1 +35:18.890 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +35:18.891 [INFO] TestAgent.server: Updating LAN server: server="Node-8e2a5c66-b401-e747-fb91-1de55d2273a9 (Addr: tcp/127.0.0.1:15111) (DC: dc1)" +35:18.891 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-8e2a5c66-b401-e747-fb91-1de55d2273a9.dc1 area=wan +35:18.919 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +35:18.919 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +35:18.919 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +35:18.919 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +35:18.919 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +35:18.919 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8e2a5c66-b401-e747-fb91-1de55d2273a9 +35:18.919 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8e2a5c66-b401-e747-fb91-1de55d2273a9 +35:18.920 [INFO] TestAgent.server: federation state anti-entropy synced +35:18.927 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8e2a5c66-b401-e747-fb91-1de55d2273a9 +35:18.934 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +35:18.935 [INFO] TestAgent: Synced node info +35:18.944 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:37850 latency=2.068104ms +35:18.947 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:37850 latency=1.452924ms +35:18.951 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/65dc1c00-1ed5-de2b-78f8-33169d3f3947/clone from=127.0.0.1:37852 latency=1.241523ms +35:18.955 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/65dc1c00-1ed5-de2b-78f8-33169d3f3947/clone from=127.0.0.1:37854 latency=1.051222ms +35:18.956 [INFO] TestAgent: Requesting shutdown +35:18.956 [INFO] TestAgent.server: shutting down server +35:18.956 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +35:18.956 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +35:18.956 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +35:18.956 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +35:18.956 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +35:18.956 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +35:18.956 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +35:18.956 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +35:18.956 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +35:18.956 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +35:18.956 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +35:18.956 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +35:18.956 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +35:18.956 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +35:18.956 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:35:18.956Z [INFO] agent.router.manager: shutting down +2021-08-13T15:35:18.956Z [INFO] agent.router.manager: shutting down +35:18.956 [INFO] TestAgent: consul server down +35:18.956 [INFO] TestAgent: shutdown complete +35:18.957 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15106 network=tcp +35:18.957 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15106 network=udp +35:18.957 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:15107 network=tcp +35:19.457 [INFO] TestAgent: Waiting for endpoints to shut down +35:19.457 [INFO] TestAgent: Endpoints down +--- PASS: TestTokenCloneCommand_JSON (0.72s) --- PASS: TestTokenCloneCommand_JSON/Description (0.00s) --- PASS: TestTokenCloneCommand_JSON/Without_Description (0.00s) -2022-09-15T21:42:58.331Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:58.356Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:58.360 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:23397ce3-d7ea-9f2d-c38d-b8ec70b05386 Address:127.0.0.1:10013}]" -42:58.360 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:10013 [Follower]" leader= -42:58.361 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-23397ce3-d7ea-9f2d-c38d-b8ec70b05386.dc1 127.0.0.1 -42:58.362 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-23397ce3-d7ea-9f2d-c38d-b8ec70b05386 127.0.0.1 -2022-09-15T21:42:58.362Z [INFO] agent.router: Initializing LAN area manager -42:58.363 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10008 network=udp -42:58.363 [INFO] TestAgent.server: Adding LAN server: server="Node-23397ce3-d7ea-9f2d-c38d-b8ec70b05386 (Addr: tcp/127.0.0.1:10013) (DC: dc1)" -42:58.363 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-23397ce3-d7ea-9f2d-c38d-b8ec70b05386.dc1 area=wan -42:58.363 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10008 network=tcp -42:58.364 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:10009 network=tcp -42:58.364 [INFO] TestAgent: started state syncer -42:58.364 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:10014 network=tcp -42:58.413 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:58.413 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:10013 [Candidate]" term=2 -42:58.413 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:58.413 [DEBUG] TestAgent.server.raft: vote granted: from=23397ce3-d7ea-9f2d-c38d-b8ec70b05386 term=2 tally=1 -42:58.413 [INFO] TestAgent.server.raft: election won: tally=1 -42:58.413 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:10013 [Leader]" -42:58.413 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -42:58.413 [INFO] TestAgent.server: cluster leadership acquired -42:58.413 [INFO] TestAgent.server: initializing acls -42:58.414 [INFO] TestAgent.server: New leader elected: payload=Node-23397ce3-d7ea-9f2d-c38d-b8ec70b05386 -42:58.414 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:58.414 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:58.415 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:58.415 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:58.415 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:58.415 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:58.416 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-23397ce3-d7ea-9f2d-c38d-b8ec70b05386 -42:58.416 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-23397ce3-d7ea-9f2d-c38d-b8ec70b05386.dc1 -42:58.417 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:58.418 [INFO] TestAgent.server: Updating LAN server: server="Node-23397ce3-d7ea-9f2d-c38d-b8ec70b05386 (Addr: tcp/127.0.0.1:10013) (DC: dc1)" -42:58.418 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-23397ce3-d7ea-9f2d-c38d-b8ec70b05386.dc1 area=wan -42:58.447 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:58.447 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:58.447 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:58.447 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:58.448 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:58.448 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-23397ce3-d7ea-9f2d-c38d-b8ec70b05386 -42:58.448 [INFO] TestAgent.server: member joined, marking health alive: member=Node-23397ce3-d7ea-9f2d-c38d-b8ec70b05386 -42:58.450 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-23397ce3-d7ea-9f2d-c38d-b8ec70b05386 -42:58.450 [INFO] TestAgent.server: federation state anti-entropy synced -42:58.547 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -42:58.548 [INFO] TestAgent: Synced node info -42:58.548 [DEBUG] TestAgent: Node info in sync -42:58.737 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:54112 latency=1.4753ms -42:58.742 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:54112 latency=4.09628ms -42:58.749 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/41d8cd26-2b13-d15e-7c7c-e0e9578c9165/clone from=127.0.0.1:54114 latency=4.04476ms -42:58.752 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/96a3e0e3-dfd7-dedc-e925-f0840418f94b from=127.0.0.1:54112 latency=518.28µs -42:58.758 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/41d8cd26-2b13-d15e-7c7c-e0e9578c9165/clone from=127.0.0.1:54116 latency=1.45802ms -42:58.760 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/48fed159-a1d9-af83-bdcd-1b0929c87199 from=127.0.0.1:54112 latency=313.3µs -42:58.760 [INFO] TestAgent: Requesting shutdown -42:58.760 [INFO] TestAgent.server: shutting down server -42:58.761 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:58.761 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:58.761 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:58.761 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:58.761 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:58.761 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:58.761 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:58.761 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:58.761 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:58.761 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:58.761 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:58.761 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:58.761 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:58.761 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:58.761 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:58.761Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:58.761Z [INFO] agent.router.manager: shutting down -42:58.761 [INFO] TestAgent: consul server down -42:58.761 [INFO] TestAgent: shutdown complete -42:58.761 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10008 network=tcp -42:58.761 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10008 network=udp -42:58.761 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:10009 network=tcp -42:59.262 [INFO] TestAgent: Waiting for endpoints to shut down -42:59.262 [INFO] TestAgent: Endpoints down ---- PASS: TestTokenCloneCommand_Pretty (1.01s) - --- PASS: TestTokenCloneCommand_Pretty/Description (0.01s) - --- PASS: TestTokenCloneCommand_Pretty/Without_Description (0.01s) +2021-08-13T15:35:18.812Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:35:18.837Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +35:18.840 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:741e0be3-91f7-c453-6617-7c7b74aa680c Address:127.0.0.1:15118}]" +35:18.840 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:15118 [Follower]" leader= +35:18.841 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-741e0be3-91f7-c453-6617-7c7b74aa680c.dc1 127.0.0.1 +35:18.842 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-741e0be3-91f7-c453-6617-7c7b74aa680c 127.0.0.1 +2021-08-13T15:35:18.842Z [INFO] agent.router: Initializing LAN area manager +35:18.842 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15113 network=udp +35:18.842 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-741e0be3-91f7-c453-6617-7c7b74aa680c.dc1 area=wan +35:18.843 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15113 network=tcp +35:18.844 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:15114 network=tcp +35:18.843 [INFO] TestAgent.server: Adding LAN server: server="Node-741e0be3-91f7-c453-6617-7c7b74aa680c (Addr: tcp/127.0.0.1:15118) (DC: dc1)" +35:18.844 [INFO] TestAgent: started state syncer +35:18.844 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:15119 network=tcp +35:18.895 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +35:18.899 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +35:18.899 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:15118 [Candidate]" term=2 +35:18.899 [DEBUG] TestAgent.server.raft: votes: needed=1 +35:18.899 [DEBUG] TestAgent.server.raft: vote granted: from=741e0be3-91f7-c453-6617-7c7b74aa680c term=2 tally=1 +35:18.899 [INFO] TestAgent.server.raft: election won: tally=1 +35:18.899 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:15118 [Leader]" +35:18.899 [INFO] TestAgent.server: cluster leadership acquired +35:18.900 [INFO] TestAgent.server: initializing acls +35:18.900 [INFO] TestAgent.server: Created ACL 'global-management' policy +35:18.900 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +35:18.901 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +35:18.901 [INFO] TestAgent.server: Created ACL anonymous token from configuration +35:18.901 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +35:18.901 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +35:18.902 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-741e0be3-91f7-c453-6617-7c7b74aa680c +35:18.902 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-741e0be3-91f7-c453-6617-7c7b74aa680c.dc1 +35:18.903 [INFO] TestAgent.server: New leader elected: payload=Node-741e0be3-91f7-c453-6617-7c7b74aa680c +35:18.903 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +35:18.903 [INFO] TestAgent.server: Updating LAN server: server="Node-741e0be3-91f7-c453-6617-7c7b74aa680c (Addr: tcp/127.0.0.1:15118) (DC: dc1)" +35:18.903 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-741e0be3-91f7-c453-6617-7c7b74aa680c.dc1 area=wan +35:18.916 [INFO] TestAgent: Synced node info +35:18.916 [DEBUG] TestAgent: Node info in sync +35:18.926 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +35:18.926 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +35:18.926 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +35:18.926 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +35:18.926 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +35:18.926 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-741e0be3-91f7-c453-6617-7c7b74aa680c +35:18.926 [INFO] TestAgent.server: member joined, marking health alive: member=Node-741e0be3-91f7-c453-6617-7c7b74aa680c +35:18.928 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-741e0be3-91f7-c453-6617-7c7b74aa680c +35:18.928 [INFO] TestAgent.server: federation state anti-entropy synced +35:19.005 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:34660 latency=2.327325ms +35:19.012 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:34660 latency=1.289763ms +35:19.015 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/1ba85765-2ee5-e903-ed37-4f6cc563c451/clone from=127.0.0.1:34662 latency=1.101143ms +35:19.017 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/798fdd44-8fc4-cd9a-65dc-fc322a78096d from=127.0.0.1:34660 latency=281.661µs +35:19.020 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/1ba85765-2ee5-e903-ed37-4f6cc563c451/clone from=127.0.0.1:34664 latency=1.054302ms +35:19.022 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/86bbc7e3-5a87-647e-70c1-f69b0ed92ceb from=127.0.0.1:34660 latency=218.24µs +35:19.022 [INFO] TestAgent: Requesting shutdown +35:19.022 [INFO] TestAgent.server: shutting down server +35:19.022 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +35:19.022 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +35:19.022 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +35:19.022 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +35:19.023 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +35:19.023 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +35:19.023 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +35:19.023 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +35:19.023 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +35:19.023 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +35:19.023 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +35:19.023 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +35:19.023 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +35:19.023 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +35:19.023 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:35:19.023Z [INFO] agent.router.manager: shutting down +2021-08-13T15:35:19.023Z [INFO] agent.router.manager: shutting down +35:19.023 [INFO] TestAgent: consul server down +35:19.023 [INFO] TestAgent: shutdown complete +35:19.023 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15113 network=tcp +35:19.023 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15113 network=udp +35:19.023 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:15114 network=tcp +35:19.524 [INFO] TestAgent: Waiting for endpoints to shut down +35:19.524 [INFO] TestAgent: Endpoints down +--- PASS: TestTokenCloneCommand_Pretty (0.78s) + --- PASS: TestTokenCloneCommand_Pretty/Description (0.00s) + --- PASS: TestTokenCloneCommand_Pretty/Without_Description (0.00s) PASS -ok github.com/hashicorp/consul/command/acl/token/clone 1.196s +ok github.com/hashicorp/consul/command/acl/token/clone 0.905s === RUN TestTokenCreateCommand_noTabs === PAUSE TestTokenCreateCommand_noTabs === RUN TestTokenCreateCommand_Pretty @@ -77210,261 +77207,261 @@ === PAUSE TestTokenCreateCommand_JSON === CONT TestTokenCreateCommand_noTabs === CONT TestTokenCreateCommand_JSON -=== CONT TestTokenCreateCommand_Pretty [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... ---- PASS: TestTokenCreateCommand_noTabs (0.01s) +=== CONT TestTokenCreateCommand_Pretty +--- PASS: TestTokenCreateCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range === RUN TestTokenCreateCommand_Pretty/policy-name === RUN TestTokenCreateCommand_Pretty/policy-id === RUN TestTokenCreateCommand_Pretty/node-identity === RUN TestTokenCreateCommand_Pretty/predefined-ids -2022-09-15T21:42:58.045Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:58.076Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:58.080 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f95b95f6-0d3b-dc31-1ca2-b9988caad9ea Address:127.0.0.1:31454}]" -42:58.080 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31454 [Follower]" leader= -42:58.081 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -42:58.088 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2022-09-15T21:42:58.088Z [INFO] agent.router: Initializing LAN area manager -42:58.088 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31449 network=udp -42:58.088 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -42:58.088 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:31454) (DC: dc1)" -42:58.088 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31449 network=tcp -42:58.089 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31450 network=tcp -42:58.089 [INFO] TestAgent: started state syncer -42:58.090 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31455 network=tcp -42:58.138 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -42:58.170 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:58.170 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31454 [Candidate]" term=2 -42:58.170 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:58.171 [DEBUG] TestAgent.server.raft: vote granted: from=f95b95f6-0d3b-dc31-1ca2-b9988caad9ea term=2 tally=1 -42:58.171 [INFO] TestAgent.server.raft: election won: tally=1 -42:58.171 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31454 [Leader]" -42:58.171 [INFO] TestAgent.server: cluster leadership acquired -42:58.171 [INFO] TestAgent.server: initializing acls -42:58.171 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:58.171 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:58.172 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:58.173 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:58.173 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:58.173 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:58.173 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: test-node -42:58.173 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: test-node.dc1 -42:58.174 [INFO] TestAgent.server: New leader elected: payload=test-node -42:58.174 [INFO] TestAgent.server: Updating LAN server: server="test-node (Addr: tcp/127.0.0.1:31454) (DC: dc1)" -42:58.174 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=test-node.dc1 area=wan -42:58.179 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:58.200 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:58.200 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:58.200 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:58.200 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:58.200 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:58.200 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -42:58.200 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -42:58.203 [INFO] TestAgent.server: federation state anti-entropy synced -42:58.203 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -42:58.211 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:50946 latency=6.42876ms -42:58.220 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:50948 latency=1.73606ms -42:58.225 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:50950 latency=1.22694ms -42:58.234 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:50952 latency=2.21988ms -42:58.242 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/nodes from=127.0.0.1:50954 latency=1.1679ms -42:58.248 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:50956 latency=1.30684ms -42:58.249 [INFO] TestAgent: Requesting shutdown -42:58.249 [INFO] TestAgent.server: shutting down server -42:58.249 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:58.249 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:58.249 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:58.249 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:58.249 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:58.249 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:58.249 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:58.250 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:58.250 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:58.250 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -42:58.250 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:58.250 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:58.250 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:58.250 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:58.250 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:58.250 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2022-09-15T21:42:58.250Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:58.250Z [INFO] agent.router.manager: shutting down -42:58.250 [INFO] TestAgent: consul server down -42:58.250 [INFO] TestAgent: shutdown complete -42:58.250 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31449 network=tcp -42:58.250 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31449 network=udp -42:58.250 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31450 network=tcp -42:58.751 [INFO] TestAgent: Waiting for endpoints to shut down -42:58.751 [INFO] TestAgent: Endpoints down ---- PASS: TestTokenCreateCommand_Pretty (0.79s) - --- PASS: TestTokenCreateCommand_Pretty/policy-name (0.01s) +2021-08-13T15:35:23.627Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:35:23.651Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +35:23.656 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f2c72510-9026-9609-906b-cb5b5c8fed01 Address:127.0.0.1:30426}]" +35:23.656 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30426 [Follower]" leader= +35:23.657 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f2c72510-9026-9609-906b-cb5b5c8fed01.dc1 127.0.0.1 +35:23.657 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f2c72510-9026-9609-906b-cb5b5c8fed01 127.0.0.1 +2021-08-13T15:35:23.657Z [INFO] agent.router: Initializing LAN area manager +35:23.658 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=udp +35:23.658 [INFO] TestAgent.server: Adding LAN server: server="Node-f2c72510-9026-9609-906b-cb5b5c8fed01 (Addr: tcp/127.0.0.1:30426) (DC: dc1)" +35:23.658 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f2c72510-9026-9609-906b-cb5b5c8fed01.dc1 area=wan +35:23.658 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=tcp +35:23.659 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30422 network=tcp +35:23.659 [INFO] TestAgent: started state syncer +35:23.659 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30427 network=tcp +35:23.702 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +35:23.703 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30426 [Candidate]" term=2 +35:23.703 [DEBUG] TestAgent.server.raft: votes: needed=1 +35:23.703 [DEBUG] TestAgent.server.raft: vote granted: from=f2c72510-9026-9609-906b-cb5b5c8fed01 term=2 tally=1 +35:23.703 [INFO] TestAgent.server.raft: election won: tally=1 +35:23.703 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30426 [Leader]" +35:23.703 [INFO] TestAgent.server: cluster leadership acquired +35:23.703 [INFO] TestAgent.server: New leader elected: payload=Node-f2c72510-9026-9609-906b-cb5b5c8fed01 +35:23.703 [INFO] TestAgent.server: initializing acls +35:23.704 [INFO] TestAgent.server: Created ACL 'global-management' policy +35:23.704 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +35:23.705 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +35:23.705 [INFO] TestAgent.server: Created ACL anonymous token from configuration +35:23.705 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +35:23.705 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +35:23.705 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-f2c72510-9026-9609-906b-cb5b5c8fed01 +35:23.705 [INFO] TestAgent.server: Updating LAN server: server="Node-f2c72510-9026-9609-906b-cb5b5c8fed01 (Addr: tcp/127.0.0.1:30426) (DC: dc1)" +35:23.705 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-f2c72510-9026-9609-906b-cb5b5c8fed01.dc1 +35:23.706 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-f2c72510-9026-9609-906b-cb5b5c8fed01.dc1 area=wan +35:23.707 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +35:23.737 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +35:23.737 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +35:23.737 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +35:23.737 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +35:23.737 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +35:23.737 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f2c72510-9026-9609-906b-cb5b5c8fed01 +35:23.737 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f2c72510-9026-9609-906b-cb5b5c8fed01 +35:23.738 [INFO] TestAgent.server: federation state anti-entropy synced +35:23.738 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f2c72510-9026-9609-906b-cb5b5c8fed01 +35:23.820 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:48800 latency=1.808124ms +35:23.823 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:48802 latency=1.417324ms +35:23.824 [INFO] TestAgent: Requesting shutdown +35:23.824 [INFO] TestAgent.server: shutting down server +35:23.824 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +35:23.824 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +35:23.824 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +35:23.824 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +35:23.824 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +35:23.824 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +35:23.824 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +35:23.824 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +35:23.824 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +35:23.824 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +35:23.824 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +35:23.824 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +35:23.824 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +35:23.824 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +35:23.824 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +35:23.824 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:35:23.824Z [INFO] agent.router.manager: shutting down +2021-08-13T15:35:23.824Z [INFO] agent.router.manager: shutting down +35:23.824 [INFO] TestAgent: consul server down +35:23.825 [INFO] TestAgent: shutdown complete +35:23.825 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=tcp +35:23.825 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=udp +35:23.825 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30422 network=tcp +35:24.325 [INFO] TestAgent: Waiting for endpoints to shut down +35:24.325 [INFO] TestAgent: Endpoints down +--- PASS: TestTokenCreateCommand_JSON (0.76s) +2021-08-13T15:35:23.632Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:35:23.653Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +35:23.656 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b2234cfc-1e14-0d85-f19b-5d8d7eec441e Address:127.0.0.1:30433}]" +35:23.656 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30433 [Follower]" leader= +35:23.657 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +35:23.657 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2021-08-13T15:35:23.657Z [INFO] agent.router: Initializing LAN area manager +35:23.658 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:30433) (DC: dc1)" +35:23.658 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30428 network=udp +35:23.658 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +35:23.659 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30428 network=tcp +35:23.659 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30429 network=tcp +35:23.659 [INFO] TestAgent: started state syncer +35:23.660 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30434 network=tcp +35:23.709 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +35:23.726 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +35:23.726 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30433 [Candidate]" term=2 +35:23.726 [DEBUG] TestAgent.server.raft: votes: needed=1 +35:23.726 [DEBUG] TestAgent.server.raft: vote granted: from=b2234cfc-1e14-0d85-f19b-5d8d7eec441e term=2 tally=1 +35:23.726 [INFO] TestAgent.server.raft: election won: tally=1 +35:23.726 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30433 [Leader]" +35:23.726 [INFO] TestAgent.server: cluster leadership acquired +35:23.726 [INFO] TestAgent.server: New leader elected: payload=test-node +35:23.726 [INFO] TestAgent.server: initializing acls +35:23.727 [INFO] TestAgent.server: Created ACL 'global-management' policy +35:23.727 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +35:23.728 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +35:23.728 [INFO] TestAgent.server: Created ACL anonymous token from configuration +35:23.728 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +35:23.728 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +35:23.728 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: test-node +35:23.728 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: test-node.dc1 +35:23.728 [INFO] TestAgent.server: Updating LAN server: server="test-node (Addr: tcp/127.0.0.1:30433) (DC: dc1)" +35:23.729 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=test-node.dc1 area=wan +35:23.731 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +35:23.734 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +35:23.734 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +35:23.734 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +35:23.734 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +35:23.734 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +35:23.734 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +35:23.735 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +35:23.737 [INFO] TestAgent.server: federation state anti-entropy synced +35:23.737 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +35:23.759 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +35:23.760 [INFO] TestAgent: Synced node info +35:24.008 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:46868 latency=1.575523ms +35:24.012 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:46870 latency=1.219623ms +35:24.015 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:46872 latency=1.092003ms +35:24.021 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:46874 latency=3.219748ms +35:24.026 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/nodes from=127.0.0.1:46876 latency=1.285723ms +35:24.033 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:46878 latency=1.089683ms +35:24.034 [INFO] TestAgent: Requesting shutdown +35:24.034 [INFO] TestAgent.server: shutting down server +35:24.034 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +35:24.034 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +35:24.034 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +35:24.034 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +35:24.034 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +35:24.034 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +35:24.034 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +35:24.034 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +35:24.034 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +35:24.034 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +35:24.034 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +35:24.034 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +35:24.034 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +35:24.034 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +35:24.034 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:35:24.034Z [INFO] agent.router.manager: shutting down +2021-08-13T15:35:24.034Z [INFO] agent.router.manager: shutting down +35:24.035 [INFO] TestAgent: consul server down +35:24.035 [INFO] TestAgent: shutdown complete +35:24.035 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30428 network=tcp +35:24.035 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30428 network=udp +35:24.035 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30429 network=tcp +35:24.535 [INFO] TestAgent: Waiting for endpoints to shut down +35:24.535 [INFO] TestAgent: Endpoints down +--- PASS: TestTokenCreateCommand_Pretty (0.97s) + --- PASS: TestTokenCreateCommand_Pretty/policy-name (0.00s) --- PASS: TestTokenCreateCommand_Pretty/policy-id (0.00s) - --- PASS: TestTokenCreateCommand_Pretty/node-identity (0.02s) + --- PASS: TestTokenCreateCommand_Pretty/node-identity (0.01s) --- PASS: TestTokenCreateCommand_Pretty/predefined-ids (0.01s) -2022-09-15T21:42:58.043Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:42:58.082Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -42:58.085 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c3d6aeec-960b-b2e3-1102-5ac8ea7ad2ed Address:127.0.0.1:31447}]" -42:58.085 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31447 [Follower]" leader= -42:58.086 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c3d6aeec-960b-b2e3-1102-5ac8ea7ad2ed.dc1 127.0.0.1 -42:58.087 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c3d6aeec-960b-b2e3-1102-5ac8ea7ad2ed 127.0.0.1 -2022-09-15T21:42:58.087Z [INFO] agent.router: Initializing LAN area manager -42:58.087 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c3d6aeec-960b-b2e3-1102-5ac8ea7ad2ed.dc1 area=wan -42:58.087 [INFO] TestAgent.server: Adding LAN server: server="Node-c3d6aeec-960b-b2e3-1102-5ac8ea7ad2ed (Addr: tcp/127.0.0.1:31447) (DC: dc1)" -42:58.087 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31442 network=tcp -42:58.090 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31442 network=udp -42:58.091 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31443 network=tcp -42:58.091 [INFO] TestAgent: started state syncer -42:58.091 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31448 network=tcp -42:58.120 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -42:58.120 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31447 [Candidate]" term=2 -42:58.121 [DEBUG] TestAgent.server.raft: votes: needed=1 -42:58.121 [DEBUG] TestAgent.server.raft: vote granted: from=c3d6aeec-960b-b2e3-1102-5ac8ea7ad2ed term=2 tally=1 -42:58.121 [INFO] TestAgent.server.raft: election won: tally=1 -42:58.121 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31447 [Leader]" -42:58.121 [INFO] TestAgent.server: cluster leadership acquired -42:58.121 [INFO] TestAgent.server: initializing acls -42:58.122 [INFO] TestAgent.server: Created ACL 'global-management' policy -42:58.122 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -42:58.123 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -42:58.126 [INFO] TestAgent.server: New leader elected: payload=Node-c3d6aeec-960b-b2e3-1102-5ac8ea7ad2ed -42:58.127 [INFO] TestAgent.server: Created ACL anonymous token from configuration -42:58.127 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -42:58.127 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -42:58.127 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-c3d6aeec-960b-b2e3-1102-5ac8ea7ad2ed -42:58.127 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-c3d6aeec-960b-b2e3-1102-5ac8ea7ad2ed.dc1 -42:58.129 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -42:58.130 [INFO] TestAgent.server: Updating LAN server: server="Node-c3d6aeec-960b-b2e3-1102-5ac8ea7ad2ed (Addr: tcp/127.0.0.1:31447) (DC: dc1)" -42:58.130 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-c3d6aeec-960b-b2e3-1102-5ac8ea7ad2ed.dc1 area=wan -42:58.202 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -42:58.202 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -42:58.202 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -42:58.202 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -42:58.202 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -42:58.202 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c3d6aeec-960b-b2e3-1102-5ac8ea7ad2ed -42:58.202 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c3d6aeec-960b-b2e3-1102-5ac8ea7ad2ed -42:58.203 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c3d6aeec-960b-b2e3-1102-5ac8ea7ad2ed -42:58.203 [INFO] TestAgent.server: federation state anti-entropy synced -42:58.307 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:59260 latency=1.7176ms -42:58.310 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:59262 latency=1.13878ms -42:58.311 [INFO] TestAgent: Requesting shutdown -42:58.311 [INFO] TestAgent.server: shutting down server -42:58.311 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -42:58.311 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -42:58.311 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -42:58.311 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -42:58.311 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -42:58.311 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -42:58.311 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -42:58.311 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -42:58.311 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -42:58.311 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -42:58.311 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -42:58.311 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -42:58.311 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -42:58.311 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -42:58.311 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -42:58.312 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:42:58.312Z [INFO] agent.router.manager: shutting down -2022-09-15T21:42:58.312Z [INFO] agent.router.manager: shutting down -42:58.312 [INFO] TestAgent: consul server down -42:58.312 [INFO] TestAgent: shutdown complete -42:58.312 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31442 network=tcp -42:58.312 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31442 network=udp -42:58.312 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31443 network=tcp -42:58.812 [INFO] TestAgent: Waiting for endpoints to shut down -42:58.812 [INFO] TestAgent: Endpoints down ---- PASS: TestTokenCreateCommand_JSON (0.85s) PASS -ok github.com/hashicorp/consul/command/acl/token/create 1.000s +ok github.com/hashicorp/consul/command/acl/token/create 1.043s === RUN TestTokenDeleteCommand_noTabs === PAUSE TestTokenDeleteCommand_noTabs === RUN TestTokenDeleteCommand === PAUSE TestTokenDeleteCommand === CONT TestTokenDeleteCommand_noTabs ---- PASS: TestTokenDeleteCommand_noTabs (0.00s) === CONT TestTokenDeleteCommand [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestTokenDeleteCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:43:00.718Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:00.744Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:00.748 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fe5ac550-8d04-4edf-5841-bf218fa6aa6f Address:127.0.0.1:20216}]" -43:00.749 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fe5ac550-8d04-4edf-5841-bf218fa6aa6f.dc1 127.0.0.1 -43:00.750 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:20216 [Follower]" leader= -43:00.751 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fe5ac550-8d04-4edf-5841-bf218fa6aa6f 127.0.0.1 -2022-09-15T21:43:00.751Z [INFO] agent.router: Initializing LAN area manager -43:00.751 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20211 network=udp -43:00.751 [INFO] TestAgent.server: Adding LAN server: server="Node-fe5ac550-8d04-4edf-5841-bf218fa6aa6f (Addr: tcp/127.0.0.1:20216) (DC: dc1)" -43:00.751 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fe5ac550-8d04-4edf-5841-bf218fa6aa6f.dc1 area=wan -43:00.752 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20211 network=tcp -43:00.753 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:20212 network=tcp -43:00.753 [INFO] TestAgent: started state syncer -43:00.753 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:20217 network=tcp -43:00.805 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -43:00.808 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:00.808 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:20216 [Candidate]" term=2 -43:00.808 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:00.808 [DEBUG] TestAgent.server.raft: vote granted: from=fe5ac550-8d04-4edf-5841-bf218fa6aa6f term=2 tally=1 -43:00.808 [INFO] TestAgent.server.raft: election won: tally=1 -43:00.808 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:20216 [Leader]" -43:00.808 [INFO] TestAgent.server: cluster leadership acquired -43:00.809 [INFO] TestAgent.server: initializing acls -43:00.809 [INFO] TestAgent.server: Created ACL 'global-management' policy -43:00.809 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -43:00.810 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -43:00.811 [INFO] TestAgent.server: Created ACL anonymous token from configuration -43:00.811 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -43:00.811 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -43:00.811 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-fe5ac550-8d04-4edf-5841-bf218fa6aa6f -43:00.811 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-fe5ac550-8d04-4edf-5841-bf218fa6aa6f.dc1 -43:00.812 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:00.826 [INFO] TestAgent.server: New leader elected: payload=Node-fe5ac550-8d04-4edf-5841-bf218fa6aa6f -43:00.826 [INFO] TestAgent.server: Updating LAN server: server="Node-fe5ac550-8d04-4edf-5841-bf218fa6aa6f (Addr: tcp/127.0.0.1:20216) (DC: dc1)" -43:00.827 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-fe5ac550-8d04-4edf-5841-bf218fa6aa6f.dc1 area=wan -43:00.829 [INFO] TestAgent: Synced node info -43:00.829 [DEBUG] TestAgent: Node info in sync -43:00.853 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:00.853 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:00.853 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:00.853 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:00.853 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:00.853 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fe5ac550-8d04-4edf-5841-bf218fa6aa6f -43:00.853 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fe5ac550-8d04-4edf-5841-bf218fa6aa6f -43:00.855 [INFO] TestAgent.server: federation state anti-entropy synced -43:00.855 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fe5ac550-8d04-4edf-5841-bf218fa6aa6f -43:01.086 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:56114 latency=2.80338ms -43:01.090 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/token/6b4254c5-a1e1-a887-9faa-71066a09b54a from=127.0.0.1:56116 latency=1.12924ms -43:01.091 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/token/6b4254c5-a1e1-a887-9faa-71066a09b54a from=127.0.0.1:56114 error="ACL not found" -43:01.091 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/6b4254c5-a1e1-a887-9faa-71066a09b54a from=127.0.0.1:56114 latency=757.54µs -43:01.092 [INFO] TestAgent: Requesting shutdown -43:01.092 [INFO] TestAgent.server: shutting down server -43:01.092 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:01.092 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:01.092 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:01.092 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:01.092 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -43:01.092 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -43:01.092 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:01.092 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:01.092 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:01.092 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:01.092 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:01.092 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:01.092 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -43:01.092 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -43:01.092 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:01.092Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:01.092Z [INFO] agent.router.manager: shutting down -43:01.093 [INFO] TestAgent: consul server down -43:01.093 [INFO] TestAgent: shutdown complete -43:01.093 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20211 network=tcp -43:01.093 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20211 network=udp -43:01.093 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:20212 network=tcp -43:01.593 [INFO] TestAgent: Waiting for endpoints to shut down -43:01.593 [INFO] TestAgent: Endpoints down ---- PASS: TestTokenDeleteCommand (0.97s) +2021-08-13T15:35:24.709Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:35:24.726Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +35:24.732 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:73159ce4-033b-0528-a6c5-f17129c22d5f Address:127.0.0.1:30426}]" +35:24.736 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30426 [Follower]" leader= +35:24.751 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-73159ce4-033b-0528-a6c5-f17129c22d5f.dc1 127.0.0.1 +35:24.752 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-73159ce4-033b-0528-a6c5-f17129c22d5f 127.0.0.1 +2021-08-13T15:35:24.752Z [INFO] agent.router: Initializing LAN area manager +35:24.753 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=udp +35:24.753 [INFO] TestAgent.server: Adding LAN server: server="Node-73159ce4-033b-0528-a6c5-f17129c22d5f (Addr: tcp/127.0.0.1:30426) (DC: dc1)" +35:24.753 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-73159ce4-033b-0528-a6c5-f17129c22d5f.dc1 area=wan +35:24.754 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=tcp +35:24.754 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30422 network=tcp +35:24.755 [INFO] TestAgent: started state syncer +35:24.755 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30427 network=tcp +35:24.780 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +35:24.780 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30426 [Candidate]" term=2 +35:24.780 [DEBUG] TestAgent.server.raft: votes: needed=1 +35:24.780 [DEBUG] TestAgent.server.raft: vote granted: from=73159ce4-033b-0528-a6c5-f17129c22d5f term=2 tally=1 +35:24.780 [INFO] TestAgent.server.raft: election won: tally=1 +35:24.780 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30426 [Leader]" +35:24.780 [INFO] TestAgent.server: cluster leadership acquired +35:24.781 [INFO] TestAgent.server: initializing acls +35:24.781 [INFO] TestAgent.server: Created ACL 'global-management' policy +35:24.781 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +35:24.782 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +35:24.783 [INFO] TestAgent.server: Created ACL anonymous token from configuration +35:24.783 [INFO] TestAgent.server: New leader elected: payload=Node-73159ce4-033b-0528-a6c5-f17129c22d5f +35:24.784 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +35:24.784 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +35:24.784 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-73159ce4-033b-0528-a6c5-f17129c22d5f +35:24.785 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-73159ce4-033b-0528-a6c5-f17129c22d5f.dc1 +35:24.785 [INFO] TestAgent.server: Updating LAN server: server="Node-73159ce4-033b-0528-a6c5-f17129c22d5f (Addr: tcp/127.0.0.1:30426) (DC: dc1)" +35:24.785 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-73159ce4-033b-0528-a6c5-f17129c22d5f.dc1 area=wan +35:24.786 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +35:24.809 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +35:24.809 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +35:24.809 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +35:24.809 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +35:24.809 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +35:24.809 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-73159ce4-033b-0528-a6c5-f17129c22d5f +35:24.809 [INFO] TestAgent.server: member joined, marking health alive: member=Node-73159ce4-033b-0528-a6c5-f17129c22d5f +35:24.811 [INFO] TestAgent.server: federation state anti-entropy synced +35:24.811 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-73159ce4-033b-0528-a6c5-f17129c22d5f +35:24.923 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +35:24.924 [INFO] TestAgent: Synced node info +35:25.195 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:48816 latency=2.001045ms +35:25.198 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/token/8e082e6d-8743-1d7e-0c45-c58befbd0c08 from=127.0.0.1:48818 latency=1.662584ms +35:25.203 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/token/8e082e6d-8743-1d7e-0c45-c58befbd0c08 from=127.0.0.1:48816 error="ACL not found" +35:25.203 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/8e082e6d-8743-1d7e-0c45-c58befbd0c08 from=127.0.0.1:48816 latency=483.701µs +35:25.204 [INFO] TestAgent: Requesting shutdown +35:25.204 [INFO] TestAgent.server: shutting down server +35:25.204 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +35:25.204 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +35:25.204 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +35:25.204 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +35:25.204 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +35:25.204 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +35:25.204 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +35:25.204 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +35:25.204 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +35:25.204 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +35:25.204 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +35:25.204 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +35:25.204 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +35:25.204 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +35:25.204 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:35:25.204Z [INFO] agent.router.manager: shutting down +2021-08-13T15:35:25.204Z [INFO] agent.router.manager: shutting down +35:25.204 [INFO] TestAgent: consul server down +35:25.204 [INFO] TestAgent: shutdown complete +35:25.205 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=tcp +35:25.205 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=udp +35:25.205 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30422 network=tcp +35:25.705 [INFO] TestAgent: Waiting for endpoints to shut down +35:25.705 [INFO] TestAgent: Endpoints down +--- PASS: TestTokenDeleteCommand (1.06s) PASS -ok github.com/hashicorp/consul/command/acl/token/delete 1.080s +ok github.com/hashicorp/consul/command/acl/token/delete 1.130s === RUN TestTokenListCommand_noTabs === PAUSE TestTokenListCommand_noTabs === RUN TestTokenListCommand_Pretty @@ -77472,172 +77469,171 @@ === RUN TestTokenListCommand_JSON === PAUSE TestTokenListCommand_JSON === CONT TestTokenListCommand_noTabs ---- PASS: TestTokenListCommand_noTabs (0.00s) +=== CONT TestTokenListCommand_Pretty === CONT TestTokenListCommand_JSON [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... -=== CONT TestTokenListCommand_Pretty +--- PASS: TestTokenListCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:43:13.027Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:13.070Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:13.086 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8ca3d7f0-64f5-e595-058b-c34de1591744 Address:127.0.0.1:26342}]" -43:13.087 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8ca3d7f0-64f5-e595-058b-c34de1591744.dc1 127.0.0.1 -43:13.088 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8ca3d7f0-64f5-e595-058b-c34de1591744 127.0.0.1 -2022-09-15T21:43:13.088Z [INFO] agent.router: Initializing LAN area manager -43:13.089 [INFO] TestAgent: Started DNS server: address=127.0.0.1:26337 network=udp -43:13.089 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:26342 [Follower]" leader= -43:13.090 [INFO] TestAgent.server: Adding LAN server: server="Node-8ca3d7f0-64f5-e595-058b-c34de1591744 (Addr: tcp/127.0.0.1:26342) (DC: dc1)" -43:13.090 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8ca3d7f0-64f5-e595-058b-c34de1591744.dc1 area=wan -43:13.092 [INFO] TestAgent: Started DNS server: address=127.0.0.1:26337 network=tcp -43:13.094 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:26338 network=tcp -43:13.094 [INFO] TestAgent: started state syncer -43:13.097 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:26343 network=tcp -43:13.141 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -43:13.154 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:13.159 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:26342 [Candidate]" term=2 -43:13.159 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:13.159 [DEBUG] TestAgent.server.raft: vote granted: from=8ca3d7f0-64f5-e595-058b-c34de1591744 term=2 tally=1 -43:13.159 [INFO] TestAgent.server.raft: election won: tally=1 -43:13.159 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:26342 [Leader]" -43:13.159 [INFO] TestAgent.server: cluster leadership acquired -43:13.160 [INFO] TestAgent.server: initializing acls -43:13.160 [INFO] TestAgent.server: New leader elected: payload=Node-8ca3d7f0-64f5-e595-058b-c34de1591744 -43:13.161 [INFO] TestAgent.server: Created ACL 'global-management' policy -43:13.161 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -43:13.166 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -43:13.168 [INFO] TestAgent: Synced node info -43:13.168 [INFO] TestAgent.server: Created ACL anonymous token from configuration -43:13.168 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -43:13.168 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -43:13.169 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-8ca3d7f0-64f5-e595-058b-c34de1591744 -43:13.169 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-8ca3d7f0-64f5-e595-058b-c34de1591744.dc1 -43:13.170 [INFO] TestAgent.server: Updating LAN server: server="Node-8ca3d7f0-64f5-e595-058b-c34de1591744 (Addr: tcp/127.0.0.1:26342) (DC: dc1)" -43:13.170 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-8ca3d7f0-64f5-e595-058b-c34de1591744.dc1 area=wan -43:13.171 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:13.200 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:13.200 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:13.200 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:13.200 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:13.200 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:13.200 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8ca3d7f0-64f5-e595-058b-c34de1591744 -43:13.200 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8ca3d7f0-64f5-e595-058b-c34de1591744 -43:13.201 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8ca3d7f0-64f5-e595-058b-c34de1591744 -43:13.202 [INFO] TestAgent.server: federation state anti-entropy synced -43:13.368 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:37070 latency=2.3108ms -43:13.374 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:37070 latency=5.01722ms -43:13.376 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:37070 latency=954.06µs -43:13.377 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:37070 latency=827.76µs -43:13.379 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:37070 latency=748µs -43:13.381 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/tokens from=127.0.0.1:37072 latency=732.2µs -43:13.383 [INFO] TestAgent: Requesting shutdown -43:13.383 [INFO] TestAgent.server: shutting down server -43:13.383 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:13.383 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:13.383 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -43:13.383 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -43:13.383 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:13.383 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:13.383 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:13.383 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:13.383 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:13.383 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:13.383 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:13.383 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -43:13.384 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -43:13.384 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:13.398 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:13.399Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:13.399Z [INFO] agent.router.manager: shutting down -43:13.399 [INFO] TestAgent: consul server down -43:13.399 [INFO] TestAgent: shutdown complete -43:13.399 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:26337 network=tcp -43:13.399 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:26337 network=udp -43:13.399 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:26338 network=tcp -43:13.899 [INFO] TestAgent: Waiting for endpoints to shut down -43:13.899 [INFO] TestAgent: Endpoints down ---- PASS: TestTokenListCommand_JSON (0.99s) -2022-09-15T21:43:13.040Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:13.091Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:13.095 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9cce5e93-9691-9108-d148-6a4406bfc252 Address:127.0.0.1:26349}]" -43:13.096 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9cce5e93-9691-9108-d148-6a4406bfc252.dc1 127.0.0.1 -43:13.097 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9cce5e93-9691-9108-d148-6a4406bfc252 127.0.0.1 -2022-09-15T21:43:13.097Z [INFO] agent.router: Initializing LAN area manager -43:13.097 [INFO] TestAgent: Started DNS server: address=127.0.0.1:26344 network=udp -43:13.097 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:26349 [Follower]" leader= -43:13.098 [INFO] TestAgent: Started DNS server: address=127.0.0.1:26344 network=tcp -43:13.099 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:26345 network=tcp -43:13.099 [INFO] TestAgent: started state syncer -43:13.099 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:26350 network=tcp -43:13.099 [INFO] TestAgent.server: Adding LAN server: server="Node-9cce5e93-9691-9108-d148-6a4406bfc252 (Addr: tcp/127.0.0.1:26349) (DC: dc1)" -43:13.100 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9cce5e93-9691-9108-d148-6a4406bfc252.dc1 area=wan -43:13.150 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -43:13.154 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:13.154 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:26349 [Candidate]" term=2 -43:13.158 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:13.158 [DEBUG] TestAgent.server.raft: vote granted: from=9cce5e93-9691-9108-d148-6a4406bfc252 term=2 tally=1 -43:13.159 [INFO] TestAgent.server.raft: election won: tally=1 -43:13.159 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:26349 [Leader]" -43:13.159 [INFO] TestAgent.server: cluster leadership acquired -43:13.159 [INFO] TestAgent.server: initializing acls -43:13.160 [INFO] TestAgent.server: Created ACL 'global-management' policy -43:13.160 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -43:13.160 [INFO] TestAgent.server: New leader elected: payload=Node-9cce5e93-9691-9108-d148-6a4406bfc252 -43:13.161 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -43:13.162 [INFO] TestAgent.server: Created ACL anonymous token from configuration -43:13.162 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -43:13.162 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -43:13.163 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-9cce5e93-9691-9108-d148-6a4406bfc252 -43:13.163 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-9cce5e93-9691-9108-d148-6a4406bfc252.dc1 -43:13.163 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-9cce5e93-9691-9108-d148-6a4406bfc252.dc1 area=wan -43:13.164 [INFO] TestAgent.server: Updating LAN server: server="Node-9cce5e93-9691-9108-d148-6a4406bfc252 (Addr: tcp/127.0.0.1:26349) (DC: dc1)" -43:13.165 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:13.199 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:13.199 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:13.199 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:13.199 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:13.199 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:13.199 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9cce5e93-9691-9108-d148-6a4406bfc252 -43:13.199 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9cce5e93-9691-9108-d148-6a4406bfc252 -43:13.200 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9cce5e93-9691-9108-d148-6a4406bfc252 -43:13.201 [INFO] TestAgent.server: federation state anti-entropy synced -43:13.270 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:13.271 [INFO] TestAgent: Synced node info -43:13.271 [DEBUG] TestAgent: Node info in sync -43:13.396 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:46670 latency=1.66372ms -43:13.398 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:46670 latency=1.11116ms -43:13.412 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:46670 latency=8.92534ms -43:13.416 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:46670 latency=1.38802ms -43:13.419 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:46670 latency=1.28332ms -43:13.422 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/tokens from=127.0.0.1:46672 latency=534.18µs -43:13.423 [INFO] TestAgent: Requesting shutdown -43:13.423 [INFO] TestAgent.server: shutting down server -43:13.423 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:13.423 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:13.423 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:13.423 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:13.423 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -43:13.423 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -43:13.423 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:13.424 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:13.424 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:13.424 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:13.424 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:13.424 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:13.424 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -43:13.424 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -43:13.424 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:13.424Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:13.424Z [INFO] agent.router.manager: shutting down -43:13.424 [INFO] TestAgent: consul server down -43:13.424 [INFO] TestAgent: shutdown complete -43:13.424 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:26344 network=tcp -43:13.424 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:26344 network=udp -43:13.424 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:26345 network=tcp -43:13.926 [INFO] TestAgent: Waiting for endpoints to shut down -43:13.926 [INFO] TestAgent: Endpoints down ---- PASS: TestTokenListCommand_Pretty (1.02s) +2021-08-13T15:35:25.583Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:35:25.604Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +35:25.607 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9179aa4c-1e32-4a8e-5357-882132003e58 Address:127.0.0.1:19195}]" +35:25.607 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19195 [Follower]" leader= +35:25.607 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9179aa4c-1e32-4a8e-5357-882132003e58.dc1 127.0.0.1 +35:25.610 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9179aa4c-1e32-4a8e-5357-882132003e58 127.0.0.1 +2021-08-13T15:35:25.610Z [INFO] agent.router: Initializing LAN area manager +35:25.611 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=udp +35:25.611 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9179aa4c-1e32-4a8e-5357-882132003e58.dc1 area=wan +35:25.611 [INFO] TestAgent.server: Adding LAN server: server="Node-9179aa4c-1e32-4a8e-5357-882132003e58 (Addr: tcp/127.0.0.1:19195) (DC: dc1)" +35:25.612 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=tcp +35:25.613 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19191 network=tcp +35:25.613 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19196 network=tcp +35:25.613 [INFO] TestAgent: started state syncer +35:25.662 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +35:25.673 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +35:25.673 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19195 [Candidate]" term=2 +35:25.674 [DEBUG] TestAgent.server.raft: votes: needed=1 +35:25.674 [DEBUG] TestAgent.server.raft: vote granted: from=9179aa4c-1e32-4a8e-5357-882132003e58 term=2 tally=1 +35:25.674 [INFO] TestAgent.server.raft: election won: tally=1 +35:25.674 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19195 [Leader]" +35:25.674 [INFO] TestAgent.server: cluster leadership acquired +35:25.674 [INFO] TestAgent.server: New leader elected: payload=Node-9179aa4c-1e32-4a8e-5357-882132003e58 +35:25.674 [INFO] TestAgent.server: initializing acls +35:25.674 [INFO] TestAgent.server: Created ACL 'global-management' policy +35:25.674 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +35:25.675 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +35:25.675 [INFO] TestAgent.server: Created ACL anonymous token from configuration +35:25.675 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +35:25.675 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +35:25.676 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-9179aa4c-1e32-4a8e-5357-882132003e58 +35:25.676 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-9179aa4c-1e32-4a8e-5357-882132003e58.dc1 +35:25.676 [INFO] TestAgent.server: Updating LAN server: server="Node-9179aa4c-1e32-4a8e-5357-882132003e58 (Addr: tcp/127.0.0.1:19195) (DC: dc1)" +35:25.676 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-9179aa4c-1e32-4a8e-5357-882132003e58.dc1 area=wan +35:25.677 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +35:25.679 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +35:25.679 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +35:25.679 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +35:25.679 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +35:25.679 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +35:25.679 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9179aa4c-1e32-4a8e-5357-882132003e58 +35:25.680 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9179aa4c-1e32-4a8e-5357-882132003e58 +35:25.680 [INFO] TestAgent.server: federation state anti-entropy synced +35:25.681 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9179aa4c-1e32-4a8e-5357-882132003e58 +35:25.864 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:35442 latency=2.054265ms +35:25.866 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:35442 latency=818.862µs +35:25.868 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:35442 latency=764.602µs +35:25.869 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:35442 latency=686.182µs +35:25.870 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:35442 latency=712.161µs +35:25.876 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/tokens from=127.0.0.1:35444 latency=5.131571ms +35:25.880 [INFO] TestAgent: Requesting shutdown +35:25.880 [INFO] TestAgent.server: shutting down server +35:25.880 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +35:25.880 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +35:25.880 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +35:25.880 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +35:25.880 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +35:25.880 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +35:25.880 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +35:25.881 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +35:25.881 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +35:25.881 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +35:25.881 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +35:25.881 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +35:25.881 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +35:25.881 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +2021-08-13T15:35:25.881Z [INFO] agent.router.manager: shutting down +2021-08-13T15:35:25.881Z [INFO] agent.router.manager: shutting down +35:25.881 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +35:25.881 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +35:25.881 [INFO] TestAgent: consul server down +35:25.881 [INFO] TestAgent: shutdown complete +35:25.881 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=tcp +35:25.881 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=udp +35:25.881 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19191 network=tcp +35:26.381 [INFO] TestAgent: Waiting for endpoints to shut down +35:26.381 [INFO] TestAgent: Endpoints down +--- PASS: TestTokenListCommand_JSON (0.86s) +2021-08-13T15:35:25.581Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:35:25.601Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +35:25.604 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b70d8ae0-64be-ad86-18dd-180682c1fed1 Address:127.0.0.1:19202}]" +35:25.605 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19202 [Follower]" leader= +35:25.605 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b70d8ae0-64be-ad86-18dd-180682c1fed1.dc1 127.0.0.1 +35:25.606 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b70d8ae0-64be-ad86-18dd-180682c1fed1 127.0.0.1 +2021-08-13T15:35:25.606Z [INFO] agent.router: Initializing LAN area manager +35:25.606 [INFO] TestAgent.server: Adding LAN server: server="Node-b70d8ae0-64be-ad86-18dd-180682c1fed1 (Addr: tcp/127.0.0.1:19202) (DC: dc1)" +35:25.607 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b70d8ae0-64be-ad86-18dd-180682c1fed1.dc1 area=wan +35:25.607 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19197 network=udp +35:25.607 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19197 network=tcp +35:25.609 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19198 network=tcp +35:25.609 [INFO] TestAgent: started state syncer +35:25.609 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19203 network=tcp +35:25.650 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +35:25.650 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19202 [Candidate]" term=2 +35:25.651 [DEBUG] TestAgent.server.raft: votes: needed=1 +35:25.651 [DEBUG] TestAgent.server.raft: vote granted: from=b70d8ae0-64be-ad86-18dd-180682c1fed1 term=2 tally=1 +35:25.651 [INFO] TestAgent.server.raft: election won: tally=1 +35:25.651 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19202 [Leader]" +35:25.651 [INFO] TestAgent.server: cluster leadership acquired +35:25.651 [INFO] TestAgent.server: New leader elected: payload=Node-b70d8ae0-64be-ad86-18dd-180682c1fed1 +35:25.651 [INFO] TestAgent.server: initializing acls +35:25.652 [INFO] TestAgent.server: Created ACL 'global-management' policy +35:25.652 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +35:25.652 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +35:25.653 [INFO] TestAgent.server: Created ACL anonymous token from configuration +35:25.653 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +35:25.653 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +35:25.653 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-b70d8ae0-64be-ad86-18dd-180682c1fed1 +35:25.653 [INFO] TestAgent.server: Updating LAN server: server="Node-b70d8ae0-64be-ad86-18dd-180682c1fed1 (Addr: tcp/127.0.0.1:19202) (DC: dc1)" +35:25.653 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-b70d8ae0-64be-ad86-18dd-180682c1fed1.dc1 +35:25.654 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-b70d8ae0-64be-ad86-18dd-180682c1fed1.dc1 area=wan +35:25.655 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +35:25.677 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +35:25.677 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +35:25.677 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +35:25.677 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +35:25.677 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +35:25.678 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b70d8ae0-64be-ad86-18dd-180682c1fed1 +35:25.678 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b70d8ae0-64be-ad86-18dd-180682c1fed1 +35:25.678 [INFO] TestAgent.server: federation state anti-entropy synced +35:25.680 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b70d8ae0-64be-ad86-18dd-180682c1fed1 +35:25.923 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +35:25.924 [INFO] TestAgent: Synced node info +35:25.924 [DEBUG] TestAgent: Node info in sync +35:26.038 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:48764 latency=1.484443ms +35:26.042 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:48764 latency=863.522µs +35:26.043 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:48764 latency=685.901µs +35:26.045 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:48764 latency=641.521µs +35:26.046 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:48764 latency=655.201µs +35:26.047 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/tokens from=127.0.0.1:48766 latency=320.801µs +35:26.048 [INFO] TestAgent: Requesting shutdown +35:26.048 [INFO] TestAgent.server: shutting down server +35:26.048 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +35:26.048 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +35:26.048 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +35:26.048 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +35:26.048 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +35:26.048 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +35:26.048 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +35:26.048 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +35:26.048 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +35:26.048 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +35:26.048 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +35:26.048 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +35:26.048 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +35:26.048 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +35:26.049 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:35:26.049Z [INFO] agent.router.manager: shutting down +2021-08-13T15:35:26.049Z [INFO] agent.router.manager: shutting down +35:26.049 [INFO] TestAgent: consul server down +35:26.049 [INFO] TestAgent: shutdown complete +35:26.049 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19197 network=tcp +35:26.049 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19197 network=udp +35:26.049 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19198 network=tcp +35:26.549 [INFO] TestAgent: Waiting for endpoints to shut down +35:26.549 [INFO] TestAgent: Endpoints down +--- PASS: TestTokenListCommand_Pretty (1.03s) PASS -ok github.com/hashicorp/consul/command/acl/token/list 1.250s +ok github.com/hashicorp/consul/command/acl/token/list 1.092s === RUN TestTokenReadCommand_noTabs === PAUSE TestTokenReadCommand_noTabs === RUN TestTokenReadCommand_Pretty @@ -77646,169 +77642,161 @@ === PAUSE TestTokenReadCommand_JSON === CONT TestTokenReadCommand_noTabs === CONT TestTokenReadCommand_JSON +=== CONT TestTokenReadCommand_Pretty [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... --- PASS: TestTokenReadCommand_noTabs (0.00s) -=== CONT TestTokenReadCommand_Pretty [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:43:14.059Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:14.127Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:14.132 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:828b4df8-4bbc-b9e5-94d1-03a62edf6a86 Address:127.0.0.1:12048}]" -43:14.132 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:12048 [Follower]" leader= -43:14.139 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-828b4df8-4bbc-b9e5-94d1-03a62edf6a86.dc1 127.0.0.1 -43:14.140 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-828b4df8-4bbc-b9e5-94d1-03a62edf6a86 127.0.0.1 -2022-09-15T21:43:14.140Z [INFO] agent.router: Initializing LAN area manager -43:14.140 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-828b4df8-4bbc-b9e5-94d1-03a62edf6a86.dc1 area=wan -43:14.140 [INFO] TestAgent.server: Adding LAN server: server="Node-828b4df8-4bbc-b9e5-94d1-03a62edf6a86 (Addr: tcp/127.0.0.1:12048) (DC: dc1)" -43:14.140 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12043 network=udp -43:14.141 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12043 network=tcp -43:14.141 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:12044 network=tcp -43:14.142 [INFO] TestAgent: started state syncer -43:14.142 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:12049 network=tcp -43:14.182 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:14.182 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:12048 [Candidate]" term=2 -43:14.187 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:14.187 [DEBUG] TestAgent.server.raft: vote granted: from=828b4df8-4bbc-b9e5-94d1-03a62edf6a86 term=2 tally=1 -43:14.187 [INFO] TestAgent.server.raft: election won: tally=1 -43:14.187 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:12048 [Leader]" -43:14.194 [INFO] TestAgent.server: cluster leadership acquired -43:14.194 [INFO] TestAgent.server: initializing acls -43:14.195 [INFO] TestAgent.server: New leader elected: payload=Node-828b4df8-4bbc-b9e5-94d1-03a62edf6a86 -43:14.198 [INFO] TestAgent.server: initializing acls -43:14.198 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -43:14.198 [INFO] TestAgent.server: Created ACL 'global-management' policy -43:14.198 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -43:14.215 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -43:14.216 [INFO] TestAgent.server: Created ACL anonymous token from configuration -43:14.216 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -43:14.216 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -43:14.217 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-828b4df8-4bbc-b9e5-94d1-03a62edf6a86 -43:14.217 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-828b4df8-4bbc-b9e5-94d1-03a62edf6a86.dc1 -43:14.218 [INFO] TestAgent.server: Updating LAN server: server="Node-828b4df8-4bbc-b9e5-94d1-03a62edf6a86 (Addr: tcp/127.0.0.1:12048) (DC: dc1)" -43:14.218 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-828b4df8-4bbc-b9e5-94d1-03a62edf6a86.dc1 area=wan -43:14.219 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:14.222 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -43:14.222 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode -43:14.223 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-828b4df8-4bbc-b9e5-94d1-03a62edf6a86 -43:14.223 [INFO] TestAgent.server: Updating LAN server: server="Node-828b4df8-4bbc-b9e5-94d1-03a62edf6a86 (Addr: tcp/127.0.0.1:12048) (DC: dc1)" -43:14.223 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-828b4df8-4bbc-b9e5-94d1-03a62edf6a86.dc1 -43:14.223 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-828b4df8-4bbc-b9e5-94d1-03a62edf6a86.dc1 area=wan -43:14.273 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:14.273 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:14.273 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:14.273 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:14.273 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:14.273 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-828b4df8-4bbc-b9e5-94d1-03a62edf6a86 -43:14.273 [INFO] TestAgent.server: member joined, marking health alive: member=Node-828b4df8-4bbc-b9e5-94d1-03a62edf6a86 -43:14.274 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-828b4df8-4bbc-b9e5-94d1-03a62edf6a86 -43:14.274 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-828b4df8-4bbc-b9e5-94d1-03a62edf6a86 -43:14.274 [INFO] TestAgent.server: federation state anti-entropy synced -43:14.281 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:43824 latency=2.4808ms -43:14.284 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/4d966514-d6c6-05bf-9eb1-b57aa2797a4c from=127.0.0.1:43826 latency=525.96µs -43:14.285 [INFO] TestAgent: Requesting shutdown -43:14.285 [INFO] TestAgent.server: shutting down server -43:14.285 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:14.285 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:14.285 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:14.285 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:14.285 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -43:14.285 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -43:14.285 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:14.285 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:14.285 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:14.285 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:14.285 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -43:14.285 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:14.285 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:14.285 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:14.285 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -43:14.285 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -2022-09-15T21:43:14.285Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:14.285Z [INFO] agent.router.manager: shutting down -43:14.286 [INFO] TestAgent: consul server down -43:14.286 [INFO] TestAgent: shutdown complete -43:14.286 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12043 network=tcp -43:14.286 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12043 network=udp -43:14.286 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:12044 network=tcp -43:14.786 [INFO] TestAgent: Waiting for endpoints to shut down -43:14.786 [INFO] TestAgent: Endpoints down ---- PASS: TestTokenReadCommand_JSON (0.90s) -2022-09-15T21:43:14.053Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:14.121Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:14.132 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:47e97ab5-467a-4275-ea43-9ae5bfc2ba1a Address:127.0.0.1:12055}]" -43:14.132 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:12055 [Follower]" leader= -43:14.133 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-47e97ab5-467a-4275-ea43-9ae5bfc2ba1a.dc1 127.0.0.1 -43:14.134 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-47e97ab5-467a-4275-ea43-9ae5bfc2ba1a 127.0.0.1 -2022-09-15T21:43:14.134Z [INFO] agent.router: Initializing LAN area manager -43:14.135 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12050 network=udp -43:14.135 [INFO] TestAgent.server: Adding LAN server: server="Node-47e97ab5-467a-4275-ea43-9ae5bfc2ba1a (Addr: tcp/127.0.0.1:12055) (DC: dc1)" -43:14.135 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-47e97ab5-467a-4275-ea43-9ae5bfc2ba1a.dc1 area=wan -43:14.135 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12050 network=tcp -43:14.136 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:12051 network=tcp -43:14.137 [INFO] TestAgent: started state syncer -43:14.137 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:12056 network=tcp -43:14.194 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -43:14.201 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:14.201 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:12055 [Candidate]" term=2 -43:14.201 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:14.201 [DEBUG] TestAgent.server.raft: vote granted: from=47e97ab5-467a-4275-ea43-9ae5bfc2ba1a term=2 tally=1 -43:14.201 [INFO] TestAgent.server.raft: election won: tally=1 -43:14.201 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:12055 [Leader]" -43:14.206 [INFO] TestAgent.server: cluster leadership acquired -43:14.206 [INFO] TestAgent.server: New leader elected: payload=Node-47e97ab5-467a-4275-ea43-9ae5bfc2ba1a -43:14.215 [INFO] TestAgent.server: initializing acls -43:14.224 [INFO] TestAgent.server: Created ACL 'global-management' policy -43:14.224 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -43:14.225 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -43:14.226 [INFO] TestAgent.server: Created ACL anonymous token from configuration -43:14.226 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -43:14.226 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -43:14.227 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-47e97ab5-467a-4275-ea43-9ae5bfc2ba1a -43:14.227 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-47e97ab5-467a-4275-ea43-9ae5bfc2ba1a.dc1 -43:14.229 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:14.229 [INFO] TestAgent.server: Updating LAN server: server="Node-47e97ab5-467a-4275-ea43-9ae5bfc2ba1a (Addr: tcp/127.0.0.1:12055) (DC: dc1)" -43:14.229 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-47e97ab5-467a-4275-ea43-9ae5bfc2ba1a.dc1 area=wan -43:14.262 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:14.262 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:14.262 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:14.262 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:14.262 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:14.262 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-47e97ab5-467a-4275-ea43-9ae5bfc2ba1a -43:14.262 [INFO] TestAgent.server: member joined, marking health alive: member=Node-47e97ab5-467a-4275-ea43-9ae5bfc2ba1a -43:14.264 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-47e97ab5-467a-4275-ea43-9ae5bfc2ba1a -43:14.264 [INFO] TestAgent.server: federation state anti-entropy synced -43:14.498 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:54318 latency=9.92774ms -43:14.504 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/647df184-af8c-b767-ee95-5d23c2c59bff from=127.0.0.1:54320 latency=538.76µs -43:14.505 [INFO] TestAgent: Requesting shutdown -43:14.505 [INFO] TestAgent.server: shutting down server -43:14.505 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:14.505 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:14.505 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:14.505 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:14.505 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -43:14.505 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -43:14.505 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:14.505 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:14.505 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:14.505 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:14.505 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:14.505 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:14.505 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:14.505 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -43:14.505 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -43:14.505 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:14.505Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:14.505Z [INFO] agent.router.manager: shutting down -43:14.506 [INFO] TestAgent: consul server down -43:14.506 [INFO] TestAgent: shutdown complete -43:14.506 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12050 network=tcp -43:14.506 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12050 network=udp -43:14.506 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:12051 network=tcp -43:15.010 [INFO] TestAgent: Waiting for endpoints to shut down -43:15.010 [INFO] TestAgent: Endpoints down ---- PASS: TestTokenReadCommand_Pretty (1.13s) +2021-08-13T15:35:41.762Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:35:41.784Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +35:41.788 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8a36b1d5-4eed-156d-3054-2efde1dfea2f Address:127.0.0.1:23279}]" +35:41.789 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23279 [Follower]" leader= +35:41.790 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8a36b1d5-4eed-156d-3054-2efde1dfea2f.dc1 127.0.0.1 +35:41.796 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8a36b1d5-4eed-156d-3054-2efde1dfea2f 127.0.0.1 +2021-08-13T15:35:41.796Z [INFO] agent.router: Initializing LAN area manager +35:41.797 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23274 network=udp +35:41.806 [INFO] TestAgent.server: Adding LAN server: server="Node-8a36b1d5-4eed-156d-3054-2efde1dfea2f (Addr: tcp/127.0.0.1:23279) (DC: dc1)" +35:41.806 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8a36b1d5-4eed-156d-3054-2efde1dfea2f.dc1 area=wan +35:41.807 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23274 network=tcp +35:41.808 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23275 network=tcp +35:41.808 [INFO] TestAgent: started state syncer +35:41.808 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23280 network=tcp +35:41.847 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +35:41.847 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23279 [Candidate]" term=2 +35:41.847 [DEBUG] TestAgent.server.raft: votes: needed=1 +35:41.847 [DEBUG] TestAgent.server.raft: vote granted: from=8a36b1d5-4eed-156d-3054-2efde1dfea2f term=2 tally=1 +35:41.847 [INFO] TestAgent.server.raft: election won: tally=1 +35:41.847 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23279 [Leader]" +35:41.847 [INFO] TestAgent.server: cluster leadership acquired +35:41.848 [INFO] TestAgent.server: initializing acls +35:41.848 [INFO] TestAgent.server: Created ACL 'global-management' policy +35:41.848 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +35:41.849 [INFO] TestAgent.server: New leader elected: payload=Node-8a36b1d5-4eed-156d-3054-2efde1dfea2f +35:41.849 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +35:41.850 [INFO] TestAgent.server: Created ACL anonymous token from configuration +35:41.850 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +35:41.850 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +35:41.850 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-8a36b1d5-4eed-156d-3054-2efde1dfea2f +35:41.850 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-8a36b1d5-4eed-156d-3054-2efde1dfea2f.dc1 +35:41.850 [INFO] TestAgent.server: Updating LAN server: server="Node-8a36b1d5-4eed-156d-3054-2efde1dfea2f (Addr: tcp/127.0.0.1:23279) (DC: dc1)" +35:41.850 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-8a36b1d5-4eed-156d-3054-2efde1dfea2f.dc1 area=wan +35:41.853 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +35:41.876 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +35:41.876 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +35:41.876 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +35:41.876 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +35:41.876 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +35:41.876 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8a36b1d5-4eed-156d-3054-2efde1dfea2f +35:41.876 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8a36b1d5-4eed-156d-3054-2efde1dfea2f +35:41.877 [INFO] TestAgent.server: federation state anti-entropy synced +35:41.877 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8a36b1d5-4eed-156d-3054-2efde1dfea2f +35:41.954 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +35:41.955 [INFO] TestAgent: Synced node info +35:42.175 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:46384 latency=2.284885ms +35:42.177 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/f4474646-ad2f-a6f7-ea01-ddf1f6d95f69 from=127.0.0.1:46386 latency=291.801µs +35:42.177 [INFO] TestAgent: Requesting shutdown +35:42.177 [INFO] TestAgent.server: shutting down server +35:42.177 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +35:42.177 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +35:42.177 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +35:42.177 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +35:42.177 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +35:42.177 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +35:42.177 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +35:42.177 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +35:42.177 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +35:42.177 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +35:42.177 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +35:42.177 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +35:42.177 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +35:42.177 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +35:42.177 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:35:42.178Z [INFO] agent.router.manager: shutting down +2021-08-13T15:35:42.178Z [INFO] agent.router.manager: shutting down +35:42.178 [INFO] TestAgent: consul server down +35:42.178 [INFO] TestAgent: shutdown complete +35:42.178 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23274 network=tcp +35:42.178 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23274 network=udp +35:42.178 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23275 network=tcp +35:42.678 [INFO] TestAgent: Waiting for endpoints to shut down +35:42.678 [INFO] TestAgent: Endpoints down +--- PASS: TestTokenReadCommand_Pretty (0.98s) +2021-08-13T15:35:41.764Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:35:41.787Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +35:41.789 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ab4fd2e1-3a90-21be-6b92-a02ff8ae133f Address:127.0.0.1:23286}]" +35:41.789 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23286 [Follower]" leader= +35:41.792 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ab4fd2e1-3a90-21be-6b92-a02ff8ae133f.dc1 127.0.0.1 +35:41.794 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ab4fd2e1-3a90-21be-6b92-a02ff8ae133f 127.0.0.1 +2021-08-13T15:35:41.794Z [INFO] agent.router: Initializing LAN area manager +35:41.794 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23281 network=udp +35:41.795 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ab4fd2e1-3a90-21be-6b92-a02ff8ae133f.dc1 area=wan +35:41.795 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23281 network=tcp +35:41.797 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23282 network=tcp +35:41.797 [INFO] TestAgent: started state syncer +35:41.797 [INFO] TestAgent.server: Adding LAN server: server="Node-ab4fd2e1-3a90-21be-6b92-a02ff8ae133f (Addr: tcp/127.0.0.1:23286) (DC: dc1)" +35:41.807 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23287 network=tcp +35:41.847 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +35:41.849 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +35:41.849 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23286 [Candidate]" term=2 +35:41.850 [DEBUG] TestAgent.server.raft: votes: needed=1 +35:41.850 [DEBUG] TestAgent.server.raft: vote granted: from=ab4fd2e1-3a90-21be-6b92-a02ff8ae133f term=2 tally=1 +35:41.850 [INFO] TestAgent.server.raft: election won: tally=1 +35:41.850 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23286 [Leader]" +35:41.850 [INFO] TestAgent.server: cluster leadership acquired +35:41.850 [INFO] TestAgent.server: initializing acls +35:41.850 [INFO] TestAgent.server: New leader elected: payload=Node-ab4fd2e1-3a90-21be-6b92-a02ff8ae133f +35:41.851 [INFO] TestAgent.server: Created ACL 'global-management' policy +35:41.851 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +35:41.851 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +35:41.852 [INFO] TestAgent.server: Created ACL anonymous token from configuration +35:41.852 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +35:41.852 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +35:41.852 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-ab4fd2e1-3a90-21be-6b92-a02ff8ae133f +35:41.852 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-ab4fd2e1-3a90-21be-6b92-a02ff8ae133f.dc1 +35:41.852 [INFO] TestAgent.server: Updating LAN server: server="Node-ab4fd2e1-3a90-21be-6b92-a02ff8ae133f (Addr: tcp/127.0.0.1:23286) (DC: dc1)" +35:41.852 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-ab4fd2e1-3a90-21be-6b92-a02ff8ae133f.dc1 area=wan +35:41.853 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +35:41.876 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +35:41.876 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +35:41.876 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +35:41.876 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +35:41.876 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +35:41.876 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ab4fd2e1-3a90-21be-6b92-a02ff8ae133f +35:41.876 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ab4fd2e1-3a90-21be-6b92-a02ff8ae133f +35:41.877 [INFO] TestAgent.server: federation state anti-entropy synced +35:41.877 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ab4fd2e1-3a90-21be-6b92-a02ff8ae133f +35:42.184 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:60246 latency=3.448668ms +35:42.186 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/8cfca1e3-e77c-32fe-ec0a-f8ed3c918342 from=127.0.0.1:60248 latency=587.442µs +35:42.187 [INFO] TestAgent: Requesting shutdown +35:42.188 [INFO] TestAgent.server: shutting down server +35:42.188 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +35:42.188 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +35:42.188 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +35:42.188 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +35:42.188 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +35:42.188 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +35:42.188 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +35:42.188 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +35:42.188 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +35:42.188 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +35:42.188 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +35:42.188 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +35:42.188 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +35:42.188 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +35:42.188 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +35:42.188 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:35:42.188Z [INFO] agent.router.manager: shutting down +2021-08-13T15:35:42.188Z [INFO] agent.router.manager: shutting down +35:42.189 [INFO] TestAgent: consul server down +35:42.189 [INFO] TestAgent: shutdown complete +35:42.189 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23281 network=tcp +35:42.189 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23281 network=udp +35:42.189 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23282 network=tcp +35:42.689 [INFO] TestAgent: Waiting for endpoints to shut down +35:42.689 [INFO] TestAgent: Endpoints down +--- PASS: TestTokenReadCommand_JSON (0.99s) PASS -ok github.com/hashicorp/consul/command/acl/token/read 1.351s +ok github.com/hashicorp/consul/command/acl/token/read 1.055s === RUN TestTokenUpdateCommand_noTabs === PAUSE TestTokenUpdateCommand_noTabs === RUN TestTokenUpdateCommand @@ -77816,10 +77804,10 @@ === RUN TestTokenUpdateCommand_JSON === PAUSE TestTokenUpdateCommand_JSON === CONT TestTokenUpdateCommand_noTabs ---- PASS: TestTokenUpdateCommand_noTabs (0.00s) === CONT TestTokenUpdateCommand_JSON [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestTokenUpdateCommand +--- PASS: TestTokenUpdateCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range === RUN TestTokenUpdateCommand_JSON/update_with_policy_by_name @@ -77828,366 +77816,369 @@ === RUN TestTokenUpdateCommand/policy-name === RUN TestTokenUpdateCommand/policy-id === RUN TestTokenUpdateCommand/merge-description -2022-09-15T21:43:17.030Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:17.087Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:17.095 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ea80f597-0681-fd67-ff29-0331b2c2002b Address:127.0.0.1:21237}]" -43:17.106 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:21237 [Follower]" leader= -43:17.121 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ea80f597-0681-fd67-ff29-0331b2c2002b.dc1 127.0.0.1 -43:17.122 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ea80f597-0681-fd67-ff29-0331b2c2002b 127.0.0.1 -2022-09-15T21:43:17.122Z [INFO] agent.router: Initializing LAN area manager -43:17.123 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21232 network=udp -43:17.126 [INFO] TestAgent.server: Adding LAN server: server="Node-ea80f597-0681-fd67-ff29-0331b2c2002b (Addr: tcp/127.0.0.1:21237) (DC: dc1)" -43:17.126 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ea80f597-0681-fd67-ff29-0331b2c2002b.dc1 area=wan -43:17.127 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21232 network=tcp -43:17.128 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:21233 network=tcp -43:17.128 [INFO] TestAgent: started state syncer -43:17.128 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:21238 network=tcp -43:17.175 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:17.175 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:21237 [Candidate]" term=2 -43:17.175 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:17.175 [DEBUG] TestAgent.server.raft: vote granted: from=ea80f597-0681-fd67-ff29-0331b2c2002b term=2 tally=1 -43:17.175 [INFO] TestAgent.server.raft: election won: tally=1 -43:17.175 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:21237 [Leader]" -43:17.176 [INFO] TestAgent.server: cluster leadership acquired -43:17.176 [INFO] TestAgent.server: initializing acls -43:17.177 [INFO] TestAgent.server: Created ACL 'global-management' policy -43:17.177 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -43:17.177 [INFO] TestAgent.server: New leader elected: payload=Node-ea80f597-0681-fd67-ff29-0331b2c2002b -43:17.177 [INFO] TestAgent.server: initializing acls -43:17.177 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -43:17.180 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -43:17.180 [INFO] TestAgent.server: Created ACL anonymous token from configuration -43:17.180 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -43:17.180 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -43:17.180 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode -43:17.180 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-ea80f597-0681-fd67-ff29-0331b2c2002b -43:17.181 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-ea80f597-0681-fd67-ff29-0331b2c2002b.dc1 -43:17.181 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-ea80f597-0681-fd67-ff29-0331b2c2002b.dc1 area=wan -43:17.181 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -43:17.181 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-ea80f597-0681-fd67-ff29-0331b2c2002b -43:17.181 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-ea80f597-0681-fd67-ff29-0331b2c2002b.dc1 -43:17.182 [INFO] TestAgent.server: Updating LAN server: server="Node-ea80f597-0681-fd67-ff29-0331b2c2002b (Addr: tcp/127.0.0.1:21237) (DC: dc1)" -43:17.182 [INFO] TestAgent.server: Updating LAN server: server="Node-ea80f597-0681-fd67-ff29-0331b2c2002b (Addr: tcp/127.0.0.1:21237) (DC: dc1)" -43:17.182 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-ea80f597-0681-fd67-ff29-0331b2c2002b.dc1 area=wan -43:17.183 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:17.221 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:17.221 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:17.221 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:17.221 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:17.221 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:17.221 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ea80f597-0681-fd67-ff29-0331b2c2002b -43:17.221 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ea80f597-0681-fd67-ff29-0331b2c2002b -43:17.225 [INFO] TestAgent.server: federation state anti-entropy synced -43:17.225 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ea80f597-0681-fd67-ff29-0331b2c2002b -43:17.225 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ea80f597-0681-fd67-ff29-0331b2c2002b -43:17.243 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:57084 latency=2.60636ms -43:17.248 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:57084 latency=2.03122ms -43:17.252 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/b33f4fbd-c711-4ff0-e086-dbca71e1f771 from=127.0.0.1:57086 latency=473.76µs -43:17.254 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/b33f4fbd-c711-4ff0-e086-dbca71e1f771 from=127.0.0.1:57086 latency=991.38µs -43:17.254 [INFO] TestAgent: Requesting shutdown -43:17.254 [INFO] TestAgent.server: shutting down server -43:17.254 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -43:17.254 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:17.254 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:17.254 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:17.254 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:17.254 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -43:17.255 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:17.255 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:17.255 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:17.255 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:17.255 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -43:17.255 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:17.255 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:17.255 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:17.255 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -43:17.255 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:17.255Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:17.255Z [INFO] agent.router.manager: shutting down -43:17.255 [INFO] TestAgent: consul server down -43:17.255 [INFO] TestAgent: shutdown complete -43:17.255 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21232 network=tcp -43:17.255 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21232 network=udp -43:17.255 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:21233 network=tcp -43:17.756 [INFO] TestAgent: Waiting for endpoints to shut down -43:17.756 [INFO] TestAgent: Endpoints down ---- PASS: TestTokenUpdateCommand_JSON (0.81s) +2021-08-13T15:35:39.098Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:35:39.117Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +35:39.122 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d0c52aec-f820-4ca5-c1c3-a28d49edbc64 Address:127.0.0.1:16132}]" +35:39.122 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16132 [Follower]" leader= +35:39.124 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d0c52aec-f820-4ca5-c1c3-a28d49edbc64.dc1 127.0.0.1 +35:39.125 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d0c52aec-f820-4ca5-c1c3-a28d49edbc64 127.0.0.1 +2021-08-13T15:35:39.125Z [INFO] agent.router: Initializing LAN area manager +35:39.125 [INFO] TestAgent.server: Adding LAN server: server="Node-d0c52aec-f820-4ca5-c1c3-a28d49edbc64 (Addr: tcp/127.0.0.1:16132) (DC: dc1)" +35:39.125 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d0c52aec-f820-4ca5-c1c3-a28d49edbc64.dc1 area=wan +35:39.125 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16127 network=udp +35:39.125 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16127 network=tcp +35:39.126 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16128 network=tcp +35:39.126 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16133 network=tcp +35:39.126 [INFO] TestAgent: started state syncer +35:39.175 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +35:39.193 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +35:39.193 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16132 [Candidate]" term=2 +35:39.193 [DEBUG] TestAgent.server.raft: votes: needed=1 +35:39.193 [DEBUG] TestAgent.server.raft: vote granted: from=d0c52aec-f820-4ca5-c1c3-a28d49edbc64 term=2 tally=1 +35:39.193 [INFO] TestAgent.server.raft: election won: tally=1 +35:39.193 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16132 [Leader]" +35:39.193 [INFO] TestAgent.server: cluster leadership acquired +35:39.193 [INFO] TestAgent.server: New leader elected: payload=Node-d0c52aec-f820-4ca5-c1c3-a28d49edbc64 +35:39.193 [INFO] TestAgent.server: initializing acls +35:39.194 [INFO] TestAgent.server: Created ACL 'global-management' policy +35:39.194 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +35:39.194 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +35:39.195 [INFO] TestAgent.server: Created ACL anonymous token from configuration +35:39.195 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +35:39.195 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +35:39.195 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-d0c52aec-f820-4ca5-c1c3-a28d49edbc64 +35:39.195 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-d0c52aec-f820-4ca5-c1c3-a28d49edbc64.dc1 +35:39.195 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-d0c52aec-f820-4ca5-c1c3-a28d49edbc64.dc1 area=wan +35:39.196 [INFO] TestAgent.server: Updating LAN server: server="Node-d0c52aec-f820-4ca5-c1c3-a28d49edbc64 (Addr: tcp/127.0.0.1:16132) (DC: dc1)" +35:39.196 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +35:39.209 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +35:39.209 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +35:39.209 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +35:39.209 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +35:39.209 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +35:39.209 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d0c52aec-f820-4ca5-c1c3-a28d49edbc64 +35:39.209 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d0c52aec-f820-4ca5-c1c3-a28d49edbc64 +35:39.212 [INFO] TestAgent.server: federation state anti-entropy synced +35:39.212 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d0c52aec-f820-4ca5-c1c3-a28d49edbc64 +35:39.249 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:47094 latency=1.681703ms +35:39.252 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:47094 latency=1.264783ms +35:39.255 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/fdc29782-0e89-ab89-67a0-506fbe6c4f37 from=127.0.0.1:47098 latency=388.941µs +35:39.261 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/fdc29782-0e89-ab89-67a0-506fbe6c4f37 from=127.0.0.1:47098 latency=1.399064ms +35:39.262 [INFO] TestAgent: Requesting shutdown +35:39.262 [INFO] TestAgent.server: shutting down server +35:39.262 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +35:39.262 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +35:39.262 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +35:39.262 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +35:39.262 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +35:39.262 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +35:39.262 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +35:39.262 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +35:39.262 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +35:39.262 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +35:39.263 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +35:39.263 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +35:39.263 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +35:39.263 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +35:39.263 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +35:39.263 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:35:39.263Z [INFO] agent.router.manager: shutting down +2021-08-13T15:35:39.263Z [INFO] agent.router.manager: shutting down +35:39.263 [INFO] TestAgent: consul server down +35:39.263 [INFO] TestAgent: shutdown complete +35:39.263 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16127 network=tcp +35:39.263 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16127 network=udp +35:39.263 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16128 network=tcp +35:39.763 [INFO] TestAgent: Waiting for endpoints to shut down +35:39.763 [INFO] TestAgent: Endpoints down +--- PASS: TestTokenUpdateCommand_JSON (0.72s) --- PASS: TestTokenUpdateCommand_JSON/update_with_policy_by_name (0.01s) === RUN TestTokenUpdateCommand/legacy-upgrade -2022-09-15T21:43:17.063Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:17.117Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:17.124 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0c81365c-96be-c1e3-9df6-6fdc2abdb592 Address:127.0.0.1:21244}]" -43:17.128 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:21244 [Follower]" leader= -43:17.134 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0c81365c-96be-c1e3-9df6-6fdc2abdb592.dc1 127.0.0.1 -43:17.135 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0c81365c-96be-c1e3-9df6-6fdc2abdb592 127.0.0.1 -2022-09-15T21:43:17.135Z [INFO] agent.router: Initializing LAN area manager -43:17.135 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21239 network=udp -43:17.136 [INFO] TestAgent.server: Adding LAN server: server="Node-0c81365c-96be-c1e3-9df6-6fdc2abdb592 (Addr: tcp/127.0.0.1:21244) (DC: dc1)" -43:17.136 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0c81365c-96be-c1e3-9df6-6fdc2abdb592.dc1 area=wan -43:17.136 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21239 network=tcp -43:17.137 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:21240 network=tcp -43:17.137 [INFO] TestAgent: started state syncer -43:17.137 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:21245 network=tcp -43:17.182 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:17.182 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:21244 [Candidate]" term=2 -43:17.182 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:17.182 [DEBUG] TestAgent.server.raft: vote granted: from=0c81365c-96be-c1e3-9df6-6fdc2abdb592 term=2 tally=1 -43:17.182 [INFO] TestAgent.server.raft: election won: tally=1 -43:17.182 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:21244 [Leader]" -43:17.182 [INFO] TestAgent.server: cluster leadership acquired -43:17.183 [INFO] TestAgent.server: New leader elected: payload=Node-0c81365c-96be-c1e3-9df6-6fdc2abdb592 -43:17.183 [INFO] TestAgent.server: initializing acls -43:17.186 [INFO] TestAgent.server: Created ACL 'global-management' policy -43:17.186 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -43:17.187 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -43:17.187 [INFO] TestAgent.server: Created ACL anonymous token from configuration -43:17.187 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -43:17.187 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -43:17.188 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-0c81365c-96be-c1e3-9df6-6fdc2abdb592 -43:17.188 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-0c81365c-96be-c1e3-9df6-6fdc2abdb592.dc1 -43:17.190 [INFO] TestAgent.server: Updating LAN server: server="Node-0c81365c-96be-c1e3-9df6-6fdc2abdb592 (Addr: tcp/127.0.0.1:21244) (DC: dc1)" -43:17.190 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-0c81365c-96be-c1e3-9df6-6fdc2abdb592.dc1 area=wan -43:17.198 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:17.225 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:17.225 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:17.225 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:17.225 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:17.225 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:17.225 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0c81365c-96be-c1e3-9df6-6fdc2abdb592 -43:17.225 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0c81365c-96be-c1e3-9df6-6fdc2abdb592 -43:17.230 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0c81365c-96be-c1e3-9df6-6fdc2abdb592 -43:17.231 [INFO] TestAgent.server: federation state anti-entropy synced -43:17.506 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:17.507 [INFO] TestAgent: Synced node info -43:17.507 [DEBUG] TestAgent: Node info in sync -43:17.508 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:45870 latency=2.1551ms -43:17.509 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:45870 latency=1.00494ms -43:17.512 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/create from=127.0.0.1:45870 latency=1.00768ms -43:17.519 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/8ee1c076-29d6-6d48-94fd-36eef5c701a3 from=127.0.0.1:45872 latency=410.64µs -43:17.521 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/8ee1c076-29d6-6d48-94fd-36eef5c701a3 from=127.0.0.1:45872 latency=1.04706ms -43:17.525 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/8ee1c076-29d6-6d48-94fd-36eef5c701a3 from=127.0.0.1:45874 latency=470.16µs -43:17.527 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/8ee1c076-29d6-6d48-94fd-36eef5c701a3 from=127.0.0.1:45874 latency=1.09806ms -43:17.531 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/8ee1c076-29d6-6d48-94fd-36eef5c701a3 from=127.0.0.1:45876 latency=417.92µs -43:17.533 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/8ee1c076-29d6-6d48-94fd-36eef5c701a3 from=127.0.0.1:45876 latency=1.0256ms -43:17.537 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/8ee1c076-29d6-6d48-94fd-36eef5c701a3 from=127.0.0.1:45878 latency=373.02µs -43:17.539 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/8ee1c076-29d6-6d48-94fd-36eef5c701a3 from=127.0.0.1:45878 latency=1.013ms -43:17.552 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/8ee1c076-29d6-6d48-94fd-36eef5c701a3 from=127.0.0.1:45880 latency=4.11228ms -43:17.556 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/8ee1c076-29d6-6d48-94fd-36eef5c701a3 from=127.0.0.1:45880 latency=1.62182ms -43:17.562 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:45870 latency=390.44µs -43:17.594 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:45870 latency=370.44µs -43:17.622 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:45870 latency=457.92µs -43:17.650 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:45870 latency=319.2µs -43:17.677 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:45870 latency=233.22µs -43:17.706 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:45870 latency=254.92µs -43:17.735 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:45870 latency=420.56µs -43:17.762 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:45870 latency=242.74µs -43:17.794 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:45870 latency=255.92µs -43:17.823 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:45870 latency=326.82µs -43:17.855 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:45870 latency=268.34µs -43:17.883 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:45870 latency=353.72µs -43:17.911 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:45870 latency=318.44µs -43:17.942 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:45870 latency=259.12µs -43:17.983 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:45870 latency=254.26µs -43:18.012 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:45870 latency=249.62µs -43:18.041 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:45870 latency=241.94µs -43:18.068 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:45870 latency=255.46µs -43:18.094 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:45870 latency=248.7µs -43:18.122 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:45870 latency=252.48µs -43:18.154 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:45870 latency=277.88µs -43:18.180 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:45870 latency=259.3µs -43:18.207 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:45870 latency=248.44µs -43:18.213 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/ee1e6392-ca61-2262-04d7-b920a96432df from=127.0.0.1:45884 latency=394.28µs -43:18.215 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/ee1e6392-ca61-2262-04d7-b920a96432df from=127.0.0.1:45884 latency=1.23712ms -43:18.216 [INFO] TestAgent: Requesting shutdown -43:18.216 [INFO] TestAgent.server: shutting down server -43:18.216 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:18.216 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:18.216 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -43:18.216 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -43:18.216 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:18.216 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:18.216 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:18.217 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:18.217 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:18.217 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:18.217 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:18.217 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -43:18.217 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -43:18.217 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:18.217 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:18.217Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:18.217Z [INFO] agent.router.manager: shutting down -43:18.217 [INFO] TestAgent: consul server down -43:18.217 [INFO] TestAgent: shutdown complete -43:18.217 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21239 network=tcp -43:18.217 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21239 network=udp -43:18.217 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:21240 network=tcp -43:18.717 [INFO] TestAgent: Waiting for endpoints to shut down -43:18.717 [INFO] TestAgent: Endpoints down ---- PASS: TestTokenUpdateCommand (1.76s) +2021-08-13T15:35:39.099Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:35:39.120Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +35:39.123 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:67df601d-59d1-f9bb-c416-f7b9acb6520b Address:127.0.0.1:16139}]" +35:39.123 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16139 [Follower]" leader= +35:39.124 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-67df601d-59d1-f9bb-c416-f7b9acb6520b.dc1 127.0.0.1 +35:39.125 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-67df601d-59d1-f9bb-c416-f7b9acb6520b 127.0.0.1 +2021-08-13T15:35:39.125Z [INFO] agent.router: Initializing LAN area manager +35:39.125 [INFO] TestAgent.server: Adding LAN server: server="Node-67df601d-59d1-f9bb-c416-f7b9acb6520b (Addr: tcp/127.0.0.1:16139) (DC: dc1)" +35:39.125 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-67df601d-59d1-f9bb-c416-f7b9acb6520b.dc1 area=wan +35:39.125 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16134 network=tcp +35:39.140 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16134 network=udp +35:39.141 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16135 network=tcp +35:39.141 [INFO] TestAgent: started state syncer +35:39.141 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16140 network=tcp +35:39.175 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +35:39.179 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +35:39.179 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16139 [Candidate]" term=2 +35:39.179 [DEBUG] TestAgent.server.raft: votes: needed=1 +35:39.179 [DEBUG] TestAgent.server.raft: vote granted: from=67df601d-59d1-f9bb-c416-f7b9acb6520b term=2 tally=1 +35:39.179 [INFO] TestAgent.server.raft: election won: tally=1 +35:39.179 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16139 [Leader]" +35:39.180 [INFO] TestAgent.server: cluster leadership acquired +35:39.180 [INFO] TestAgent.server: New leader elected: payload=Node-67df601d-59d1-f9bb-c416-f7b9acb6520b +35:39.180 [INFO] TestAgent.server: initializing acls +35:39.181 [INFO] TestAgent.server: Created ACL 'global-management' policy +35:39.181 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +35:39.181 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +35:39.182 [INFO] TestAgent.server: Created ACL anonymous token from configuration +35:39.182 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +35:39.182 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +35:39.182 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-67df601d-59d1-f9bb-c416-f7b9acb6520b +35:39.183 [INFO] TestAgent.server: Updating LAN server: server="Node-67df601d-59d1-f9bb-c416-f7b9acb6520b (Addr: tcp/127.0.0.1:16139) (DC: dc1)" +35:39.183 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-67df601d-59d1-f9bb-c416-f7b9acb6520b.dc1 +35:39.183 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-67df601d-59d1-f9bb-c416-f7b9acb6520b.dc1 area=wan +35:39.184 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +35:39.196 [INFO] TestAgent: Synced node info +35:39.211 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +35:39.211 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +35:39.211 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +35:39.211 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +35:39.211 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +35:39.211 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-67df601d-59d1-f9bb-c416-f7b9acb6520b +35:39.211 [INFO] TestAgent.server: member joined, marking health alive: member=Node-67df601d-59d1-f9bb-c416-f7b9acb6520b +35:39.212 [INFO] TestAgent.server: federation state anti-entropy synced +35:39.212 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-67df601d-59d1-f9bb-c416-f7b9acb6520b +35:39.252 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:51144 latency=1.371863ms +35:39.254 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:51144 latency=1.007982ms +35:39.257 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/create from=127.0.0.1:51144 latency=2.529366ms +35:39.261 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/b567d7d6-1200-533d-2ebd-0c97dee874cd from=127.0.0.1:51148 latency=315.76µs +35:39.263 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/b567d7d6-1200-533d-2ebd-0c97dee874cd from=127.0.0.1:51148 latency=1.522604ms +35:39.267 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/b567d7d6-1200-533d-2ebd-0c97dee874cd from=127.0.0.1:51150 latency=306.221µs +35:39.268 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/b567d7d6-1200-533d-2ebd-0c97dee874cd from=127.0.0.1:51150 latency=877.602µs +35:39.271 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/b567d7d6-1200-533d-2ebd-0c97dee874cd from=127.0.0.1:51152 latency=346.521µs +35:39.273 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/b567d7d6-1200-533d-2ebd-0c97dee874cd from=127.0.0.1:51152 latency=955.522µs +35:39.276 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/b567d7d6-1200-533d-2ebd-0c97dee874cd from=127.0.0.1:51154 latency=361.081µs +35:39.278 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/b567d7d6-1200-533d-2ebd-0c97dee874cd from=127.0.0.1:51154 latency=900.943µs +35:39.280 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/b567d7d6-1200-533d-2ebd-0c97dee874cd from=127.0.0.1:51156 latency=365.901µs +35:39.282 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/b567d7d6-1200-533d-2ebd-0c97dee874cd from=127.0.0.1:51156 latency=903.862µs +35:39.283 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:51144 latency=140.46µs +35:39.309 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:51144 latency=244.781µs +35:39.336 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:51144 latency=222.82µs +35:39.363 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:51144 latency=327.521µs +35:39.390 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:51144 latency=294.86µs +35:39.416 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:51144 latency=217.681µs +35:39.443 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:51144 latency=344.681µs +35:39.470 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:51144 latency=222.66µs +35:39.496 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:51144 latency=253.581µs +35:39.523 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:51144 latency=244.261µs +35:39.550 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:51144 latency=306.481µs +35:39.576 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:51144 latency=221.92µs +35:39.603 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:51144 latency=251.78µs +35:39.629 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:51144 latency=224.14µs +35:39.656 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:51144 latency=229.12µs +35:39.683 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:51144 latency=218.04µs +35:39.709 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:51144 latency=216.641µs +35:39.736 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:51144 latency=226.04µs +35:39.762 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:51144 latency=218.62µs +35:39.789 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:51144 latency=225.901µs +35:39.816 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:51144 latency=262.84µs +35:39.842 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:51144 latency=231.581µs +35:39.869 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:51144 latency=223.04µs +35:39.896 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:51144 latency=215.001µs +35:39.922 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:51144 latency=253.481µs +35:39.949 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:51144 latency=273.98µs +35:39.976 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:51144 latency=230.321µs +35:40.003 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:51144 latency=231.54µs +35:40.029 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:51144 latency=224.48µs +35:40.056 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:51144 latency=249.9µs +35:40.083 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:51144 latency=230.4µs +35:40.109 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:51144 latency=223.48µs +35:40.136 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:51144 latency=211.701µs +35:40.162 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:51144 latency=227.3µs +35:40.189 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:51144 latency=201.501µs +35:40.192 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/a1deb7c6-ebb1-e292-42a1-fcd67f2c12bc from=127.0.0.1:51158 latency=353.601µs +35:40.193 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/a1deb7c6-ebb1-e292-42a1-fcd67f2c12bc from=127.0.0.1:51158 latency=969.682µs +35:40.194 [INFO] TestAgent: Requesting shutdown +35:40.194 [INFO] TestAgent.server: shutting down server +35:40.194 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +35:40.194 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +35:40.194 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +35:40.194 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +35:40.194 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +35:40.194 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +35:40.194 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +35:40.194 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +35:40.194 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +35:40.194 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +35:40.194 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +35:40.194 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +35:40.194 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +35:40.194 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +35:40.194 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:35:40.195Z [INFO] agent.router.manager: shutting down +2021-08-13T15:35:40.195Z [INFO] agent.router.manager: shutting down +35:40.195 [INFO] TestAgent: consul server down +35:40.195 [INFO] TestAgent: shutdown complete +35:40.195 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16134 network=tcp +35:40.195 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16134 network=udp +35:40.195 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16135 network=tcp +35:40.695 [INFO] TestAgent: Waiting for endpoints to shut down +35:40.695 [INFO] TestAgent: Endpoints down +--- PASS: TestTokenUpdateCommand (1.65s) --- PASS: TestTokenUpdateCommand/node-identity (0.01s) - --- PASS: TestTokenUpdateCommand/node-identity-merge (0.01s) - --- PASS: TestTokenUpdateCommand/policy-name (0.01s) - --- PASS: TestTokenUpdateCommand/policy-id (0.01s) - --- PASS: TestTokenUpdateCommand/merge-description (0.02s) - --- PASS: TestTokenUpdateCommand/legacy-upgrade (0.01s) + --- PASS: TestTokenUpdateCommand/node-identity-merge (0.00s) + --- PASS: TestTokenUpdateCommand/policy-name (0.00s) + --- PASS: TestTokenUpdateCommand/policy-id (0.00s) + --- PASS: TestTokenUpdateCommand/merge-description (0.00s) + --- PASS: TestTokenUpdateCommand/legacy-upgrade (0.00s) PASS -ok github.com/hashicorp/consul/command/acl/token/update 1.927s +ok github.com/hashicorp/consul/command/acl/token/update 1.720s === RUN TestConfigFail === PAUSE TestConfigFail === RUN TestRetryJoin [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:43:28.490Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:28.519Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:28.523 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:713d91f2-0826-846c-befd-35d201e181a4 Address:127.0.0.1:19202}]" -43:28.524 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19202 [Follower]" leader= -43:28.525 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-713d91f2-0826-846c-befd-35d201e181a4.dc1 127.0.0.1 -43:28.527 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-713d91f2-0826-846c-befd-35d201e181a4 127.0.0.1 -2022-09-15T21:43:28.527Z [INFO] agent.router: Initializing LAN area manager -43:28.527 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19197 network=udp -43:28.535 [INFO] TestAgent.server: Adding LAN server: server="Node-713d91f2-0826-846c-befd-35d201e181a4 (Addr: tcp/127.0.0.1:19202) (DC: dc1)" -43:28.535 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-713d91f2-0826-846c-befd-35d201e181a4.dc1 area=wan -43:28.535 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19197 network=tcp -43:28.536 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19198 network=tcp -43:28.536 [INFO] TestAgent: started state syncer -43:28.536 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19203 network=tcp -43:28.536 [INFO] TestAgent: Retry join is supported for the following discovery methods: cluster=LAN discovery_methods="aliyun aws gce mdns os packet" -43:28.536 [INFO] TestAgent: Joining cluster...: cluster=LAN -43:28.536 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:19193] -43:28.537 [INFO] TestAgent: Retry join is supported for the following discovery methods: cluster=WAN discovery_methods="aliyun aws gce mdns os packet" -43:28.537 [INFO] TestAgent: Joining cluster...: cluster=WAN -43:28.537 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:19194] -43:28.537 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:19193 -43:28.537 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:19194 -43:28.538 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-058829c4-8d92-08e1-ca88-096e613a5a95.dc1 127.0.0.1 -43:28.538 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 -43:28.539 [INFO] TestAgent: Join cluster completed. Synced with initial agents: cluster=WAN num_agents=1 -43:28.539 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-058829c4-8d92-08e1-ca88-096e613a5a95.dc1 area=wan -43:28.539 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-058829c4-8d92-08e1-ca88-096e613a5a95 127.0.0.1 -43:28.539 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 -43:28.539 [DEBUG] TestAgent: systemd notify failed: error="No socket" -43:28.539 [INFO] TestAgent: Join cluster completed. Synced with initial agents: cluster=LAN num_agents=1 -43:28.539 [INFO] TestAgent.server: Adding LAN server: server="Node-058829c4-8d92-08e1-ca88-096e613a5a95 (Addr: tcp/127.0.0.1:19195) (DC: dc1)" -43:28.562 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:28.562 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19202 [Candidate]" term=2 -43:28.563 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:28.563 [DEBUG] TestAgent.server.raft: vote granted: from=713d91f2-0826-846c-befd-35d201e181a4 term=2 tally=1 -43:28.563 [INFO] TestAgent.server.raft: election won: tally=1 -43:28.563 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19202 [Leader]" -43:28.563 [INFO] TestAgent.server: cluster leadership acquired -43:28.563 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:19202 -43:28.565 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:28.567 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:28.567 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:28.567 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:28.567 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:28.567 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:28.567 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-713d91f2-0826-846c-befd-35d201e181a4 other=Node-058829c4-8d92-08e1-ca88-096e613a5a95 -43:28.568 [INFO] TestAgent.server: member joined, marking health alive: member=Node-713d91f2-0826-846c-befd-35d201e181a4 -43:28.568 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-058829c4-8d92-08e1-ca88-096e613a5a95 other=Node-713d91f2-0826-846c-befd-35d201e181a4 -43:28.568 [INFO] TestAgent.server: member joined, marking health alive: member=Node-058829c4-8d92-08e1-ca88-096e613a5a95 -43:28.569 [INFO] TestAgent.server: New leader elected: payload=Node-713d91f2-0826-846c-befd-35d201e181a4 -43:28.574 [INFO] TestAgent.server: federation state anti-entropy synced -43:28.707 [DEBUG] TestAgent.server.serf.lan: serf: messageUserEventType: consul:new-leader -43:28.850 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:28.851 [INFO] TestAgent: Synced node info -43:28.851 [DEBUG] TestAgent: Node info in sync -43:28.905 [DEBUG] TestAgent.server.serf.lan: serf: messageUserEventType: consul:new-leader -43:28.905 [DEBUG] TestAgent.server.serf.lan: serf: messageUserEventType: consul:new-leader -43:28.905 [DEBUG] TestAgent.server.serf.lan: serf: messageJoinType: Node-713d91f2-0826-846c-befd-35d201e181a4 -43:28.915 [INFO] TestAgent: Requesting shutdown -43:28.916 [INFO] TestAgent.server: shutting down server -43:28.916 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:28.916 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:28.916 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:28.916 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:28.916 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:28.916 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:28.916 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:28.916 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:28.916 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:28.916 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:28.916 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:28.916Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:28.916Z [INFO] agent.router.manager: shutting down -43:28.916 [INFO] TestAgent: consul server down -43:28.916 [INFO] TestAgent: shutdown complete -43:28.916 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19197 network=tcp -43:28.916 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19197 network=udp -43:28.917 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19198 network=tcp -43:29.417 [INFO] TestAgent: Waiting for endpoints to shut down -43:29.417 [INFO] TestAgent: Endpoints down -2022-09-15T21:43:28.072Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:28.097Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:28.101 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:058829c4-8d92-08e1-ca88-096e613a5a95 Address:127.0.0.1:19195}]" -43:28.103 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-058829c4-8d92-08e1-ca88-096e613a5a95.dc1 127.0.0.1 -43:28.104 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-058829c4-8d92-08e1-ca88-096e613a5a95 127.0.0.1 -2022-09-15T21:43:28.104Z [INFO] agent.router: Initializing LAN area manager -43:28.105 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=udp -43:28.105 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19195 [Follower]" leader= -43:28.106 [INFO] TestAgent.server: Adding LAN server: server="Node-058829c4-8d92-08e1-ca88-096e613a5a95 (Addr: tcp/127.0.0.1:19195) (DC: dc1)" -43:28.106 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-058829c4-8d92-08e1-ca88-096e613a5a95.dc1 area=wan -43:28.106 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=tcp -43:28.107 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19191 network=tcp -43:28.107 [INFO] TestAgent: started state syncer -43:28.108 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19196 network=tcp -43:28.171 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:28.171 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19195 [Candidate]" term=2 -43:28.171 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:28.171 [DEBUG] TestAgent.server.raft: vote granted: from=058829c4-8d92-08e1-ca88-096e613a5a95 term=2 tally=1 -43:28.171 [INFO] TestAgent.server.raft: election won: tally=1 -43:28.171 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19195 [Leader]" -43:28.171 [INFO] TestAgent.server: cluster leadership acquired -43:28.172 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:19195 -43:28.182 [INFO] TestAgent.server: New leader elected: payload=Node-058829c4-8d92-08e1-ca88-096e613a5a95 -43:28.184 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:28.219 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:28.219 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:28.219 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:28.219 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:28.219 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:28.219 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-058829c4-8d92-08e1-ca88-096e613a5a95 -43:28.219 [INFO] TestAgent.server: member joined, marking health alive: member=Node-058829c4-8d92-08e1-ca88-096e613a5a95 -43:28.220 [INFO] TestAgent.server: federation state anti-entropy synced -43:28.366 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:28.367 [INFO] TestAgent: Synced node info -43:28.537 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:36688 -43:28.537 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:53384 -43:28.538 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-713d91f2-0826-846c-befd-35d201e181a4.dc1 127.0.0.1 -43:28.538 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-713d91f2-0826-846c-befd-35d201e181a4 127.0.0.1 -43:28.539 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-713d91f2-0826-846c-befd-35d201e181a4.dc1 area=wan -43:28.539 [INFO] TestAgent.server: Adding LAN server: server="Node-713d91f2-0826-846c-befd-35d201e181a4 (Addr: tcp/127.0.0.1:19202) (DC: dc1)" -43:28.539 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-713d91f2-0826-846c-befd-35d201e181a4 other=Node-058829c4-8d92-08e1-ca88-096e613a5a95 -43:28.539 [INFO] TestAgent.server: member joined, marking health alive: member=Node-713d91f2-0826-846c-befd-35d201e181a4 -43:28.728 [DEBUG] TestAgent.server.serf.lan: serf: messageUserEventType: consul:new-leader -43:28.728 [DEBUG] TestAgent.server.serf.lan: serf: messageJoinType: Node-713d91f2-0826-846c-befd-35d201e181a4 -43:28.728 [INFO] TestAgent.server: New leader elected: payload=Node-713d91f2-0826-846c-befd-35d201e181a4 -43:29.417 [INFO] TestAgent: Requesting shutdown -43:29.417 [INFO] TestAgent.server: shutting down server -43:29.417 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:29.417 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:29.417 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:29.417 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:29.417 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:29.417 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:29.417 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:29.417 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:29.417 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:29.417 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:29.417 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:29.417Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:29.418Z [INFO] agent.router.manager: shutting down -43:29.418 [INFO] TestAgent: consul server down -43:29.418 [INFO] TestAgent: shutdown complete -43:29.418 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=tcp -43:29.418 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=udp -43:29.418 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19191 network=tcp -43:29.605 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Failed ping: Node-713d91f2-0826-846c-befd-35d201e181a4 (timeout reached) -43:29.919 [INFO] TestAgent: Waiting for endpoints to shut down -43:29.919 [INFO] TestAgent: Endpoints down ---- PASS: TestRetryJoin (1.91s) +2021-08-13T15:35:42.334Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:35:42.353Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +35:42.356 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d1b15d67-778c-ee51-c8e7-ed141bf5cc9b Address:127.0.0.1:22265}]" +35:42.356 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22265 [Follower]" leader= +35:42.357 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d1b15d67-778c-ee51-c8e7-ed141bf5cc9b.dc1 127.0.0.1 +35:42.358 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d1b15d67-778c-ee51-c8e7-ed141bf5cc9b 127.0.0.1 +2021-08-13T15:35:42.358Z [INFO] agent.router: Initializing LAN area manager +35:42.358 [INFO] TestAgent.server: Adding LAN server: server="Node-d1b15d67-778c-ee51-c8e7-ed141bf5cc9b (Addr: tcp/127.0.0.1:22265) (DC: dc1)" +35:42.358 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d1b15d67-778c-ee51-c8e7-ed141bf5cc9b.dc1 area=wan +35:42.358 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22260 network=udp +35:42.358 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22260 network=tcp +35:42.359 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22261 network=tcp +35:42.359 [INFO] TestAgent: started state syncer +35:42.359 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22266 network=tcp +35:42.359 [INFO] TestAgent: Retry join is supported for the following discovery methods: cluster=LAN discovery_methods="aliyun aws gce mdns os packet" +35:42.359 [INFO] TestAgent: Joining cluster...: cluster=LAN +35:42.359 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:22256] +35:42.359 [INFO] TestAgent: Retry join is supported for the following discovery methods: cluster=WAN discovery_methods="aliyun aws gce mdns os packet" +35:42.360 [INFO] TestAgent: Joining cluster...: cluster=WAN +35:42.360 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:22257] +35:42.360 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:22256 +35:42.360 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:22257 +35:42.361 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d22ff358-30a8-3441-51b9-02c46180db2a 127.0.0.1 +35:42.361 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d22ff358-30a8-3441-51b9-02c46180db2a.dc1 127.0.0.1 +35:42.361 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 +35:42.361 [DEBUG] TestAgent: systemd notify failed: error="No socket" +35:42.361 [INFO] TestAgent: Join cluster completed. Synced with initial agents: cluster=LAN num_agents=1 +35:42.361 [INFO] TestAgent.server: Adding LAN server: server="Node-d22ff358-30a8-3441-51b9-02c46180db2a (Addr: tcp/127.0.0.1:22258) (DC: dc1)" +35:42.361 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 +35:42.361 [INFO] TestAgent: Join cluster completed. Synced with initial agents: cluster=WAN num_agents=1 +35:42.361 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d22ff358-30a8-3441-51b9-02c46180db2a.dc1 area=wan +35:42.370 [DEBUG] TestAgent.server.serf.lan: serf: messageUserEventType: consul:new-leader +35:42.394 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +35:42.394 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22265 [Candidate]" term=2 +35:42.394 [DEBUG] TestAgent.server.raft: votes: needed=1 +35:42.394 [DEBUG] TestAgent.server.raft: vote granted: from=d1b15d67-778c-ee51-c8e7-ed141bf5cc9b term=2 tally=1 +35:42.394 [INFO] TestAgent.server.raft: election won: tally=1 +35:42.394 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22265 [Leader]" +35:42.394 [INFO] TestAgent.server: cluster leadership acquired +35:42.394 [INFO] TestAgent.server: New leader elected: payload=Node-d1b15d67-778c-ee51-c8e7-ed141bf5cc9b +35:42.394 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22265 +35:42.395 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +35:42.398 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +35:42.398 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +35:42.398 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +35:42.398 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +35:42.398 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +35:42.398 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-d22ff358-30a8-3441-51b9-02c46180db2a other=Node-d1b15d67-778c-ee51-c8e7-ed141bf5cc9b +35:42.398 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d22ff358-30a8-3441-51b9-02c46180db2a +35:42.399 [INFO] TestAgent.server: federation state anti-entropy synced +35:42.399 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-d1b15d67-778c-ee51-c8e7-ed141bf5cc9b other=Node-d22ff358-30a8-3441-51b9-02c46180db2a +35:42.399 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d1b15d67-778c-ee51-c8e7-ed141bf5cc9b +35:42.571 [DEBUG] TestAgent.server.serf.lan: serf: messageUserEventType: consul:new-leader +35:42.571 [DEBUG] TestAgent.server.serf.lan: serf: messageUserEventType: consul:new-leader +35:42.571 [DEBUG] TestAgent.server.serf.lan: serf: messageJoinType: Node-d1b15d67-778c-ee51-c8e7-ed141bf5cc9b +35:42.640 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +35:42.641 [INFO] TestAgent: Synced node info +35:42.641 [DEBUG] TestAgent: Node info in sync +35:42.734 [INFO] TestAgent: Requesting shutdown +35:42.734 [INFO] TestAgent.server: shutting down server +35:42.734 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +35:42.734 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +35:42.734 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +35:42.734 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +35:42.734 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +35:42.734 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +35:42.734 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +35:42.734 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +35:42.734 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +35:42.734 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +35:42.734 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:35:42.734Z [INFO] agent.router.manager: shutting down +2021-08-13T15:35:42.734Z [INFO] agent.router.manager: shutting down +35:42.734 [INFO] TestAgent: consul server down +35:42.734 [INFO] TestAgent: shutdown complete +35:42.734 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22260 network=tcp +35:42.734 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22260 network=udp +35:42.734 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22261 network=tcp +35:43.234 [INFO] TestAgent: Waiting for endpoints to shut down +35:43.235 [INFO] TestAgent: Endpoints down +2021-08-13T15:35:42.148Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:35:42.163Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +35:42.167 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d22ff358-30a8-3441-51b9-02c46180db2a Address:127.0.0.1:22258}]" +35:42.167 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22258 [Follower]" leader= +35:42.168 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d22ff358-30a8-3441-51b9-02c46180db2a.dc1 127.0.0.1 +35:42.169 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d22ff358-30a8-3441-51b9-02c46180db2a 127.0.0.1 +2021-08-13T15:35:42.169Z [INFO] agent.router: Initializing LAN area manager +35:42.169 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22253 network=udp +35:42.170 [INFO] TestAgent.server: Adding LAN server: server="Node-d22ff358-30a8-3441-51b9-02c46180db2a (Addr: tcp/127.0.0.1:22258) (DC: dc1)" +35:42.170 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d22ff358-30a8-3441-51b9-02c46180db2a.dc1 area=wan +35:42.170 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22253 network=tcp +35:42.171 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22254 network=tcp +35:42.171 [INFO] TestAgent: started state syncer +35:42.171 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22259 network=tcp +35:42.221 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +35:42.221 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22258 [Candidate]" term=2 +35:42.221 [DEBUG] TestAgent.server.raft: votes: needed=1 +35:42.221 [DEBUG] TestAgent.server.raft: vote granted: from=d22ff358-30a8-3441-51b9-02c46180db2a term=2 tally=1 +35:42.221 [INFO] TestAgent.server.raft: election won: tally=1 +35:42.221 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22258 [Leader]" +35:42.221 [INFO] TestAgent.server: cluster leadership acquired +35:42.221 [INFO] TestAgent.server: New leader elected: payload=Node-d22ff358-30a8-3441-51b9-02c46180db2a +35:42.222 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22258 +35:42.227 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +35:42.237 [INFO] TestAgent: Synced node info +35:42.237 [DEBUG] TestAgent: Node info in sync +35:42.250 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +35:42.250 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +35:42.250 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +35:42.250 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +35:42.250 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +35:42.250 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d22ff358-30a8-3441-51b9-02c46180db2a +35:42.250 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d22ff358-30a8-3441-51b9-02c46180db2a +35:42.251 [INFO] TestAgent.server: federation state anti-entropy synced +35:42.360 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:34208 +35:42.360 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:42770 +35:42.361 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d1b15d67-778c-ee51-c8e7-ed141bf5cc9b 127.0.0.1 +35:42.361 [INFO] TestAgent.server: Adding LAN server: server="Node-d1b15d67-778c-ee51-c8e7-ed141bf5cc9b (Addr: tcp/127.0.0.1:22265) (DC: dc1)" +35:42.361 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-d1b15d67-778c-ee51-c8e7-ed141bf5cc9b other=Node-d22ff358-30a8-3441-51b9-02c46180db2a +35:42.361 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d1b15d67-778c-ee51-c8e7-ed141bf5cc9b +35:42.361 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d1b15d67-778c-ee51-c8e7-ed141bf5cc9b.dc1 127.0.0.1 +35:42.362 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d1b15d67-778c-ee51-c8e7-ed141bf5cc9b.dc1 area=wan +35:42.559 [DEBUG] TestAgent.server.serf.lan: serf: messageUserEventType: consul:new-leader +35:42.559 [DEBUG] TestAgent.server.serf.lan: serf: messageJoinType: Node-d1b15d67-778c-ee51-c8e7-ed141bf5cc9b +35:42.559 [INFO] TestAgent.server: New leader elected: payload=Node-d1b15d67-778c-ee51-c8e7-ed141bf5cc9b +35:43.235 [INFO] TestAgent: Requesting shutdown +35:43.235 [INFO] TestAgent.server: shutting down server +35:43.235 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +35:43.235 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +35:43.235 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +35:43.235 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +35:43.235 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +35:43.235 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +35:43.235 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +35:43.235 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +35:43.235 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +35:43.235 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +35:43.235 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:35:43.235Z [INFO] agent.router.manager: shutting down +2021-08-13T15:35:43.235Z [INFO] agent.router.manager: shutting down +35:43.235 [INFO] TestAgent: consul server down +35:43.235 [INFO] TestAgent: shutdown complete +35:43.235 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22253 network=tcp +35:43.235 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22253 network=udp +35:43.236 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22254 network=tcp +35:43.669 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Failed ping: Node-d1b15d67-778c-ee51-c8e7-ed141bf5cc9b (timeout reached) +35:43.736 [INFO] TestAgent: Waiting for endpoints to shut down +35:43.736 [INFO] TestAgent: Endpoints down +--- PASS: TestRetryJoin (1.64s) === RUN TestRetryJoinFail === PAUSE TestRetryJoinFail === RUN TestRetryJoinWanFail @@ -78198,40 +78189,40 @@ === PAUSE TestBadDataDirPermissions === CONT TestConfigFail === RUN TestConfigFail/agent_-server_-bind=10.0.0.1_-datacenter= +=== CONT TestProtectDataDir === CONT TestRetryJoinWanFail === CONT TestRetryJoinFail -=== CONT TestProtectDataDir === CONT TestBadDataDirPermissions ---- PASS: TestBadDataDirPermissions (0.04s) +--- PASS: TestBadDataDirPermissions (0.05s) --- PASS: TestProtectDataDir (0.06s) ---- PASS: TestRetryJoinWanFail (0.10s) +--- PASS: TestRetryJoinFail (0.08s) === RUN TestConfigFail/agent_-server_-bind=10.0.0.1_-datacenter=foo_some-other-arg === RUN TestConfigFail/agent_-server_-bind=10.0.0.1 -=== RUN TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul684415962_-advertise_0.0.0.0_-bind_10.0.0.1 -=== RUN TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul684415962_-advertise_::_-bind_10.0.0.1 ---- PASS: TestRetryJoinFail (0.61s) -=== RUN TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul684415962_-advertise_[::]_-bind_10.0.0.1 -=== RUN TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul684415962_-advertise-wan_0.0.0.0_-bind_10.0.0.1 -=== RUN TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul684415962_-advertise-wan_::_-bind_10.0.0.1 -=== RUN TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul684415962_-advertise-wan_[::]_-bind_10.0.0.1 ---- PASS: TestConfigFail (1.11s) - --- PASS: TestConfigFail/agent_-server_-bind=10.0.0.1_-datacenter= (0.18s) - --- PASS: TestConfigFail/agent_-server_-bind=10.0.0.1_-datacenter=foo_some-other-arg (0.11s) +=== RUN TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul933988338_-advertise_0.0.0.0_-bind_10.0.0.1 +=== RUN TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul933988338_-advertise_::_-bind_10.0.0.1 +=== RUN TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul933988338_-advertise_[::]_-bind_10.0.0.1 +=== RUN TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul933988338_-advertise-wan_0.0.0.0_-bind_10.0.0.1 +--- PASS: TestRetryJoinWanFail (0.59s) +=== RUN TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul933988338_-advertise-wan_::_-bind_10.0.0.1 +=== RUN TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul933988338_-advertise-wan_[::]_-bind_10.0.0.1 +--- PASS: TestConfigFail (0.86s) + --- PASS: TestConfigFail/agent_-server_-bind=10.0.0.1_-datacenter= (0.17s) + --- PASS: TestConfigFail/agent_-server_-bind=10.0.0.1_-datacenter=foo_some-other-arg (0.07s) --- PASS: TestConfigFail/agent_-server_-bind=10.0.0.1 (0.10s) - --- PASS: TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul684415962_-advertise_0.0.0.0_-bind_10.0.0.1 (0.13s) - --- PASS: TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul684415962_-advertise_::_-bind_10.0.0.1 (0.11s) - --- PASS: TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul684415962_-advertise_[::]_-bind_10.0.0.1 (0.10s) - --- PASS: TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul684415962_-advertise-wan_0.0.0.0_-bind_10.0.0.1 (0.10s) - --- PASS: TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul684415962_-advertise-wan_::_-bind_10.0.0.1 (0.12s) - --- PASS: TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul684415962_-advertise-wan_[::]_-bind_10.0.0.1 (0.17s) + --- PASS: TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul933988338_-advertise_0.0.0.0_-bind_10.0.0.1 (0.08s) + --- PASS: TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul933988338_-advertise_::_-bind_10.0.0.1 (0.08s) + --- PASS: TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul933988338_-advertise_[::]_-bind_10.0.0.1 (0.08s) + --- PASS: TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul933988338_-advertise-wan_0.0.0.0_-bind_10.0.0.1 (0.08s) + --- PASS: TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul933988338_-advertise-wan_::_-bind_10.0.0.1 (0.09s) + --- PASS: TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul933988338_-advertise-wan_[::]_-bind_10.0.0.1 (0.11s) PASS -ok github.com/hashicorp/consul/command/agent 3.172s +ok github.com/hashicorp/consul/command/agent 2.564s === RUN TestCatalogCommand_noTabs === PAUSE TestCatalogCommand_noTabs === CONT TestCatalogCommand_noTabs --- PASS: TestCatalogCommand_noTabs (0.00s) PASS -ok github.com/hashicorp/consul/command/catalog 0.032s +ok github.com/hashicorp/consul/command/catalog 0.007s === RUN TestCatalogListDatacentersCommand_noTabs === PAUSE TestCatalogListDatacentersCommand_noTabs === RUN TestCatalogListDatacentersCommand_Validation @@ -78240,71 +78231,72 @@ === PAUSE TestCatalogListDatacentersCommand === CONT TestCatalogListDatacentersCommand_noTabs === CONT TestCatalogListDatacentersCommand -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestCatalogListDatacentersCommand_Validation +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... --- PASS: TestCatalogListDatacentersCommand_Validation (0.00s) --- PASS: TestCatalogListDatacentersCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:43:30.027Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:30.056Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:30.060 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:758a4cdb-9bf3-c335-73b2-1a295e679ae8 Address:127.0.0.1:15111}]" -43:30.061 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:15111 [Follower]" leader= -43:30.062 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-758a4cdb-9bf3-c335-73b2-1a295e679ae8.dc1 127.0.0.1 -43:30.063 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-758a4cdb-9bf3-c335-73b2-1a295e679ae8 127.0.0.1 -2022-09-15T21:43:30.063Z [INFO] agent.router: Initializing LAN area manager -43:30.063 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15106 network=udp -43:30.064 [INFO] TestAgent.server: Adding LAN server: server="Node-758a4cdb-9bf3-c335-73b2-1a295e679ae8 (Addr: tcp/127.0.0.1:15111) (DC: dc1)" -43:30.064 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-758a4cdb-9bf3-c335-73b2-1a295e679ae8.dc1 area=wan -43:30.064 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15106 network=tcp -43:30.065 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:15107 network=tcp -43:30.065 [INFO] TestAgent: started state syncer -43:30.066 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:15112 network=tcp -43:30.126 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:30.126 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:15111 [Candidate]" term=2 -43:30.126 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:30.126 [DEBUG] TestAgent.server.raft: vote granted: from=758a4cdb-9bf3-c335-73b2-1a295e679ae8 term=2 tally=1 -43:30.126 [INFO] TestAgent.server.raft: election won: tally=1 -43:30.126 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:15111 [Leader]" -43:30.126 [INFO] TestAgent.server: cluster leadership acquired -43:30.127 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:15111 -43:30.128 [INFO] TestAgent.server: New leader elected: payload=Node-758a4cdb-9bf3-c335-73b2-1a295e679ae8 -43:30.128 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:30.154 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:30.154 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:30.154 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:30.154 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:30.154 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:30.154 [INFO] TestAgent.server: federation state anti-entropy synced -43:30.155 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-758a4cdb-9bf3-c335-73b2-1a295e679ae8 -43:30.155 [INFO] TestAgent.server: member joined, marking health alive: member=Node-758a4cdb-9bf3-c335-73b2-1a295e679ae8 -43:30.216 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/datacenters from=127.0.0.1:37808 latency=2.50826ms -43:30.217 [INFO] TestAgent: Requesting shutdown -43:30.217 [INFO] TestAgent.server: shutting down server -43:30.218 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:30.218 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:30.218 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:30.218 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:30.218 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:30.218 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:30.218 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:30.218 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:30.218 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:30.218 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:30.218 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:30.218 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:30.218Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:30.218Z [INFO] agent.router.manager: shutting down -43:30.219 [INFO] TestAgent: consul server down -43:30.219 [INFO] TestAgent: shutdown complete -43:30.219 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15106 network=tcp -43:30.225 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15106 network=udp -43:30.225 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:15107 network=tcp -43:30.725 [INFO] TestAgent: Waiting for endpoints to shut down -43:30.725 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogListDatacentersCommand (0.80s) +2021-08-13T15:35:40.545Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:35:40.560Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +35:40.566 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6f9029c1-6926-68ec-8236-c50dd1c551d2 Address:127.0.0.1:12048}]" +35:40.566 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:12048 [Follower]" leader= +35:40.567 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6f9029c1-6926-68ec-8236-c50dd1c551d2.dc1 127.0.0.1 +35:40.568 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6f9029c1-6926-68ec-8236-c50dd1c551d2 127.0.0.1 +2021-08-13T15:35:40.568Z [INFO] agent.router: Initializing LAN area manager +35:40.569 [INFO] TestAgent.server: Adding LAN server: server="Node-6f9029c1-6926-68ec-8236-c50dd1c551d2 (Addr: tcp/127.0.0.1:12048) (DC: dc1)" +35:40.569 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6f9029c1-6926-68ec-8236-c50dd1c551d2.dc1 area=wan +35:40.569 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12043 network=tcp +35:40.569 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12043 network=udp +35:40.570 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:12044 network=tcp +35:40.570 [INFO] TestAgent: started state syncer +35:40.570 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:12049 network=tcp +35:40.607 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +35:40.607 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:12048 [Candidate]" term=2 +35:40.608 [DEBUG] TestAgent.server.raft: votes: needed=1 +35:40.608 [DEBUG] TestAgent.server.raft: vote granted: from=6f9029c1-6926-68ec-8236-c50dd1c551d2 term=2 tally=1 +35:40.608 [INFO] TestAgent.server.raft: election won: tally=1 +35:40.608 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:12048 [Leader]" +35:40.608 [INFO] TestAgent.server: cluster leadership acquired +35:40.608 [INFO] TestAgent.server: New leader elected: payload=Node-6f9029c1-6926-68ec-8236-c50dd1c551d2 +35:40.608 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:12048 +35:40.610 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +35:40.633 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +35:40.633 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +35:40.633 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +35:40.633 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +35:40.633 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +35:40.633 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6f9029c1-6926-68ec-8236-c50dd1c551d2 +35:40.633 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6f9029c1-6926-68ec-8236-c50dd1c551d2 +35:40.634 [INFO] TestAgent.server: federation state anti-entropy synced +35:40.697 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +35:40.697 [INFO] TestAgent: Synced node info +35:40.783 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/datacenters from=127.0.0.1:47944 latency=373.941µs +35:40.784 [INFO] TestAgent: Requesting shutdown +35:40.784 [INFO] TestAgent.server: shutting down server +35:40.784 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +35:40.784 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +35:40.784 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +35:40.784 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +35:40.784 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +35:40.784 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +35:40.784 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +35:40.784 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +35:40.784 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +35:40.784 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +35:40.784 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:35:40.784Z [INFO] agent.router.manager: shutting down +2021-08-13T15:35:40.784Z [INFO] agent.router.manager: shutting down +35:40.784 [INFO] TestAgent: consul server down +35:40.784 [INFO] TestAgent: shutdown complete +35:40.784 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12043 network=tcp +35:40.784 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12043 network=udp +35:40.784 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:12044 network=tcp +35:41.285 [INFO] TestAgent: Waiting for endpoints to shut down +35:41.285 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogListDatacentersCommand (0.79s) PASS -ok github.com/hashicorp/consul/command/catalog/list/dc 0.890s +ok github.com/hashicorp/consul/command/catalog/list/dc 0.854s === RUN TestCatalogListNodesCommand_noTabs === PAUSE TestCatalogListNodesCommand_noTabs === RUN TestCatalogListNodesCommand_Validation @@ -78315,11 +78307,11 @@ catalog_list_nodes_test.go:168: DM-skipped --- SKIP: TestCatalogListNodesCommand_verticalBar (0.00s) === CONT TestCatalogListNodesCommand_noTabs ---- PASS: TestCatalogListNodesCommand_noTabs (0.00s) === CONT TestCatalogListNodesCommand -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestCatalogListNodesCommand_Validation ---- PASS: TestCatalogListNodesCommand_Validation (0.02s) +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestCatalogListNodesCommand_noTabs (0.00s) +--- PASS: TestCatalogListNodesCommand_Validation (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range === RUN TestCatalogListNodesCommand/simple @@ -78329,80 +78321,78 @@ === RUN TestCatalogListNodesCommand/near === RUN TestCatalogListNodesCommand/service_present === RUN TestCatalogListNodesCommand/service_missing -2022-09-15T21:43:31.723Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:31.764Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:31.769 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d113661b-e1b2-bb3d-f6f6-666848ede617 Address:127.0.0.1:27363}]" -43:31.770 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:27363 [Follower]" leader= -43:31.771 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d113661b-e1b2-bb3d-f6f6-666848ede617.dc1 127.0.0.1 -43:31.772 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d113661b-e1b2-bb3d-f6f6-666848ede617 127.0.0.1 -2022-09-15T21:43:31.772Z [INFO] agent.router: Initializing LAN area manager -43:31.772 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27358 network=udp -43:31.773 [INFO] TestAgent.server: Adding LAN server: server="Node-d113661b-e1b2-bb3d-f6f6-666848ede617 (Addr: tcp/127.0.0.1:27363) (DC: dc1)" -43:31.773 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d113661b-e1b2-bb3d-f6f6-666848ede617.dc1 area=wan -43:31.773 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27358 network=tcp -43:31.774 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:27359 network=tcp -43:31.775 [INFO] TestAgent: started state syncer -43:31.775 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:27364 network=tcp -43:31.834 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:31.834 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:27363 [Candidate]" term=2 -43:31.834 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:31.834 [DEBUG] TestAgent.server.raft: vote granted: from=d113661b-e1b2-bb3d-f6f6-666848ede617 term=2 tally=1 -43:31.834 [INFO] TestAgent.server.raft: election won: tally=1 -43:31.834 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:27363 [Leader]" -43:31.834 [INFO] TestAgent.server: cluster leadership acquired -43:31.835 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:27363 -43:31.837 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:31.838 [INFO] TestAgent.server: New leader elected: payload=Node-d113661b-e1b2-bb3d-f6f6-666848ede617 -43:31.904 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:31.904 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:31.904 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:31.904 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:31.904 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:31.904 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d113661b-e1b2-bb3d-f6f6-666848ede617 -43:31.904 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d113661b-e1b2-bb3d-f6f6-666848ede617 -43:31.914 [INFO] TestAgent.server: federation state anti-entropy synced -43:32.143 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:32.144 [INFO] TestAgent: Synced node info -43:32.144 [DEBUG] TestAgent: Node info in sync -43:32.168 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/nodes from=127.0.0.1:56628 latency=888.7µs -43:32.178 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/nodes from=127.0.0.1:56630 latency=462.88µs -43:32.181 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/nodes?node-meta=foo%3Abar from=127.0.0.1:56632 latency=228.24µs -43:32.191 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/nodes?filter=Meta.foo+%3D%3D+bar from=127.0.0.1:56634 latency=821.34µs -43:32.194 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/nodes?near=_agent from=127.0.0.1:56636 latency=568.86µs -43:32.198 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/service/consul from=127.0.0.1:56638 latency=1.55432ms -43:32.201 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/service/this-service-will-literally-never-exist from=127.0.0.1:56640 latency=264µs -43:32.202 [INFO] TestAgent: Requesting shutdown -43:32.202 [INFO] TestAgent.server: shutting down server -43:32.202 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:32.202 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:32.202 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:32.202 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:32.202 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:32.202 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:32.202 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:32.202 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:32.202 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:32.202 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:32.203 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:32.203Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:32.203Z [INFO] agent.router.manager: shutting down -43:32.203 [INFO] TestAgent: consul server down -43:32.203 [INFO] TestAgent: shutdown complete -43:32.203 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27358 network=tcp -43:32.203 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27358 network=udp -43:32.203 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:27359 network=tcp -43:32.704 [INFO] TestAgent: Waiting for endpoints to shut down -43:32.704 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogListNodesCommand (1.06s) - --- PASS: TestCatalogListNodesCommand/simple (0.00s) - --- PASS: TestCatalogListNodesCommand/detailed (0.01s) - --- PASS: TestCatalogListNodesCommand/node-meta (0.00s) +2021-08-13T15:35:53.235Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:35:53.251Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +35:53.258 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:098c130e-e584-2d4a-6cd2-53c319743486 Address:127.0.0.1:17153}]" +35:53.258 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17153 [Follower]" leader= +35:53.259 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-098c130e-e584-2d4a-6cd2-53c319743486.dc1 127.0.0.1 +35:53.260 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-098c130e-e584-2d4a-6cd2-53c319743486 127.0.0.1 +2021-08-13T15:35:53.260Z [INFO] agent.router: Initializing LAN area manager +35:53.260 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-098c130e-e584-2d4a-6cd2-53c319743486.dc1 area=wan +35:53.260 [INFO] TestAgent.server: Adding LAN server: server="Node-098c130e-e584-2d4a-6cd2-53c319743486 (Addr: tcp/127.0.0.1:17153) (DC: dc1)" +35:53.260 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17148 network=tcp +35:53.260 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17148 network=udp +35:53.261 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17149 network=tcp +35:53.261 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17154 network=tcp +35:53.261 [INFO] TestAgent: started state syncer +35:53.315 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +35:53.315 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17153 [Candidate]" term=2 +35:53.315 [DEBUG] TestAgent.server.raft: votes: needed=1 +35:53.315 [DEBUG] TestAgent.server.raft: vote granted: from=098c130e-e584-2d4a-6cd2-53c319743486 term=2 tally=1 +35:53.315 [INFO] TestAgent.server.raft: election won: tally=1 +35:53.315 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17153 [Leader]" +35:53.315 [INFO] TestAgent.server: cluster leadership acquired +35:53.316 [INFO] TestAgent.server: New leader elected: payload=Node-098c130e-e584-2d4a-6cd2-53c319743486 +35:53.316 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17153 +35:53.317 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +35:53.341 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +35:53.341 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +35:53.341 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +35:53.341 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +35:53.341 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +35:53.341 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-098c130e-e584-2d4a-6cd2-53c319743486 +35:53.341 [INFO] TestAgent.server: member joined, marking health alive: member=Node-098c130e-e584-2d4a-6cd2-53c319743486 +35:53.341 [INFO] TestAgent.server: federation state anti-entropy synced +35:53.350 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/nodes from=127.0.0.1:37640 latency=488.161µs +35:53.367 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/nodes from=127.0.0.1:37642 latency=566.301µs +35:53.374 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/nodes?node-meta=foo%3Abar from=127.0.0.1:37644 latency=3.222848ms +35:53.386 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/nodes?filter=Meta.foo+%3D%3D+bar from=127.0.0.1:37646 latency=1.007502ms +35:53.390 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/nodes?near=_agent from=127.0.0.1:37648 latency=442.581µs +35:53.394 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/service/consul from=127.0.0.1:37650 latency=1.249363ms +35:53.397 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/service/this-service-will-literally-never-exist from=127.0.0.1:37652 latency=324.18µs +35:53.397 [INFO] TestAgent: Requesting shutdown +35:53.397 [INFO] TestAgent.server: shutting down server +35:53.397 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +35:53.397 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +35:53.397 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +35:53.397 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +35:53.397 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +35:53.398 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +35:53.398 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +35:53.398 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +35:53.398 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +35:53.398 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +35:53.398 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +35:53.398 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:35:53.398Z [INFO] agent.router.manager: shutting down +2021-08-13T15:35:53.398Z [INFO] agent.router.manager: shutting down +35:53.398 [INFO] TestAgent: consul server down +35:53.398 [INFO] TestAgent: shutdown complete +35:53.398 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17148 network=tcp +35:53.398 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17148 network=udp +35:53.398 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17149 network=tcp +35:53.899 [INFO] TestAgent: Waiting for endpoints to shut down +35:53.899 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogListNodesCommand (0.73s) + --- PASS: TestCatalogListNodesCommand/simple (0.02s) + --- PASS: TestCatalogListNodesCommand/detailed (0.00s) + --- PASS: TestCatalogListNodesCommand/node-meta (0.01s) --- PASS: TestCatalogListNodesCommand/filter (0.01s) --- PASS: TestCatalogListNodesCommand/near (0.00s) --- PASS: TestCatalogListNodesCommand/service_present (0.00s) --- PASS: TestCatalogListNodesCommand/service_missing (0.00s) PASS -ok github.com/hashicorp/consul/command/catalog/list/nodes 1.223s +ok github.com/hashicorp/consul/command/catalog/list/nodes 0.802s === RUN TestCatalogListServicesCommand_noTabs === PAUSE TestCatalogListServicesCommand_noTabs === RUN TestCatalogListServicesCommand_Validation @@ -78410,10 +78400,10 @@ === RUN TestCatalogListServicesCommand === PAUSE TestCatalogListServicesCommand === CONT TestCatalogListServicesCommand_noTabs ---- PASS: TestCatalogListServicesCommand_noTabs (0.00s) === CONT TestCatalogListServicesCommand [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestCatalogListServicesCommand_Validation +--- PASS: TestCatalogListServicesCommand_noTabs (0.00s) --- PASS: TestCatalogListServicesCommand_Validation (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range @@ -78422,77 +78412,77 @@ === RUN TestCatalogListServicesCommand/node_missing === RUN TestCatalogListServicesCommand/node_present === RUN TestCatalogListServicesCommand/node-meta -2022-09-15T21:43:42.476Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:42.529Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:42.539 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:84b4ffcf-ebe0-be53-b669-64acb63f8e40 Address:127.0.0.1:14090}]" -43:42.541 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-84b4ffcf-ebe0-be53-b669-64acb63f8e40.dc1 127.0.0.1 -43:42.541 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-84b4ffcf-ebe0-be53-b669-64acb63f8e40 127.0.0.1 -43:42.542 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14090 [Follower]" leader= -2022-09-15T21:43:42.543Z [INFO] agent.router: Initializing LAN area manager -43:42.543 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14085 network=udp -43:42.543 [INFO] TestAgent.server: Adding LAN server: server="Node-84b4ffcf-ebe0-be53-b669-64acb63f8e40 (Addr: tcp/127.0.0.1:14090) (DC: dc1)" -43:42.543 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-84b4ffcf-ebe0-be53-b669-64acb63f8e40.dc1 area=wan -43:42.544 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14085 network=tcp -43:42.545 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14086 network=tcp -43:42.545 [INFO] TestAgent: started state syncer -43:42.545 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14091 network=tcp -43:42.582 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:42.582 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14090 [Candidate]" term=2 -43:42.582 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:42.582 [DEBUG] TestAgent.server.raft: vote granted: from=84b4ffcf-ebe0-be53-b669-64acb63f8e40 term=2 tally=1 -43:42.582 [INFO] TestAgent.server.raft: election won: tally=1 -43:42.582 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14090 [Leader]" -43:42.583 [INFO] TestAgent.server: cluster leadership acquired -43:42.583 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14090 -43:42.585 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:42.586 [INFO] TestAgent.server: New leader elected: payload=Node-84b4ffcf-ebe0-be53-b669-64acb63f8e40 -43:42.630 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:42.630 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:42.630 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:42.630 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:42.630 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:42.630 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-84b4ffcf-ebe0-be53-b669-64acb63f8e40 -43:42.630 [INFO] TestAgent.server: member joined, marking health alive: member=Node-84b4ffcf-ebe0-be53-b669-64acb63f8e40 -43:42.632 [INFO] TestAgent.server: federation state anti-entropy synced -43:42.645 [INFO] TestAgent: Synced node info -43:42.645 [INFO] TestAgent: Synced service: service=testing -43:42.645 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:57400 latency=2.9231ms -43:42.662 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/services from=127.0.0.1:57402 latency=14.43068ms -43:42.673 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/services from=127.0.0.1:57404 latency=2.61268ms -43:42.715 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/node/not-a-real-node from=127.0.0.1:57406 latency=728.36µs -43:42.723 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/node/Node-84b4ffcf-ebe0-be53-b669-64acb63f8e40 from=127.0.0.1:57408 latency=578.3µs -43:42.731 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/services?node-meta=foo%3Abar from=127.0.0.1:57410 latency=477.76µs -43:42.731 [INFO] TestAgent: Requesting shutdown -43:42.731 [INFO] TestAgent.server: shutting down server -43:42.731 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:42.731 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:42.731 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:42.731 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:42.731 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:42.731 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:42.731 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:42.732 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -43:42.732 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:42.732 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:42.732 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:42.732 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:42.732Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:42.732Z [INFO] agent.router.manager: shutting down -43:42.732 [INFO] TestAgent: consul server down -43:42.732 [INFO] TestAgent: shutdown complete -43:42.732 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14085 network=tcp -43:42.732 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14085 network=udp -43:42.732 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14086 network=tcp -43:43.232 [INFO] TestAgent: Waiting for endpoints to shut down -43:43.232 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogListServicesCommand (0.85s) - --- PASS: TestCatalogListServicesCommand/simple (0.02s) - --- PASS: TestCatalogListServicesCommand/tags (0.02s) - --- PASS: TestCatalogListServicesCommand/node_missing (0.03s) +2021-08-13T15:35:56.681Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:35:56.702Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +35:56.706 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:dfb72411-8c13-9166-a75d-4e0937d797d3 Address:127.0.0.1:25321}]" +35:56.706 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25321 [Follower]" leader= +35:56.707 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-dfb72411-8c13-9166-a75d-4e0937d797d3.dc1 127.0.0.1 +35:56.708 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-dfb72411-8c13-9166-a75d-4e0937d797d3 127.0.0.1 +2021-08-13T15:35:56.708Z [INFO] agent.router: Initializing LAN area manager +35:56.708 [INFO] TestAgent.server: Adding LAN server: server="Node-dfb72411-8c13-9166-a75d-4e0937d797d3 (Addr: tcp/127.0.0.1:25321) (DC: dc1)" +35:56.708 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-dfb72411-8c13-9166-a75d-4e0937d797d3.dc1 area=wan +35:56.708 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25316 network=udp +35:56.708 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25316 network=tcp +35:56.709 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25317 network=tcp +35:56.709 [INFO] TestAgent: started state syncer +35:56.710 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25322 network=tcp +35:56.774 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +35:56.774 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25321 [Candidate]" term=2 +35:56.774 [DEBUG] TestAgent.server.raft: votes: needed=1 +35:56.775 [DEBUG] TestAgent.server.raft: vote granted: from=dfb72411-8c13-9166-a75d-4e0937d797d3 term=2 tally=1 +35:56.775 [INFO] TestAgent.server.raft: election won: tally=1 +35:56.775 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25321 [Leader]" +35:56.775 [INFO] TestAgent.server: cluster leadership acquired +35:56.775 [INFO] TestAgent.server: New leader elected: payload=Node-dfb72411-8c13-9166-a75d-4e0937d797d3 +35:56.775 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25321 +35:56.783 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +35:56.808 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +35:56.808 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +35:56.808 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +35:56.808 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +35:56.808 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +35:56.808 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-dfb72411-8c13-9166-a75d-4e0937d797d3 +35:56.808 [INFO] TestAgent.server: member joined, marking health alive: member=Node-dfb72411-8c13-9166-a75d-4e0937d797d3 +35:56.808 [INFO] TestAgent.server: federation state anti-entropy synced +35:56.906 [INFO] TestAgent: Synced node info +35:56.907 [INFO] TestAgent: Synced service: service=testing +35:56.907 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:41332 latency=2.463026ms +35:56.908 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/services from=127.0.0.1:41334 latency=319.821µs +35:56.910 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/services from=127.0.0.1:41336 latency=218.24µs +35:56.913 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/node/not-a-real-node from=127.0.0.1:41338 latency=555.322µs +35:56.922 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/node/Node-dfb72411-8c13-9166-a75d-4e0937d797d3 from=127.0.0.1:41340 latency=557.061µs +35:56.926 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/services?node-meta=foo%3Abar from=127.0.0.1:41342 latency=319.901µs +35:56.927 [INFO] TestAgent: Requesting shutdown +35:56.927 [INFO] TestAgent.server: shutting down server +35:56.927 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +35:56.927 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +35:56.927 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +35:56.927 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +35:56.927 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +35:56.927 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +35:56.927 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +35:56.927 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +35:56.927 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +35:56.927 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +35:56.927 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +35:56.927 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:35:56.927Z [INFO] agent.router.manager: shutting down +2021-08-13T15:35:56.927Z [INFO] agent.router.manager: shutting down +35:56.927 [INFO] TestAgent: consul server down +35:56.927 [INFO] TestAgent: shutdown complete +35:56.927 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25316 network=tcp +35:56.928 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25316 network=udp +35:56.928 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25317 network=tcp +35:57.428 [INFO] TestAgent: Waiting for endpoints to shut down +35:57.428 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogListServicesCommand (0.80s) + --- PASS: TestCatalogListServicesCommand/simple (0.00s) + --- PASS: TestCatalogListServicesCommand/tags (0.00s) + --- PASS: TestCatalogListServicesCommand/node_missing (0.00s) --- PASS: TestCatalogListServicesCommand/node_present (0.01s) - --- PASS: TestCatalogListServicesCommand/node-meta (0.01s) + --- PASS: TestCatalogListServicesCommand/node-meta (0.00s) PASS -ok github.com/hashicorp/consul/command/catalog/list/services 1.136s +ok github.com/hashicorp/consul/command/catalog/list/services 0.874s ? github.com/hashicorp/consul/command/config [no test files] === RUN TestConfigDelete_noTabs === PAUSE TestConfigDelete_noTabs @@ -78501,157 +78491,157 @@ === RUN TestConfigDelete_InvalidArgs === PAUSE TestConfigDelete_InvalidArgs === CONT TestConfigDelete_noTabs ---- PASS: TestConfigDelete_noTabs (0.00s) === CONT TestConfigDelete_InvalidArgs === RUN TestConfigDelete_InvalidArgs/no_kind +--- PASS: TestConfigDelete_noTabs (0.00s) === CONT TestConfigDelete === RUN TestConfigDelete_InvalidArgs/no_name [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... ---- PASS: TestConfigDelete_InvalidArgs (0.02s) - --- PASS: TestConfigDelete_InvalidArgs/no_kind (0.01s) +--- PASS: TestConfigDelete_InvalidArgs (0.00s) + --- PASS: TestConfigDelete_InvalidArgs/no_kind (0.00s) --- PASS: TestConfigDelete_InvalidArgs/no_name (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:43:49.352Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:49.418Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:49.434 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b4252f47-ec84-b954-e648-ed3ca1ce07d4 Address:127.0.0.1:31447}]" -43:49.435 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b4252f47-ec84-b954-e648-ed3ca1ce07d4.dc1 127.0.0.1 -43:49.436 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b4252f47-ec84-b954-e648-ed3ca1ce07d4 127.0.0.1 -2022-09-15T21:43:49.436Z [INFO] agent.router: Initializing LAN area manager -43:49.437 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31442 network=udp -43:49.437 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31447 [Follower]" leader= -43:49.437 [INFO] TestAgent.server: Adding LAN server: server="Node-b4252f47-ec84-b954-e648-ed3ca1ce07d4 (Addr: tcp/127.0.0.1:31447) (DC: dc1)" -43:49.437 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b4252f47-ec84-b954-e648-ed3ca1ce07d4.dc1 area=wan -43:49.442 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31442 network=tcp -43:49.443 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31443 network=tcp -43:49.443 [INFO] TestAgent: started state syncer -43:49.443 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31448 network=tcp -43:49.486 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:49.486 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31447 [Candidate]" term=2 -43:49.486 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:49.486 [DEBUG] TestAgent.server.raft: vote granted: from=b4252f47-ec84-b954-e648-ed3ca1ce07d4 term=2 tally=1 -43:49.486 [INFO] TestAgent.server.raft: election won: tally=1 -43:49.486 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31447 [Leader]" -43:49.486 [INFO] TestAgent.server: cluster leadership acquired -43:49.487 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:31447 -43:49.488 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:49.490 [INFO] TestAgent.server: New leader elected: payload=Node-b4252f47-ec84-b954-e648-ed3ca1ce07d4 -43:49.532 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:49.532 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:49.532 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:49.532 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:49.532 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:49.533 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b4252f47-ec84-b954-e648-ed3ca1ce07d4 -43:49.533 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b4252f47-ec84-b954-e648-ed3ca1ce07d4 -43:49.542 [INFO] TestAgent.server: federation state anti-entropy synced -43:49.653 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:59426 latency=3.84758ms -43:49.671 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/config/service-defaults/web from=127.0.0.1:59428 latency=3.37612ms -43:49.686 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:49.686 [ERROR] TestAgent.http: Request error: method=GET url=/v1/config/service-defaults/web from=127.0.0.1:59426 error="Config entry not found for "service-defaults" / "web"" -43:49.687 [INFO] TestAgent: Synced node info -43:49.687 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/service-defaults/web from=127.0.0.1:59426 latency=483.82µs -43:49.687 [INFO] TestAgent: Requesting shutdown -43:49.687 [INFO] TestAgent.server: shutting down server -43:49.687 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:49.687 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:49.687 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:49.687 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:49.687 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:49.687 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:49.687 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:49.687 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:49.687 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:49.687 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:49.688 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:49.688Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:49.688Z [INFO] agent.router.manager: shutting down -43:49.688 [INFO] TestAgent: consul server down -43:49.688 [INFO] TestAgent: shutdown complete -43:49.688 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31442 network=tcp -43:49.688 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31442 network=udp -43:49.688 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31443 network=tcp -43:50.189 [INFO] TestAgent: Waiting for endpoints to shut down -43:50.189 [INFO] TestAgent: Endpoints down +2021-08-13T15:36:03.187Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:36:03.204Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +36:03.211 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:35d731a8-eae1-5749-2772-b5e991e794d9 Address:127.0.0.1:19195}]" +36:03.212 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19195 [Follower]" leader= +36:03.212 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-35d731a8-eae1-5749-2772-b5e991e794d9.dc1 127.0.0.1 +36:03.213 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-35d731a8-eae1-5749-2772-b5e991e794d9 127.0.0.1 +2021-08-13T15:36:03.213Z [INFO] agent.router: Initializing LAN area manager +36:03.213 [INFO] TestAgent.server: Adding LAN server: server="Node-35d731a8-eae1-5749-2772-b5e991e794d9 (Addr: tcp/127.0.0.1:19195) (DC: dc1)" +36:03.213 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-35d731a8-eae1-5749-2772-b5e991e794d9.dc1 area=wan +36:03.214 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=tcp +36:03.214 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=udp +36:03.215 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19191 network=tcp +36:03.215 [INFO] TestAgent: started state syncer +36:03.215 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19196 network=tcp +36:03.278 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +36:03.278 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19195 [Candidate]" term=2 +36:03.278 [DEBUG] TestAgent.server.raft: votes: needed=1 +36:03.278 [DEBUG] TestAgent.server.raft: vote granted: from=35d731a8-eae1-5749-2772-b5e991e794d9 term=2 tally=1 +36:03.278 [INFO] TestAgent.server.raft: election won: tally=1 +36:03.278 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19195 [Leader]" +36:03.278 [INFO] TestAgent.server: cluster leadership acquired +36:03.279 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:19195 +36:03.280 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +36:03.283 [INFO] TestAgent.server: New leader elected: payload=Node-35d731a8-eae1-5749-2772-b5e991e794d9 +36:03.303 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +36:03.303 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +36:03.303 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +36:03.303 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +36:03.303 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +36:03.303 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-35d731a8-eae1-5749-2772-b5e991e794d9 +36:03.303 [INFO] TestAgent.server: member joined, marking health alive: member=Node-35d731a8-eae1-5749-2772-b5e991e794d9 +36:03.304 [INFO] TestAgent.server: federation state anti-entropy synced +36:03.499 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +36:03.499 [INFO] TestAgent: Synced node info +36:03.499 [DEBUG] TestAgent: Node info in sync +36:03.565 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:35518 latency=1.486084ms +36:03.567 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/config/service-defaults/web from=127.0.0.1:35520 latency=1.633443ms +36:03.572 [ERROR] TestAgent.http: Request error: method=GET url=/v1/config/service-defaults/web from=127.0.0.1:35518 error="Config entry not found for "service-defaults" / "web"" +36:03.572 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/service-defaults/web from=127.0.0.1:35518 latency=456.522µs +36:03.572 [INFO] TestAgent: Requesting shutdown +36:03.573 [INFO] TestAgent.server: shutting down server +36:03.573 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +36:03.573 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +36:03.573 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +36:03.573 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +36:03.573 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +36:03.573 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +36:03.573 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +36:03.573 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +36:03.573 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +36:03.573 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +36:03.573 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:36:03.573Z [INFO] agent.router.manager: shutting down +2021-08-13T15:36:03.573Z [INFO] agent.router.manager: shutting down +36:03.573 [INFO] TestAgent: consul server down +36:03.573 [INFO] TestAgent: shutdown complete +36:03.573 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=tcp +36:03.573 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=udp +36:03.573 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19191 network=tcp +36:04.074 [INFO] TestAgent: Waiting for endpoints to shut down +36:04.074 [INFO] TestAgent: Endpoints down --- PASS: TestConfigDelete (0.95s) PASS -ok github.com/hashicorp/consul/command/config/delete 1.195s +ok github.com/hashicorp/consul/command/config/delete 1.034s === RUN TestConfigList_noTabs === PAUSE TestConfigList_noTabs === RUN TestConfigList [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:43:48.810Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:48.840Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:48.846 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:66b3ff2a-cddb-f77b-d837-42d180a949ff Address:127.0.0.1:12048}]" -43:48.847 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-66b3ff2a-cddb-f77b-d837-42d180a949ff.dc1 127.0.0.1 -43:48.848 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:12048 [Follower]" leader= -43:48.849 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-66b3ff2a-cddb-f77b-d837-42d180a949ff 127.0.0.1 -2022-09-15T21:43:48.849Z [INFO] agent.router: Initializing LAN area manager -43:48.850 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12043 network=udp -43:48.850 [INFO] TestAgent.server: Adding LAN server: server="Node-66b3ff2a-cddb-f77b-d837-42d180a949ff (Addr: tcp/127.0.0.1:12048) (DC: dc1)" -43:48.851 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-66b3ff2a-cddb-f77b-d837-42d180a949ff.dc1 area=wan -43:48.858 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12043 network=tcp -43:48.859 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:12044 network=tcp -43:48.859 [INFO] TestAgent: started state syncer -43:48.859 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:12049 network=tcp -43:48.899 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:48.900 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:12048 [Candidate]" term=2 -43:48.900 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:48.900 [DEBUG] TestAgent.server.raft: vote granted: from=66b3ff2a-cddb-f77b-d837-42d180a949ff term=2 tally=1 -43:48.900 [INFO] TestAgent.server.raft: election won: tally=1 -43:48.900 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:12048 [Leader]" -43:48.900 [INFO] TestAgent.server: cluster leadership acquired -43:48.900 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:12048 -43:48.902 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:48.906 [INFO] TestAgent.server: New leader elected: payload=Node-66b3ff2a-cddb-f77b-d837-42d180a949ff -43:48.951 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:48.951 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:48.951 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:48.951 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:48.951 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:48.951 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-66b3ff2a-cddb-f77b-d837-42d180a949ff -43:48.951 [INFO] TestAgent.server: member joined, marking health alive: member=Node-66b3ff2a-cddb-f77b-d837-42d180a949ff -43:48.953 [INFO] TestAgent.server: federation state anti-entropy synced -43:49.058 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:49.059 [INFO] TestAgent: Synced node info -43:49.102 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:43898 latency=1.80136ms -43:49.104 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:43898 latency=1.2445ms -43:49.105 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:43898 latency=916.32µs -43:49.115 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/service-defaults from=127.0.0.1:43900 latency=660.64µs -43:49.118 [INFO] TestAgent: Requesting shutdown -43:49.118 [INFO] TestAgent.server: shutting down server -43:49.118 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:49.118 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:49.118 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:49.118 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:49.119 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:49.119 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:49.119 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:49.119 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:49.119 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:49.119 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:49.119 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:49.119Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:49.119Z [INFO] agent.router.manager: shutting down -43:49.119 [INFO] TestAgent: consul server down -43:49.119 [INFO] TestAgent: shutdown complete -43:49.119 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12043 network=tcp -43:49.119 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12043 network=udp -43:49.119 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:12044 network=tcp -43:49.620 [INFO] TestAgent: Waiting for endpoints to shut down -43:49.620 [INFO] TestAgent: Endpoints down ---- PASS: TestConfigList (0.97s) +2021-08-13T15:35:59.126Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:35:59.145Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +35:59.152 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f321d45f-74ce-b8e7-b9c4-000bc9faf432 Address:127.0.0.1:10006}]" +35:59.153 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f321d45f-74ce-b8e7-b9c4-000bc9faf432.dc1 127.0.0.1 +35:59.154 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f321d45f-74ce-b8e7-b9c4-000bc9faf432 127.0.0.1 +2021-08-13T15:35:59.154Z [INFO] agent.router: Initializing LAN area manager +35:59.155 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10001 network=udp +35:59.155 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:10006 [Follower]" leader= +35:59.155 [INFO] TestAgent.server: Adding LAN server: server="Node-f321d45f-74ce-b8e7-b9c4-000bc9faf432 (Addr: tcp/127.0.0.1:10006) (DC: dc1)" +35:59.156 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f321d45f-74ce-b8e7-b9c4-000bc9faf432.dc1 area=wan +35:59.156 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10001 network=tcp +35:59.157 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:10002 network=tcp +35:59.157 [INFO] TestAgent: started state syncer +35:59.157 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:10007 network=tcp +35:59.201 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +35:59.202 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:10006 [Candidate]" term=2 +35:59.202 [DEBUG] TestAgent.server.raft: votes: needed=1 +35:59.202 [DEBUG] TestAgent.server.raft: vote granted: from=f321d45f-74ce-b8e7-b9c4-000bc9faf432 term=2 tally=1 +35:59.202 [INFO] TestAgent.server.raft: election won: tally=1 +35:59.202 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:10006 [Leader]" +35:59.207 [INFO] TestAgent.server: cluster leadership acquired +35:59.208 [INFO] TestAgent.server: New leader elected: payload=Node-f321d45f-74ce-b8e7-b9c4-000bc9faf432 +35:59.211 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:10006 +35:59.213 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +35:59.243 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +35:59.243 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +35:59.243 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +35:59.243 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +35:59.243 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +35:59.243 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f321d45f-74ce-b8e7-b9c4-000bc9faf432 +35:59.243 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f321d45f-74ce-b8e7-b9c4-000bc9faf432 +35:59.245 [INFO] TestAgent.server: federation state anti-entropy synced +35:59.350 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:59996 latency=1.775545ms +35:59.351 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:59996 latency=1.130883ms +35:59.353 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:59996 latency=1.154643ms +35:59.355 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/service-defaults from=127.0.0.1:59998 latency=675.021µs +35:59.355 [INFO] TestAgent: Requesting shutdown +35:59.355 [INFO] TestAgent.server: shutting down server +35:59.355 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +35:59.355 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +35:59.355 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +35:59.355 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +35:59.356 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +35:59.356 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +35:59.356 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +35:59.356 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +35:59.356 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +35:59.356 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +35:59.356 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +35:59.356 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:35:59.356Z [INFO] agent.router.manager: shutting down +2021-08-13T15:35:59.356Z [INFO] agent.router.manager: shutting down +35:59.356 [INFO] TestAgent: consul server down +35:59.356 [INFO] TestAgent: shutdown complete +35:59.356 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10001 network=tcp +35:59.356 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10001 network=udp +35:59.356 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:10002 network=tcp +35:59.856 [INFO] TestAgent: Waiting for endpoints to shut down +35:59.856 [INFO] TestAgent: Endpoints down +--- PASS: TestConfigList (0.80s) === RUN TestConfigList_InvalidArgs === PAUSE TestConfigList_InvalidArgs === CONT TestConfigList_noTabs --- PASS: TestConfigList_noTabs (0.00s) === CONT TestConfigList_InvalidArgs === RUN TestConfigList_InvalidArgs/no_kind ---- PASS: TestConfigList_InvalidArgs (0.02s) - --- PASS: TestConfigList_InvalidArgs/no_kind (0.01s) +--- PASS: TestConfigList_InvalidArgs (0.00s) + --- PASS: TestConfigList_InvalidArgs/no_kind (0.00s) PASS -ok github.com/hashicorp/consul/command/config/list 1.225s +ok github.com/hashicorp/consul/command/config/list 0.864s === RUN TestConfigRead_noTabs === PAUSE TestConfigRead_noTabs === RUN TestConfigRead @@ -78661,77 +78651,75 @@ === CONT TestConfigRead_noTabs --- PASS: TestConfigRead_noTabs (0.00s) === CONT TestConfigRead_InvalidArgs -=== RUN TestConfigRead_InvalidArgs/no_name === RUN TestConfigRead_InvalidArgs/no_kind +=== RUN TestConfigRead_InvalidArgs/no_name --- PASS: TestConfigRead_InvalidArgs (0.00s) - --- PASS: TestConfigRead_InvalidArgs/no_name (0.00s) --- PASS: TestConfigRead_InvalidArgs/no_kind (0.00s) + --- PASS: TestConfigRead_InvalidArgs/no_name (0.00s) === CONT TestConfigRead [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:43:50.759Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:43:50.798Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -43:50.802 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c48cedd1-a08f-fee4-c0d6-aa3bb3141883 Address:127.0.0.1:31447}]" -43:50.803 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c48cedd1-a08f-fee4-c0d6-aa3bb3141883.dc1 127.0.0.1 -43:50.804 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c48cedd1-a08f-fee4-c0d6-aa3bb3141883 127.0.0.1 -2022-09-15T21:43:50.804Z [INFO] agent.router: Initializing LAN area manager -43:50.804 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31442 network=udp -43:50.804 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31447 [Follower]" leader= -43:50.805 [INFO] TestAgent.server: Adding LAN server: server="Node-c48cedd1-a08f-fee4-c0d6-aa3bb3141883 (Addr: tcp/127.0.0.1:31447) (DC: dc1)" -43:50.805 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c48cedd1-a08f-fee4-c0d6-aa3bb3141883.dc1 area=wan -43:50.805 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31442 network=tcp -43:50.806 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31443 network=tcp -43:50.806 [INFO] TestAgent: started state syncer -43:50.807 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31448 network=tcp -43:50.846 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -43:50.847 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31447 [Candidate]" term=2 -43:50.847 [DEBUG] TestAgent.server.raft: votes: needed=1 -43:50.847 [DEBUG] TestAgent.server.raft: vote granted: from=c48cedd1-a08f-fee4-c0d6-aa3bb3141883 term=2 tally=1 -43:50.847 [INFO] TestAgent.server.raft: election won: tally=1 -43:50.847 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31447 [Leader]" -43:50.847 [INFO] TestAgent.server: cluster leadership acquired -43:50.848 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:31447 -43:50.848 [INFO] TestAgent.server: New leader elected: payload=Node-c48cedd1-a08f-fee4-c0d6-aa3bb3141883 -43:50.849 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -43:50.876 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -43:50.876 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -43:50.876 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -43:50.876 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -43:50.876 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -43:50.876 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c48cedd1-a08f-fee4-c0d6-aa3bb3141883 -43:50.876 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c48cedd1-a08f-fee4-c0d6-aa3bb3141883 -43:50.877 [INFO] TestAgent.server: federation state anti-entropy synced -43:51.023 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -43:51.035 [INFO] TestAgent: Synced node info -43:51.035 [DEBUG] TestAgent: Node info in sync -43:51.100 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:59430 latency=1.72024ms -43:51.116 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/service-defaults/web from=127.0.0.1:59432 latency=718.7µs -43:51.117 [INFO] TestAgent: Requesting shutdown -43:51.118 [INFO] TestAgent.server: shutting down server -43:51.118 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -43:51.118 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -43:51.118 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -43:51.118 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -43:51.118 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -43:51.118 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -43:51.118 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -43:51.118 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -43:51.118 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -43:51.118 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -43:51.118 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:43:51.118Z [INFO] agent.router.manager: shutting down -2022-09-15T21:43:51.118Z [INFO] agent.router.manager: shutting down -43:51.118 [INFO] TestAgent: consul server down -43:51.118 [INFO] TestAgent: shutdown complete -43:51.119 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31442 network=tcp -43:51.119 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31442 network=udp -43:51.119 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31443 network=tcp -43:51.620 [INFO] TestAgent: Waiting for endpoints to shut down -43:51.620 [INFO] TestAgent: Endpoints down ---- PASS: TestConfigRead (0.94s) +2021-08-13T15:36:10.984Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:36:11.001Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +36:11.006 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:eb1d9700-0eec-0ae7-10da-4ebd8d508730 Address:127.0.0.1:24300}]" +36:11.006 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24300 [Follower]" leader= +36:11.011 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-eb1d9700-0eec-0ae7-10da-4ebd8d508730.dc1 127.0.0.1 +36:11.013 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-eb1d9700-0eec-0ae7-10da-4ebd8d508730 127.0.0.1 +2021-08-13T15:36:11.013Z [INFO] agent.router: Initializing LAN area manager +36:11.013 [INFO] TestAgent.server: Adding LAN server: server="Node-eb1d9700-0eec-0ae7-10da-4ebd8d508730 (Addr: tcp/127.0.0.1:24300) (DC: dc1)" +36:11.013 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-eb1d9700-0eec-0ae7-10da-4ebd8d508730.dc1 area=wan +36:11.014 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24295 network=tcp +36:11.014 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24295 network=udp +36:11.016 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24296 network=tcp +36:11.016 [INFO] TestAgent: started state syncer +36:11.016 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24301 network=tcp +36:11.054 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +36:11.054 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24300 [Candidate]" term=2 +36:11.054 [DEBUG] TestAgent.server.raft: votes: needed=1 +36:11.054 [DEBUG] TestAgent.server.raft: vote granted: from=eb1d9700-0eec-0ae7-10da-4ebd8d508730 term=2 tally=1 +36:11.054 [INFO] TestAgent.server.raft: election won: tally=1 +36:11.054 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24300 [Leader]" +36:11.054 [INFO] TestAgent.server: cluster leadership acquired +36:11.055 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24300 +36:11.056 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +36:11.059 [INFO] TestAgent.server: New leader elected: payload=Node-eb1d9700-0eec-0ae7-10da-4ebd8d508730 +36:11.080 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +36:11.080 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +36:11.080 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +36:11.080 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +36:11.080 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +36:11.080 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-eb1d9700-0eec-0ae7-10da-4ebd8d508730 +36:11.080 [INFO] TestAgent.server: member joined, marking health alive: member=Node-eb1d9700-0eec-0ae7-10da-4ebd8d508730 +36:11.082 [INFO] TestAgent.server: federation state anti-entropy synced +36:11.110 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:34874 latency=1.618663ms +36:11.112 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/service-defaults/web from=127.0.0.1:34876 latency=632.861µs +36:11.112 [INFO] TestAgent: Requesting shutdown +36:11.112 [INFO] TestAgent.server: shutting down server +36:11.112 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +36:11.112 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +36:11.112 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +36:11.112 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +36:11.112 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +36:11.112 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +36:11.112 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +36:11.112 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +36:11.112 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +36:11.113 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +36:11.113 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +36:11.113 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:36:11.113Z [INFO] agent.router.manager: shutting down +2021-08-13T15:36:11.113Z [INFO] agent.router.manager: shutting down +36:11.113 [INFO] TestAgent: consul server down +36:11.113 [INFO] TestAgent: shutdown complete +36:11.113 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24295 network=tcp +36:11.113 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24295 network=udp +36:11.113 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24296 network=tcp +36:11.613 [INFO] TestAgent: Waiting for endpoints to shut down +36:11.613 [INFO] TestAgent: Endpoints down +--- PASS: TestConfigRead (0.69s) PASS -ok github.com/hashicorp/consul/command/config/read 1.077s +ok github.com/hashicorp/consul/command/config/read 0.764s === RUN TestConfigWrite_noTabs === PAUSE TestConfigWrite_noTabs === RUN TestConfigWrite @@ -78739,28 +78727,28 @@ === RUN TestParseConfigEntry === PAUSE TestParseConfigEntry === CONT TestConfigWrite_noTabs ---- PASS: TestConfigWrite_noTabs (0.00s) === CONT TestParseConfigEntry +=== CONT TestConfigWrite === RUN TestParseConfigEntry/proxy-defaults:_extra_fields_or_typo_(hcl_snake_case) === RUN TestParseConfigEntry/proxy-defaults:_extra_fields_or_typo_(hcl_camel_case) -=== CONT TestConfigWrite -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === RUN TestParseConfigEntry/proxy-defaults:_extra_fields_or_typo_(json_snake_case) === RUN TestParseConfigEntry/proxy-defaults:_extra_fields_or_typo_(json_camel_case) === RUN TestParseConfigEntry/proxy-defaults_(hcl_snake_case) === RUN TestParseConfigEntry/proxy-defaults_(hcl_camel_case) +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === RUN TestParseConfigEntry/proxy-defaults_(json_snake_case) === RUN TestParseConfigEntry/proxy-defaults_(json_camel_case) === RUN TestParseConfigEntry/terminating-gateway_(hcl_snake_case) === RUN TestParseConfigEntry/terminating-gateway_(hcl_camel_case) === RUN TestParseConfigEntry/terminating-gateway_(json_snake_case) +[INFO] freeport: detected ephemeral port range of [32768, 60999] +[INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range +--- PASS: TestConfigWrite_noTabs (0.01s) === RUN TestParseConfigEntry/terminating-gateway_(json_camel_case) === RUN TestParseConfigEntry/service-defaults_(hcl_snake_case) === RUN TestParseConfigEntry/service-defaults_(hcl_camel_case) === RUN TestParseConfigEntry/service-defaults_(json_snake_case) === RUN TestParseConfigEntry/service-defaults_(json_camel_case) -[INFO] freeport: detected ephemeral port range of [32768, 60999] -[INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range === RUN TestParseConfigEntry/service-router:_kitchen_sink_(hcl_snake_case) === RUN TestParseConfigEntry/service-router:_kitchen_sink_(hcl_camel_case) === RUN TestParseConfigEntry/service-router:_kitchen_sink_(json_snake_case) @@ -78793,7 +78781,7 @@ === RUN TestParseConfigEntry/ingress-gateway:_kitchen_sink_(hcl_camel_case) === RUN TestParseConfigEntry/ingress-gateway:_kitchen_sink_(json_snake_case) === RUN TestParseConfigEntry/ingress-gateway:_kitchen_sink_(json_camel_case) ---- PASS: TestParseConfigEntry (0.07s) +--- PASS: TestParseConfigEntry (0.04s) --- PASS: TestParseConfigEntry/proxy-defaults:_extra_fields_or_typo_(hcl_snake_case) (0.00s) --- PASS: TestParseConfigEntry/proxy-defaults:_extra_fields_or_typo_(hcl_camel_case) (0.00s) --- PASS: TestParseConfigEntry/proxy-defaults:_extra_fields_or_typo_(json_snake_case) (0.00s) @@ -78811,9 +78799,9 @@ --- PASS: TestParseConfigEntry/service-defaults_(json_snake_case) (0.00s) --- PASS: TestParseConfigEntry/service-defaults_(json_camel_case) (0.00s) --- PASS: TestParseConfigEntry/service-router:_kitchen_sink_(hcl_snake_case) (0.00s) - --- PASS: TestParseConfigEntry/service-router:_kitchen_sink_(hcl_camel_case) (0.01s) + --- PASS: TestParseConfigEntry/service-router:_kitchen_sink_(hcl_camel_case) (0.00s) --- PASS: TestParseConfigEntry/service-router:_kitchen_sink_(json_snake_case) (0.00s) - --- PASS: TestParseConfigEntry/service-router:_kitchen_sink_(json_camel_case) (0.01s) + --- PASS: TestParseConfigEntry/service-router:_kitchen_sink_(json_camel_case) (0.00s) --- PASS: TestParseConfigEntry/service-splitter:_kitchen_sink_(hcl_snake_case) (0.00s) --- PASS: TestParseConfigEntry/service-splitter:_kitchen_sink_(hcl_camel_case) (0.00s) --- PASS: TestParseConfigEntry/service-splitter:_kitchen_sink_(json_snake_case) (0.00s) @@ -78845,82 +78833,85 @@ === RUN TestConfigWrite/File === RUN TestConfigWrite/Stdin === RUN TestConfigWrite/No_config -2022-09-15T21:44:00.544Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:44:00.577Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -44:00.586 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:05fa4020-1ee2-6326-217f-bf7f39606cef Address:127.0.0.1:24300}]" -44:00.587 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24300 [Follower]" leader= -44:00.587 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-05fa4020-1ee2-6326-217f-bf7f39606cef.dc1 127.0.0.1 -44:00.588 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-05fa4020-1ee2-6326-217f-bf7f39606cef 127.0.0.1 -2022-09-15T21:44:00.588Z [INFO] agent.router: Initializing LAN area manager -44:00.589 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24295 network=udp -44:00.589 [INFO] TestAgent.server: Adding LAN server: server="Node-05fa4020-1ee2-6326-217f-bf7f39606cef (Addr: tcp/127.0.0.1:24300) (DC: dc1)" -44:00.590 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-05fa4020-1ee2-6326-217f-bf7f39606cef.dc1 area=wan -44:00.590 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24295 network=tcp -44:00.591 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24296 network=tcp -44:00.591 [INFO] TestAgent: started state syncer -44:00.592 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24301 network=tcp -44:00.646 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -44:00.646 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24300 [Candidate]" term=2 -44:00.646 [DEBUG] TestAgent.server.raft: votes: needed=1 -44:00.646 [DEBUG] TestAgent.server.raft: vote granted: from=05fa4020-1ee2-6326-217f-bf7f39606cef term=2 tally=1 -44:00.646 [INFO] TestAgent.server.raft: election won: tally=1 -44:00.646 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24300 [Leader]" -44:00.646 [INFO] TestAgent.server: cluster leadership acquired -44:00.647 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24300 -44:00.648 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -44:00.660 [INFO] TestAgent.server: New leader elected: payload=Node-05fa4020-1ee2-6326-217f-bf7f39606cef -44:00.707 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -44:00.708 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -44:00.708 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -44:00.708 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -44:00.708 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -44:00.708 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-05fa4020-1ee2-6326-217f-bf7f39606cef -44:00.708 [INFO] TestAgent.server: member joined, marking health alive: member=Node-05fa4020-1ee2-6326-217f-bf7f39606cef -44:00.709 [INFO] TestAgent.server: federation state anti-entropy synced -44:00.739 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:55204 latency=1.88862ms -44:00.741 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/service-defaults/web from=127.0.0.1:55206 latency=737.42µs -44:00.746 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:55208 latency=1.73314ms -44:00.760 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/proxy-defaults/global from=127.0.0.1:55206 latency=2.27188ms -44:00.765 [INFO] TestAgent: Requesting shutdown -44:00.765 [INFO] TestAgent.server: shutting down server -44:00.765 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -44:00.765 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -44:00.767 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -44:00.767 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -44:00.767 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -44:00.766 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -44:00.767 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -44:00.767 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -44:00.767 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -44:00.767 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -44:00.767 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:44:00.767Z [INFO] agent.router.manager: shutting down -2022-09-15T21:44:00.768Z [INFO] agent.router.manager: shutting down -44:00.768 [INFO] TestAgent: consul server down -44:00.768 [INFO] TestAgent: shutdown complete -44:00.768 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24295 network=tcp -44:00.768 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24295 network=udp -44:00.769 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24296 network=tcp -44:01.269 [INFO] TestAgent: Waiting for endpoints to shut down -44:01.269 [INFO] TestAgent: Endpoints down ---- PASS: TestConfigWrite (0.83s) +2021-08-13T15:36:15.987Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:36:16.011Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +36:16.015 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:afb96549-8830-980e-5360-13286edca1b6 Address:127.0.0.1:19195}]" +36:16.015 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19195 [Follower]" leader= +36:16.016 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-afb96549-8830-980e-5360-13286edca1b6.dc1 127.0.0.1 +36:16.017 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-afb96549-8830-980e-5360-13286edca1b6 127.0.0.1 +2021-08-13T15:36:16.017Z [INFO] agent.router: Initializing LAN area manager +36:16.019 [INFO] TestAgent.server: Adding LAN server: server="Node-afb96549-8830-980e-5360-13286edca1b6 (Addr: tcp/127.0.0.1:19195) (DC: dc1)" +36:16.023 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=udp +36:16.023 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=tcp +36:16.023 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-afb96549-8830-980e-5360-13286edca1b6.dc1 area=wan +36:16.025 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19191 network=tcp +36:16.025 [INFO] TestAgent: started state syncer +36:16.025 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19196 network=tcp +36:16.081 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +36:16.081 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19195 [Candidate]" term=2 +36:16.082 [DEBUG] TestAgent.server.raft: votes: needed=1 +36:16.082 [DEBUG] TestAgent.server.raft: vote granted: from=afb96549-8830-980e-5360-13286edca1b6 term=2 tally=1 +36:16.082 [INFO] TestAgent.server.raft: election won: tally=1 +36:16.082 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19195 [Leader]" +36:16.082 [INFO] TestAgent.server: cluster leadership acquired +36:16.082 [INFO] TestAgent.server: New leader elected: payload=Node-afb96549-8830-980e-5360-13286edca1b6 +36:16.082 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:19195 +36:16.084 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +36:16.109 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +36:16.109 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +36:16.109 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +36:16.109 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +36:16.109 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +36:16.109 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-afb96549-8830-980e-5360-13286edca1b6 +36:16.109 [INFO] TestAgent.server: member joined, marking health alive: member=Node-afb96549-8830-980e-5360-13286edca1b6 +36:16.110 [INFO] TestAgent.server: federation state anti-entropy synced +36:16.298 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +36:16.298 [INFO] TestAgent: Synced node info +36:16.298 [DEBUG] TestAgent: Node info in sync +36:16.376 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:35526 latency=1.784924ms +36:16.378 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/service-defaults/web from=127.0.0.1:35528 latency=400.481µs +36:16.387 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:35530 latency=7.162476ms +36:16.390 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/proxy-defaults/global from=127.0.0.1:35528 latency=458.641µs +36:16.391 [INFO] TestAgent: Requesting shutdown +36:16.391 [INFO] TestAgent.server: shutting down server +36:16.391 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +36:16.391 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +36:16.391 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +36:16.391 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +36:16.391 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +36:16.391 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +36:16.391 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +36:16.391 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +36:16.391 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +36:16.391 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +36:16.392 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:36:16.392Z [INFO] agent.router.manager: shutting down +2021-08-13T15:36:16.392Z [INFO] agent.router.manager: shutting down +36:16.392 [INFO] TestAgent: consul server down +36:16.392 [INFO] TestAgent: shutdown complete +36:16.392 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=tcp +36:16.392 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=udp +36:16.392 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19191 network=tcp +36:16.892 [INFO] TestAgent: Waiting for endpoints to shut down +36:16.892 [INFO] TestAgent: Endpoints down +--- PASS: TestConfigWrite (1.02s) --- PASS: TestConfigWrite/File (0.01s) - --- PASS: TestConfigWrite/Stdin (0.02s) + --- PASS: TestConfigWrite/Stdin (0.01s) --- PASS: TestConfigWrite/No_config (0.00s) PASS -ok github.com/hashicorp/consul/command/config/write 1.030s +ok github.com/hashicorp/consul/command/config/write 1.122s === RUN TestConnectCommand_noTabs === PAUSE TestConnectCommand_noTabs === CONT TestConnectCommand_noTabs --- PASS: TestConnectCommand_noTabs (0.00s) PASS -ok github.com/hashicorp/consul/command/connect 0.016s +ok github.com/hashicorp/consul/command/connect 0.006s === RUN TestCatalogCommand_noTabs === PAUSE TestCatalogCommand_noTabs === CONT TestCatalogCommand_noTabs --- PASS: TestCatalogCommand_noTabs (0.00s) PASS -ok github.com/hashicorp/consul/command/connect/ca 0.026s +ok github.com/hashicorp/consul/command/connect/ca 0.006s === RUN TestConnectCAGetConfigCommand_noTabs === PAUSE TestConnectCAGetConfigCommand_noTabs === RUN TestConnectCAGetConfigCommand @@ -78931,137 +78922,139 @@ [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:44:07.500Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:44:07.539Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -44:07.544 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:47ee95a2-b27e-26b2-00d2-e8f97b043091 Address:127.0.0.1:29405}]" -44:07.544 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29405 [Follower]" leader= -44:07.546 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-47ee95a2-b27e-26b2-00d2-e8f97b043091.dc1 127.0.0.1 -44:07.547 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-47ee95a2-b27e-26b2-00d2-e8f97b043091 127.0.0.1 -2022-09-15T21:44:07.547Z [INFO] agent.router: Initializing LAN area manager -44:07.547 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29400 network=udp -44:07.548 [INFO] TestAgent.server: Adding LAN server: server="Node-47ee95a2-b27e-26b2-00d2-e8f97b043091 (Addr: tcp/127.0.0.1:29405) (DC: dc1)" -44:07.548 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-47ee95a2-b27e-26b2-00d2-e8f97b043091.dc1 area=wan -44:07.548 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29400 network=tcp -44:07.549 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29401 network=tcp -44:07.549 [INFO] TestAgent: started state syncer -44:07.549 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29406 network=tcp -44:07.594 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -44:07.594 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29405 [Candidate]" term=2 -44:07.594 [DEBUG] TestAgent.server.raft: votes: needed=1 -44:07.594 [DEBUG] TestAgent.server.raft: vote granted: from=47ee95a2-b27e-26b2-00d2-e8f97b043091 term=2 tally=1 -44:07.594 [INFO] TestAgent.server.raft: election won: tally=1 -44:07.594 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29405 [Leader]" -44:07.594 [INFO] TestAgent.server: cluster leadership acquired -44:07.595 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29405 -44:07.596 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -44:07.598 [INFO] TestAgent.server: New leader elected: payload=Node-47ee95a2-b27e-26b2-00d2-e8f97b043091 -44:07.638 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -44:07.638 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -44:07.638 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -44:07.638 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -44:07.638 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -44:07.638 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-47ee95a2-b27e-26b2-00d2-e8f97b043091 -44:07.638 [INFO] TestAgent.server: member joined, marking health alive: member=Node-47ee95a2-b27e-26b2-00d2-e8f97b043091 -44:07.640 [INFO] TestAgent.server: federation state anti-entropy synced -44:07.694 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -44:07.695 [INFO] TestAgent: Synced node info -44:07.695 [DEBUG] TestAgent: Node info in sync -44:07.960 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/ca/configuration from=127.0.0.1:35308 latency=9.67786ms -44:07.963 [INFO] TestAgent: Requesting shutdown -44:07.963 [INFO] TestAgent.server: shutting down server -44:07.963 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -44:07.963 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -44:07.963 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -44:07.963 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -44:07.963 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -44:07.963 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -44:07.963 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -44:07.964 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -44:07.964 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -44:07.964 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -44:07.964 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:44:07.970Z [INFO] agent.router.manager: shutting down -2022-09-15T21:44:07.972Z [INFO] agent.router.manager: shutting down -44:07.972 [INFO] TestAgent: consul server down -44:07.972 [INFO] TestAgent: shutdown complete -44:07.972 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29400 network=tcp -44:07.972 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29400 network=udp -44:07.972 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29401 network=tcp -44:08.474 [INFO] TestAgent: Waiting for endpoints to shut down -44:08.474 [INFO] TestAgent: Endpoints down ---- PASS: TestConnectCAGetConfigCommand (1.14s) +2021-08-13T15:36:21.844Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:36:21.870Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +36:21.873 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7494b261-1174-d711-a3ce-3f87bbfc4ac6 Address:127.0.0.1:28384}]" +36:21.874 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:28384 [Follower]" leader= +36:21.875 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7494b261-1174-d711-a3ce-3f87bbfc4ac6.dc1 127.0.0.1 +36:21.876 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7494b261-1174-d711-a3ce-3f87bbfc4ac6 127.0.0.1 +2021-08-13T15:36:21.876Z [INFO] agent.router: Initializing LAN area manager +36:21.876 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28379 network=udp +36:21.877 [INFO] TestAgent.server: Adding LAN server: server="Node-7494b261-1174-d711-a3ce-3f87bbfc4ac6 (Addr: tcp/127.0.0.1:28384) (DC: dc1)" +36:21.877 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7494b261-1174-d711-a3ce-3f87bbfc4ac6.dc1 area=wan +36:21.877 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28379 network=tcp +36:21.878 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:28380 network=tcp +36:21.878 [INFO] TestAgent: started state syncer +36:21.878 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:28385 network=tcp +36:21.917 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +36:21.917 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:28384 [Candidate]" term=2 +36:21.918 [DEBUG] TestAgent.server.raft: votes: needed=1 +36:21.918 [DEBUG] TestAgent.server.raft: vote granted: from=7494b261-1174-d711-a3ce-3f87bbfc4ac6 term=2 tally=1 +36:21.918 [INFO] TestAgent.server.raft: election won: tally=1 +36:21.918 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:28384 [Leader]" +36:21.918 [INFO] TestAgent.server: cluster leadership acquired +36:21.918 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:28384 +36:21.920 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +36:21.923 [INFO] TestAgent.server: New leader elected: payload=Node-7494b261-1174-d711-a3ce-3f87bbfc4ac6 +36:21.943 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +36:21.943 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +36:21.943 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +36:21.943 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +36:21.943 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +36:21.943 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7494b261-1174-d711-a3ce-3f87bbfc4ac6 +36:21.943 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7494b261-1174-d711-a3ce-3f87bbfc4ac6 +36:21.945 [INFO] TestAgent.server: federation state anti-entropy synced +36:22.083 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +36:22.084 [INFO] TestAgent: Synced node info +36:22.084 [DEBUG] TestAgent: Node info in sync +36:22.316 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/ca/configuration from=127.0.0.1:48356 latency=2.313145ms +36:22.317 [INFO] TestAgent: Requesting shutdown +36:22.318 [INFO] TestAgent.server: shutting down server +36:22.318 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +36:22.318 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +36:22.318 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +36:22.318 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +36:22.318 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +36:22.318 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +36:22.318 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +36:22.318 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +36:22.318 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +36:22.318 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +36:22.318 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:36:22.318Z [INFO] agent.router.manager: shutting down +2021-08-13T15:36:22.318Z [INFO] agent.router.manager: shutting down +36:22.318 [INFO] TestAgent: consul server down +36:22.318 [INFO] TestAgent: shutdown complete +36:22.318 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28379 network=tcp +36:22.319 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28379 network=udp +36:22.319 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:28380 network=tcp +36:22.819 [INFO] TestAgent: Waiting for endpoints to shut down +36:22.819 [INFO] TestAgent: Endpoints down +--- PASS: TestConnectCAGetConfigCommand (1.03s) PASS -ok github.com/hashicorp/consul/command/connect/ca/get 1.307s +ok github.com/hashicorp/consul/command/connect/ca/get 1.111s === RUN TestConnectCASetConfigCommand_noTabs === PAUSE TestConnectCASetConfigCommand_noTabs === RUN TestConnectCASetConfigCommand === PAUSE TestConnectCASetConfigCommand === CONT TestConnectCASetConfigCommand_noTabs === CONT TestConnectCASetConfigCommand ---- PASS: TestConnectCASetConfigCommand_noTabs (0.00s) [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestConnectCASetConfigCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:44:06.592Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:44:06.653Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -44:06.662 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d66a2ee0-1a32-dba1-ba17-83fcd80e6d7e Address:127.0.0.1:19195}]" -44:06.663 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d66a2ee0-1a32-dba1-ba17-83fcd80e6d7e.dc1 127.0.0.1 -44:06.664 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d66a2ee0-1a32-dba1-ba17-83fcd80e6d7e 127.0.0.1 -2022-09-15T21:44:06.664Z [INFO] agent.router: Initializing LAN area manager -44:06.665 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=udp -44:06.665 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19195 [Follower]" leader= -44:06.665 [INFO] TestAgent.server: Adding LAN server: server="Node-d66a2ee0-1a32-dba1-ba17-83fcd80e6d7e (Addr: tcp/127.0.0.1:19195) (DC: dc1)" -44:06.665 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d66a2ee0-1a32-dba1-ba17-83fcd80e6d7e.dc1 area=wan -44:06.670 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=tcp -44:06.671 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19191 network=tcp -44:06.671 [INFO] TestAgent: started state syncer -44:06.671 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19196 network=tcp -44:06.718 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -44:06.718 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19195 [Candidate]" term=2 -44:06.718 [DEBUG] TestAgent.server.raft: votes: needed=1 -44:06.718 [DEBUG] TestAgent.server.raft: vote granted: from=d66a2ee0-1a32-dba1-ba17-83fcd80e6d7e term=2 tally=1 -44:06.718 [INFO] TestAgent.server.raft: election won: tally=1 -44:06.718 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19195 [Leader]" -44:06.719 [INFO] TestAgent.server: cluster leadership acquired -44:06.719 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:19195 -44:06.720 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -44:06.722 [INFO] TestAgent.server: New leader elected: payload=Node-d66a2ee0-1a32-dba1-ba17-83fcd80e6d7e -44:06.772 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -44:06.772 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -44:06.773 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -44:06.773 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -44:06.773 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -44:06.773 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d66a2ee0-1a32-dba1-ba17-83fcd80e6d7e -44:06.773 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d66a2ee0-1a32-dba1-ba17-83fcd80e6d7e -44:06.790 [INFO] TestAgent.server: federation state anti-entropy synced -44:06.833 [INFO] TestAgent.server.connect: CA provider config updated -44:06.833 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/connect/ca/configuration from=127.0.0.1:40766 latency=1.2348ms -44:06.834 [INFO] TestAgent: Requesting shutdown -44:06.834 [INFO] TestAgent.server: shutting down server -44:06.834 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -44:06.834 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -44:06.834 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -44:06.834 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -44:06.834 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -44:06.834 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -44:06.834 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -44:06.834 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -44:06.834 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -44:06.834 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -44:06.834 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -44:06.834 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:44:06.835Z [INFO] agent.router.manager: shutting down -2022-09-15T21:44:06.835Z [INFO] agent.router.manager: shutting down -44:06.835 [INFO] TestAgent: consul server down -44:06.835 [INFO] TestAgent: shutdown complete -44:06.835 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=tcp -44:06.835 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=udp -44:06.835 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19191 network=tcp -44:07.338 [INFO] TestAgent: Waiting for endpoints to shut down -44:07.338 [INFO] TestAgent: Endpoints down ---- PASS: TestConnectCASetConfigCommand (0.87s) +2021-08-13T15:36:22.416Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:36:22.433Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +36:22.440 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ba3a7990-d8fe-9090-b2d1-0dbd37e5e582 Address:127.0.0.1:13069}]" +36:22.440 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13069 [Follower]" leader= +36:22.441 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ba3a7990-d8fe-9090-b2d1-0dbd37e5e582.dc1 127.0.0.1 +36:22.442 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ba3a7990-d8fe-9090-b2d1-0dbd37e5e582 127.0.0.1 +2021-08-13T15:36:22.442Z [INFO] agent.router: Initializing LAN area manager +36:22.443 [INFO] TestAgent.server: Adding LAN server: server="Node-ba3a7990-d8fe-9090-b2d1-0dbd37e5e582 (Addr: tcp/127.0.0.1:13069) (DC: dc1)" +36:22.443 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ba3a7990-d8fe-9090-b2d1-0dbd37e5e582.dc1 area=wan +36:22.443 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13064 network=udp +36:22.443 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13064 network=tcp +36:22.444 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13065 network=tcp +36:22.444 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13070 network=tcp +36:22.444 [INFO] TestAgent: started state syncer +36:22.482 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +36:22.482 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13069 [Candidate]" term=2 +36:22.482 [DEBUG] TestAgent.server.raft: votes: needed=1 +36:22.482 [DEBUG] TestAgent.server.raft: vote granted: from=ba3a7990-d8fe-9090-b2d1-0dbd37e5e582 term=2 tally=1 +36:22.482 [INFO] TestAgent.server.raft: election won: tally=1 +36:22.482 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13069 [Leader]" +36:22.483 [INFO] TestAgent.server: cluster leadership acquired +36:22.483 [INFO] TestAgent.server: New leader elected: payload=Node-ba3a7990-d8fe-9090-b2d1-0dbd37e5e582 +36:22.483 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13069 +36:22.485 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +36:22.508 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +36:22.508 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +36:22.508 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +36:22.508 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +36:22.508 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +36:22.508 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ba3a7990-d8fe-9090-b2d1-0dbd37e5e582 +36:22.508 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ba3a7990-d8fe-9090-b2d1-0dbd37e5e582 +36:22.509 [INFO] TestAgent.server: federation state anti-entropy synced +36:22.541 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +36:22.542 [INFO] TestAgent: Synced node info +36:22.542 [DEBUG] TestAgent: Node info in sync +36:22.610 [INFO] TestAgent.server.connect: CA provider config updated +36:22.610 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/connect/ca/configuration from=127.0.0.1:36080 latency=1.118503ms +36:22.610 [INFO] TestAgent: Requesting shutdown +36:22.610 [INFO] TestAgent.server: shutting down server +36:22.610 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +36:22.610 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +36:22.610 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +36:22.610 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +36:22.610 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +36:22.610 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +36:22.610 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +36:22.610 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +36:22.610 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +36:22.610 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +36:22.610 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:36:22.611Z [INFO] agent.router.manager: shutting down +2021-08-13T15:36:22.611Z [INFO] agent.router.manager: shutting down +36:22.611 [INFO] TestAgent: consul server down +36:22.611 [INFO] TestAgent: shutdown complete +36:22.611 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13064 network=tcp +36:22.611 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13064 network=udp +36:22.611 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13065 network=tcp +36:23.111 [INFO] TestAgent: Waiting for endpoints to shut down +36:23.111 [INFO] TestAgent: Endpoints down +--- PASS: TestConnectCASetConfigCommand (0.75s) PASS -ok github.com/hashicorp/consul/command/connect/ca/set 1.114s +ok github.com/hashicorp/consul/command/connect/ca/set 0.817s === RUN TestBootstrapConfig_ConfigureArgs === RUN TestBootstrapConfig_ConfigureArgs/defaults === RUN TestBootstrapConfig_ConfigureArgs/extra-stats-sinks @@ -79086,7 +79079,7 @@ === RUN TestBootstrapConfig_ConfigureArgs/ready-bind-addr === RUN TestBootstrapConfig_ConfigureArgs/ready-bind-addr-with-overrides === RUN TestBootstrapConfig_ConfigureArgs/ready-bind-addr-and-prometheus-and-stats ---- PASS: TestBootstrapConfig_ConfigureArgs (0.03s) +--- PASS: TestBootstrapConfig_ConfigureArgs (0.01s) --- PASS: TestBootstrapConfig_ConfigureArgs/defaults (0.00s) --- PASS: TestBootstrapConfig_ConfigureArgs/extra-stats-sinks (0.00s) --- PASS: TestBootstrapConfig_ConfigureArgs/simple-statsd-sink (0.00s) @@ -79109,7 +79102,7 @@ --- PASS: TestBootstrapConfig_ConfigureArgs/err-bad-dogstatsd-addr (0.00s) --- PASS: TestBootstrapConfig_ConfigureArgs/ready-bind-addr (0.00s) --- PASS: TestBootstrapConfig_ConfigureArgs/ready-bind-addr-with-overrides (0.00s) - --- PASS: TestBootstrapConfig_ConfigureArgs/ready-bind-addr-and-prometheus-and-stats (0.01s) + --- PASS: TestBootstrapConfig_ConfigureArgs/ready-bind-addr-and-prometheus-and-stats (0.00s) === RUN TestEnvoyCommand_noTabs === PAUSE TestEnvoyCommand_noTabs === RUN TestEnvoyGateway_Validation @@ -82125,32 +82118,32 @@ ] } } ---- PASS: TestGenerateConfig (0.39s) +--- PASS: TestGenerateConfig (0.20s) --- PASS: TestGenerateConfig/no-args (0.00s) - --- PASS: TestGenerateConfig/defaults (0.02s) - --- PASS: TestGenerateConfig/token-arg (0.02s) + --- PASS: TestGenerateConfig/defaults (0.01s) + --- PASS: TestGenerateConfig/token-arg (0.01s) --- PASS: TestGenerateConfig/token-env (0.01s) --- PASS: TestGenerateConfig/token-file-arg (0.01s) --- PASS: TestGenerateConfig/token-file-env (0.01s) - --- PASS: TestGenerateConfig/grpc-addr-flag (0.02s) - --- PASS: TestGenerateConfig/grpc-addr-env (0.01s) - --- PASS: TestGenerateConfig/grpc-addr-unix (0.01s) + --- PASS: TestGenerateConfig/grpc-addr-flag (0.01s) + --- PASS: TestGenerateConfig/grpc-addr-env (0.00s) + --- PASS: TestGenerateConfig/grpc-addr-unix (0.00s) --- PASS: TestGenerateConfig/grpc-addr-config (0.01s) --- PASS: TestGenerateConfig/access-log-path (0.01s) --- PASS: TestGenerateConfig/missing-ca-file (0.00s) --- PASS: TestGenerateConfig/existing-ca-file (0.01s) - --- PASS: TestGenerateConfig/missing-ca-path (0.02s) - --- PASS: TestGenerateConfig/existing-ca-path (0.02s) - --- PASS: TestGenerateConfig/custom-bootstrap (0.02s) + --- PASS: TestGenerateConfig/missing-ca-path (0.00s) + --- PASS: TestGenerateConfig/existing-ca-path (0.01s) + --- PASS: TestGenerateConfig/custom-bootstrap (0.01s) --- PASS: TestGenerateConfig/extra_-single (0.01s) --- PASS: TestGenerateConfig/extra_-multiple (0.01s) --- PASS: TestGenerateConfig/stats-config-override (0.01s) - --- PASS: TestGenerateConfig/zipkin-tracing-config (0.02s) - --- PASS: TestGenerateConfig/CONSUL_HTTP_ADDR-with-https-scheme-enables-tls (0.02s) - --- PASS: TestGenerateConfig/ingress-gateway (0.02s) + --- PASS: TestGenerateConfig/zipkin-tracing-config (0.01s) + --- PASS: TestGenerateConfig/CONSUL_HTTP_ADDR-with-https-scheme-enables-tls (0.01s) + --- PASS: TestGenerateConfig/ingress-gateway (0.01s) --- PASS: TestGenerateConfig/ingress-gateway-address-specified (0.01s) - --- PASS: TestGenerateConfig/ingress-gateway-register-with-service-without-proxy-id (0.04s) - --- PASS: TestGenerateConfig/ingress-gateway-register-with-service-and-proxy-id (0.03s) + --- PASS: TestGenerateConfig/ingress-gateway-register-with-service-without-proxy-id (0.01s) + --- PASS: TestGenerateConfig/ingress-gateway-register-with-service-and-proxy-id (0.01s) --- PASS: TestGenerateConfig/ingress-gateway-no-auto-register (0.01s) === RUN TestEnvoy_GatewayRegistration === PAUSE TestEnvoy_GatewayRegistration @@ -82162,12 +82155,12 @@ === RUN TestExecEnvoy/hot-restart-version === RUN TestExecEnvoy/hot-restart-version#01 === RUN TestExecEnvoy/hot-restart-version#02 ---- PASS: TestExecEnvoy (2.14s) - --- PASS: TestExecEnvoy/default (0.40s) - --- PASS: TestExecEnvoy/hot-restart-epoch (0.50s) - --- PASS: TestExecEnvoy/hot-restart-version (0.50s) - --- PASS: TestExecEnvoy/hot-restart-version#01 (0.39s) - --- PASS: TestExecEnvoy/hot-restart-version#02 (0.36s) +--- PASS: TestExecEnvoy (0.77s) + --- PASS: TestExecEnvoy/default (0.21s) + --- PASS: TestExecEnvoy/hot-restart-epoch (0.14s) + --- PASS: TestExecEnvoy/hot-restart-version (0.13s) + --- PASS: TestExecEnvoy/hot-restart-version#01 (0.14s) + --- PASS: TestExecEnvoy/hot-restart-version#02 (0.14s) === RUN TestHelperProcess --- PASS: TestHelperProcess (0.00s) === RUN TestServiceAddressValue_Value @@ -82193,7 +82186,7 @@ === RUN TestServiceAddressValue_Set/invalid_addres === RUN TestServiceAddressValue_Set/invalid_port === RUN TestServiceAddressValue_Set/invalid_address_format ---- PASS: TestServiceAddressValue_Set (0.01s) +--- PASS: TestServiceAddressValue_Set (0.00s) --- PASS: TestServiceAddressValue_Set/default_port (0.00s) --- PASS: TestServiceAddressValue_Set/valid_address (0.00s) --- PASS: TestServiceAddressValue_Set/address_with_no_port (0.00s) @@ -82201,11 +82194,7 @@ --- PASS: TestServiceAddressValue_Set/invalid_port (0.00s) --- PASS: TestServiceAddressValue_Set/invalid_address_format (0.00s) === CONT TestEnvoyCommand_noTabs ---- PASS: TestEnvoyCommand_noTabs (0.01s) -=== CONT TestEnvoy_GatewayRegistration -=== CONT TestEnvoyGateway_Validation -=== RUN TestEnvoyGateway_Validation/-register_for_non-gateway -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestEnvoyCommand_noTabs (0.00s) === CONT TestEnvoyCommand_canBindInternal === RUN TestEnvoyCommand_canBindInternal/IPNet === RUN TestEnvoyCommand_canBindInternal/IPNet/ipv4 @@ -82214,11 +82203,11 @@ === RUN TestEnvoyCommand_canBindInternal/IPNet/ipv4_not_found === RUN TestEnvoyCommand_canBindInternal/IPNet/ipv6_not_found === RUN TestEnvoyCommand_canBindInternal/IPAddr -=== RUN TestEnvoyCommand_canBindInternal/IPAddr/ipv4 === RUN TestEnvoyCommand_canBindInternal/IPAddr/secondary_ipv4 === RUN TestEnvoyCommand_canBindInternal/IPAddr/ipv6 === RUN TestEnvoyCommand_canBindInternal/IPAddr/ipv4_not_found === RUN TestEnvoyCommand_canBindInternal/IPAddr/ipv6_not_found +=== RUN TestEnvoyCommand_canBindInternal/IPAddr/ipv4 --- PASS: TestEnvoyCommand_canBindInternal (0.00s) --- PASS: TestEnvoyCommand_canBindInternal/IPNet (0.00s) --- PASS: TestEnvoyCommand_canBindInternal/IPNet/ipv4 (0.00s) @@ -82227,15 +82216,19 @@ --- PASS: TestEnvoyCommand_canBindInternal/IPNet/ipv4_not_found (0.00s) --- PASS: TestEnvoyCommand_canBindInternal/IPNet/ipv6_not_found (0.00s) --- PASS: TestEnvoyCommand_canBindInternal/IPAddr (0.00s) - --- PASS: TestEnvoyCommand_canBindInternal/IPAddr/ipv4 (0.00s) --- PASS: TestEnvoyCommand_canBindInternal/IPAddr/secondary_ipv4 (0.00s) --- PASS: TestEnvoyCommand_canBindInternal/IPAddr/ipv6 (0.00s) --- PASS: TestEnvoyCommand_canBindInternal/IPAddr/ipv4_not_found (0.00s) --- PASS: TestEnvoyCommand_canBindInternal/IPAddr/ipv6_not_found (0.00s) -=== RUN TestEnvoyGateway_Validation/-mesh-gateway_and_-gateway_cannot_be_combined -=== RUN TestEnvoyGateway_Validation/no_proxy_registration_specified_nor_discovered + --- PASS: TestEnvoyCommand_canBindInternal/IPAddr/ipv4 (0.00s) +=== CONT TestEnvoy_GatewayRegistration +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +=== CONT TestEnvoyGateway_Validation +=== RUN TestEnvoyGateway_Validation/-register_for_non-gateway [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range +=== RUN TestEnvoyGateway_Validation/-mesh-gateway_and_-gateway_cannot_be_combined +=== RUN TestEnvoyGateway_Validation/no_proxy_registration_specified_nor_discovered --- PASS: TestEnvoyGateway_Validation (0.01s) --- PASS: TestEnvoyGateway_Validation/-register_for_non-gateway (0.00s) --- PASS: TestEnvoyGateway_Validation/-mesh-gateway_and_-gateway_cannot_be_combined (0.00s) @@ -82270,7 +82263,7 @@ { "socket_address": { "address": "127.0.0.1", - "port_value": 28385 + "port_value": 14091 } } ] @@ -82460,7 +82453,7 @@ { "socket_address": { "address": "127.0.0.1", - "port_value": 28385 + "port_value": 14091 } } ] @@ -82650,7 +82643,7 @@ { "socket_address": { "address": "127.0.0.1", - "port_value": 28385 + "port_value": 14091 } } ] @@ -82840,7 +82833,7 @@ { "socket_address": { "address": "127.0.0.1", - "port_value": 28385 + "port_value": 14091 } } ] @@ -83000,150 +82993,121 @@ ] } } -2022-09-15T21:44:12.483Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:44:12.517Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -44:12.525 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6eaf990e-3cca-0d1b-f693-ba5dd22f2fba Address:127.0.0.1:28384}]" -44:12.531 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6eaf990e-3cca-0d1b-f693-ba5dd22f2fba.dc1 127.0.0.1 -44:12.532 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6eaf990e-3cca-0d1b-f693-ba5dd22f2fba 127.0.0.1 -2022-09-15T21:44:12.532Z [INFO] agent.router: Initializing LAN area manager -44:12.533 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28379 network=udp -44:12.533 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:28384 [Follower]" leader= -44:12.533 [INFO] TestAgent.server: Adding LAN server: server="Node-6eaf990e-3cca-0d1b-f693-ba5dd22f2fba (Addr: tcp/127.0.0.1:28384) (DC: dc1)" -44:12.534 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6eaf990e-3cca-0d1b-f693-ba5dd22f2fba.dc1 area=wan -44:12.534 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28379 network=tcp -44:12.535 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:28380 network=tcp -44:12.535 [INFO] TestAgent: started state syncer -44:12.535 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:28385 network=tcp -44:12.574 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -44:12.574 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:28384 [Candidate]" term=2 -44:12.574 [DEBUG] TestAgent.server.raft: votes: needed=1 -44:12.574 [DEBUG] TestAgent.server.raft: vote granted: from=6eaf990e-3cca-0d1b-f693-ba5dd22f2fba term=2 tally=1 -44:12.574 [INFO] TestAgent.server.raft: election won: tally=1 -44:12.574 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:28384 [Leader]" -44:12.574 [INFO] TestAgent.server: cluster leadership acquired -44:12.575 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:28384 -44:12.586 [INFO] TestAgent.server: New leader elected: payload=Node-6eaf990e-3cca-0d1b-f693-ba5dd22f2fba -44:12.595 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -44:12.642 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -44:12.650 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -44:12.650 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -44:12.650 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -44:12.650 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -44:12.650 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6eaf990e-3cca-0d1b-f693-ba5dd22f2fba -44:12.650 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6eaf990e-3cca-0d1b-f693-ba5dd22f2fba -44:12.652 [INFO] TestAgent.server: federation state anti-entropy synced -44:12.866 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -44:12.867 [INFO] TestAgent: Synced node info -44:12.867 [DEBUG] TestAgent: Node info in sync -44:12.955 [WARN] TestAgent.server.catalog: no terminating-gateway or ingress-gateway associated with this gateway: gateway=us-ingress -44:12.955 [WARN] TestAgent.server.catalog: no terminating-gateway or ingress-gateway associated with this gateway: gateway=us-ingress -44:12.955 [ERROR] TestAgent.proxycfg: watch error: service_id=us-ingress-1 id=gateway-config error="invalid type for config entry: " -44:12.963 [DEBUG] TestAgent: Node info in sync -44:12.964 [INFO] TestAgent: Synced service: service=us-ingress-1 -44:12.964 [DEBUG] TestAgent: Check in sync: check=service:us-ingress-1 -44:12.964 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:56992 latency=11.59422ms -44:12.967 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:56992 latency=1.97496ms -44:12.972 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/us-ingress-1 from=127.0.0.1:56992 latency=428.22µs -44:12.975 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/us-ingress-1 from=127.0.0.1:56994 latency=369.7µs -44:12.990 [DEBUG] TestAgent: Node info in sync -44:12.990 [DEBUG] TestAgent: Service in sync: service=us-ingress-1 -44:12.992 [INFO] TestAgent: Synced service: service=us-ingress -44:12.992 [DEBUG] TestAgent: Check in sync: check=service:us-ingress-1 -44:12.992 [DEBUG] TestAgent: Check in sync: check=service:us-ingress -44:12.992 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:56996 latency=3.05394ms -44:13.007 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:56996 latency=13.95806ms -44:13.015 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/us-ingress from=127.0.0.1:56996 latency=278.6µs -44:13.018 [ERROR] TestAgent.proxycfg: watch error: service_id=us-ingress id=gateway-config error="invalid type for config entry: " -44:13.018 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/us-ingress from=127.0.0.1:56994 latency=307.08µs -44:13.027 [DEBUG] TestAgent: Node info in sync -44:13.027 [DEBUG] TestAgent: Service in sync: service=us-ingress-1 -44:13.027 [DEBUG] TestAgent: Service in sync: service=us-ingress -44:13.028 [INFO] TestAgent: Synced service: service=ingress-gateway -44:13.028 [DEBUG] TestAgent: Check in sync: check=service:us-ingress-1 -44:13.028 [DEBUG] TestAgent: Check in sync: check=service:us-ingress -44:13.028 [DEBUG] TestAgent: Check in sync: check=service:ingress-gateway -44:13.028 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:56998 latency=2.01062ms -44:13.029 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -44:13.029 [DEBUG] TestAgent: Node info in sync -44:13.029 [DEBUG] TestAgent: Service in sync: service=us-ingress -44:13.029 [INFO] TestAgent: Synced service: service=ingress-gateway -44:13.029 [DEBUG] TestAgent: Service in sync: service=us-ingress-1 -44:13.029 [DEBUG] TestAgent: Check in sync: check=service:us-ingress-1 -44:13.029 [DEBUG] TestAgent: Check in sync: check=service:us-ingress -44:13.029 [DEBUG] TestAgent: Check in sync: check=service:ingress-gateway -44:13.029 [DEBUG] TestAgent: Node info in sync -44:13.029 [DEBUG] TestAgent: Service in sync: service=ingress-gateway -44:13.029 [DEBUG] TestAgent: Service in sync: service=us-ingress-1 -44:13.029 [DEBUG] TestAgent: Service in sync: service=us-ingress -44:13.029 [DEBUG] TestAgent: Check in sync: check=service:us-ingress -44:13.029 [DEBUG] TestAgent: Check in sync: check=service:ingress-gateway -44:13.029 [DEBUG] TestAgent: Check in sync: check=service:us-ingress-1 -44:13.030 [WARN] TestAgent.server.catalog: no terminating-gateway or ingress-gateway associated with this gateway: gateway=ingress-gateway -44:13.030 [WARN] TestAgent.server.catalog: no terminating-gateway or ingress-gateway associated with this gateway: gateway=ingress-gateway -44:13.031 [ERROR] TestAgent.proxycfg: watch error: service_id=ingress-gateway id=gateway-config error="invalid type for config entry: " -44:13.035 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:56998 latency=1.7247ms -44:13.039 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/ingress-gateway from=127.0.0.1:56998 latency=233.86µs -44:13.041 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/ingress-gateway from=127.0.0.1:56994 latency=184.52µs -44:13.047 [DEBUG] TestAgent: Node info in sync -44:13.048 [INFO] TestAgent: Synced service: service=us-ingress-1 -44:13.048 [DEBUG] TestAgent: Service in sync: service=us-ingress -44:13.048 [DEBUG] TestAgent: Service in sync: service=ingress-gateway -44:13.048 [DEBUG] TestAgent: Check in sync: check=service:ingress-gateway -44:13.048 [DEBUG] TestAgent: Check in sync: check=service:us-ingress-1 -44:13.048 [DEBUG] TestAgent: Check in sync: check=service:us-ingress -44:13.048 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:57000 latency=2.14418ms -44:13.048 [DEBUG] TestAgent: Node info in sync -44:13.048 [DEBUG] TestAgent: Service in sync: service=us-ingress-1 -44:13.048 [DEBUG] TestAgent: Service in sync: service=us-ingress -44:13.048 [DEBUG] TestAgent: Service in sync: service=ingress-gateway -44:13.048 [DEBUG] TestAgent: Check in sync: check=service:us-ingress-1 -44:13.048 [DEBUG] TestAgent: Check in sync: check=service:us-ingress -44:13.048 [DEBUG] TestAgent: Check in sync: check=service:ingress-gateway -44:13.048 [DEBUG] TestAgent: Node info in sync -44:13.048 [DEBUG] TestAgent: Service in sync: service=us-ingress-1 -44:13.048 [DEBUG] TestAgent: Service in sync: service=us-ingress -44:13.048 [DEBUG] TestAgent: Service in sync: service=ingress-gateway -44:13.048 [DEBUG] TestAgent: Check in sync: check=service:us-ingress -44:13.048 [DEBUG] TestAgent: Check in sync: check=service:ingress-gateway -44:13.048 [DEBUG] TestAgent: Check in sync: check=service:us-ingress-1 -44:13.051 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:57000 latency=1.81788ms -44:13.058 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/us-ingress-1 from=127.0.0.1:57000 latency=1.2987ms -44:13.060 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/us-ingress-1 from=127.0.0.1:56994 latency=193.3µs -44:13.060 [INFO] TestAgent: Requesting shutdown -44:13.061 [INFO] TestAgent.server: shutting down server -44:13.061 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -44:13.061 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -44:13.061 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -44:13.061 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -44:13.061 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -44:13.061 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -44:13.061 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -44:13.061 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -44:13.061 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -44:13.061 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -44:13.061 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:44:13.061Z [INFO] agent.router.manager: shutting down -2022-09-15T21:44:13.061Z [INFO] agent.router.manager: shutting down -44:13.062 [INFO] TestAgent: consul server down -44:13.062 [INFO] TestAgent: shutdown complete -44:13.062 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28379 network=tcp -44:13.062 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28379 network=udp -44:13.062 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:28380 network=tcp -44:13.563 [INFO] TestAgent: Waiting for endpoints to shut down -44:13.563 [INFO] TestAgent: Endpoints down ---- PASS: TestEnvoy_GatewayRegistration (1.22s) - --- PASS: TestEnvoy_GatewayRegistration/register_gateway_with_proxy-id_and_name (0.03s) - --- PASS: TestEnvoy_GatewayRegistration/register_gateway_without_proxy-id_with_name (0.04s) - --- PASS: TestEnvoy_GatewayRegistration/register_gateway_without_proxy-id_and_without_name (0.02s) - --- PASS: TestEnvoy_GatewayRegistration/register_gateway_with_proxy-id_without_name (0.02s) +2021-08-13T15:36:29.667Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:36:29.685Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +36:29.689 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a4dc018a-5b8a-1249-a280-678ce0302396 Address:127.0.0.1:14090}]" +36:29.689 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14090 [Follower]" leader= +36:29.690 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a4dc018a-5b8a-1249-a280-678ce0302396.dc1 127.0.0.1 +36:29.691 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a4dc018a-5b8a-1249-a280-678ce0302396 127.0.0.1 +2021-08-13T15:36:29.691Z [INFO] agent.router: Initializing LAN area manager +36:29.691 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14085 network=udp +36:29.691 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a4dc018a-5b8a-1249-a280-678ce0302396.dc1 area=wan +36:29.691 [INFO] TestAgent.server: Adding LAN server: server="Node-a4dc018a-5b8a-1249-a280-678ce0302396 (Addr: tcp/127.0.0.1:14090) (DC: dc1)" +36:29.692 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14085 network=tcp +36:29.692 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14086 network=tcp +36:29.692 [INFO] TestAgent: started state syncer +36:29.693 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14091 network=tcp +36:29.728 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +36:29.728 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14090 [Candidate]" term=2 +36:29.729 [DEBUG] TestAgent.server.raft: votes: needed=1 +36:29.729 [DEBUG] TestAgent.server.raft: vote granted: from=a4dc018a-5b8a-1249-a280-678ce0302396 term=2 tally=1 +36:29.729 [INFO] TestAgent.server.raft: election won: tally=1 +36:29.729 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14090 [Leader]" +36:29.729 [INFO] TestAgent.server: cluster leadership acquired +36:29.729 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14090 +36:29.730 [INFO] TestAgent.server: New leader elected: payload=Node-a4dc018a-5b8a-1249-a280-678ce0302396 +36:29.731 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +36:29.756 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +36:29.756 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +36:29.756 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +36:29.756 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +36:29.756 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +36:29.756 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a4dc018a-5b8a-1249-a280-678ce0302396 +36:29.756 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a4dc018a-5b8a-1249-a280-678ce0302396 +36:29.757 [INFO] TestAgent.server: federation state anti-entropy synced +36:30.052 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +36:30.053 [INFO] TestAgent: Synced node info +36:30.053 [DEBUG] TestAgent: Node info in sync +36:30.087 [WARN] TestAgent.server.catalog: no terminating-gateway or ingress-gateway associated with this gateway: gateway=us-ingress +36:30.088 [WARN] TestAgent.server.catalog: no terminating-gateway or ingress-gateway associated with this gateway: gateway=us-ingress +36:30.088 [ERROR] TestAgent.proxycfg: watch error: service_id=us-ingress-1 id=gateway-config error="invalid type for config entry: " +36:30.088 [DEBUG] TestAgent: Node info in sync +36:30.089 [INFO] TestAgent: Synced service: service=us-ingress-1 +36:30.089 [DEBUG] TestAgent: Check in sync: check=service:us-ingress-1 +36:30.089 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:49344 latency=2.978147ms +36:30.096 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:49344 latency=6.093875ms +36:30.099 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/us-ingress-1 from=127.0.0.1:49344 latency=242.32µs +36:30.102 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/us-ingress-1 from=127.0.0.1:49346 latency=320.421µs +36:30.105 [DEBUG] TestAgent: Node info in sync +36:30.105 [DEBUG] TestAgent: Service in sync: service=us-ingress-1 +36:30.106 [INFO] TestAgent: Synced service: service=us-ingress +36:30.106 [DEBUG] TestAgent: Check in sync: check=service:us-ingress-1 +36:30.106 [DEBUG] TestAgent: Check in sync: check=service:us-ingress +36:30.106 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:49348 latency=1.679104ms +36:30.107 [ERROR] TestAgent.proxycfg: watch error: service_id=us-ingress id=gateway-config error="invalid type for config entry: " +36:30.109 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:49348 latency=1.481684ms +36:30.112 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/us-ingress from=127.0.0.1:49348 latency=176.58µs +36:30.114 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/us-ingress from=127.0.0.1:49346 latency=155.36µs +36:30.117 [DEBUG] TestAgent: Node info in sync +36:30.117 [DEBUG] TestAgent: Service in sync: service=us-ingress-1 +36:30.117 [DEBUG] TestAgent: Service in sync: service=us-ingress +36:30.118 [INFO] TestAgent: Synced service: service=ingress-gateway +36:30.118 [DEBUG] TestAgent: Check in sync: check=service:us-ingress-1 +36:30.118 [DEBUG] TestAgent: Check in sync: check=service:us-ingress +36:30.118 [DEBUG] TestAgent: Check in sync: check=service:ingress-gateway +36:30.118 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:49350 latency=1.579364ms +36:30.119 [WARN] TestAgent.server.catalog: no terminating-gateway or ingress-gateway associated with this gateway: gateway=ingress-gateway +36:30.119 [WARN] TestAgent.server.catalog: no terminating-gateway or ingress-gateway associated with this gateway: gateway=ingress-gateway +36:30.124 [ERROR] TestAgent.proxycfg: watch error: service_id=ingress-gateway id=gateway-config error="invalid type for config entry: " +36:30.143 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:49350 latency=23.727435ms +36:30.147 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/ingress-gateway from=127.0.0.1:49350 latency=352.661µs +36:30.149 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/ingress-gateway from=127.0.0.1:49346 latency=312.66µs +36:30.153 [DEBUG] TestAgent: Node info in sync +36:30.154 [INFO] TestAgent: Synced service: service=us-ingress-1 +36:30.154 [DEBUG] TestAgent: Service in sync: service=us-ingress +36:30.154 [DEBUG] TestAgent: Service in sync: service=ingress-gateway +36:30.154 [DEBUG] TestAgent: Check in sync: check=service:ingress-gateway +36:30.154 [DEBUG] TestAgent: Check in sync: check=service:us-ingress-1 +36:30.154 [DEBUG] TestAgent: Check in sync: check=service:us-ingress +36:30.154 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:49352 latency=1.974065ms +36:30.156 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:49352 latency=1.578404ms +36:30.159 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/us-ingress-1 from=127.0.0.1:49352 latency=187.701µs +36:30.161 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/us-ingress-1 from=127.0.0.1:49346 latency=211.321µs +36:30.162 [INFO] TestAgent: Requesting shutdown +36:30.162 [INFO] TestAgent.server: shutting down server +36:30.162 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +36:30.162 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +36:30.162 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +36:30.162 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +36:30.162 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +36:30.162 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +36:30.162 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +36:30.162 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +36:30.162 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +36:30.162 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +36:30.162 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:36:30.163Z [INFO] agent.router.manager: shutting down +2021-08-13T15:36:30.163Z [INFO] agent.router.manager: shutting down +36:30.163 [INFO] TestAgent: consul server down +36:30.163 [INFO] TestAgent: shutdown complete +36:30.163 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14085 network=tcp +36:30.163 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14085 network=udp +36:30.163 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14086 network=tcp +36:30.667 [INFO] TestAgent: Waiting for endpoints to shut down +36:30.667 [INFO] TestAgent: Endpoints down +--- PASS: TestEnvoy_GatewayRegistration (1.05s) + --- PASS: TestEnvoy_GatewayRegistration/register_gateway_with_proxy-id_and_name (0.02s) + --- PASS: TestEnvoy_GatewayRegistration/register_gateway_without_proxy-id_with_name (0.01s) + --- PASS: TestEnvoy_GatewayRegistration/register_gateway_without_proxy-id_and_without_name (0.04s) + --- PASS: TestEnvoy_GatewayRegistration/register_gateway_with_proxy-id_without_name (0.01s) PASS -ok github.com/hashicorp/consul/command/connect/envoy 3.914s +ok github.com/hashicorp/consul/command/connect/envoy 2.109s === RUN TestConnectEnvoyPipeBootstrapCommand_noTabs === PAUSE TestConnectEnvoyPipeBootstrapCommand_noTabs === CONT TestConnectEnvoyPipeBootstrapCommand_noTabs --- PASS: TestConnectEnvoyPipeBootstrapCommand_noTabs (0.00s) PASS -ok github.com/hashicorp/consul/command/connect/envoy/pipe-bootstrap 0.035s +ok github.com/hashicorp/consul/command/connect/envoy/pipe-bootstrap 0.007s === RUN TestConnectExpose === PAUSE TestConnectExpose === RUN TestConnectExpose_invalidFlags @@ -83151,223 +83115,222 @@ === RUN TestConnectExpose_existingConfig === PAUSE TestConnectExpose_existingConfig === CONT TestConnectExpose -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestConnectExpose_existingConfig +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestConnectExpose_invalidFlags [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range === RUN TestConnectExpose_invalidFlags/missing_service === RUN TestConnectExpose_invalidFlags/missing_gateway === RUN TestConnectExpose_invalidFlags/missing_port -2022-09-15T21:44:19.782Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:44:19.942Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -44:19.945 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9f49e444-6b9c-8519-ad7f-7f9c09d59e27 Address:127.0.0.1:12062}]" -44:19.946 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:12062 [Follower]" leader= -44:19.947 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9f49e444-6b9c-8519-ad7f-7f9c09d59e27.dc1 127.0.0.1 -44:19.966 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9f49e444-6b9c-8519-ad7f-7f9c09d59e27 127.0.0.1 -2022-09-15T21:44:19.966Z [INFO] agent.router: Initializing LAN area manager -44:19.967 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12057 network=udp -44:19.967 [INFO] TestAgent.server: Adding LAN server: server="Node-9f49e444-6b9c-8519-ad7f-7f9c09d59e27 (Addr: tcp/127.0.0.1:12062) (DC: dc1)" -44:19.967 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9f49e444-6b9c-8519-ad7f-7f9c09d59e27.dc1 area=wan -44:19.967 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12057 network=tcp -44:19.969 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:12058 network=tcp -44:19.969 [INFO] TestAgent: started state syncer -44:19.969 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:12063 network=tcp -44:19.998 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -44:19.998 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:12062 [Candidate]" term=2 -44:19.998 [DEBUG] TestAgent.server.raft: votes: needed=1 -44:19.998 [DEBUG] TestAgent.server.raft: vote granted: from=9f49e444-6b9c-8519-ad7f-7f9c09d59e27 term=2 tally=1 -44:19.998 [INFO] TestAgent.server.raft: election won: tally=1 -44:19.998 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:12062 [Leader]" -44:19.999 [INFO] TestAgent.server: cluster leadership acquired -44:19.999 [INFO] TestAgent.server: New leader elected: payload=Node-9f49e444-6b9c-8519-ad7f-7f9c09d59e27 -44:20.000 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:12062 -44:20.001 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -44:20.021 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -44:20.021 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -44:20.021 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -44:20.021 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -44:20.021 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -44:20.021 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9f49e444-6b9c-8519-ad7f-7f9c09d59e27 -44:20.021 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9f49e444-6b9c-8519-ad7f-7f9c09d59e27 -44:20.022 [INFO] TestAgent.server: federation state anti-entropy synced -44:20.048 [INFO] TestAgent: Requesting shutdown -44:20.048 [INFO] TestAgent.server: shutting down server -44:20.048 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -44:20.048 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -44:20.048 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -44:20.049 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -44:20.049 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -44:20.049 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -44:20.049 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -44:20.049 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -44:20.049 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -44:20.049 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -44:20.049 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -44:20.049 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:44:20.049Z [INFO] agent.router.manager: shutting down -2022-09-15T21:44:20.049Z [INFO] agent.router.manager: shutting down -44:20.049 [INFO] TestAgent: consul server down -44:20.049 [INFO] TestAgent: shutdown complete -44:20.049 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12057 network=tcp -44:20.049 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12057 network=udp -44:20.049 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:12058 network=tcp -44:20.550 [INFO] TestAgent: Waiting for endpoints to shut down -44:20.550 [INFO] TestAgent: Endpoints down ---- PASS: TestConnectExpose_invalidFlags (0.93s) +2021-08-13T15:36:41.196Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:36:41.236Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +36:41.243 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:50cb679b-a18d-e351-37df-be9550094c9d Address:127.0.0.1:20223}]" +36:41.244 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-50cb679b-a18d-e351-37df-be9550094c9d.dc1 127.0.0.1 +36:41.244 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:20223 [Follower]" leader= +36:41.245 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-50cb679b-a18d-e351-37df-be9550094c9d 127.0.0.1 +2021-08-13T15:36:41.245Z [INFO] agent.router: Initializing LAN area manager +36:41.245 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-50cb679b-a18d-e351-37df-be9550094c9d.dc1 area=wan +36:41.245 [INFO] TestAgent.server: Adding LAN server: server="Node-50cb679b-a18d-e351-37df-be9550094c9d (Addr: tcp/127.0.0.1:20223) (DC: dc1)" +36:41.245 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20218 network=udp +36:41.245 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20218 network=tcp +36:41.246 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:20219 network=tcp +36:41.246 [INFO] TestAgent: started state syncer +36:41.246 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:20224 network=tcp +36:41.311 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +36:41.311 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:20223 [Candidate]" term=2 +36:41.311 [DEBUG] TestAgent.server.raft: votes: needed=1 +36:41.311 [DEBUG] TestAgent.server.raft: vote granted: from=50cb679b-a18d-e351-37df-be9550094c9d term=2 tally=1 +36:41.312 [INFO] TestAgent.server.raft: election won: tally=1 +36:41.312 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:20223 [Leader]" +36:41.312 [INFO] TestAgent.server: cluster leadership acquired +36:41.312 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:20223 +36:41.313 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +36:41.315 [INFO] TestAgent.server: New leader elected: payload=Node-50cb679b-a18d-e351-37df-be9550094c9d +36:41.318 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +36:41.318 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +36:41.318 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +36:41.318 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +36:41.318 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +36:41.318 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-50cb679b-a18d-e351-37df-be9550094c9d +36:41.318 [INFO] TestAgent.server: member joined, marking health alive: member=Node-50cb679b-a18d-e351-37df-be9550094c9d +36:41.320 [INFO] TestAgent.server: federation state anti-entropy synced +36:41.346 [INFO] TestAgent: Requesting shutdown +36:41.346 [INFO] TestAgent.server: shutting down server +36:41.346 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +36:41.346 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +36:41.346 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +36:41.346 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +36:41.346 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +36:41.346 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +36:41.346 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +36:41.346 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +36:41.346 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +36:41.346 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +36:41.346 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +36:41.346 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:36:41.346Z [INFO] agent.router.manager: shutting down +2021-08-13T15:36:41.346Z [INFO] agent.router.manager: shutting down +36:41.347 [INFO] TestAgent: consul server down +36:41.347 [INFO] TestAgent: shutdown complete +36:41.347 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20218 network=tcp +36:41.347 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20218 network=udp +36:41.347 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:20219 network=tcp +36:41.847 [INFO] TestAgent: Waiting for endpoints to shut down +36:41.847 [INFO] TestAgent: Endpoints down +--- PASS: TestConnectExpose_invalidFlags (0.73s) --- PASS: TestConnectExpose_invalidFlags/missing_service (0.00s) --- PASS: TestConnectExpose_invalidFlags/missing_gateway (0.00s) --- PASS: TestConnectExpose_invalidFlags/missing_port (0.00s) -2022-09-15T21:44:19.737Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:44:19.816Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -44:19.833 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2268a881-0b5f-a0df-696f-e050bbae7bb3 Address:127.0.0.1:12055}]" -44:19.833 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:12055 [Follower]" leader= -44:19.843 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2268a881-0b5f-a0df-696f-e050bbae7bb3.dc1 127.0.0.1 -44:19.861 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2268a881-0b5f-a0df-696f-e050bbae7bb3 127.0.0.1 -2022-09-15T21:44:19.861Z [INFO] agent.router: Initializing LAN area manager -44:19.862 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12050 network=udp -44:19.866 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2268a881-0b5f-a0df-696f-e050bbae7bb3.dc1 area=wan -44:19.866 [INFO] TestAgent.server: Adding LAN server: server="Node-2268a881-0b5f-a0df-696f-e050bbae7bb3 (Addr: tcp/127.0.0.1:12055) (DC: dc1)" -44:19.870 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12050 network=tcp -44:19.903 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:12051 network=tcp -44:19.903 [INFO] TestAgent: started state syncer -44:19.907 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:12056 network=tcp -44:19.922 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -44:19.922 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:12055 [Candidate]" term=2 -44:19.922 [DEBUG] TestAgent.server.raft: votes: needed=1 -44:19.922 [DEBUG] TestAgent.server.raft: vote granted: from=2268a881-0b5f-a0df-696f-e050bbae7bb3 term=2 tally=1 -44:19.922 [INFO] TestAgent.server.raft: election won: tally=1 -44:19.922 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:12055 [Leader]" -44:19.922 [INFO] TestAgent.server: cluster leadership acquired -44:19.923 [INFO] TestAgent.server: New leader elected: payload=Node-2268a881-0b5f-a0df-696f-e050bbae7bb3 -44:19.923 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:12055 -44:19.932 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -44:19.996 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -44:19.996 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -44:19.996 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -44:19.996 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -44:19.996 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -44:19.996 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2268a881-0b5f-a0df-696f-e050bbae7bb3 -44:19.996 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2268a881-0b5f-a0df-696f-e050bbae7bb3 -44:19.999 [INFO] TestAgent.server: federation state anti-entropy synced -44:20.087 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:54472 latency=1.85204ms -44:20.096 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:54472 latency=1.5223ms -44:20.099 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:54472 latency=1.77026ms -44:20.114 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:54474 latency=12.83828ms -44:20.117 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config?cas=14 from=127.0.0.1:54474 latency=1.9749ms -44:20.123 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:54474 latency=445.34µs -44:20.125 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:54474 latency=1.05972ms -44:20.126 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:54472 latency=182.3µs -44:20.134 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:54476 latency=250.72µs -44:20.138 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config?cas=15 from=127.0.0.1:54476 latency=2.60874ms -44:20.139 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:54476 latency=337.12µs -44:20.141 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:54476 latency=865.3µs -44:20.150 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:54472 latency=241.6µs -44:20.167 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:54478 latency=250.48µs -44:20.177 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config?cas=17 from=127.0.0.1:54478 latency=8.93006ms -44:20.194 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:54478 latency=553.66µs -44:20.197 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:54478 latency=1.13626ms -44:20.198 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:54472 latency=342.46µs -44:20.199 [INFO] TestAgent: Requesting shutdown -44:20.199 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -44:20.200 [INFO] TestAgent: Synced node info -44:20.200 [DEBUG] TestAgent: Node info in sync -44:20.199 [INFO] TestAgent.server: shutting down server -44:20.200 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -44:20.200 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -44:20.200 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -44:20.200 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -44:20.200 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -44:20.200 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -44:20.200 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -44:20.200 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -44:20.200 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -44:20.200 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -44:20.200 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:44:20.200Z [INFO] agent.router.manager: shutting down -2022-09-15T21:44:20.200Z [INFO] agent.router.manager: shutting down -44:20.201 [INFO] TestAgent: consul server down -44:20.201 [INFO] TestAgent: shutdown complete -44:20.201 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12050 network=tcp -44:20.201 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12050 network=udp -44:20.201 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:12051 network=tcp -44:20.704 [INFO] TestAgent: Waiting for endpoints to shut down -44:20.704 [INFO] TestAgent: Endpoints down ---- PASS: TestConnectExpose_existingConfig (1.09s) -2022-09-15T21:44:19.744Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:44:19.874Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -44:19.922 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2ea8c88a-37c9-e533-e496-1a994b2f3b05 Address:127.0.0.1:12048}]" -44:19.922 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:12048 [Follower]" leader= -44:19.923 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2ea8c88a-37c9-e533-e496-1a994b2f3b05.dc1 127.0.0.1 -44:19.934 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2ea8c88a-37c9-e533-e496-1a994b2f3b05 127.0.0.1 -2022-09-15T21:44:19.934Z [INFO] agent.router: Initializing LAN area manager -44:19.935 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12043 network=udp -44:19.935 [INFO] TestAgent.server: Adding LAN server: server="Node-2ea8c88a-37c9-e533-e496-1a994b2f3b05 (Addr: tcp/127.0.0.1:12048) (DC: dc1)" -44:19.935 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2ea8c88a-37c9-e533-e496-1a994b2f3b05.dc1 area=wan -44:19.935 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12043 network=tcp -44:19.936 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:12044 network=tcp -44:19.936 [INFO] TestAgent: started state syncer -44:19.936 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:12049 network=tcp -44:19.966 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -44:19.966 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:12048 [Candidate]" term=2 -44:19.966 [DEBUG] TestAgent.server.raft: votes: needed=1 -44:19.966 [DEBUG] TestAgent.server.raft: vote granted: from=2ea8c88a-37c9-e533-e496-1a994b2f3b05 term=2 tally=1 -44:19.966 [INFO] TestAgent.server.raft: election won: tally=1 -44:19.966 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:12048 [Leader]" -44:19.966 [INFO] TestAgent.server: cluster leadership acquired -44:19.967 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:12048 -44:19.968 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -44:19.969 [INFO] TestAgent.server: New leader elected: payload=Node-2ea8c88a-37c9-e533-e496-1a994b2f3b05 -44:19.997 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -44:19.997 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -44:19.997 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -44:19.997 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -44:19.997 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -44:19.997 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2ea8c88a-37c9-e533-e496-1a994b2f3b05 -44:19.997 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2ea8c88a-37c9-e533-e496-1a994b2f3b05 -44:19.999 [INFO] TestAgent.server: federation state anti-entropy synced -44:20.174 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -44:20.175 [INFO] TestAgent: Synced node info -44:20.270 [ERROR] TestAgent.http: Request error: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:43990 error="Config entry not found for "ingress-gateway" / "ingress"" -44:20.271 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:43990 latency=402.04µs -44:20.273 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config?cas=0 from=127.0.0.1:43990 latency=1.59816ms -44:20.274 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:43990 latency=129.02µs -44:20.275 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:43990 latency=742.26µs -44:20.277 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:43992 latency=405.9µs -44:20.278 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:43992 latency=294.46µs -44:20.281 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:43994 latency=157.46µs -44:20.285 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config?cas=13 from=127.0.0.1:43994 latency=3.74362ms -44:20.286 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:43994 latency=189.42µs -44:20.287 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:43992 latency=131.26µs -44:20.288 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:43992 latency=137.52µs -44:20.294 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:43996 latency=210.06µs -44:20.295 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:43992 latency=135.88µs -44:20.296 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:43992 latency=182.54µs -44:20.297 [INFO] TestAgent: Requesting shutdown -44:20.297 [INFO] TestAgent.server: shutting down server -44:20.297 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -44:20.297 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -44:20.297 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -44:20.297 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -44:20.297 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -44:20.297 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -44:20.297 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -44:20.297 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -44:20.297 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -44:20.297 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -44:20.297 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:44:20.297Z [INFO] agent.router.manager: shutting down -2022-09-15T21:44:20.297Z [INFO] agent.router.manager: shutting down -44:20.298 [INFO] TestAgent: consul server down -44:20.298 [INFO] TestAgent: shutdown complete -44:20.298 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12043 network=tcp -44:20.298 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12043 network=udp -44:20.298 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:12044 network=tcp -44:20.802 [INFO] TestAgent: Waiting for endpoints to shut down -44:20.802 [INFO] TestAgent: Endpoints down ---- PASS: TestConnectExpose (1.18s) +2021-08-13T15:36:41.193Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:36:41.228Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +36:41.232 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:90687763-a349-e833-77b0-93cd0413ee67 Address:127.0.0.1:20230}]" +36:41.233 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-90687763-a349-e833-77b0-93cd0413ee67.dc1 127.0.0.1 +36:41.238 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-90687763-a349-e833-77b0-93cd0413ee67 127.0.0.1 +2021-08-13T15:36:41.239Z [INFO] agent.router: Initializing LAN area manager +36:41.239 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:20230 [Follower]" leader= +36:41.240 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-90687763-a349-e833-77b0-93cd0413ee67.dc1 area=wan +36:41.240 [INFO] TestAgent.server: Adding LAN server: server="Node-90687763-a349-e833-77b0-93cd0413ee67 (Addr: tcp/127.0.0.1:20230) (DC: dc1)" +36:41.241 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20225 network=udp +36:41.253 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20225 network=tcp +36:41.254 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:20226 network=tcp +36:41.254 [INFO] TestAgent: started state syncer +36:41.254 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:20231 network=tcp +36:41.279 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +36:41.279 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:20230 [Candidate]" term=2 +36:41.280 [DEBUG] TestAgent.server.raft: votes: needed=1 +36:41.280 [DEBUG] TestAgent.server.raft: vote granted: from=90687763-a349-e833-77b0-93cd0413ee67 term=2 tally=1 +36:41.280 [INFO] TestAgent.server.raft: election won: tally=1 +36:41.280 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:20230 [Leader]" +36:41.280 [INFO] TestAgent.server: cluster leadership acquired +36:41.280 [INFO] TestAgent.server: New leader elected: payload=Node-90687763-a349-e833-77b0-93cd0413ee67 +36:41.280 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:20230 +36:41.281 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +36:41.317 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +36:41.317 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +36:41.317 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +36:41.317 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +36:41.317 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +36:41.317 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-90687763-a349-e833-77b0-93cd0413ee67 +36:41.317 [INFO] TestAgent.server: member joined, marking health alive: member=Node-90687763-a349-e833-77b0-93cd0413ee67 +36:41.319 [INFO] TestAgent.server: federation state anti-entropy synced +36:41.543 [ERROR] TestAgent.http: Request error: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:39532 error="Config entry not found for "ingress-gateway" / "ingress"" +36:41.544 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:39532 latency=389.161µs +36:41.546 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config?cas=0 from=127.0.0.1:39532 latency=1.607984ms +36:41.551 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:39532 latency=4.31875ms +36:41.552 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:39532 latency=797.521µs +36:41.554 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:39534 latency=286.3µs +36:41.554 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:39534 latency=127.08µs +36:41.556 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:39536 latency=236.001µs +36:41.559 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config?cas=12 from=127.0.0.1:39536 latency=1.342763ms +36:41.559 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:39536 latency=275.661µs +36:41.561 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:39534 latency=187.641µs +36:41.561 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:39534 latency=120.06µs +36:41.563 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:39538 latency=115.36µs +36:41.564 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:39534 latency=105.04µs +36:41.565 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:39534 latency=126.44µs +36:41.565 [INFO] TestAgent: Requesting shutdown +36:41.565 [INFO] TestAgent.server: shutting down server +36:41.565 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +36:41.565 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +36:41.565 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +36:41.565 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +36:41.565 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +36:41.565 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +36:41.565 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +36:41.565 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +36:41.565 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +36:41.565 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +36:41.565 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +36:41.565 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:36:41.566Z [INFO] agent.router.manager: shutting down +2021-08-13T15:36:41.566Z [INFO] agent.router.manager: shutting down +36:41.566 [INFO] TestAgent: consul server down +36:41.566 [INFO] TestAgent: shutdown complete +36:41.566 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20225 network=tcp +36:41.566 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20225 network=udp +36:41.566 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:20226 network=tcp +36:42.066 [INFO] TestAgent: Waiting for endpoints to shut down +36:42.066 [INFO] TestAgent: Endpoints down +--- PASS: TestConnectExpose (0.95s) +2021-08-13T15:36:41.201Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:36:41.244Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +36:41.247 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:534f6966-70d6-3f73-ab5d-59abb1646285 Address:127.0.0.1:20216}]" +36:41.247 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:20216 [Follower]" leader= +36:41.248 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-534f6966-70d6-3f73-ab5d-59abb1646285.dc1 127.0.0.1 +36:41.249 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-534f6966-70d6-3f73-ab5d-59abb1646285 127.0.0.1 +2021-08-13T15:36:41.249Z [INFO] agent.router: Initializing LAN area manager +36:41.249 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20211 network=udp +36:41.249 [INFO] TestAgent.server: Adding LAN server: server="Node-534f6966-70d6-3f73-ab5d-59abb1646285 (Addr: tcp/127.0.0.1:20216) (DC: dc1)" +36:41.249 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-534f6966-70d6-3f73-ab5d-59abb1646285.dc1 area=wan +36:41.249 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20211 network=tcp +36:41.250 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:20212 network=tcp +36:41.250 [INFO] TestAgent: started state syncer +36:41.250 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:20217 network=tcp +36:41.290 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +36:41.290 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:20216 [Candidate]" term=2 +36:41.291 [DEBUG] TestAgent.server.raft: votes: needed=1 +36:41.291 [DEBUG] TestAgent.server.raft: vote granted: from=534f6966-70d6-3f73-ab5d-59abb1646285 term=2 tally=1 +36:41.291 [INFO] TestAgent.server.raft: election won: tally=1 +36:41.291 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:20216 [Leader]" +36:41.291 [INFO] TestAgent.server: cluster leadership acquired +36:41.291 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:20216 +36:41.292 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +36:41.292 [INFO] TestAgent.server: New leader elected: payload=Node-534f6966-70d6-3f73-ab5d-59abb1646285 +36:41.309 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +36:41.309 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +36:41.309 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +36:41.309 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +36:41.309 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +36:41.309 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-534f6966-70d6-3f73-ab5d-59abb1646285 +36:41.309 [INFO] TestAgent.server: member joined, marking health alive: member=Node-534f6966-70d6-3f73-ab5d-59abb1646285 +36:41.321 [INFO] TestAgent.server: federation state anti-entropy synced +36:41.509 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +36:41.509 [INFO] TestAgent: Synced node info +36:41.509 [DEBUG] TestAgent: Node info in sync +36:41.585 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:56672 latency=1.063762ms +36:41.586 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:56672 latency=759.182µs +36:41.601 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:56672 latency=1.872644ms +36:41.606 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:56674 latency=179.04µs +36:41.609 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config?cas=15 from=127.0.0.1:56674 latency=2.133665ms +36:41.610 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:56674 latency=126.441µs +36:41.615 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:56674 latency=4.49305ms +36:41.616 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:56672 latency=139.32µs +36:41.618 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:56676 latency=117µs +36:41.620 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config?cas=16 from=127.0.0.1:56676 latency=1.995324ms +36:41.621 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:56676 latency=135.281µs +36:41.622 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:56676 latency=585.582µs +36:41.622 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:56672 latency=111.06µs +36:41.625 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:56678 latency=216.801µs +36:41.627 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config?cas=18 from=127.0.0.1:56678 latency=1.834144ms +36:41.628 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:56678 latency=141.16µs +36:41.629 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:56678 latency=727.342µs +36:41.630 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:56672 latency=185.98µs +36:41.631 [INFO] TestAgent: Requesting shutdown +36:41.631 [INFO] TestAgent.server: shutting down server +36:41.631 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +36:41.631 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +36:41.631 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +36:41.631 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +36:41.631 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +36:41.631 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +36:41.631 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +36:41.631 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +36:41.631 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +36:41.631 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +36:41.631 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:36:41.631Z [INFO] agent.router.manager: shutting down +2021-08-13T15:36:41.631Z [INFO] agent.router.manager: shutting down +36:41.632 [INFO] TestAgent: consul server down +36:41.632 [INFO] TestAgent: shutdown complete +36:41.632 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20211 network=tcp +36:41.632 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20211 network=udp +36:41.632 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:20212 network=tcp +36:42.132 [INFO] TestAgent: Waiting for endpoints to shut down +36:42.132 [INFO] TestAgent: Endpoints down +--- PASS: TestConnectExpose_existingConfig (1.02s) PASS -ok github.com/hashicorp/consul/command/connect/expose 1.322s +ok github.com/hashicorp/consul/command/connect/expose 1.115s === RUN TestFlagUpstreams_impl --- PASS: TestFlagUpstreams_impl (0.00s) === RUN TestFlagUpstreams @@ -83405,795 +83368,740 @@ --- PASS: TestCatalogCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:44:25.501Z [INFO] TestRegisterMonitor_good: registered Consul service: service=foo-proxy -2022-09-15T21:44:25.523Z [INFO] TestRegisterMonitor_good: stop request received, deregistering -2022-09-15T21:44:25.282Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:44:25.332Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -44:25.336 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:55041ce6-82e3-ea44-649e-3c929413a728 Address:127.0.0.1:10020}]" -44:25.361 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:10020 [Follower]" leader= -44:25.379 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-55041ce6-82e3-ea44-649e-3c929413a728.dc1 127.0.0.1 -44:25.380 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-55041ce6-82e3-ea44-649e-3c929413a728 127.0.0.1 -2022-09-15T21:44:25.380Z [INFO] agent.router: Initializing LAN area manager -44:25.380 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10015 network=udp -44:25.380 [INFO] TestAgent.server: Adding LAN server: server="Node-55041ce6-82e3-ea44-649e-3c929413a728 (Addr: tcp/127.0.0.1:10020) (DC: dc1)" -44:25.380 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-55041ce6-82e3-ea44-649e-3c929413a728.dc1 area=wan -44:25.381 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10015 network=tcp -44:25.386 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:10016 network=tcp -44:25.387 [INFO] TestAgent: started state syncer -44:25.387 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:10021 network=tcp -44:25.418 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -44:25.418 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:10020 [Candidate]" term=2 -44:25.418 [DEBUG] TestAgent.server.raft: votes: needed=1 -44:25.418 [DEBUG] TestAgent.server.raft: vote granted: from=55041ce6-82e3-ea44-649e-3c929413a728 term=2 tally=1 -44:25.418 [INFO] TestAgent.server.raft: election won: tally=1 -44:25.418 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:10020 [Leader]" -44:25.418 [INFO] TestAgent.server: cluster leadership acquired -44:25.419 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:10020 -44:25.420 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -44:25.420 [INFO] TestAgent.server: New leader elected: payload=Node-55041ce6-82e3-ea44-649e-3c929413a728 -44:25.471 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/service/foo-proxy?stale= from=127.0.0.1:51266 latency=2.13116ms -44:25.494 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:51264 latency=19.614ms -44:25.497 [INFO] TestAgent: Synced node info -44:25.498 [INFO] TestAgent: Synced service: service=foo-proxy -44:25.498 [DEBUG] TestAgent: Check in sync: check=foo-proxy-ttl -44:25.498 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:51266 latency=24.58368ms -44:25.499 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -44:25.499 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -44:25.499 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -44:25.499 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -44:25.499 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -44:25.499 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-55041ce6-82e3-ea44-649e-3c929413a728 -44:25.499 [INFO] TestAgent.server: member joined, marking health alive: member=Node-55041ce6-82e3-ea44-649e-3c929413a728 -44:25.501 [INFO] TestAgent.server: federation state anti-entropy synced -44:25.501 [ERROR] TestAgent.proxycfg: watch error: service_id=foo-proxy id=leaf error="error filling agent cache: internal error: CA root is nil" -44:25.501 [ERROR] TestAgent.proxycfg: watch error: service_id=foo-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" -44:25.501 [ERROR] TestAgent.proxycfg: watch error: service_id=foo-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" -44:25.501 [ERROR] TestAgent.proxycfg: watch error: service_id=foo-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" -44:25.501 [ERROR] TestAgent.proxycfg: watch error: service_id=foo-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" -44:25.522 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:51266 latency=411.42µs -44:25.523 [DEBUG] TestAgent: removed check: check=foo-proxy-ttl -44:25.523 [DEBUG] TestAgent: removed service: service=foo-proxy -44:25.523 [DEBUG] TestAgent: Node info in sync -44:25.524 [INFO] TestAgent: Deregistered service: service=foo-proxy -44:25.524 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/deregister/foo-proxy from=127.0.0.1:51266 latency=1.24484ms -44:25.526 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:51266 latency=308.34µs -44:25.527 [INFO] TestAgent: Requesting shutdown -44:25.527 [INFO] TestAgent.server: shutting down server -44:25.527 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -44:25.527 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -44:25.527 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -44:25.527 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -44:25.527 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -44:25.527 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -44:25.527 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -44:25.527 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -44:25.527 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -44:25.527 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -44:25.527 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -44:25.527 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:44:25.528Z [INFO] agent.router.manager: shutting down -2022-09-15T21:44:25.528Z [INFO] agent.router.manager: shutting down -44:25.528 [INFO] TestAgent: consul server down -44:25.528 [INFO] TestAgent: shutdown complete -44:25.528 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10015 network=tcp -44:25.528 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10015 network=udp -44:25.528 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:10016 network=tcp -44:26.028 [INFO] TestAgent: Waiting for endpoints to shut down -44:26.028 [INFO] TestAgent: Endpoints down ---- PASS: TestRegisterMonitor_good (0.88s) -2022-09-15T21:44:25.278Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:44:25.381Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -44:25.385 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f7d5ef1e-458c-227f-5167-2d3f222cae2e Address:127.0.0.1:10006}]" -44:25.386 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f7d5ef1e-458c-227f-5167-2d3f222cae2e.dc1 127.0.0.1 -44:25.387 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:10006 [Follower]" leader= -44:25.388 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f7d5ef1e-458c-227f-5167-2d3f222cae2e 127.0.0.1 -2022-09-15T21:44:25.388Z [INFO] agent.router: Initializing LAN area manager -44:25.388 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10001 network=udp -44:25.389 [INFO] TestAgent.server: Adding LAN server: server="Node-f7d5ef1e-458c-227f-5167-2d3f222cae2e (Addr: tcp/127.0.0.1:10006) (DC: dc1)" -44:25.389 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f7d5ef1e-458c-227f-5167-2d3f222cae2e.dc1 area=wan -44:25.390 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10001 network=tcp -44:25.391 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:10002 network=tcp -44:25.391 [INFO] TestAgent: started state syncer -44:25.391 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:10007 network=tcp -44:25.466 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -44:25.466 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:10006 [Candidate]" term=2 -44:25.466 [DEBUG] TestAgent.server.raft: votes: needed=1 -44:25.466 [DEBUG] TestAgent.server.raft: vote granted: from=f7d5ef1e-458c-227f-5167-2d3f222cae2e term=2 tally=1 -44:25.466 [INFO] TestAgent.server.raft: election won: tally=1 -44:25.466 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:10006 [Leader]" -44:25.466 [INFO] TestAgent.server: cluster leadership acquired -44:25.467 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:10006 -44:25.468 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -44:25.469 [INFO] TestAgent.server: New leader elected: payload=Node-f7d5ef1e-458c-227f-5167-2d3f222cae2e -44:25.493 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -44:25.493 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -44:25.493 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -44:25.493 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -44:25.493 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -44:25.496 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f7d5ef1e-458c-227f-5167-2d3f222cae2e -44:25.496 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f7d5ef1e-458c-227f-5167-2d3f222cae2e -44:25.506 [INFO] TestAgent.server: federation state anti-entropy synced -44:25.625 [INFO] TestAgent: Requesting shutdown -44:25.625 [INFO] TestAgent.server: shutting down server -44:25.625 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -44:25.625 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -44:25.625 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -44:25.625 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -44:25.625 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -44:25.625 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -44:25.625 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -44:25.625 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -44:25.625 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -44:25.625 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -44:25.625 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -44:25.625 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:44:25.626Z [INFO] agent.router.manager: shutting down -2022-09-15T21:44:25.626Z [INFO] agent.router.manager: shutting down -44:25.626 [INFO] TestAgent: consul server down -44:25.626 [INFO] TestAgent: shutdown complete -44:25.626 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10001 network=tcp -44:25.627 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10001 network=udp -44:25.627 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:10002 network=tcp -44:26.130 [INFO] TestAgent: Waiting for endpoints to shut down -44:26.130 [INFO] TestAgent: Endpoints down +2021-08-13T15:36:34.945Z [INFO] TestRegisterMonitor_heartbeat: registered Consul service: service=foo-proxy +2021-08-13T15:36:35.040Z [INFO] TestRegisterMonitor_heartbeat: stop request received, deregistering +2021-08-13T15:36:34.792Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:36:34.815Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +36:34.819 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:462d8050-e9fa-52f3-94cd-2728e7db272e Address:127.0.0.1:22265}]" +36:34.820 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-462d8050-e9fa-52f3-94cd-2728e7db272e.dc1 127.0.0.1 +36:34.821 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-462d8050-e9fa-52f3-94cd-2728e7db272e 127.0.0.1 +2021-08-13T15:36:34.821Z [INFO] agent.router: Initializing LAN area manager +36:34.821 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22260 network=udp +36:34.823 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22265 [Follower]" leader= +36:34.824 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-462d8050-e9fa-52f3-94cd-2728e7db272e.dc1 area=wan +36:34.825 [INFO] TestAgent.server: Adding LAN server: server="Node-462d8050-e9fa-52f3-94cd-2728e7db272e (Addr: tcp/127.0.0.1:22265) (DC: dc1)" +36:34.825 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22260 network=tcp +36:34.827 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22261 network=tcp +36:34.827 [INFO] TestAgent: started state syncer +36:34.827 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22266 network=tcp +36:34.860 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +36:34.860 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22265 [Candidate]" term=2 +36:34.860 [DEBUG] TestAgent.server.raft: votes: needed=1 +36:34.860 [DEBUG] TestAgent.server.raft: vote granted: from=462d8050-e9fa-52f3-94cd-2728e7db272e term=2 tally=1 +36:34.860 [INFO] TestAgent.server.raft: election won: tally=1 +36:34.860 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22265 [Leader]" +36:34.860 [INFO] TestAgent.server: cluster leadership acquired +36:34.860 [INFO] TestAgent.server: New leader elected: payload=Node-462d8050-e9fa-52f3-94cd-2728e7db272e +36:34.860 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22265 +36:34.862 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +36:34.890 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +36:34.891 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +36:34.891 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +36:34.891 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +36:34.891 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +36:34.891 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-462d8050-e9fa-52f3-94cd-2728e7db272e +36:34.891 [INFO] TestAgent.server: member joined, marking health alive: member=Node-462d8050-e9fa-52f3-94cd-2728e7db272e +36:34.896 [INFO] TestAgent.server: federation state anti-entropy synced +36:34.939 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/service/foo-proxy?stale= from=127.0.0.1:51274 latency=1.185163ms +36:34.940 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:51272 latency=821.742µs +36:34.942 [INFO] TestAgent: Synced node info +36:34.943 [INFO] TestAgent: Synced service: service=foo-proxy +36:34.943 [DEBUG] TestAgent: Check in sync: check=foo-proxy-ttl +36:34.943 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:51274 latency=3.283888ms +36:34.946 [ERROR] TestAgent.proxycfg: watch error: service_id=foo-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" +36:34.946 [ERROR] TestAgent.proxycfg: watch error: service_id=foo-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" +36:34.946 [ERROR] TestAgent.proxycfg: watch error: service_id=foo-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" +36:34.946 [ERROR] TestAgent.proxycfg: watch error: service_id=foo-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" +36:34.971 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:51274 latency=411.441µs +36:34.973 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/checks from=127.0.0.1:51274 latency=154.62µs +36:34.974 [DEBUG] TestAgent: Check status updated: check=foo-proxy-ttl status=critical +36:34.974 [DEBUG] TestAgent: Node info in sync +36:34.974 [DEBUG] TestAgent: Service in sync: service=foo-proxy +36:34.975 [INFO] TestAgent: Synced check: check=foo-proxy-ttl +36:34.975 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/check/fail/foo-proxy-ttl?note= from=127.0.0.1:51274 latency=1.517144ms +36:34.979 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/checks from=127.0.0.1:51274 latency=325.161µs +36:35.006 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/checks from=127.0.0.1:51274 latency=290.741µs +36:35.013 [DEBUG] TestAgent: Check status updated: check=foo-proxy-ttl status=passing +36:35.013 [DEBUG] TestAgent: Node info in sync +36:35.013 [DEBUG] TestAgent: Service in sync: service=foo-proxy +36:35.014 [INFO] TestAgent: Synced check: check=foo-proxy-ttl +36:35.014 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/check/pass/foo-proxy-ttl?note= from=127.0.0.1:51274 latency=1.459683ms +36:35.039 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/checks from=127.0.0.1:51274 latency=292.301µs +36:35.041 [DEBUG] TestAgent: removed check: check=foo-proxy-ttl +36:35.041 [DEBUG] TestAgent: removed service: service=foo-proxy +36:35.041 [DEBUG] TestAgent: Node info in sync +36:35.042 [INFO] TestAgent: Deregistered service: service=foo-proxy +36:35.042 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/deregister/foo-proxy from=127.0.0.1:51274 latency=1.394223ms +36:35.042 [INFO] TestAgent: Requesting shutdown +36:35.043 [INFO] TestAgent.server: shutting down server +36:35.043 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +36:35.043 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +36:35.043 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +36:35.043 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +36:35.043 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +36:35.043 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +36:35.043 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +36:35.043 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +36:35.043 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +36:35.043 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +36:35.043 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +36:35.043 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +2021-08-13T15:36:35.043Z [INFO] agent.router.manager: shutting down +2021-08-13T15:36:35.043Z [INFO] agent.router.manager: shutting down +36:35.043 [INFO] TestAgent: consul server down +36:35.043 [INFO] TestAgent: shutdown complete +36:35.043 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22260 network=tcp +36:35.043 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22260 network=udp +36:35.043 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22261 network=tcp +36:35.544 [INFO] TestAgent: Waiting for endpoints to shut down +36:35.544 [INFO] TestAgent: Endpoints down +--- PASS: TestRegisterMonitor_heartbeat (0.83s) +2021-08-13T15:36:34.809Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:36:34.836Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +36:34.838 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c327288d-eeb6-6eb1-19d8-aa525d01f635 Address:127.0.0.1:22258}]" +36:34.839 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22258 [Follower]" leader= +36:34.839 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c327288d-eeb6-6eb1-19d8-aa525d01f635.dc1 127.0.0.1 +36:34.840 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c327288d-eeb6-6eb1-19d8-aa525d01f635 127.0.0.1 +2021-08-13T15:36:34.840Z [INFO] agent.router: Initializing LAN area manager +36:34.840 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c327288d-eeb6-6eb1-19d8-aa525d01f635.dc1 area=wan +36:34.840 [INFO] TestAgent.server: Adding LAN server: server="Node-c327288d-eeb6-6eb1-19d8-aa525d01f635 (Addr: tcp/127.0.0.1:22258) (DC: dc1)" +36:34.841 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22253 network=udp +36:34.841 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22253 network=tcp +36:34.842 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22254 network=tcp +36:34.842 [INFO] TestAgent: started state syncer +36:34.842 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22259 network=tcp +36:34.879 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +36:34.879 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22258 [Candidate]" term=2 +36:34.879 [DEBUG] TestAgent.server.raft: votes: needed=1 +36:34.879 [DEBUG] TestAgent.server.raft: vote granted: from=c327288d-eeb6-6eb1-19d8-aa525d01f635 term=2 tally=1 +36:34.879 [INFO] TestAgent.server.raft: election won: tally=1 +36:34.879 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22258 [Leader]" +36:34.879 [INFO] TestAgent.server: cluster leadership acquired +36:34.879 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22258 +36:34.882 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +36:34.883 [INFO] TestAgent.server: New leader elected: payload=Node-c327288d-eeb6-6eb1-19d8-aa525d01f635 +36:34.890 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +36:34.890 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +36:34.890 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +36:34.890 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +36:34.890 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +36:34.890 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c327288d-eeb6-6eb1-19d8-aa525d01f635 +36:34.890 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c327288d-eeb6-6eb1-19d8-aa525d01f635 +36:34.901 [INFO] TestAgent.server: federation state anti-entropy synced +36:35.055 [INFO] TestAgent: Requesting shutdown +36:35.067 [INFO] TestAgent.server: shutting down server +36:35.067 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +36:35.067 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +36:35.067 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +36:35.067 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +36:35.067 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +36:35.067 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +36:35.067 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +36:35.067 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +36:35.068 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +36:35.068 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +36:35.068 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +36:35.068 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:36:35.068Z [INFO] agent.router.manager: shutting down +2021-08-13T15:36:35.068Z [INFO] agent.router.manager: shutting down +36:35.077 [INFO] TestAgent: consul server down +36:35.077 [INFO] TestAgent: shutdown complete +36:35.077 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22253 network=tcp +36:35.077 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22253 network=udp +36:35.077 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22254 network=tcp +36:35.577 [INFO] TestAgent: Waiting for endpoints to shut down +36:35.577 [INFO] TestAgent: Endpoints down === RUN TestCommandConfigWatcher/-service_flag_with_upstreams -2022-09-15T21:44:25.551Z [INFO] TestRegisterMonitor_heartbeat: registered Consul service: service=foo-proxy -2022-09-15T21:44:25.647Z [INFO] TestRegisterMonitor_heartbeat: stop request received, deregistering -2022-09-15T21:44:25.276Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:44:25.335Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -44:25.360 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:70b75d73-14f0-8266-aba4-90c411c58f9c Address:127.0.0.1:10013}]" -44:25.361 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-70b75d73-14f0-8266-aba4-90c411c58f9c.dc1 127.0.0.1 -44:25.364 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:10013 [Follower]" leader= -44:25.372 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-70b75d73-14f0-8266-aba4-90c411c58f9c 127.0.0.1 -2022-09-15T21:44:25.372Z [INFO] agent.router: Initializing LAN area manager -44:25.372 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10008 network=udp -44:25.374 [INFO] TestAgent.server: Adding LAN server: server="Node-70b75d73-14f0-8266-aba4-90c411c58f9c (Addr: tcp/127.0.0.1:10013) (DC: dc1)" -44:25.374 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-70b75d73-14f0-8266-aba4-90c411c58f9c.dc1 area=wan -44:25.375 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10008 network=tcp -44:25.376 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:10009 network=tcp -44:25.376 [INFO] TestAgent: started state syncer -44:25.376 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:10014 network=tcp -44:25.414 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -44:25.414 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:10013 [Candidate]" term=2 -44:25.414 [DEBUG] TestAgent.server.raft: votes: needed=1 -44:25.414 [DEBUG] TestAgent.server.raft: vote granted: from=70b75d73-14f0-8266-aba4-90c411c58f9c term=2 tally=1 -44:25.414 [INFO] TestAgent.server.raft: election won: tally=1 -44:25.414 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:10013 [Leader]" -44:25.414 [INFO] TestAgent.server: cluster leadership acquired -44:25.415 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:10013 -44:25.416 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -44:25.418 [INFO] TestAgent.server: New leader elected: payload=Node-70b75d73-14f0-8266-aba4-90c411c58f9c -44:25.500 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -44:25.500 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -44:25.500 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -44:25.500 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -44:25.500 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -44:25.500 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-70b75d73-14f0-8266-aba4-90c411c58f9c -44:25.500 [INFO] TestAgent.server: member joined, marking health alive: member=Node-70b75d73-14f0-8266-aba4-90c411c58f9c -44:25.507 [INFO] TestAgent.server: federation state anti-entropy synced -44:25.538 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/service/foo-proxy?stale= from=127.0.0.1:54370 latency=298.22µs -44:25.538 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:54368 latency=317.84µs -44:25.547 [INFO] TestAgent: Synced node info -44:25.548 [INFO] TestAgent: Synced service: service=foo-proxy -44:25.548 [DEBUG] TestAgent: Check in sync: check=foo-proxy-ttl -44:25.548 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:54370 latency=2.5987ms -44:25.550 [ERROR] TestAgent.proxycfg: watch error: service_id=foo-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" -44:25.550 [ERROR] TestAgent.proxycfg: watch error: service_id=foo-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" -44:25.550 [ERROR] TestAgent.proxycfg: watch error: service_id=foo-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" -44:25.550 [ERROR] TestAgent.proxycfg: watch error: service_id=foo-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" -44:25.573 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:54370 latency=6.93782ms -44:25.575 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/checks from=127.0.0.1:54370 latency=314.12µs -44:25.584 [DEBUG] TestAgent: Check status updated: check=foo-proxy-ttl status=critical -44:25.585 [DEBUG] TestAgent: Node info in sync -44:25.585 [DEBUG] TestAgent: Service in sync: service=foo-proxy -44:25.587 [INFO] TestAgent: Synced check: check=foo-proxy-ttl -44:25.587 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/check/fail/foo-proxy-ttl?note= from=127.0.0.1:54370 latency=9.95346ms -44:25.591 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/checks from=127.0.0.1:54370 latency=213.98µs -44:25.619 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/checks from=127.0.0.1:54370 latency=418.76µs -44:25.619 [DEBUG] TestAgent: Check status updated: check=foo-proxy-ttl status=passing -44:25.620 [DEBUG] TestAgent: Node info in sync -44:25.620 [DEBUG] TestAgent: Service in sync: service=foo-proxy -44:25.621 [INFO] TestAgent: Synced check: check=foo-proxy-ttl -44:25.621 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/check/pass/foo-proxy-ttl?note= from=127.0.0.1:54370 latency=1.32572ms -44:25.647 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/checks from=127.0.0.1:54370 latency=282.08µs -44:25.648 [DEBUG] TestAgent: removed check: check=foo-proxy-ttl -44:25.648 [DEBUG] TestAgent: removed service: service=foo-proxy -44:25.648 [DEBUG] TestAgent: Node info in sync -44:25.649 [INFO] TestAgent: Deregistered service: service=foo-proxy -44:25.649 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/deregister/foo-proxy from=127.0.0.1:54370 latency=1.1607ms -44:25.649 [INFO] TestAgent: Requesting shutdown -44:25.649 [INFO] TestAgent.server: shutting down server -44:25.649 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -44:25.649 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -44:25.649 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -44:25.649 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -44:25.649 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -44:25.650 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -44:25.650 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -44:25.650 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -44:25.650 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:44:25.650Z [INFO] agent.router.manager: shutting down -2022-09-15T21:44:25.650Z [INFO] agent.router.manager: shutting down -44:25.650 [INFO] TestAgent: consul server down -44:25.650 [INFO] TestAgent: shutdown complete -44:25.650 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10008 network=tcp -44:25.650 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10008 network=udp -44:25.650 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:10009 network=tcp -44:25.652 [DEBUG] TestAgent.server: shutting down leader loop -44:25.652 [INFO] TestAgent.server: cluster leadership lost -44:25.652 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -44:25.652 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -44:25.652 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -44:26.154 [INFO] TestAgent: Waiting for endpoints to shut down -44:26.154 [INFO] TestAgent: Endpoints down ---- PASS: TestRegisterMonitor_heartbeat (1.01s) -2022-09-15T21:44:26.168Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:44:26.192Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -44:26.195 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:26b4659d-c3d3-2118-e32a-faa9ce3de635 Address:127.0.0.1:10027}]" -44:26.198 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:10027 [Follower]" leader= -44:26.199 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-26b4659d-c3d3-2118-e32a-faa9ce3de635.dc1 127.0.0.1 -44:26.199 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-26b4659d-c3d3-2118-e32a-faa9ce3de635 127.0.0.1 -2022-09-15T21:44:26.199Z [INFO] agent.router: Initializing LAN area manager -44:26.200 [INFO] TestAgent.server: Adding LAN server: server="Node-26b4659d-c3d3-2118-e32a-faa9ce3de635 (Addr: tcp/127.0.0.1:10027) (DC: dc1)" -44:26.200 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10022 network=udp -44:26.201 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10022 network=tcp -44:26.201 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:10023 network=tcp -44:26.202 [INFO] TestAgent: started state syncer -44:26.202 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:10028 network=tcp -44:26.202 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-26b4659d-c3d3-2118-e32a-faa9ce3de635.dc1 area=wan -44:26.254 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -44:26.254 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:10027 [Candidate]" term=2 -44:26.254 [DEBUG] TestAgent.server.raft: votes: needed=1 -44:26.254 [DEBUG] TestAgent.server.raft: vote granted: from=26b4659d-c3d3-2118-e32a-faa9ce3de635 term=2 tally=1 -44:26.254 [INFO] TestAgent.server.raft: election won: tally=1 -44:26.254 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:10027 [Leader]" -44:26.254 [INFO] TestAgent.server: cluster leadership acquired -44:26.255 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:10027 -44:26.256 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -44:26.258 [INFO] TestAgent.server: New leader elected: payload=Node-26b4659d-c3d3-2118-e32a-faa9ce3de635 -44:26.259 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -44:26.259 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -44:26.259 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -44:26.259 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -44:26.259 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -44:26.259 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-26b4659d-c3d3-2118-e32a-faa9ce3de635 -44:26.259 [INFO] TestAgent.server: member joined, marking health alive: member=Node-26b4659d-c3d3-2118-e32a-faa9ce3de635 -44:26.275 [INFO] TestAgent.server: federation state anti-entropy synced -44:26.454 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -44:26.455 [INFO] TestAgent: Synced node info -44:26.455 [INFO] TestAgent: Synced service: service=two-sidecars -44:26.456 [INFO] TestAgent: Synced service: service=two-sidecars-sidecar-proxy -44:26.457 [INFO] TestAgent: Synced service: service=other-sidecar-for-two-sidecars -44:26.457 [INFO] TestAgent: Synced service: service=no-sidecar -44:26.458 [INFO] TestAgent: Synced service: service=one-sidecar -44:26.459 [INFO] TestAgent: Synced service: service=one-sidecar-sidecar-proxy -44:26.459 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:1 -44:26.459 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:2 -44:26.459 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:1 -44:26.459 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:2 -44:26.534 [INFO] TestAgent: Requesting shutdown -44:26.534 [INFO] TestAgent.server: shutting down server -44:26.534 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -44:26.534 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -44:26.534 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -44:26.534 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -44:26.534 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -44:26.534 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -44:26.534 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -44:26.535 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -44:26.535 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -44:26.535 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -44:26.535 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:44:26.535Z [INFO] agent.router.manager: shutting down -2022-09-15T21:44:26.535Z [INFO] agent.router.manager: shutting down -44:26.535 [INFO] TestAgent: consul server down -44:26.535 [INFO] TestAgent: shutdown complete -44:26.535 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10022 network=tcp -44:26.535 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10022 network=udp -44:26.535 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:10023 network=tcp -44:27.036 [INFO] TestAgent: Waiting for endpoints to shut down -44:27.036 [INFO] TestAgent: Endpoints down +2021-08-13T15:36:35.190Z [INFO] TestRegisterMonitor_good: registered Consul service: service=foo-proxy +2021-08-13T15:36:35.221Z [INFO] TestRegisterMonitor_good: stop request received, deregistering +2021-08-13T15:36:34.792Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:36:34.819Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +36:34.821 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3ff850ae-69b8-7bdb-3dc4-221c03a8d3b3 Address:127.0.0.1:22272}]" +36:34.827 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22272 [Follower]" leader= +36:34.828 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3ff850ae-69b8-7bdb-3dc4-221c03a8d3b3.dc1 127.0.0.1 +36:34.829 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3ff850ae-69b8-7bdb-3dc4-221c03a8d3b3 127.0.0.1 +2021-08-13T15:36:34.829Z [INFO] agent.router: Initializing LAN area manager +36:34.830 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22267 network=udp +36:34.830 [INFO] TestAgent.server: Adding LAN server: server="Node-3ff850ae-69b8-7bdb-3dc4-221c03a8d3b3 (Addr: tcp/127.0.0.1:22272) (DC: dc1)" +36:34.830 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3ff850ae-69b8-7bdb-3dc4-221c03a8d3b3.dc1 area=wan +36:34.830 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22267 network=tcp +36:34.831 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22268 network=tcp +36:34.831 [INFO] TestAgent: started state syncer +36:34.831 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22273 network=tcp +36:34.865 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +36:34.865 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22272 [Candidate]" term=2 +36:34.865 [DEBUG] TestAgent.server.raft: votes: needed=1 +36:34.865 [DEBUG] TestAgent.server.raft: vote granted: from=3ff850ae-69b8-7bdb-3dc4-221c03a8d3b3 term=2 tally=1 +36:34.865 [INFO] TestAgent.server.raft: election won: tally=1 +36:34.865 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22272 [Leader]" +36:34.865 [INFO] TestAgent.server: cluster leadership acquired +36:34.865 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22272 +36:34.866 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +36:34.867 [INFO] TestAgent.server: New leader elected: payload=Node-3ff850ae-69b8-7bdb-3dc4-221c03a8d3b3 +36:34.894 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +36:34.894 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +36:34.894 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +36:34.894 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +36:34.894 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +36:34.894 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3ff850ae-69b8-7bdb-3dc4-221c03a8d3b3 +36:34.894 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3ff850ae-69b8-7bdb-3dc4-221c03a8d3b3 +36:34.899 [INFO] TestAgent.server: federation state anti-entropy synced +36:34.951 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +36:34.952 [INFO] TestAgent: Synced node info +36:35.187 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:34384 latency=370.241µs +36:35.187 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/service/foo-proxy?stale= from=127.0.0.1:34386 latency=383.601µs +36:35.188 [DEBUG] TestAgent: Node info in sync +36:35.190 [INFO] TestAgent: Synced service: service=foo-proxy +36:35.190 [DEBUG] TestAgent: Check in sync: check=foo-proxy-ttl +36:35.190 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:34386 latency=1.922884ms +36:35.191 [ERROR] TestAgent.proxycfg: watch error: service_id=foo-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" +36:35.191 [ERROR] TestAgent.proxycfg: watch error: service_id=foo-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" +36:35.191 [ERROR] TestAgent.proxycfg: watch error: service_id=foo-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" +36:35.191 [ERROR] TestAgent.proxycfg: watch error: service_id=foo-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" +36:35.220 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:34386 latency=353.721µs +36:35.221 [DEBUG] TestAgent: removed check: check=foo-proxy-ttl +36:35.221 [DEBUG] TestAgent: removed service: service=foo-proxy +36:35.221 [DEBUG] TestAgent: Node info in sync +36:35.222 [INFO] TestAgent: Deregistered service: service=foo-proxy +36:35.222 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/deregister/foo-proxy from=127.0.0.1:34386 latency=1.166522ms +36:35.223 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:34386 latency=147.5µs +36:35.223 [INFO] TestAgent: Requesting shutdown +36:35.223 [INFO] TestAgent.server: shutting down server +36:35.223 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +36:35.223 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +36:35.223 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +36:35.223 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +36:35.223 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +36:35.224 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +36:35.224 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +36:35.224 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +36:35.224 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +36:35.224 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +36:35.224 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:36:35.224Z [INFO] agent.router.manager: shutting down +2021-08-13T15:36:35.224Z [INFO] agent.router.manager: shutting down +36:35.224 [INFO] TestAgent: consul server down +36:35.224 [INFO] TestAgent: shutdown complete +36:35.224 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22267 network=tcp +36:35.224 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22267 network=udp +36:35.224 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22268 network=tcp +36:35.725 [INFO] TestAgent: Waiting for endpoints to shut down +36:35.725 [INFO] TestAgent: Endpoints down +--- PASS: TestRegisterMonitor_good (1.01s) +2021-08-13T15:36:35.597Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:36:35.615Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +36:35.618 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:57b2bc61-3e76-341a-d749-815e1a494683 Address:127.0.0.1:22279}]" +36:35.618 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22279 [Follower]" leader= +36:35.619 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-57b2bc61-3e76-341a-d749-815e1a494683.dc1 127.0.0.1 +36:35.620 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-57b2bc61-3e76-341a-d749-815e1a494683 127.0.0.1 +2021-08-13T15:36:35.620Z [INFO] agent.router: Initializing LAN area manager +36:35.620 [INFO] TestAgent.server: Adding LAN server: server="Node-57b2bc61-3e76-341a-d749-815e1a494683 (Addr: tcp/127.0.0.1:22279) (DC: dc1)" +36:35.620 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-57b2bc61-3e76-341a-d749-815e1a494683.dc1 area=wan +36:35.621 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22274 network=udp +36:35.621 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22274 network=tcp +36:35.622 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22275 network=tcp +36:35.622 [INFO] TestAgent: started state syncer +36:35.622 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22280 network=tcp +36:35.654 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +36:35.654 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22279 [Candidate]" term=2 +36:35.654 [DEBUG] TestAgent.server.raft: votes: needed=1 +36:35.654 [DEBUG] TestAgent.server.raft: vote granted: from=57b2bc61-3e76-341a-d749-815e1a494683 term=2 tally=1 +36:35.654 [INFO] TestAgent.server.raft: election won: tally=1 +36:35.654 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22279 [Leader]" +36:35.654 [INFO] TestAgent.server: cluster leadership acquired +36:35.654 [INFO] TestAgent.server: New leader elected: payload=Node-57b2bc61-3e76-341a-d749-815e1a494683 +36:35.655 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22279 +36:35.656 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +36:35.659 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +36:35.659 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +36:35.659 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +36:35.659 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +36:35.659 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +36:35.659 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-57b2bc61-3e76-341a-d749-815e1a494683 +36:35.659 [INFO] TestAgent.server: member joined, marking health alive: member=Node-57b2bc61-3e76-341a-d749-815e1a494683 +36:35.660 [INFO] TestAgent.server: federation state anti-entropy synced +36:35.662 [INFO] TestAgent: Requesting shutdown +36:35.662 [INFO] TestAgent.server: shutting down server +36:35.662 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +36:35.662 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +36:35.662 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +36:35.662 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +36:35.662 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +36:35.662 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +36:35.662 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +36:35.662 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +36:35.662 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +36:35.662 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +36:35.662 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +36:35.662 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:36:35.662Z [INFO] agent.router.manager: shutting down +2021-08-13T15:36:35.662Z [INFO] agent.router.manager: shutting down +36:35.662 [INFO] TestAgent: consul server down +36:35.662 [INFO] TestAgent: shutdown complete +36:35.663 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22274 network=tcp +36:35.663 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22274 network=udp +36:35.663 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22275 network=tcp +36:36.163 [INFO] TestAgent: Waiting for endpoints to shut down +36:36.163 [INFO] TestAgent: Endpoints down === RUN TestCommandConfigWatcher/-service_flag_with_-service-addr -2022-09-15T21:44:27.098Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:44:27.153Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -44:27.173 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ef4073b5-719d-3f1e-b8fa-e7b862c3e461 Address:127.0.0.1:10034}]" -44:27.175 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:10034 [Follower]" leader= -44:27.182 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ef4073b5-719d-3f1e-b8fa-e7b862c3e461.dc1 127.0.0.1 -44:27.183 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ef4073b5-719d-3f1e-b8fa-e7b862c3e461 127.0.0.1 -2022-09-15T21:44:27.183Z [INFO] agent.router: Initializing LAN area manager -44:27.184 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10029 network=udp -44:27.184 [INFO] TestAgent.server: Adding LAN server: server="Node-ef4073b5-719d-3f1e-b8fa-e7b862c3e461 (Addr: tcp/127.0.0.1:10034) (DC: dc1)" -44:27.184 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ef4073b5-719d-3f1e-b8fa-e7b862c3e461.dc1 area=wan -44:27.185 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10029 network=tcp -44:27.198 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:10030 network=tcp -44:27.199 [INFO] TestAgent: started state syncer -44:27.199 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:10035 network=tcp -44:27.221 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -44:27.221 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:10034 [Candidate]" term=2 -44:27.221 [DEBUG] TestAgent.server.raft: votes: needed=1 -44:27.221 [DEBUG] TestAgent.server.raft: vote granted: from=ef4073b5-719d-3f1e-b8fa-e7b862c3e461 term=2 tally=1 -44:27.221 [INFO] TestAgent.server.raft: election won: tally=1 -44:27.221 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:10034 [Leader]" -44:27.221 [INFO] TestAgent.server: cluster leadership acquired -44:27.221 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:10034 -44:27.222 [INFO] TestAgent.server: New leader elected: payload=Node-ef4073b5-719d-3f1e-b8fa-e7b862c3e461 -44:27.223 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -44:27.225 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -44:27.238 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -44:27.238 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -44:27.238 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -44:27.238 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -44:27.238 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ef4073b5-719d-3f1e-b8fa-e7b862c3e461 -44:27.238 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ef4073b5-719d-3f1e-b8fa-e7b862c3e461 -44:27.239 [INFO] TestAgent: Synced node info -44:27.240 [INFO] TestAgent: Synced service: service=other-sidecar-for-two-sidecars -44:27.240 [INFO] TestAgent: Synced service: service=no-sidecar -44:27.241 [INFO] TestAgent: Synced service: service=one-sidecar -44:27.260 [INFO] TestAgent: Synced service: service=one-sidecar-sidecar-proxy -44:27.261 [INFO] TestAgent: Synced service: service=two-sidecars -44:27.289 [INFO] TestAgent: Synced service: service=two-sidecars-sidecar-proxy -44:27.289 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:2 -44:27.289 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:1 -44:27.289 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:2 -44:27.289 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:1 -44:27.290 [DEBUG] TestAgent: Node info in sync -44:27.290 [DEBUG] TestAgent: Service in sync: service=no-sidecar -44:27.290 [DEBUG] TestAgent: Service in sync: service=one-sidecar -44:27.290 [DEBUG] TestAgent: Service in sync: service=one-sidecar-sidecar-proxy -44:27.290 [DEBUG] TestAgent: Service in sync: service=two-sidecars -44:27.290 [DEBUG] TestAgent: Service in sync: service=two-sidecars-sidecar-proxy -44:27.290 [DEBUG] TestAgent: Service in sync: service=other-sidecar-for-two-sidecars -44:27.290 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:2 -44:27.290 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:1 -44:27.290 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:2 -44:27.290 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:1 -44:27.331 [INFO] TestAgent: Requesting shutdown -44:27.332 [INFO] TestAgent.server: shutting down server -44:27.332 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -44:27.332 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -44:27.332 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -44:27.332 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -44:27.332 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -44:27.332 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="shutdown waiting for leader" -44:27.332 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -44:27.332 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -44:27.332 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -44:27.332 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -44:27.332 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:44:27.333Z [INFO] agent.router.manager: shutting down -2022-09-15T21:44:27.333Z [INFO] agent.router.manager: shutting down -44:27.333 [INFO] TestAgent: consul server down -44:27.333 [INFO] TestAgent: shutdown complete -44:27.333 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10029 network=tcp -44:27.333 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10029 network=udp -44:27.333 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:10030 network=tcp -44:27.834 [INFO] TestAgent: Waiting for endpoints to shut down -44:27.834 [INFO] TestAgent: Endpoints down +2021-08-13T15:36:36.182Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:36:36.199Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +36:36.202 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:98d864b1-5bc6-e2af-aab1-bcea6143d8d6 Address:127.0.0.1:22286}]" +36:36.202 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22286 [Follower]" leader= +36:36.203 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-98d864b1-5bc6-e2af-aab1-bcea6143d8d6.dc1 127.0.0.1 +36:36.204 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-98d864b1-5bc6-e2af-aab1-bcea6143d8d6 127.0.0.1 +2021-08-13T15:36:36.204Z [INFO] agent.router: Initializing LAN area manager +36:36.204 [INFO] TestAgent.server: Adding LAN server: server="Node-98d864b1-5bc6-e2af-aab1-bcea6143d8d6 (Addr: tcp/127.0.0.1:22286) (DC: dc1)" +36:36.204 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-98d864b1-5bc6-e2af-aab1-bcea6143d8d6.dc1 area=wan +36:36.205 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22281 network=udp +36:36.205 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22281 network=tcp +36:36.205 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22282 network=tcp +36:36.206 [INFO] TestAgent: started state syncer +36:36.206 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22287 network=tcp +36:36.245 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +36:36.246 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22286 [Candidate]" term=2 +36:36.246 [DEBUG] TestAgent.server.raft: votes: needed=1 +36:36.246 [DEBUG] TestAgent.server.raft: vote granted: from=98d864b1-5bc6-e2af-aab1-bcea6143d8d6 term=2 tally=1 +36:36.246 [INFO] TestAgent.server.raft: election won: tally=1 +36:36.246 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22286 [Leader]" +36:36.246 [INFO] TestAgent.server: cluster leadership acquired +36:36.246 [INFO] TestAgent.server: New leader elected: payload=Node-98d864b1-5bc6-e2af-aab1-bcea6143d8d6 +36:36.246 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22286 +36:36.248 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +36:36.250 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +36:36.250 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +36:36.250 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +36:36.250 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +36:36.250 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +36:36.250 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-98d864b1-5bc6-e2af-aab1-bcea6143d8d6 +36:36.250 [INFO] TestAgent.server: member joined, marking health alive: member=Node-98d864b1-5bc6-e2af-aab1-bcea6143d8d6 +36:36.251 [INFO] TestAgent.server: federation state anti-entropy synced +36:36.470 [INFO] TestAgent: Requesting shutdown +36:36.470 [INFO] TestAgent.server: shutting down server +36:36.470 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +36:36.470 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +36:36.470 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +36:36.470 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +36:36.470 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +36:36.470 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +36:36.470 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +36:36.470 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +36:36.470 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +36:36.470 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +36:36.470 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +36:36.470 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:36:36.471Z [INFO] agent.router.manager: shutting down +2021-08-13T15:36:36.471Z [INFO] agent.router.manager: shutting down +36:36.471 [INFO] TestAgent: consul server down +36:36.471 [INFO] TestAgent: shutdown complete +36:36.471 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22281 network=tcp +36:36.471 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22281 network=udp +36:36.471 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22282 network=tcp +36:36.971 [INFO] TestAgent: Waiting for endpoints to shut down +36:36.971 [INFO] TestAgent: Endpoints down === RUN TestCommandConfigWatcher/-service,_-service-addr,_-listen -2022-09-15T21:44:27.915Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:44:27.951Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -44:27.954 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a82251d0-d3bb-8d60-e56c-911a5e7b9441 Address:127.0.0.1:10041}]" -44:27.955 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:10041 [Follower]" leader= -44:27.955 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a82251d0-d3bb-8d60-e56c-911a5e7b9441.dc1 127.0.0.1 -44:27.956 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a82251d0-d3bb-8d60-e56c-911a5e7b9441 127.0.0.1 -2022-09-15T21:44:27.956Z [INFO] agent.router: Initializing LAN area manager -44:27.957 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10036 network=udp -44:27.957 [INFO] TestAgent.server: Adding LAN server: server="Node-a82251d0-d3bb-8d60-e56c-911a5e7b9441 (Addr: tcp/127.0.0.1:10041) (DC: dc1)" -44:27.957 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a82251d0-d3bb-8d60-e56c-911a5e7b9441.dc1 area=wan -44:27.958 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10036 network=tcp -44:27.959 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:10037 network=tcp -44:27.959 [INFO] TestAgent: started state syncer -44:27.959 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:10042 network=tcp -44:28.006 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -44:28.006 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:10041 [Candidate]" term=2 -44:28.006 [DEBUG] TestAgent.server.raft: votes: needed=1 -44:28.006 [DEBUG] TestAgent.server.raft: vote granted: from=a82251d0-d3bb-8d60-e56c-911a5e7b9441 term=2 tally=1 -44:28.006 [INFO] TestAgent.server.raft: election won: tally=1 -44:28.006 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:10041 [Leader]" -44:28.007 [INFO] TestAgent.server: cluster leadership acquired -44:28.007 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:10041 -44:28.008 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -44:28.010 [INFO] TestAgent.server: New leader elected: payload=Node-a82251d0-d3bb-8d60-e56c-911a5e7b9441 -44:28.011 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -44:28.011 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -44:28.011 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -44:28.011 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -44:28.011 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -44:28.012 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a82251d0-d3bb-8d60-e56c-911a5e7b9441 -44:28.012 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a82251d0-d3bb-8d60-e56c-911a5e7b9441 -44:28.013 [INFO] TestAgent.server: federation state anti-entropy synced -44:28.135 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -44:28.136 [INFO] TestAgent: Synced node info -44:28.136 [INFO] TestAgent: Synced service: service=one-sidecar -44:28.137 [INFO] TestAgent: Synced service: service=one-sidecar-sidecar-proxy -44:28.139 [INFO] TestAgent: Synced service: service=two-sidecars -44:28.140 [INFO] TestAgent: Synced service: service=two-sidecars-sidecar-proxy -44:28.141 [INFO] TestAgent: Synced service: service=other-sidecar-for-two-sidecars -44:28.141 [INFO] TestAgent: Synced service: service=no-sidecar -44:28.141 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:1 -44:28.141 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:2 -44:28.141 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:1 -44:28.141 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:2 -44:28.397 [INFO] TestAgent: Requesting shutdown -44:28.397 [INFO] TestAgent.server: shutting down server -44:28.397 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -44:28.397 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -44:28.397 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -44:28.397 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -44:28.397 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -44:28.397 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -44:28.397 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -44:28.398 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -44:28.398 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -44:28.398 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -44:28.398 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:44:28.398Z [INFO] agent.router.manager: shutting down -2022-09-15T21:44:28.398Z [INFO] agent.router.manager: shutting down -44:28.398 [INFO] TestAgent: consul server down -44:28.398 [INFO] TestAgent: shutdown complete -44:28.398 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10036 network=tcp -44:28.398 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10036 network=udp -44:28.398 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:10037 network=tcp -44:28.900 [INFO] TestAgent: Waiting for endpoints to shut down -44:28.900 [INFO] TestAgent: Endpoints down +2021-08-13T15:36:36.989Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:36:37.012Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +36:37.015 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8713df0d-6c52-637c-b6a7-59c932bff36c Address:127.0.0.1:22293}]" +36:37.015 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22293 [Follower]" leader= +36:37.016 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8713df0d-6c52-637c-b6a7-59c932bff36c.dc1 127.0.0.1 +36:37.017 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8713df0d-6c52-637c-b6a7-59c932bff36c 127.0.0.1 +2021-08-13T15:36:37.017Z [INFO] agent.router: Initializing LAN area manager +36:37.017 [INFO] TestAgent.server: Adding LAN server: server="Node-8713df0d-6c52-637c-b6a7-59c932bff36c (Addr: tcp/127.0.0.1:22293) (DC: dc1)" +36:37.017 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8713df0d-6c52-637c-b6a7-59c932bff36c.dc1 area=wan +36:37.017 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22288 network=tcp +36:37.017 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22288 network=udp +36:37.018 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22289 network=tcp +36:37.018 [INFO] TestAgent: started state syncer +36:37.018 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22294 network=tcp +36:37.067 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +36:37.067 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22293 [Candidate]" term=2 +36:37.067 [DEBUG] TestAgent.server.raft: votes: needed=1 +36:37.068 [DEBUG] TestAgent.server.raft: vote granted: from=8713df0d-6c52-637c-b6a7-59c932bff36c term=2 tally=1 +36:37.068 [INFO] TestAgent.server.raft: election won: tally=1 +36:37.068 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22293 [Leader]" +36:37.068 [INFO] TestAgent.server: cluster leadership acquired +36:37.068 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22293 +36:37.068 [INFO] TestAgent.server: New leader elected: payload=Node-8713df0d-6c52-637c-b6a7-59c932bff36c +36:37.069 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +36:37.072 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +36:37.072 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +36:37.072 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +36:37.072 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +36:37.072 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +36:37.072 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8713df0d-6c52-637c-b6a7-59c932bff36c +36:37.072 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8713df0d-6c52-637c-b6a7-59c932bff36c +36:37.073 [INFO] TestAgent.server: federation state anti-entropy synced +36:37.127 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +36:37.128 [INFO] TestAgent: Synced node info +36:37.128 [INFO] TestAgent: Synced service: service=no-sidecar +36:37.129 [INFO] TestAgent: Synced service: service=one-sidecar +36:37.130 [INFO] TestAgent: Synced service: service=one-sidecar-sidecar-proxy +36:37.131 [INFO] TestAgent: Synced service: service=two-sidecars +36:37.132 [INFO] TestAgent: Synced service: service=two-sidecars-sidecar-proxy +36:37.133 [INFO] TestAgent: Synced service: service=other-sidecar-for-two-sidecars +36:37.133 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:1 +36:37.133 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:2 +36:37.133 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:1 +36:37.133 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:2 +36:37.133 [DEBUG] TestAgent: Node info in sync +36:37.133 [DEBUG] TestAgent: Service in sync: service=no-sidecar +36:37.133 [DEBUG] TestAgent: Service in sync: service=one-sidecar +36:37.133 [DEBUG] TestAgent: Service in sync: service=one-sidecar-sidecar-proxy +36:37.133 [DEBUG] TestAgent: Service in sync: service=two-sidecars +36:37.133 [DEBUG] TestAgent: Service in sync: service=two-sidecars-sidecar-proxy +36:37.133 [DEBUG] TestAgent: Service in sync: service=other-sidecar-for-two-sidecars +36:37.133 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:1 +36:37.133 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:2 +36:37.133 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:1 +36:37.133 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:2 +36:37.320 [INFO] TestAgent: Requesting shutdown +36:37.320 [INFO] TestAgent.server: shutting down server +36:37.320 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +36:37.320 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +36:37.320 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +36:37.320 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +36:37.320 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +36:37.320 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +36:37.320 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +36:37.320 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +36:37.320 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +36:37.320 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +36:37.320 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:36:37.321Z [INFO] agent.router.manager: shutting down +2021-08-13T15:36:37.321Z [INFO] agent.router.manager: shutting down +36:37.321 [INFO] TestAgent: consul server down +36:37.321 [INFO] TestAgent: shutdown complete +36:37.321 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22288 network=tcp +36:37.321 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22288 network=udp +36:37.321 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22289 network=tcp +36:37.821 [INFO] TestAgent: Waiting for endpoints to shut down +36:37.821 [INFO] TestAgent: Endpoints down === RUN TestCommandConfigWatcher/-sidecar-for,_no_sidecar -2022-09-15T21:44:28.947Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:44:29.034Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -44:29.037 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:56c2e422-69e2-6753-0efc-2d9b0358c5ae Address:127.0.0.1:10048}]" -44:29.037 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:10048 [Follower]" leader= -44:29.038 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-56c2e422-69e2-6753-0efc-2d9b0358c5ae.dc1 127.0.0.1 -44:29.039 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-56c2e422-69e2-6753-0efc-2d9b0358c5ae 127.0.0.1 -2022-09-15T21:44:29.039Z [INFO] agent.router: Initializing LAN area manager -44:29.040 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10043 network=udp -44:29.040 [INFO] TestAgent.server: Adding LAN server: server="Node-56c2e422-69e2-6753-0efc-2d9b0358c5ae (Addr: tcp/127.0.0.1:10048) (DC: dc1)" -44:29.040 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-56c2e422-69e2-6753-0efc-2d9b0358c5ae.dc1 area=wan -44:29.041 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10043 network=tcp -44:29.049 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:10044 network=tcp -44:29.049 [INFO] TestAgent: started state syncer -44:29.049 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:10049 network=tcp -44:29.082 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -44:29.082 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:10048 [Candidate]" term=2 -44:29.082 [DEBUG] TestAgent.server.raft: votes: needed=1 -44:29.082 [DEBUG] TestAgent.server.raft: vote granted: from=56c2e422-69e2-6753-0efc-2d9b0358c5ae term=2 tally=1 -44:29.082 [INFO] TestAgent.server.raft: election won: tally=1 -44:29.082 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:10048 [Leader]" -44:29.082 [INFO] TestAgent.server: cluster leadership acquired -44:29.083 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:10048 -44:29.084 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -44:29.085 [INFO] TestAgent.server: New leader elected: payload=Node-56c2e422-69e2-6753-0efc-2d9b0358c5ae -44:29.087 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -44:29.087 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -44:29.087 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -44:29.087 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -44:29.087 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -44:29.087 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-56c2e422-69e2-6753-0efc-2d9b0358c5ae -44:29.087 [INFO] TestAgent.server: member joined, marking health alive: member=Node-56c2e422-69e2-6753-0efc-2d9b0358c5ae -44:29.089 [INFO] TestAgent.server: federation state anti-entropy synced -44:29.113 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -44:29.113 [INFO] TestAgent: Synced node info -44:29.114 [INFO] TestAgent: Synced service: service=no-sidecar -44:29.114 [INFO] TestAgent: Synced service: service=one-sidecar -44:29.115 [INFO] TestAgent: Synced service: service=one-sidecar-sidecar-proxy -44:29.116 [INFO] TestAgent: Synced service: service=two-sidecars -44:29.117 [INFO] TestAgent: Synced service: service=two-sidecars-sidecar-proxy -44:29.118 [INFO] TestAgent: Synced service: service=other-sidecar-for-two-sidecars -44:29.118 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:1 -44:29.118 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:2 -44:29.118 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:1 -44:29.118 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:2 -44:29.231 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:41578 latency=389.68µs -44:29.232 [INFO] TestAgent: Requesting shutdown -44:29.233 [INFO] TestAgent.server: shutting down server -44:29.233 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -44:29.233 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -44:29.233 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -44:29.233 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -44:29.233 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -44:29.233 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -44:29.233 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -44:29.233 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -44:29.233 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -44:29.233 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -44:29.233 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:44:29.233Z [INFO] agent.router.manager: shutting down -2022-09-15T21:44:29.233Z [INFO] agent.router.manager: shutting down -44:29.233 [INFO] TestAgent: consul server down -44:29.234 [INFO] TestAgent: shutdown complete -44:29.234 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10043 network=tcp -44:29.234 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10043 network=udp -44:29.234 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:10044 network=tcp -44:29.736 [INFO] TestAgent: Waiting for endpoints to shut down -44:29.736 [INFO] TestAgent: Endpoints down +2021-08-13T15:36:37.847Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:36:37.864Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +36:37.867 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e07a3abf-24d5-a437-5b64-4b54d8a972fd Address:127.0.0.1:22300}]" +36:37.867 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22300 [Follower]" leader= +36:37.868 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e07a3abf-24d5-a437-5b64-4b54d8a972fd.dc1 127.0.0.1 +36:37.869 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e07a3abf-24d5-a437-5b64-4b54d8a972fd 127.0.0.1 +2021-08-13T15:36:37.869Z [INFO] agent.router: Initializing LAN area manager +36:37.869 [INFO] TestAgent.server: Adding LAN server: server="Node-e07a3abf-24d5-a437-5b64-4b54d8a972fd (Addr: tcp/127.0.0.1:22300) (DC: dc1)" +36:37.869 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e07a3abf-24d5-a437-5b64-4b54d8a972fd.dc1 area=wan +36:37.870 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22295 network=udp +36:37.870 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22295 network=tcp +36:37.871 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22296 network=tcp +36:37.871 [INFO] TestAgent: started state syncer +36:37.871 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22301 network=tcp +36:37.930 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +36:37.930 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22300 [Candidate]" term=2 +36:37.930 [DEBUG] TestAgent.server.raft: votes: needed=1 +36:37.930 [DEBUG] TestAgent.server.raft: vote granted: from=e07a3abf-24d5-a437-5b64-4b54d8a972fd term=2 tally=1 +36:37.930 [INFO] TestAgent.server.raft: election won: tally=1 +36:37.930 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22300 [Leader]" +36:37.931 [INFO] TestAgent.server: cluster leadership acquired +36:37.931 [INFO] TestAgent.server: New leader elected: payload=Node-e07a3abf-24d5-a437-5b64-4b54d8a972fd +36:37.931 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22300 +36:37.933 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +36:37.935 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +36:37.935 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +36:37.935 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +36:37.935 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +36:37.935 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +36:37.936 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e07a3abf-24d5-a437-5b64-4b54d8a972fd +36:37.936 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e07a3abf-24d5-a437-5b64-4b54d8a972fd +36:37.937 [INFO] TestAgent.server: federation state anti-entropy synced +36:37.973 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +36:37.974 [INFO] TestAgent: Synced node info +36:37.975 [INFO] TestAgent: Synced service: service=no-sidecar +36:37.975 [INFO] TestAgent: Synced service: service=one-sidecar +36:37.976 [INFO] TestAgent: Synced service: service=one-sidecar-sidecar-proxy +36:37.977 [INFO] TestAgent: Synced service: service=two-sidecars +36:37.978 [INFO] TestAgent: Synced service: service=two-sidecars-sidecar-proxy +36:37.979 [INFO] TestAgent: Synced service: service=other-sidecar-for-two-sidecars +36:37.979 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:1 +36:37.979 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:2 +36:37.979 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:1 +36:37.979 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:2 +36:38.083 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:33038 latency=2.302545ms +36:38.088 [INFO] TestAgent: Requesting shutdown +36:38.088 [INFO] TestAgent.server: shutting down server +36:38.088 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +36:38.088 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +36:38.088 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +36:38.088 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +36:38.088 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +36:38.088 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +36:38.088 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +36:38.088 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +36:38.088 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +36:38.088 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +36:38.088 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +2021-08-13T15:36:38.088Z [INFO] agent.router.manager: shutting down +2021-08-13T15:36:38.088Z [INFO] agent.router.manager: shutting down +36:38.089 [INFO] TestAgent: consul server down +36:38.089 [INFO] TestAgent: shutdown complete +36:38.089 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22295 network=tcp +36:38.089 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22295 network=udp +36:38.089 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22296 network=tcp +36:38.589 [INFO] TestAgent: Waiting for endpoints to shut down +36:38.589 [INFO] TestAgent: Endpoints down === RUN TestCommandConfigWatcher/-sidecar-for,_multiple_sidecars -2022-09-15T21:44:29.799Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:44:29.867Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -44:29.874 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:170cbf92-172f-06e0-1ac0-6fe91f4bb7c3 Address:127.0.0.1:10055}]" -44:29.875 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-170cbf92-172f-06e0-1ac0-6fe91f4bb7c3.dc1 127.0.0.1 -44:29.876 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-170cbf92-172f-06e0-1ac0-6fe91f4bb7c3 127.0.0.1 -2022-09-15T21:44:29.876Z [INFO] agent.router: Initializing LAN area manager -44:29.877 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10050 network=udp -44:29.877 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:10055 [Follower]" leader= -44:29.878 [INFO] TestAgent.server: Adding LAN server: server="Node-170cbf92-172f-06e0-1ac0-6fe91f4bb7c3 (Addr: tcp/127.0.0.1:10055) (DC: dc1)" -44:29.878 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-170cbf92-172f-06e0-1ac0-6fe91f4bb7c3.dc1 area=wan -44:29.880 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10050 network=tcp -44:29.881 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:10051 network=tcp -44:29.881 [INFO] TestAgent: started state syncer -44:29.881 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:10056 network=tcp -44:29.938 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -44:29.938 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:10055 [Candidate]" term=2 -44:29.938 [DEBUG] TestAgent.server.raft: votes: needed=1 -44:29.938 [DEBUG] TestAgent.server.raft: vote granted: from=170cbf92-172f-06e0-1ac0-6fe91f4bb7c3 term=2 tally=1 -44:29.938 [INFO] TestAgent.server.raft: election won: tally=1 -44:29.938 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:10055 [Leader]" -44:29.938 [INFO] TestAgent.server: cluster leadership acquired -44:29.939 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:10055 -44:29.940 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -44:29.942 [INFO] TestAgent.server: New leader elected: payload=Node-170cbf92-172f-06e0-1ac0-6fe91f4bb7c3 -44:29.943 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -44:29.943 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -44:29.943 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -44:29.943 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -44:29.943 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -44:29.943 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-170cbf92-172f-06e0-1ac0-6fe91f4bb7c3 -44:29.943 [INFO] TestAgent.server: member joined, marking health alive: member=Node-170cbf92-172f-06e0-1ac0-6fe91f4bb7c3 -44:29.944 [INFO] TestAgent.server: federation state anti-entropy synced -44:30.130 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -44:30.131 [INFO] TestAgent: Synced node info -44:30.131 [INFO] TestAgent: Synced service: service=two-sidecars -44:30.132 [INFO] TestAgent: Synced service: service=two-sidecars-sidecar-proxy -44:30.133 [INFO] TestAgent: Synced service: service=other-sidecar-for-two-sidecars -44:30.133 [INFO] TestAgent: Synced service: service=no-sidecar -44:30.135 [INFO] TestAgent: Synced service: service=one-sidecar -44:30.136 [INFO] TestAgent: Synced service: service=one-sidecar-sidecar-proxy -44:30.136 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:1 -44:30.136 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:2 -44:30.136 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:1 -44:30.136 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:2 -44:30.137 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:45542 latency=2.32492ms -44:30.138 [INFO] TestAgent: Requesting shutdown -44:30.138 [INFO] TestAgent.server: shutting down server -44:30.138 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -44:30.138 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -44:30.138 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -44:30.138 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -44:30.138 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -44:30.139 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -44:30.139 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -44:30.139 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -44:30.139 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -44:30.139 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -44:30.139 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:44:30.139Z [INFO] agent.router.manager: shutting down -2022-09-15T21:44:30.139Z [INFO] agent.router.manager: shutting down -44:30.139 [INFO] TestAgent: consul server down -44:30.139 [INFO] TestAgent: shutdown complete -44:30.140 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10050 network=tcp -44:30.140 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10050 network=udp -44:30.140 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:10051 network=tcp -44:30.642 [INFO] TestAgent: Waiting for endpoints to shut down -44:30.642 [INFO] TestAgent: Endpoints down +2021-08-13T15:36:38.608Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:36:38.626Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +36:38.628 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ef9bf94e-105a-3519-dd14-6489b4dae804 Address:127.0.0.1:22307}]" +36:38.629 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22307 [Follower]" leader= +36:38.630 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ef9bf94e-105a-3519-dd14-6489b4dae804.dc1 127.0.0.1 +36:38.630 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ef9bf94e-105a-3519-dd14-6489b4dae804 127.0.0.1 +2021-08-13T15:36:38.630Z [INFO] agent.router: Initializing LAN area manager +36:38.631 [INFO] TestAgent.server: Adding LAN server: server="Node-ef9bf94e-105a-3519-dd14-6489b4dae804 (Addr: tcp/127.0.0.1:22307) (DC: dc1)" +36:38.631 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ef9bf94e-105a-3519-dd14-6489b4dae804.dc1 area=wan +36:38.631 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22302 network=udp +36:38.631 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22302 network=tcp +36:38.632 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22303 network=tcp +36:38.632 [INFO] TestAgent: started state syncer +36:38.632 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22308 network=tcp +36:38.693 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +36:38.693 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22307 [Candidate]" term=2 +36:38.693 [DEBUG] TestAgent.server.raft: votes: needed=1 +36:38.693 [DEBUG] TestAgent.server.raft: vote granted: from=ef9bf94e-105a-3519-dd14-6489b4dae804 term=2 tally=1 +36:38.693 [INFO] TestAgent.server.raft: election won: tally=1 +36:38.693 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22307 [Leader]" +36:38.693 [INFO] TestAgent.server: cluster leadership acquired +36:38.694 [INFO] TestAgent.server: New leader elected: payload=Node-ef9bf94e-105a-3519-dd14-6489b4dae804 +36:38.694 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22307 +36:38.695 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +36:38.697 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +36:38.697 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +36:38.697 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +36:38.697 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +36:38.697 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +36:38.698 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ef9bf94e-105a-3519-dd14-6489b4dae804 +36:38.698 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ef9bf94e-105a-3519-dd14-6489b4dae804 +36:38.698 [INFO] TestAgent.server: federation state anti-entropy synced +36:38.742 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:46548 latency=600.321µs +36:38.742 [INFO] TestAgent: Requesting shutdown +36:38.743 [INFO] TestAgent.server: shutting down server +36:38.743 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +36:38.743 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +36:38.743 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +36:38.743 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +36:38.743 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +36:38.743 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +36:38.743 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +36:38.743 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +36:38.743 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +36:38.743 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +36:38.743 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +36:38.743 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:36:38.743Z [INFO] agent.router.manager: shutting down +2021-08-13T15:36:38.743Z [INFO] agent.router.manager: shutting down +36:38.743 [INFO] TestAgent: consul server down +36:38.743 [INFO] TestAgent: shutdown complete +36:38.743 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22302 network=tcp +36:38.743 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22302 network=udp +36:38.743 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22303 network=tcp +36:39.243 [INFO] TestAgent: Waiting for endpoints to shut down +36:39.244 [INFO] TestAgent: Endpoints down === RUN TestCommandConfigWatcher/-sidecar-for,_non-existent -2022-09-15T21:44:30.685Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:44:30.743Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -44:30.751 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:88397c60-746e-ebdc-c066-78d58be37827 Address:127.0.0.1:10062}]" -44:30.752 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-88397c60-746e-ebdc-c066-78d58be37827.dc1 127.0.0.1 -44:30.753 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:10062 [Follower]" leader= -44:30.754 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-88397c60-746e-ebdc-c066-78d58be37827 127.0.0.1 -2022-09-15T21:44:30.754Z [INFO] agent.router: Initializing LAN area manager -44:30.773 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-88397c60-746e-ebdc-c066-78d58be37827.dc1 area=wan -44:30.774 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10057 network=udp -44:30.774 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10057 network=tcp -44:30.775 [INFO] TestAgent.server: Adding LAN server: server="Node-88397c60-746e-ebdc-c066-78d58be37827 (Addr: tcp/127.0.0.1:10062) (DC: dc1)" -44:30.778 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:10058 network=tcp -44:30.778 [INFO] TestAgent: started state syncer -44:30.779 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:10063 network=tcp -44:30.800 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -44:30.800 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:10062 [Candidate]" term=2 -44:30.800 [DEBUG] TestAgent.server.raft: votes: needed=1 -44:30.800 [DEBUG] TestAgent.server.raft: vote granted: from=88397c60-746e-ebdc-c066-78d58be37827 term=2 tally=1 -44:30.800 [INFO] TestAgent.server.raft: election won: tally=1 -44:30.800 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:10062 [Leader]" -44:30.801 [INFO] TestAgent.server: cluster leadership acquired -44:30.803 [INFO] TestAgent.server: New leader elected: payload=Node-88397c60-746e-ebdc-c066-78d58be37827 -44:30.803 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:10062 -44:30.810 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -44:30.821 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -44:30.821 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -44:30.821 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -44:30.821 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -44:30.821 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -44:30.821 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-88397c60-746e-ebdc-c066-78d58be37827 -44:30.821 [INFO] TestAgent.server: member joined, marking health alive: member=Node-88397c60-746e-ebdc-c066-78d58be37827 -44:30.824 [INFO] TestAgent.server: federation state anti-entropy synced -44:30.836 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:36332 latency=437.42µs -44:30.837 [INFO] TestAgent: Requesting shutdown -44:30.838 [INFO] TestAgent.server: shutting down server -44:30.838 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -44:30.838 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -44:30.838 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -44:30.838 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -44:30.838 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -44:30.838 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -44:30.838 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -44:30.838 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -44:30.838 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -44:30.838 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -44:30.838 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -44:30.838 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2022-09-15T21:44:30.838Z [INFO] agent.router.manager: shutting down -2022-09-15T21:44:30.838Z [INFO] agent.router.manager: shutting down -44:30.838 [INFO] TestAgent: consul server down -44:30.838 [INFO] TestAgent: shutdown complete -44:30.838 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10057 network=tcp -44:30.838 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10057 network=udp -44:30.838 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:10058 network=tcp -44:31.339 [INFO] TestAgent: Waiting for endpoints to shut down -44:31.339 [INFO] TestAgent: Endpoints down +2021-08-13T15:36:39.267Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:36:39.284Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +36:39.287 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:04836bab-54a4-22a2-5253-efd7e24439ef Address:127.0.0.1:22314}]" +36:39.287 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22314 [Follower]" leader= +36:39.288 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-04836bab-54a4-22a2-5253-efd7e24439ef.dc1 127.0.0.1 +36:39.289 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-04836bab-54a4-22a2-5253-efd7e24439ef 127.0.0.1 +2021-08-13T15:36:39.289Z [INFO] agent.router: Initializing LAN area manager +36:39.289 [INFO] TestAgent.server: Adding LAN server: server="Node-04836bab-54a4-22a2-5253-efd7e24439ef (Addr: tcp/127.0.0.1:22314) (DC: dc1)" +36:39.289 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-04836bab-54a4-22a2-5253-efd7e24439ef.dc1 area=wan +36:39.290 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22309 network=udp +36:39.290 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22309 network=tcp +36:39.291 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22310 network=tcp +36:39.291 [INFO] TestAgent: started state syncer +36:39.291 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22315 network=tcp +36:39.352 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +36:39.352 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22314 [Candidate]" term=2 +36:39.352 [DEBUG] TestAgent.server.raft: votes: needed=1 +36:39.352 [DEBUG] TestAgent.server.raft: vote granted: from=04836bab-54a4-22a2-5253-efd7e24439ef term=2 tally=1 +36:39.352 [INFO] TestAgent.server.raft: election won: tally=1 +36:39.352 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22314 [Leader]" +36:39.352 [INFO] TestAgent.server: cluster leadership acquired +36:39.353 [INFO] TestAgent.server: New leader elected: payload=Node-04836bab-54a4-22a2-5253-efd7e24439ef +36:39.353 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22314 +36:39.354 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +36:39.357 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +36:39.357 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +36:39.357 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +36:39.357 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +36:39.357 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +36:39.357 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-04836bab-54a4-22a2-5253-efd7e24439ef +36:39.357 [INFO] TestAgent.server: member joined, marking health alive: member=Node-04836bab-54a4-22a2-5253-efd7e24439ef +36:39.358 [INFO] TestAgent.server: federation state anti-entropy synced +36:39.598 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:57244 latency=304.301µs +36:39.599 [INFO] TestAgent: Requesting shutdown +36:39.599 [INFO] TestAgent.server: shutting down server +36:39.599 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +36:39.599 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +36:39.599 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +36:39.599 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +36:39.599 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +36:39.599 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +36:39.599 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +36:39.600 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +36:39.600 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +36:39.600 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +36:39.600 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +36:39.600 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:36:39.600Z [INFO] agent.router.manager: shutting down +2021-08-13T15:36:39.600Z [INFO] agent.router.manager: shutting down +36:39.600 [INFO] TestAgent: consul server down +36:39.600 [INFO] TestAgent: shutdown complete +36:39.600 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22309 network=tcp +36:39.600 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22309 network=udp +36:39.600 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22310 network=tcp +36:40.100 [INFO] TestAgent: Waiting for endpoints to shut down +36:40.100 [INFO] TestAgent: Endpoints down === RUN TestCommandConfigWatcher/-sidecar-for,_one_sidecar -2022-09-15T21:44:31.399Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:44:31.427Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -44:31.430 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:761a5849-319e-5dfb-f10a-293b96332ee2 Address:127.0.0.1:10069}]" -44:31.431 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-761a5849-319e-5dfb-f10a-293b96332ee2.dc1 127.0.0.1 -44:31.432 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-761a5849-319e-5dfb-f10a-293b96332ee2 127.0.0.1 -2022-09-15T21:44:31.432Z [INFO] agent.router: Initializing LAN area manager -44:31.432 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:10069 [Follower]" leader= -44:31.433 [INFO] TestAgent.server: Adding LAN server: server="Node-761a5849-319e-5dfb-f10a-293b96332ee2 (Addr: tcp/127.0.0.1:10069) (DC: dc1)" -44:31.433 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10064 network=udp -44:31.433 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-761a5849-319e-5dfb-f10a-293b96332ee2.dc1 area=wan -44:31.433 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10064 network=tcp -44:31.435 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:10065 network=tcp -44:31.435 [INFO] TestAgent: started state syncer -44:31.435 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:10070 network=tcp -44:31.482 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -44:31.482 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:10069 [Candidate]" term=2 -44:31.483 [DEBUG] TestAgent.server.raft: votes: needed=1 -44:31.483 [DEBUG] TestAgent.server.raft: vote granted: from=761a5849-319e-5dfb-f10a-293b96332ee2 term=2 tally=1 -44:31.483 [INFO] TestAgent.server.raft: election won: tally=1 -44:31.483 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:10069 [Leader]" -44:31.483 [INFO] TestAgent.server: cluster leadership acquired -44:31.483 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:10069 -44:31.484 [INFO] TestAgent.server: New leader elected: payload=Node-761a5849-319e-5dfb-f10a-293b96332ee2 -44:31.484 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -44:31.488 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -44:31.488 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -44:31.488 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -44:31.488 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -44:31.488 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -44:31.489 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-761a5849-319e-5dfb-f10a-293b96332ee2 -44:31.489 [INFO] TestAgent.server: member joined, marking health alive: member=Node-761a5849-319e-5dfb-f10a-293b96332ee2 -44:31.494 [INFO] TestAgent.server: federation state anti-entropy synced -44:31.622 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -44:31.623 [INFO] TestAgent: Synced node info -44:31.624 [INFO] TestAgent: Synced service: service=other-sidecar-for-two-sidecars -44:31.624 [INFO] TestAgent: Synced service: service=no-sidecar -44:31.625 [INFO] TestAgent: Synced service: service=one-sidecar -44:31.626 [INFO] TestAgent: Synced service: service=one-sidecar-sidecar-proxy -44:31.627 [INFO] TestAgent: Synced service: service=two-sidecars -44:31.628 [INFO] TestAgent: Synced service: service=two-sidecars-sidecar-proxy -44:31.628 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:2 -44:31.628 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:1 -44:31.628 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:2 -44:31.628 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:1 -44:31.628 [DEBUG] TestAgent: Node info in sync -44:31.628 [DEBUG] TestAgent: Service in sync: service=one-sidecar-sidecar-proxy -44:31.628 [DEBUG] TestAgent: Service in sync: service=two-sidecars -44:31.628 [DEBUG] TestAgent: Service in sync: service=two-sidecars-sidecar-proxy -44:31.628 [DEBUG] TestAgent: Service in sync: service=other-sidecar-for-two-sidecars -44:31.628 [DEBUG] TestAgent: Service in sync: service=no-sidecar -44:31.628 [DEBUG] TestAgent: Service in sync: service=one-sidecar -44:31.628 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:1 -44:31.628 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:2 -44:31.628 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:1 -44:31.628 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:2 -44:31.743 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:50050 latency=302.06µs -44:31.745 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/one-sidecar-sidecar-proxy from=127.0.0.1:50050 latency=367.5µs -44:31.752 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/one-sidecar-sidecar-proxy from=127.0.0.1:50052 latency=6.72848ms -44:31.757 [INFO] TestAgent: Requesting shutdown -44:31.757 [INFO] TestAgent.server: shutting down server -44:31.757 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -44:31.757 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -44:31.757 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -44:31.757 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -44:31.757 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -44:31.758 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -44:31.758 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -44:31.759 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -44:31.759 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -44:31.759 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -44:31.759 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:44:31.759Z [INFO] agent.router.manager: shutting down -2022-09-15T21:44:31.759Z [INFO] agent.router.manager: shutting down -44:31.759 [INFO] TestAgent: consul server down -44:31.759 [INFO] TestAgent: shutdown complete -44:31.759 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10064 network=tcp -44:31.759 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10064 network=udp -44:31.759 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:10065 network=tcp -44:31.760 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/one-sidecar-sidecar-proxy?hash=3bafbcac3734b91 from=127.0.0.1:50052 latency=2.02692ms -44:32.262 [INFO] TestAgent: Waiting for endpoints to shut down -44:32.262 [INFO] TestAgent: Endpoints down ---- PASS: TestCommandConfigWatcher (7.12s) - --- PASS: TestCommandConfigWatcher/-service_flag_only (0.99s) - --- PASS: TestCommandConfigWatcher/-service_flag_with_upstreams (0.91s) - --- PASS: TestCommandConfigWatcher/-service_flag_with_-service-addr (0.80s) - --- PASS: TestCommandConfigWatcher/-service,_-service-addr,_-listen (1.07s) - --- PASS: TestCommandConfigWatcher/-sidecar-for,_no_sidecar (0.84s) - --- PASS: TestCommandConfigWatcher/-sidecar-for,_multiple_sidecars (0.91s) - --- PASS: TestCommandConfigWatcher/-sidecar-for,_non-existent (0.70s) - --- PASS: TestCommandConfigWatcher/-sidecar-for,_one_sidecar (0.92s) +2021-08-13T15:36:40.126Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:36:40.151Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +36:40.154 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:05c0d0c8-9e63-6953-f969-c6487043bae9 Address:127.0.0.1:22321}]" +36:40.154 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22321 [Follower]" leader= +36:40.155 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-05c0d0c8-9e63-6953-f969-c6487043bae9.dc1 127.0.0.1 +36:40.156 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-05c0d0c8-9e63-6953-f969-c6487043bae9 127.0.0.1 +2021-08-13T15:36:40.156Z [INFO] agent.router: Initializing LAN area manager +36:40.156 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-05c0d0c8-9e63-6953-f969-c6487043bae9.dc1 area=wan +36:40.156 [INFO] TestAgent.server: Adding LAN server: server="Node-05c0d0c8-9e63-6953-f969-c6487043bae9 (Addr: tcp/127.0.0.1:22321) (DC: dc1)" +36:40.157 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22316 network=udp +36:40.157 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22316 network=tcp +36:40.158 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22317 network=tcp +36:40.158 [INFO] TestAgent: started state syncer +36:40.158 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22322 network=tcp +36:40.200 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +36:40.200 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22321 [Candidate]" term=2 +36:40.200 [DEBUG] TestAgent.server.raft: votes: needed=1 +36:40.200 [DEBUG] TestAgent.server.raft: vote granted: from=05c0d0c8-9e63-6953-f969-c6487043bae9 term=2 tally=1 +36:40.200 [INFO] TestAgent.server.raft: election won: tally=1 +36:40.200 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22321 [Leader]" +36:40.200 [INFO] TestAgent.server: cluster leadership acquired +36:40.200 [INFO] TestAgent.server: New leader elected: payload=Node-05c0d0c8-9e63-6953-f969-c6487043bae9 +36:40.201 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22321 +36:40.202 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +36:40.204 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +36:40.204 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +36:40.204 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +36:40.204 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +36:40.205 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +36:40.205 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-05c0d0c8-9e63-6953-f969-c6487043bae9 +36:40.205 [INFO] TestAgent.server: member joined, marking health alive: member=Node-05c0d0c8-9e63-6953-f969-c6487043bae9 +36:40.205 [INFO] TestAgent.server: federation state anti-entropy synced +36:40.444 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:57722 latency=272.321µs +36:40.446 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/one-sidecar-sidecar-proxy from=127.0.0.1:57722 latency=255.86µs +36:40.446 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/one-sidecar-sidecar-proxy from=127.0.0.1:57724 latency=327.781µs +36:40.447 [INFO] TestAgent: Requesting shutdown +36:40.447 [INFO] TestAgent.server: shutting down server +36:40.447 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +36:40.447 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +36:40.447 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +36:40.447 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +36:40.447 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +36:40.447 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +36:40.447 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +36:40.447 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +36:40.447 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +36:40.447 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +36:40.447 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +36:40.448 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:36:40.448Z [INFO] agent.router.manager: shutting down +2021-08-13T15:36:40.448Z [INFO] agent.router.manager: shutting down +36:40.448 [INFO] TestAgent: consul server down +36:40.448 [INFO] TestAgent: shutdown complete +36:40.448 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22316 network=tcp +36:40.448 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22316 network=udp +36:40.448 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22317 network=tcp +36:40.449 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/one-sidecar-sidecar-proxy?hash=3bafbcac3734b91 from=127.0.0.1:57724 latency=1.461503ms +36:40.948 [INFO] TestAgent: Waiting for endpoints to shut down +36:40.948 [INFO] TestAgent: Endpoints down +--- PASS: TestCommandConfigWatcher (6.23s) + --- PASS: TestCommandConfigWatcher/-service_flag_only (0.86s) + --- PASS: TestCommandConfigWatcher/-service_flag_with_upstreams (0.59s) + --- PASS: TestCommandConfigWatcher/-service_flag_with_-service-addr (0.81s) + --- PASS: TestCommandConfigWatcher/-service,_-service-addr,_-listen (0.85s) + --- PASS: TestCommandConfigWatcher/-sidecar-for,_no_sidecar (0.77s) + --- PASS: TestCommandConfigWatcher/-sidecar-for,_multiple_sidecars (0.65s) + --- PASS: TestCommandConfigWatcher/-sidecar-for,_non-existent (0.86s) + --- PASS: TestCommandConfigWatcher/-sidecar-for,_one_sidecar (0.85s) PASS -ok github.com/hashicorp/consul/command/connect/proxy 7.287s +ok github.com/hashicorp/consul/command/connect/proxy 6.303s === RUN TestEventCommand_noTabs === PAUSE TestEventCommand_noTabs === RUN TestEventCommand === PAUSE TestEventCommand === CONT TestEventCommand_noTabs ---- PASS: TestEventCommand_noTabs (0.00s) === CONT TestEventCommand [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestEventCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:44:25.669Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:44:25.703Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -44:25.711 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9430f244-99ec-b9f0-512e-d55c10d01405 Address:127.0.0.1:18174}]" -44:25.712 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:18174 [Follower]" leader= -44:25.713 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9430f244-99ec-b9f0-512e-d55c10d01405.dc1 127.0.0.1 -44:25.718 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9430f244-99ec-b9f0-512e-d55c10d01405 127.0.0.1 -2022-09-15T21:44:25.718Z [INFO] agent.router: Initializing LAN area manager -44:25.719 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18169 network=udp -44:25.719 [INFO] TestAgent.server: Adding LAN server: server="Node-9430f244-99ec-b9f0-512e-d55c10d01405 (Addr: tcp/127.0.0.1:18174) (DC: dc1)" -44:25.719 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9430f244-99ec-b9f0-512e-d55c10d01405.dc1 area=wan -44:25.719 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18169 network=tcp -44:25.720 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:18170 network=tcp -44:25.720 [INFO] TestAgent: started state syncer -44:25.721 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:18175 network=tcp -44:25.782 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -44:25.782 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:18174 [Candidate]" term=2 -44:25.782 [DEBUG] TestAgent.server.raft: votes: needed=1 -44:25.782 [DEBUG] TestAgent.server.raft: vote granted: from=9430f244-99ec-b9f0-512e-d55c10d01405 term=2 tally=1 -44:25.782 [INFO] TestAgent.server.raft: election won: tally=1 -44:25.782 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:18174 [Leader]" -44:25.782 [INFO] TestAgent.server: cluster leadership acquired -44:25.783 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:18174 -44:25.784 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -44:25.786 [INFO] TestAgent.server: New leader elected: payload=Node-9430f244-99ec-b9f0-512e-d55c10d01405 -44:25.832 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -44:25.832 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -44:25.832 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -44:25.833 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -44:25.833 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -44:25.833 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9430f244-99ec-b9f0-512e-d55c10d01405 -44:25.833 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9430f244-99ec-b9f0-512e-d55c10d01405 -44:25.839 [INFO] TestAgent.server: federation state anti-entropy synced -44:25.866 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:54344 latency=6.7536ms -44:25.876 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/event/fire/cmd from=127.0.0.1:54344 latency=696.02µs -44:25.876 [DEBUG] TestAgent.server: User event: event=cmd -44:25.876 [DEBUG] TestAgent: new event: event_name=cmd event_id=d6f84ab0-347e-f5c3-055f-e152d6d5560b -44:25.877 [INFO] TestAgent: Requesting shutdown -44:25.877 [INFO] TestAgent.server: shutting down server -44:25.877 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -44:25.877 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -44:25.877 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -44:25.877 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -44:25.877 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -44:25.877 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -44:25.877 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -44:25.877 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -44:25.877 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -44:25.877 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -44:25.877 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -44:25.877 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:44:25.879Z [INFO] agent.router.manager: shutting down -2022-09-15T21:44:25.879Z [INFO] agent.router.manager: shutting down -44:25.879 [INFO] TestAgent: consul server down -44:25.879 [INFO] TestAgent: shutdown complete -44:25.879 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18169 network=tcp -44:25.879 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18169 network=udp -44:25.879 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:18170 network=tcp -44:26.380 [INFO] TestAgent: Waiting for endpoints to shut down -44:26.380 [INFO] TestAgent: Endpoints down ---- PASS: TestEventCommand (0.84s) +2021-08-13T15:36:39.022Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:36:39.038Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +36:39.043 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a31fe76c-d906-3939-ed3a-2ab1eb02e9f9 Address:127.0.0.1:27363}]" +36:39.044 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:27363 [Follower]" leader= +36:39.045 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a31fe76c-d906-3939-ed3a-2ab1eb02e9f9.dc1 127.0.0.1 +36:39.045 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a31fe76c-d906-3939-ed3a-2ab1eb02e9f9 127.0.0.1 +2021-08-13T15:36:39.045Z [INFO] agent.router: Initializing LAN area manager +36:39.046 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a31fe76c-d906-3939-ed3a-2ab1eb02e9f9.dc1 area=wan +36:39.046 [INFO] TestAgent.server: Adding LAN server: server="Node-a31fe76c-d906-3939-ed3a-2ab1eb02e9f9 (Addr: tcp/127.0.0.1:27363) (DC: dc1)" +36:39.046 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27358 network=tcp +36:39.046 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27358 network=udp +36:39.047 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:27359 network=tcp +36:39.047 [INFO] TestAgent: started state syncer +36:39.047 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:27364 network=tcp +36:39.079 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +36:39.079 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:27363 [Candidate]" term=2 +36:39.080 [DEBUG] TestAgent.server.raft: votes: needed=1 +36:39.080 [DEBUG] TestAgent.server.raft: vote granted: from=a31fe76c-d906-3939-ed3a-2ab1eb02e9f9 term=2 tally=1 +36:39.080 [INFO] TestAgent.server.raft: election won: tally=1 +36:39.080 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:27363 [Leader]" +36:39.080 [INFO] TestAgent.server: cluster leadership acquired +36:39.080 [INFO] TestAgent.server: New leader elected: payload=Node-a31fe76c-d906-3939-ed3a-2ab1eb02e9f9 +36:39.080 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:27363 +36:39.082 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +36:39.105 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +36:39.105 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +36:39.105 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +36:39.105 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +36:39.105 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +36:39.105 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a31fe76c-d906-3939-ed3a-2ab1eb02e9f9 +36:39.105 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a31fe76c-d906-3939-ed3a-2ab1eb02e9f9 +36:39.106 [INFO] TestAgent.server: federation state anti-entropy synced +36:39.314 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:34246 latency=2.220145ms +36:39.323 [DEBUG] TestAgent.server: User event: event=cmd +36:39.323 [DEBUG] TestAgent: new event: event_name=cmd event_id=4d6a877f-6c2d-5f5d-371c-fd923d090d50 +36:39.323 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/event/fire/cmd from=127.0.0.1:34246 latency=439.961µs +36:39.323 [INFO] TestAgent: Requesting shutdown +36:39.323 [INFO] TestAgent.server: shutting down server +36:39.323 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +36:39.323 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +36:39.323 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +36:39.324 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +36:39.324 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +36:39.324 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +36:39.324 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +36:39.324 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +36:39.324 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +36:39.324 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +36:39.324 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +36:39.324 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:36:39.324Z [INFO] agent.router.manager: shutting down +2021-08-13T15:36:39.324Z [INFO] agent.router.manager: shutting down +36:39.324 [INFO] TestAgent: consul server down +36:39.324 [INFO] TestAgent: shutdown complete +36:39.324 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27358 network=tcp +36:39.324 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27358 network=udp +36:39.324 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:27359 network=tcp +36:39.824 [INFO] TestAgent: Waiting for endpoints to shut down +36:39.824 [INFO] TestAgent: Endpoints down +--- PASS: TestEventCommand (0.86s) PASS -ok github.com/hashicorp/consul/command/event 1.119s +ok github.com/hashicorp/consul/command/event 0.915s === RUN TestExecCommand_noTabs === PAUSE TestExecCommand_noTabs === RUN TestExecCommand @@ -84213,571 +84121,573 @@ === RUN TestExecCommand_StreamResults === PAUSE TestExecCommand_StreamResults === CONT TestExecCommand_noTabs ---- PASS: TestExecCommand_noTabs (0.01s) === CONT TestExecCommand_Sessions === CONT TestExecCommand_StreamResults +=== CONT TestExecCommand_CrossDC [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... -=== CONT TestExecCommand_NoShell +=== CONT TestExecCommand_Validate +--- PASS: TestExecCommand_Validate (0.00s) === CONT TestExecCommand -=== CONT TestExecCommand_CrossDC === CONT TestExecCommand_UploadDestroy +=== CONT TestExecCommand_NoShell +--- PASS: TestExecCommand_noTabs (0.00s) === CONT TestExecCommand_Sessions_Foreign -=== CONT TestExecCommand_Validate ---- PASS: TestExecCommand_Validate (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:44:30.375Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:44:30.508Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -44:30.552 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:76b98126-86f4-5fb7-9513-e205e0aa6c77 Address:127.0.0.1:22286}]" -44:30.552 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22286 [Follower]" leader= -44:30.583 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-76b98126-86f4-5fb7-9513-e205e0aa6c77.dc1 127.0.0.1 -44:30.590 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-76b98126-86f4-5fb7-9513-e205e0aa6c77 127.0.0.1 -2022-09-15T21:44:30.590Z [INFO] agent.router: Initializing LAN area manager -44:30.590 [INFO] TestAgent.server: Adding LAN server: server="Node-76b98126-86f4-5fb7-9513-e205e0aa6c77 (Addr: tcp/127.0.0.1:22286) (DC: dc1)" -44:30.590 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-76b98126-86f4-5fb7-9513-e205e0aa6c77.dc1 area=wan -44:30.591 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22281 network=udp -44:30.591 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22281 network=tcp -44:30.592 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22282 network=tcp -44:30.592 [INFO] TestAgent: started state syncer -44:30.592 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22287 network=tcp -44:30.614 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -44:30.614 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22286 [Candidate]" term=2 -44:30.614 [DEBUG] TestAgent.server.raft: votes: needed=1 -44:30.614 [DEBUG] TestAgent.server.raft: vote granted: from=76b98126-86f4-5fb7-9513-e205e0aa6c77 term=2 tally=1 -44:30.614 [INFO] TestAgent.server.raft: election won: tally=1 -44:30.614 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22286 [Leader]" -44:30.614 [INFO] TestAgent.server: cluster leadership acquired -44:30.614 [INFO] TestAgent.server: New leader elected: payload=Node-76b98126-86f4-5fb7-9513-e205e0aa6c77 -44:30.622 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22286 -44:30.710 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -44:30.771 [INFO] TestAgent: Synced node info -44:30.771 [DEBUG] TestAgent: Node info in sync -44:30.818 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -44:30.818 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -44:30.818 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -44:30.818 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -44:30.818 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -44:30.819 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-76b98126-86f4-5fb7-9513-e205e0aa6c77 -44:30.819 [INFO] TestAgent.server: member joined, marking health alive: member=Node-76b98126-86f4-5fb7-9513-e205e0aa6c77 -44:30.821 [INFO] TestAgent.server: federation state anti-entropy synced -44:30.833 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:48020 latency=2.15246ms -44:30.869 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/_rexec/594210b5-fb9c-77b9-9d62-262d157dd0f9/job?acquire=594210b5-fb9c-77b9-9d62-262d157dd0f9 from=127.0.0.1:48020 latency=30.63606ms -44:30.872 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/594210b5-fb9c-77b9-9d62-262d157dd0f9/job from=127.0.0.1:48026 latency=433.84µs -44:30.874 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/_rexec/594210b5-fb9c-77b9-9d62-262d157dd0f9?recurse= from=127.0.0.1:48020 latency=562.72µs -44:30.875 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/594210b5-fb9c-77b9-9d62-262d157dd0f9/job from=127.0.0.1:48028 latency=93.28µs -44:30.875 [INFO] TestAgent: Requesting shutdown -44:30.875 [INFO] TestAgent.server: shutting down server -44:30.875 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -44:30.875 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -44:30.875 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -44:30.875 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -44:30.875 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -44:30.876 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -44:30.876 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -44:30.876 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -44:30.876 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -44:30.876 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -44:30.876 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:44:30.876Z [INFO] agent.router.manager: shutting down -2022-09-15T21:44:30.876Z [INFO] agent.router.manager: shutting down -44:30.876 [INFO] TestAgent: consul server down -44:30.876 [INFO] TestAgent: shutdown complete -44:30.876 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22281 network=tcp -44:30.876 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22281 network=udp -44:30.876 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22282 network=tcp -44:31.378 [INFO] TestAgent: Waiting for endpoints to shut down -44:31.378 [INFO] TestAgent: Endpoints down ---- PASS: TestExecCommand_UploadDestroy (1.50s) -2022-09-15T21:44:30.153Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:44:30.357Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -44:30.363 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2d929b4e-c2b1-a042-58d1-2f0bde9a3cad Address:127.0.0.1:22293}]" -44:30.366 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2d929b4e-c2b1-a042-58d1-2f0bde9a3cad.dc1 127.0.0.1 -44:30.367 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2d929b4e-c2b1-a042-58d1-2f0bde9a3cad 127.0.0.1 -2022-09-15T21:44:30.367Z [INFO] agent.router: Initializing LAN area manager -44:30.390 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22293 [Follower]" leader= -44:30.391 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22288 network=udp -44:30.391 [INFO] TestAgent.server: Adding LAN server: server="Node-2d929b4e-c2b1-a042-58d1-2f0bde9a3cad (Addr: tcp/127.0.0.1:22293) (DC: dc1)" -44:30.391 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2d929b4e-c2b1-a042-58d1-2f0bde9a3cad.dc1 area=wan -44:30.391 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22288 network=tcp -44:30.392 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22289 network=tcp -44:30.392 [INFO] TestAgent: started state syncer -44:30.392 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22294 network=tcp -44:30.466 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -44:30.466 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22293 [Candidate]" term=2 -44:30.466 [DEBUG] TestAgent.server.raft: votes: needed=1 -44:30.466 [DEBUG] TestAgent.server.raft: vote granted: from=2d929b4e-c2b1-a042-58d1-2f0bde9a3cad term=2 tally=1 -44:30.466 [INFO] TestAgent.server.raft: election won: tally=1 -44:30.466 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22293 [Leader]" -44:30.558 [INFO] TestAgent.server: cluster leadership acquired -44:30.590 [INFO] TestAgent.server: New leader elected: payload=Node-2d929b4e-c2b1-a042-58d1-2f0bde9a3cad -44:30.590 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22293 -44:30.610 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -44:30.687 [INFO] TestAgent: Synced node info -44:30.687 [DEBUG] TestAgent: Node info in sync -44:30.730 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -44:30.730 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -44:30.730 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -44:30.730 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -44:30.730 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -44:30.730 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2d929b4e-c2b1-a042-58d1-2f0bde9a3cad -44:30.730 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2d929b4e-c2b1-a042-58d1-2f0bde9a3cad -44:30.770 [INFO] TestAgent.server: federation state anti-entropy synced -44:30.869 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/health/service/consul?passing=1 from=127.0.0.1:35128 latency=62.8273ms -44:30.889 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:35128 latency=904.34µs -44:30.892 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/session/info/31869208-a814-9e2f-28b3-760e7a635785 from=127.0.0.1:35142 latency=444.94µs -44:30.896 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/31869208-a814-9e2f-28b3-760e7a635785 from=127.0.0.1:35128 latency=551µs -44:30.897 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/session/info/31869208-a814-9e2f-28b3-760e7a635785 from=127.0.0.1:35144 latency=133.94µs -44:30.897 [INFO] TestAgent: Requesting shutdown -44:30.897 [INFO] TestAgent.server: shutting down server -44:30.897 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -44:30.897 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -44:30.897 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -44:30.897 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -44:30.897 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -44:30.898 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -44:30.898 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -44:30.898 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -44:30.898 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -44:30.898 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -44:30.898 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:44:30.898Z [INFO] agent.router.manager: shutting down -2022-09-15T21:44:30.898Z [INFO] agent.router.manager: shutting down -44:30.898 [INFO] TestAgent: consul server down -44:30.898 [INFO] TestAgent: shutdown complete -44:30.898 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22288 network=tcp -44:30.898 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22288 network=udp -44:30.898 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22289 network=tcp -44:31.402 [INFO] TestAgent: Waiting for endpoints to shut down -44:31.402 [INFO] TestAgent: Endpoints down ---- PASS: TestExecCommand_Sessions_Foreign (1.52s) -2022-09-15T21:44:30.117Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:44:30.282Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -44:30.304 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:78488f91-44ad-0e5b-f8e7-b1112f530ce5 Address:127.0.0.1:22300}]" -44:30.305 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-78488f91-44ad-0e5b-f8e7-b1112f530ce5.dc1 127.0.0.1 -44:30.310 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22300 [Follower]" leader= -44:30.318 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-78488f91-44ad-0e5b-f8e7-b1112f530ce5 127.0.0.1 -2022-09-15T21:44:30.318Z [INFO] agent.router: Initializing LAN area manager -44:30.319 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22295 network=udp -44:30.319 [INFO] TestAgent.server: Adding LAN server: server="Node-78488f91-44ad-0e5b-f8e7-b1112f530ce5 (Addr: tcp/127.0.0.1:22300) (DC: dc1)" -44:30.319 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-78488f91-44ad-0e5b-f8e7-b1112f530ce5.dc1 area=wan -44:30.320 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22295 network=tcp -44:30.321 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22296 network=tcp -44:30.321 [INFO] TestAgent: started state syncer -44:30.321 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22301 network=tcp -44:30.378 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -44:30.378 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22300 [Candidate]" term=2 -44:30.378 [DEBUG] TestAgent.server.raft: votes: needed=1 -44:30.378 [DEBUG] TestAgent.server.raft: vote granted: from=78488f91-44ad-0e5b-f8e7-b1112f530ce5 term=2 tally=1 -44:30.378 [INFO] TestAgent.server.raft: election won: tally=1 -44:30.378 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22300 [Leader]" -44:30.378 [INFO] TestAgent.server: cluster leadership acquired -44:30.379 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22300 -44:30.380 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -44:30.385 [INFO] TestAgent.server: New leader elected: payload=Node-78488f91-44ad-0e5b-f8e7-b1112f530ce5 -44:30.621 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -44:30.621 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -44:30.621 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -44:30.621 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -44:30.621 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -44:30.621 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-78488f91-44ad-0e5b-f8e7-b1112f530ce5 -44:30.621 [INFO] TestAgent.server: member joined, marking health alive: member=Node-78488f91-44ad-0e5b-f8e7-b1112f530ce5 -44:30.631 [INFO] TestAgent.server: federation state anti-entropy synced -44:30.687 [INFO] TestAgent: Synced node info -44:30.875 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:53962 latency=41.31066ms -44:30.882 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/9947edad-133f-8092-45c7-d0a9898ffde5/?keys=&wait=2000ms from=127.0.0.1:53962 latency=201.72µs -44:30.907 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/_rexec/9947edad-133f-8092-45c7-d0a9898ffde5/foo/ack?acquire=9947edad-133f-8092-45c7-d0a9898ffde5 from=127.0.0.1:53978 latency=966.82µs -44:30.907 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/9947edad-133f-8092-45c7-d0a9898ffde5/?index=1&keys=&wait=2000ms from=127.0.0.1:53962 latency=24.33264ms -44:30.915 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/_rexec/9947edad-133f-8092-45c7-d0a9898ffde5/foo/exit?acquire=9947edad-133f-8092-45c7-d0a9898ffde5 from=127.0.0.1:53984 latency=946.78µs -44:30.916 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/9947edad-133f-8092-45c7-d0a9898ffde5/?index=14&keys=&wait=2000ms from=127.0.0.1:53962 latency=1.76388ms -44:30.917 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/9947edad-133f-8092-45c7-d0a9898ffde5/foo/exit from=127.0.0.1:53962 latency=143.28µs -44:30.923 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/_rexec/9947edad-133f-8092-45c7-d0a9898ffde5/foo/random?acquire=9947edad-133f-8092-45c7-d0a9898ffde5 from=127.0.0.1:53986 latency=933.7µs -44:30.923 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/9947edad-133f-8092-45c7-d0a9898ffde5/?index=15&keys=&wait=2000ms from=127.0.0.1:53962 latency=5.64898ms -44:30.943 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/_rexec/9947edad-133f-8092-45c7-d0a9898ffde5/foo/out/00000?acquire=9947edad-133f-8092-45c7-d0a9898ffde5 from=127.0.0.1:53988 latency=16.10688ms -44:30.944 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/9947edad-133f-8092-45c7-d0a9898ffde5/?index=16&keys=&wait=2000ms from=127.0.0.1:53962 latency=16.78556ms -44:30.946 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/9947edad-133f-8092-45c7-d0a9898ffde5/foo/out/00000 from=127.0.0.1:53962 latency=227µs -44:30.949 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/_rexec/9947edad-133f-8092-45c7-d0a9898ffde5/foo/out/00001?acquire=9947edad-133f-8092-45c7-d0a9898ffde5 from=127.0.0.1:53992 latency=764.2µs -44:30.949 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/9947edad-133f-8092-45c7-d0a9898ffde5/?index=17&keys=&wait=2000ms from=127.0.0.1:53962 latency=1.71684ms -44:30.954 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/9947edad-133f-8092-45c7-d0a9898ffde5/foo/out/00001 from=127.0.0.1:53962 latency=222.96µs -44:30.955 [INFO] TestAgent: Requesting shutdown -44:30.955 [INFO] TestAgent.server: shutting down server -44:30.955 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -44:30.955 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -44:30.955 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -44:30.955 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -44:30.955 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -44:30.956 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -44:30.956 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -44:30.956 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -44:30.956 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -44:30.956 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -44:30.956 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:44:30.956Z [INFO] agent.router.manager: shutting down -2022-09-15T21:44:30.956Z [INFO] agent.router.manager: shutting down -44:30.956 [INFO] TestAgent: consul server down -44:30.956 [INFO] TestAgent: shutdown complete -44:30.956 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22295 network=tcp -44:30.956 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22295 network=udp -44:30.956 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22296 network=tcp -44:31.457 [INFO] TestAgent: Waiting for endpoints to shut down -44:31.457 [INFO] TestAgent: Endpoints down ---- PASS: TestExecCommand_StreamResults (1.58s) -2022-09-15T21:44:30.285Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:44:30.504Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -44:30.515 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:550e98d1-6d92-a9a1-b6f9-b8ba227b0d0e Address:127.0.0.1:22258}]" -44:30.517 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-550e98d1-6d92-a9a1-b6f9-b8ba227b0d0e.dc1 127.0.0.1 -44:30.523 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-550e98d1-6d92-a9a1-b6f9-b8ba227b0d0e 127.0.0.1 -2022-09-15T21:44:30.523Z [INFO] agent.router: Initializing LAN area manager -44:30.586 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22258 [Follower]" leader= -44:30.588 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22253 network=udp -44:30.588 [INFO] TestAgent.server: Adding LAN server: server="Node-550e98d1-6d92-a9a1-b6f9-b8ba227b0d0e (Addr: tcp/127.0.0.1:22258) (DC: dc1)" -44:30.588 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-550e98d1-6d92-a9a1-b6f9-b8ba227b0d0e.dc1 area=wan -44:30.588 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22253 network=tcp -44:30.589 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22254 network=tcp -44:30.589 [INFO] TestAgent: started state syncer -44:30.589 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22259 network=tcp -44:30.638 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -44:30.662 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22258 [Candidate]" term=2 -44:30.686 [DEBUG] TestAgent.server.raft: votes: needed=1 -44:30.686 [DEBUG] TestAgent.server.raft: vote granted: from=550e98d1-6d92-a9a1-b6f9-b8ba227b0d0e term=2 tally=1 -44:30.686 [INFO] TestAgent.server.raft: election won: tally=1 -44:30.686 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22258 [Leader]" -44:30.686 [INFO] TestAgent.server: cluster leadership acquired -44:30.687 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22258 -44:30.687 [INFO] TestAgent.server: New leader elected: payload=Node-550e98d1-6d92-a9a1-b6f9-b8ba227b0d0e -44:30.706 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -44:30.768 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -44:30.768 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -44:30.768 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -44:30.768 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -44:30.768 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -44:30.768 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-550e98d1-6d92-a9a1-b6f9-b8ba227b0d0e -44:30.768 [INFO] TestAgent.server: member joined, marking health alive: member=Node-550e98d1-6d92-a9a1-b6f9-b8ba227b0d0e -44:30.798 [INFO] TestAgent.server: federation state anti-entropy synced -44:30.890 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -44:30.890 [INFO] TestAgent: Synced node info -44:30.890 [DEBUG] TestAgent: Node info in sync -44:30.947 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:49522 latency=1.05478ms -44:30.949 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/session/info/2a29c498-ffc9-adcb-5c9e-c0f51afff4b3 from=127.0.0.1:49526 latency=177.78µs -44:30.967 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/2a29c498-ffc9-adcb-5c9e-c0f51afff4b3 from=127.0.0.1:49522 latency=1.15306ms -44:30.979 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/session/info/2a29c498-ffc9-adcb-5c9e-c0f51afff4b3 from=127.0.0.1:49530 latency=343.46µs -44:30.980 [INFO] TestAgent: Requesting shutdown -44:30.980 [INFO] TestAgent.server: shutting down server -44:30.980 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -44:30.980 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -44:30.980 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -44:30.980 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -44:30.980 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -44:30.980 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -44:30.980 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -44:30.980 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -44:30.980 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -44:30.980 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -44:30.981 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:44:30.981Z [INFO] agent.router.manager: shutting down -2022-09-15T21:44:30.981Z [INFO] agent.router.manager: shutting down -44:30.981 [INFO] TestAgent: consul server down -44:30.981 [INFO] TestAgent: shutdown complete -44:30.981 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22253 network=tcp -44:30.981 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22253 network=udp -44:30.981 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22254 network=tcp -44:31.486 [INFO] TestAgent: Waiting for endpoints to shut down -44:31.486 [INFO] TestAgent: Endpoints down ---- PASS: TestExecCommand_Sessions (1.61s) -2022-09-15T21:44:30.172Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:44:30.277Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -44:30.398 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4bef07e8-a622-0f7e-c76a-5e70c13a6af6 Address:127.0.0.1:22265}]" -44:30.398 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22265 [Follower]" leader= -44:30.419 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4bef07e8-a622-0f7e-c76a-5e70c13a6af6.dc1 127.0.0.1 -44:30.420 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4bef07e8-a622-0f7e-c76a-5e70c13a6af6 127.0.0.1 -2022-09-15T21:44:30.421Z [INFO] agent.router: Initializing LAN area manager -44:30.421 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22260 network=udp -44:30.441 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22260 network=tcp -44:30.457 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -44:30.457 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22265 [Candidate]" term=2 -44:30.465 [WARN] TestAgent.server.raft: unable to get address for sever, using fallback address: id=4bef07e8-a622-0f7e-c76a-5e70c13a6af6 fallback=127.0.0.1:22265 error="Could not find address for server id 4bef07e8-a622-0f7e-c76a-5e70c13a6af6" -44:30.465 [DEBUG] TestAgent.server.raft: votes: needed=1 -44:30.465 [DEBUG] TestAgent.server.raft: vote granted: from=4bef07e8-a622-0f7e-c76a-5e70c13a6af6 term=2 tally=1 -44:30.465 [INFO] TestAgent.server.raft: election won: tally=1 -44:30.465 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22265 [Leader]" -44:30.460 [INFO] TestAgent.server: Adding LAN server: server="Node-4bef07e8-a622-0f7e-c76a-5e70c13a6af6 (Addr: tcp/127.0.0.1:22265) (DC: dc1)" -44:30.462 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4bef07e8-a622-0f7e-c76a-5e70c13a6af6.dc1 area=wan -44:30.550 [INFO] TestAgent.server: cluster leadership acquired -44:30.550 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22265 -44:30.551 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -44:30.552 [INFO] TestAgent.server: New leader elected: payload=Node-4bef07e8-a622-0f7e-c76a-5e70c13a6af6 -44:30.558 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22261 network=tcp -44:30.566 [INFO] TestAgent: started state syncer -44:30.580 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22266 network=tcp -44:30.583 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -44:30.583 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -44:30.583 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -44:30.583 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -44:30.583 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -44:30.583 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4bef07e8-a622-0f7e-c76a-5e70c13a6af6 -44:30.583 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4bef07e8-a622-0f7e-c76a-5e70c13a6af6 -44:30.590 [INFO] TestAgent: Synced node info -44:30.592 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4bef07e8-a622-0f7e-c76a-5e70c13a6af6 -44:30.592 [INFO] TestAgent.server: federation state anti-entropy synced -44:30.864 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:34288 latency=57.07492ms -44:30.873 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:34288 latency=1.1362ms -44:30.874 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/_rexec/13bf5060-e51a-c2d0-f940-19bc9bbf50d8/job?acquire=13bf5060-e51a-c2d0-f940-19bc9bbf50d8 from=127.0.0.1:34288 latency=537.96µs -44:31.079 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/event/fire/_rexec from=127.0.0.1:34288 latency=524.9µs -44:31.079 [DEBUG] TestAgent.server: User event: event=_rexec -44:31.079 [DEBUG] TestAgent: received remote exec event: id=b9aa1eaa-1694-d4c6-c67b-6a5dd0643fb0 -44:31.080 [INFO] TestAgent: remote exec script: script= -44:31.082 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/13bf5060-e51a-c2d0-f940-19bc9bbf50d8/?keys=&wait=1000ms from=127.0.0.1:34288 latency=444.28µs -44:31.086 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/13bf5060-e51a-c2d0-f940-19bc9bbf50d8/?index=15&keys=&wait=1000ms from=127.0.0.1:34288 latency=3.46336ms -44:31.087 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/13bf5060-e51a-c2d0-f940-19bc9bbf50d8/Node-4bef07e8-a622-0f7e-c76a-5e70c13a6af6/exit from=127.0.0.1:34288 latency=131.06µs -44:31.087 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/13bf5060-e51a-c2d0-f940-19bc9bbf50d8/Node-4bef07e8-a622-0f7e-c76a-5e70c13a6af6/out/00000 from=127.0.0.1:34288 latency=113.14µs -44:32.092 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/13bf5060-e51a-c2d0-f940-19bc9bbf50d8 from=127.0.0.1:34334 latency=1.06216ms -44:32.092 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/13bf5060-e51a-c2d0-f940-19bc9bbf50d8/?index=17&keys=&wait=1000ms from=127.0.0.1:34288 latency=1.00438958s -44:32.093 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/_rexec/13bf5060-e51a-c2d0-f940-19bc9bbf50d8?recurse= from=127.0.0.1:34334 latency=474.94µs -44:32.098 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/13bf5060-e51a-c2d0-f940-19bc9bbf50d8 from=127.0.0.1:34334 latency=490.06µs -44:32.099 [INFO] TestAgent: Requesting shutdown -44:32.099 [INFO] TestAgent.server: shutting down server -44:32.099 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -44:32.099 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -44:32.099 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -44:32.099 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -44:32.099 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -44:32.099 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -44:32.099 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -44:32.099 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -44:32.099 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -44:32.099 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -44:32.099 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:44:32.099Z [INFO] agent.router.manager: shutting down -2022-09-15T21:44:32.099Z [INFO] agent.router.manager: shutting down -44:32.099 [INFO] TestAgent: consul server down -44:32.099 [INFO] TestAgent: shutdown complete -44:32.099 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22260 network=tcp -44:32.100 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22260 network=udp -44:32.100 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22261 network=tcp -44:32.600 [INFO] TestAgent: Waiting for endpoints to shut down -44:32.600 [INFO] TestAgent: Endpoints down ---- PASS: TestExecCommand_NoShell (2.72s) -2022-09-15T21:44:30.201Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:44:30.390Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -44:30.401 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0ccf388b-05a9-976a-00de-f429bd7d8c31 Address:127.0.0.1:22272}]" -44:30.415 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0ccf388b-05a9-976a-00de-f429bd7d8c31.dc1 127.0.0.1 -44:30.427 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22272 [Follower]" leader= -44:30.439 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0ccf388b-05a9-976a-00de-f429bd7d8c31 127.0.0.1 -2022-09-15T21:44:30.440Z [INFO] agent.router: Initializing LAN area manager -44:30.440 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22267 network=udp -44:30.448 [INFO] TestAgent.server: Adding LAN server: server="Node-0ccf388b-05a9-976a-00de-f429bd7d8c31 (Addr: tcp/127.0.0.1:22272) (DC: dc1)" -44:30.448 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0ccf388b-05a9-976a-00de-f429bd7d8c31.dc1 area=wan -44:30.457 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22267 network=tcp -44:30.498 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -44:30.498 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22272 [Candidate]" term=2 -44:30.498 [DEBUG] TestAgent.server.raft: votes: needed=1 -44:30.498 [DEBUG] TestAgent.server.raft: vote granted: from=0ccf388b-05a9-976a-00de-f429bd7d8c31 term=2 tally=1 -44:30.498 [INFO] TestAgent.server.raft: election won: tally=1 -44:30.498 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22272 [Leader]" -44:30.586 [INFO] TestAgent.server: cluster leadership acquired -44:30.587 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22268 network=tcp -44:30.587 [INFO] TestAgent: started state syncer -44:30.588 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22273 network=tcp -44:30.591 [INFO] TestAgent.server: New leader elected: payload=Node-0ccf388b-05a9-976a-00de-f429bd7d8c31 -44:30.591 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22272 -44:30.592 [INFO] TestAgent: Synced node info -44:30.592 [DEBUG] TestAgent: Node info in sync -44:30.707 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -44:30.799 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -44:30.799 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -44:30.799 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -44:30.799 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -44:30.799 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -44:30.799 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0ccf388b-05a9-976a-00de-f429bd7d8c31 -44:30.799 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0ccf388b-05a9-976a-00de-f429bd7d8c31 -44:30.807 [INFO] TestAgent.server: federation state anti-entropy synced -44:30.867 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:57374 latency=31.69418ms -44:30.895 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:57374 latency=23.3377ms -44:30.897 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/_rexec/7f8d6f1a-7cd3-56f0-d280-09c8de4ea04d/job?acquire=7f8d6f1a-7cd3-56f0-d280-09c8de4ea04d from=127.0.0.1:57374 latency=522.84µs -44:31.102 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/event/fire/_rexec from=127.0.0.1:57374 latency=335.94µs -44:31.102 [DEBUG] TestAgent.server: User event: event=_rexec -44:31.103 [DEBUG] TestAgent: received remote exec event: id=50303b18-b727-bbe0-e82d-5dc525bef1c5 -44:31.103 [INFO] TestAgent: remote exec script: script=uptime -44:31.107 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/7f8d6f1a-7cd3-56f0-d280-09c8de4ea04d/?keys=&wait=1000ms from=127.0.0.1:57374 latency=335.2µs -44:31.127 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/7f8d6f1a-7cd3-56f0-d280-09c8de4ea04d/?index=15&keys=&wait=1000ms from=127.0.0.1:57374 latency=13.95268ms -44:31.133 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/7f8d6f1a-7cd3-56f0-d280-09c8de4ea04d/Node-0ccf388b-05a9-976a-00de-f429bd7d8c31/exit from=127.0.0.1:57374 latency=423µs -44:31.135 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/7f8d6f1a-7cd3-56f0-d280-09c8de4ea04d/Node-0ccf388b-05a9-976a-00de-f429bd7d8c31/out/00000 from=127.0.0.1:57374 latency=228.86µs -44:31.807 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -44:31.807 [DEBUG] TestAgent: Node info in sync -44:32.138 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/7f8d6f1a-7cd3-56f0-d280-09c8de4ea04d from=127.0.0.1:57416 latency=1.16336ms -44:32.138 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/7f8d6f1a-7cd3-56f0-d280-09c8de4ea04d/?index=17&keys=&wait=1000ms from=127.0.0.1:57374 latency=1.00272924s -44:32.139 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/_rexec/7f8d6f1a-7cd3-56f0-d280-09c8de4ea04d?recurse= from=127.0.0.1:57374 latency=454.24µs -44:32.140 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/7f8d6f1a-7cd3-56f0-d280-09c8de4ea04d from=127.0.0.1:57374 latency=350.18µs -44:32.140 [INFO] TestAgent: Requesting shutdown -44:32.140 [INFO] TestAgent.server: shutting down server -44:32.140 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -44:32.140 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -44:32.140 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -44:32.140 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -44:32.140 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -44:32.140 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -44:32.140 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -44:32.141 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -44:32.141 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -44:32.141 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -44:32.141 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:44:32.141Z [INFO] agent.router.manager: shutting down -2022-09-15T21:44:32.141Z [INFO] agent.router.manager: shutting down -44:32.141 [INFO] TestAgent: consul server down -44:32.141 [INFO] TestAgent: shutdown complete -44:32.141 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22267 network=tcp -44:32.141 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22267 network=udp -44:32.141 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22268 network=tcp -44:32.642 [INFO] TestAgent: Waiting for endpoints to shut down -44:32.642 [INFO] TestAgent: Endpoints down ---- PASS: TestExecCommand (2.76s) -2022-09-15T21:44:30.913Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:44:31.008Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -44:31.016 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3c8dfe0c-b036-474a-7834-f794e2fc8220 Address:127.0.0.1:22307}]" -44:31.017 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3c8dfe0c-b036-474a-7834-f794e2fc8220.dc2 127.0.0.1 -44:31.018 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22307 [Follower]" leader= -44:31.023 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3c8dfe0c-b036-474a-7834-f794e2fc8220 127.0.0.1 -2022-09-15T21:44:31.023Z [INFO] agent.router: Initializing LAN area manager -44:31.023 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22302 network=udp -44:31.024 [INFO] TestAgent.server: Adding LAN server: server="Node-3c8dfe0c-b036-474a-7834-f794e2fc8220 (Addr: tcp/127.0.0.1:22307) (DC: dc2)" -44:31.024 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3c8dfe0c-b036-474a-7834-f794e2fc8220.dc2 area=wan -44:31.024 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22302 network=tcp -44:31.025 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22303 network=tcp -44:31.025 [INFO] TestAgent: started state syncer -44:31.025 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22308 network=tcp -44:31.078 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -44:31.078 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22307 [Candidate]" term=2 -44:31.078 [DEBUG] TestAgent.server.raft: votes: needed=1 -44:31.078 [DEBUG] TestAgent.server.raft: vote granted: from=3c8dfe0c-b036-474a-7834-f794e2fc8220 term=2 tally=1 -44:31.078 [INFO] TestAgent.server.raft: election won: tally=1 -44:31.078 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22307 [Leader]" -44:31.078 [INFO] TestAgent.server: cluster leadership acquired -44:31.078 [INFO] TestAgent.server: New leader elected: payload=Node-3c8dfe0c-b036-474a-7834-f794e2fc8220 -44:31.079 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22307 -44:31.080 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -44:31.086 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -44:31.086 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -44:31.086 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -44:31.086 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -44:31.086 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -44:31.086 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3c8dfe0c-b036-474a-7834-f794e2fc8220 -44:31.086 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3c8dfe0c-b036-474a-7834-f794e2fc8220 -44:31.088 [INFO] TestAgent.server: federation state anti-entropy synced -44:31.344 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -44:31.345 [INFO] TestAgent: Synced node info -44:31.345 [DEBUG] TestAgent: Node info in sync -44:31.430 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:22278] -44:31.431 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:22278 -44:31.433 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-aabbb58b-1249-c615-33bf-c5d475f62c5c.dc1 127.0.0.1 -44:31.433 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 -44:31.434 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-aabbb58b-1249-c615-33bf-c5d475f62c5c.dc1 area=wan -44:31.669 [DEBUG] TestAgent.server: User event: event=_rexec -44:31.669 [DEBUG] TestAgent: received remote exec event: id=92b77c9f-c056-c571-77a5-bb192eb5fabe -44:31.669 [INFO] TestAgent: remote exec script: script=uptime -44:31.851 [DEBUG] TestAgent.server.serf.wan: serf: messageJoinType: Node-3c8dfe0c-b036-474a-7834-f794e2fc8220.dc2 -44:32.192 [INFO] TestAgent: Requesting shutdown -44:32.192 [INFO] TestAgent.server: shutting down server -44:32.192 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -44:32.192 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -44:32.192 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -44:32.192 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -44:32.192 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -44:32.192 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -44:32.192 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -44:32.192 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -44:32.192 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -44:32.192 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -44:32.192 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:44:32.193Z [INFO] agent.router.manager: shutting down -2022-09-15T21:44:32.193Z [INFO] agent.router.manager: shutting down -2022-09-15T21:44:32.193Z [INFO] agent.router.manager: shutting down -44:32.193 [INFO] TestAgent: consul server down -44:32.193 [INFO] TestAgent: shutdown complete -44:32.193 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22302 network=tcp -44:32.193 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22302 network=udp -44:32.193 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22303 network=tcp -44:32.693 [INFO] TestAgent: Waiting for endpoints to shut down -44:32.693 [INFO] TestAgent: Endpoints down -2022-09-15T21:44:30.107Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:44:30.324Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -44:30.348 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:aabbb58b-1249-c615-33bf-c5d475f62c5c Address:127.0.0.1:22279}]" -44:30.349 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-aabbb58b-1249-c615-33bf-c5d475f62c5c.dc1 127.0.0.1 -44:30.350 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22279 [Follower]" leader= -44:30.382 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-aabbb58b-1249-c615-33bf-c5d475f62c5c 127.0.0.1 -2022-09-15T21:44:30.382Z [INFO] agent.router: Initializing LAN area manager -44:30.383 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22274 network=udp -44:30.383 [INFO] TestAgent.server: Adding LAN server: server="Node-aabbb58b-1249-c615-33bf-c5d475f62c5c (Addr: tcp/127.0.0.1:22279) (DC: dc1)" -44:30.383 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-aabbb58b-1249-c615-33bf-c5d475f62c5c.dc1 area=wan -44:30.383 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22274 network=tcp -44:30.384 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22275 network=tcp -44:30.385 [INFO] TestAgent: started state syncer -44:30.385 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22280 network=tcp -44:30.443 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -44:30.462 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22279 [Candidate]" term=2 -44:30.462 [DEBUG] TestAgent.server.raft: votes: needed=1 -44:30.462 [DEBUG] TestAgent.server.raft: vote granted: from=aabbb58b-1249-c615-33bf-c5d475f62c5c term=2 tally=1 -44:30.462 [INFO] TestAgent.server.raft: election won: tally=1 -44:30.462 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22279 [Leader]" -44:30.463 [INFO] TestAgent.server: cluster leadership acquired -44:30.463 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22279 -44:30.465 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -44:30.465 [INFO] TestAgent.server: New leader elected: payload=Node-aabbb58b-1249-c615-33bf-c5d475f62c5c -44:30.592 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -44:30.592 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -44:30.592 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -44:30.592 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -44:30.592 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -44:30.592 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-aabbb58b-1249-c615-33bf-c5d475f62c5c -44:30.592 [INFO] TestAgent.server: member joined, marking health alive: member=Node-aabbb58b-1249-c615-33bf-c5d475f62c5c -44:30.594 [INFO] TestAgent.server: federation state anti-entropy synced -44:30.734 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -44:30.750 [INFO] TestAgent: Synced node info -44:30.750 [DEBUG] TestAgent: Node info in sync -44:31.053 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -44:31.053 [DEBUG] TestAgent: Node info in sync -44:31.431 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:58802 -44:31.432 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3c8dfe0c-b036-474a-7834-f794e2fc8220.dc2 127.0.0.1 -44:31.432 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3c8dfe0c-b036-474a-7834-f794e2fc8220.dc2 area=wan -44:31.456 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self?dc=dc2 from=127.0.0.1:33958 latency=20.99944ms -44:31.463 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/health/service/consul?dc=dc2&passing=1 from=127.0.0.1:33958 latency=2.57088ms -44:31.465 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create?dc=dc2 from=127.0.0.1:33958 latency=1.05546ms -44:31.467 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/_rexec/efbebb9b-1738-1702-b91f-104b5aa23202/job?acquire=efbebb9b-1738-1702-b91f-104b5aa23202&dc=dc2 from=127.0.0.1:33958 latency=881.94µs -44:31.520 [DEBUG] TestAgent.server.serf.wan: serf: messageJoinType: Node-3c8dfe0c-b036-474a-7834-f794e2fc8220.dc2 -44:31.670 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/event/fire/_rexec?dc=dc2 from=127.0.0.1:33958 latency=2.20202ms -44:31.672 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/efbebb9b-1738-1702-b91f-104b5aa23202/?dc=dc2&keys=&wait=500ms from=127.0.0.1:33958 latency=725.64µs -44:31.682 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/efbebb9b-1738-1702-b91f-104b5aa23202/?dc=dc2&index=15&keys=&wait=500ms from=127.0.0.1:33958 latency=9.36362ms -44:31.683 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/efbebb9b-1738-1702-b91f-104b5aa23202/Node-3c8dfe0c-b036-474a-7834-f794e2fc8220/exit?dc=dc2 from=127.0.0.1:33958 latency=713.72µs -44:31.685 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/efbebb9b-1738-1702-b91f-104b5aa23202/Node-3c8dfe0c-b036-474a-7834-f794e2fc8220/out/00000?dc=dc2 from=127.0.0.1:33958 latency=524.74µs -44:32.019 [DEBUG] TestAgent.server.serf.wan: serf: messageJoinType: Node-3c8dfe0c-b036-474a-7834-f794e2fc8220.dc2 -44:32.189 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/efbebb9b-1738-1702-b91f-104b5aa23202/?dc=dc2&index=17&keys=&wait=500ms from=127.0.0.1:33958 latency=503.491ms -44:32.189 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/efbebb9b-1738-1702-b91f-104b5aa23202?dc=dc2 from=127.0.0.1:33970 latency=2.30268ms -44:32.191 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/_rexec/efbebb9b-1738-1702-b91f-104b5aa23202?dc=dc2&recurse= from=127.0.0.1:33970 latency=831.16µs -44:32.192 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/efbebb9b-1738-1702-b91f-104b5aa23202?dc=dc2 from=127.0.0.1:33970 latency=665.48µs -44:32.693 [INFO] TestAgent: Requesting shutdown -44:32.693 [INFO] TestAgent.server: shutting down server -44:32.693 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -44:32.693 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -44:32.693 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -44:32.693 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -44:32.693 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -44:32.694 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -44:32.694 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -44:32.694 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -44:32.694 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -44:32.694 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -44:32.694 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:44:32.694Z [INFO] agent.router.manager: shutting down -2022-09-15T21:44:32.694Z [INFO] agent.router.manager: shutting down -2022-09-15T21:44:32.694Z [INFO] agent.router.manager: shutting down -44:32.695 [INFO] TestAgent: consul server down -44:32.695 [INFO] TestAgent: shutdown complete -44:32.695 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22274 network=tcp -44:32.695 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22274 network=udp -44:32.695 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22275 network=tcp -44:33.196 [INFO] TestAgent: Waiting for endpoints to shut down -44:33.196 [INFO] TestAgent: Endpoints down ---- PASS: TestExecCommand_CrossDC (3.32s) +2021-08-13T15:36:55.046Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:36:55.102Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +36:55.107 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c7747ed1-e642-0ffe-741d-6ca56b31272b Address:127.0.0.1:21251}]" +36:55.109 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c7747ed1-e642-0ffe-741d-6ca56b31272b.dc1 127.0.0.1 +36:55.110 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c7747ed1-e642-0ffe-741d-6ca56b31272b 127.0.0.1 +2021-08-13T15:36:55.110Z [INFO] agent.router: Initializing LAN area manager +36:55.110 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21246 network=udp +36:55.110 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:21251 [Follower]" leader= +36:55.114 [INFO] TestAgent.server: Adding LAN server: server="Node-c7747ed1-e642-0ffe-741d-6ca56b31272b (Addr: tcp/127.0.0.1:21251) (DC: dc1)" +36:55.114 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21246 network=tcp +36:55.115 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:21247 network=tcp +36:55.115 [INFO] TestAgent: started state syncer +36:55.115 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:21252 network=tcp +36:55.115 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c7747ed1-e642-0ffe-741d-6ca56b31272b.dc1 area=wan +36:55.166 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +36:55.166 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:21251 [Candidate]" term=2 +36:55.166 [DEBUG] TestAgent.server.raft: votes: needed=1 +36:55.166 [DEBUG] TestAgent.server.raft: vote granted: from=c7747ed1-e642-0ffe-741d-6ca56b31272b term=2 tally=1 +36:55.166 [INFO] TestAgent.server.raft: election won: tally=1 +36:55.166 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:21251 [Leader]" +36:55.166 [INFO] TestAgent.server: cluster leadership acquired +36:55.167 [INFO] TestAgent.server: New leader elected: payload=Node-c7747ed1-e642-0ffe-741d-6ca56b31272b +36:55.167 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:21251 +36:55.168 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +36:55.178 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +36:55.178 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +36:55.178 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +36:55.178 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +36:55.178 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +36:55.178 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c7747ed1-e642-0ffe-741d-6ca56b31272b +36:55.178 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c7747ed1-e642-0ffe-741d-6ca56b31272b +36:55.181 [INFO] TestAgent.server: federation state anti-entropy synced +36:55.282 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:46892 latency=3.112468ms +36:55.287 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/session/info/a3cb6c90-a529-6d16-2a88-faf52944b1dd from=127.0.0.1:46896 latency=532.561µs +36:55.289 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/a3cb6c90-a529-6d16-2a88-faf52944b1dd from=127.0.0.1:46892 latency=483.681µs +36:55.290 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/session/info/a3cb6c90-a529-6d16-2a88-faf52944b1dd from=127.0.0.1:46902 latency=237.26µs +36:55.290 [INFO] TestAgent: Requesting shutdown +36:55.290 [INFO] TestAgent.server: shutting down server +36:55.290 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +36:55.290 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +36:55.290 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +36:55.290 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +36:55.290 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +36:55.290 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +36:55.290 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +36:55.290 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +36:55.290 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +36:55.290 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +36:55.290 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +36:55.290 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:36:55.290Z [INFO] agent.router.manager: shutting down +2021-08-13T15:36:55.290Z [INFO] agent.router.manager: shutting down +36:55.291 [INFO] TestAgent: consul server down +36:55.291 [INFO] TestAgent: shutdown complete +36:55.291 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21246 network=tcp +36:55.291 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21246 network=udp +36:55.291 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:21247 network=tcp +36:55.791 [INFO] TestAgent: Waiting for endpoints to shut down +36:55.791 [INFO] TestAgent: Endpoints down +--- PASS: TestExecCommand_Sessions (0.81s) +2021-08-13T15:36:55.071Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:36:55.114Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +36:55.117 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f60e1434-a36e-0d0b-633e-b3c2cfa394e0 Address:127.0.0.1:21237}]" +36:55.117 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:21237 [Follower]" leader= +36:55.118 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f60e1434-a36e-0d0b-633e-b3c2cfa394e0.dc1 127.0.0.1 +36:55.118 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f60e1434-a36e-0d0b-633e-b3c2cfa394e0 127.0.0.1 +2021-08-13T15:36:55.118Z [INFO] agent.router: Initializing LAN area manager +36:55.123 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f60e1434-a36e-0d0b-633e-b3c2cfa394e0.dc1 area=wan +36:55.123 [INFO] TestAgent.server: Adding LAN server: server="Node-f60e1434-a36e-0d0b-633e-b3c2cfa394e0 (Addr: tcp/127.0.0.1:21237) (DC: dc1)" +36:55.124 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21232 network=udp +36:55.125 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21232 network=tcp +36:55.126 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:21233 network=tcp +36:55.126 [INFO] TestAgent: started state syncer +36:55.133 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:21238 network=tcp +36:55.169 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +36:55.169 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:21237 [Candidate]" term=2 +36:55.169 [DEBUG] TestAgent.server.raft: votes: needed=1 +36:55.169 [DEBUG] TestAgent.server.raft: vote granted: from=f60e1434-a36e-0d0b-633e-b3c2cfa394e0 term=2 tally=1 +36:55.169 [INFO] TestAgent.server.raft: election won: tally=1 +36:55.169 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:21237 [Leader]" +36:55.170 [INFO] TestAgent.server: cluster leadership acquired +36:55.170 [INFO] TestAgent.server: New leader elected: payload=Node-f60e1434-a36e-0d0b-633e-b3c2cfa394e0 +36:55.170 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:21237 +36:55.171 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +36:55.180 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +36:55.180 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +36:55.180 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +36:55.180 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +36:55.180 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +36:55.180 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f60e1434-a36e-0d0b-633e-b3c2cfa394e0 +36:55.180 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f60e1434-a36e-0d0b-633e-b3c2cfa394e0 +36:55.181 [INFO] TestAgent.server: federation state anti-entropy synced +36:55.285 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:42648 latency=3.742789ms +36:55.287 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/552b8388-11ae-1407-8694-655aa7cc0368/?keys=&wait=2000ms from=127.0.0.1:42648 latency=124.041µs +36:55.287 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/_rexec/552b8388-11ae-1407-8694-655aa7cc0368/foo/ack?acquire=552b8388-11ae-1407-8694-655aa7cc0368 from=127.0.0.1:42652 latency=709.102µs +36:55.288 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/552b8388-11ae-1407-8694-655aa7cc0368/?index=1&keys=&wait=2000ms from=127.0.0.1:42648 latency=984.322µs +36:55.290 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/_rexec/552b8388-11ae-1407-8694-655aa7cc0368/foo/exit?acquire=552b8388-11ae-1407-8694-655aa7cc0368 from=127.0.0.1:42654 latency=707.202µs +36:55.290 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/552b8388-11ae-1407-8694-655aa7cc0368/?index=13&keys=&wait=2000ms from=127.0.0.1:42648 latency=922.882µs +36:55.291 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/552b8388-11ae-1407-8694-655aa7cc0368/foo/exit from=127.0.0.1:42648 latency=247.64µs +36:55.292 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/_rexec/552b8388-11ae-1407-8694-655aa7cc0368/foo/random?acquire=552b8388-11ae-1407-8694-655aa7cc0368 from=127.0.0.1:42658 latency=484.441µs +36:55.292 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/552b8388-11ae-1407-8694-655aa7cc0368/?index=14&keys=&wait=2000ms from=127.0.0.1:42648 latency=1.017602ms +36:55.294 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/_rexec/552b8388-11ae-1407-8694-655aa7cc0368/foo/out/00000?acquire=552b8388-11ae-1407-8694-655aa7cc0368 from=127.0.0.1:42660 latency=540.701µs +36:55.294 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/552b8388-11ae-1407-8694-655aa7cc0368/?index=15&keys=&wait=2000ms from=127.0.0.1:42648 latency=898.822µs +36:55.295 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/552b8388-11ae-1407-8694-655aa7cc0368/foo/out/00000 from=127.0.0.1:42648 latency=104.84µs +36:55.296 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/552b8388-11ae-1407-8694-655aa7cc0368/?index=16&keys=&wait=2000ms from=127.0.0.1:42648 latency=762.482µs +36:55.296 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/_rexec/552b8388-11ae-1407-8694-655aa7cc0368/foo/out/00001?acquire=552b8388-11ae-1407-8694-655aa7cc0368 from=127.0.0.1:42662 latency=716.202µs +36:55.297 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/552b8388-11ae-1407-8694-655aa7cc0368/foo/out/00001 from=127.0.0.1:42648 latency=109.22µs +36:55.297 [INFO] TestAgent: Requesting shutdown +36:55.297 [INFO] TestAgent.server: shutting down server +36:55.297 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +36:55.297 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +36:55.297 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +36:55.297 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +36:55.297 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +36:55.297 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +36:55.297 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +36:55.297 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/552b8388-11ae-1407-8694-655aa7cc0368/?index=17&keys=&wait=2000ms from=127.0.0.1:42648 latency=144.061µs +36:55.297 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +36:55.297 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +36:55.297 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +36:55.297 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2021-08-13T15:36:55.297Z [INFO] agent.router.manager: shutting down +2021-08-13T15:36:55.298Z [INFO] agent.router.manager: shutting down +36:55.297 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +36:55.298 [INFO] TestAgent: consul server down +36:55.298 [INFO] TestAgent: shutdown complete +36:55.298 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21232 network=tcp +36:55.298 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21232 network=udp +36:55.298 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:21233 network=tcp +36:55.798 [INFO] TestAgent: Waiting for endpoints to shut down +36:55.798 [INFO] TestAgent: Endpoints down +--- PASS: TestExecCommand_StreamResults (0.82s) +2021-08-13T15:36:55.080Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:36:55.113Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +36:55.115 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b9652896-2ff0-2b6e-5aae-0641ea666b97 Address:127.0.0.1:21265}]" +36:55.116 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b9652896-2ff0-2b6e-5aae-0641ea666b97.dc1 127.0.0.1 +36:55.117 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:21265 [Follower]" leader= +36:55.118 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b9652896-2ff0-2b6e-5aae-0641ea666b97 127.0.0.1 +2021-08-13T15:36:55.118Z [INFO] agent.router: Initializing LAN area manager +36:55.118 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21260 network=udp +36:55.123 [INFO] TestAgent.server: Adding LAN server: server="Node-b9652896-2ff0-2b6e-5aae-0641ea666b97 (Addr: tcp/127.0.0.1:21265) (DC: dc1)" +36:55.123 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b9652896-2ff0-2b6e-5aae-0641ea666b97.dc1 area=wan +36:55.124 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21260 network=tcp +36:55.125 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:21261 network=tcp +36:55.125 [INFO] TestAgent: started state syncer +36:55.126 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:21266 network=tcp +36:55.177 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +36:55.178 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:21265 [Candidate]" term=2 +36:55.178 [DEBUG] TestAgent.server.raft: votes: needed=1 +36:55.178 [DEBUG] TestAgent.server.raft: vote granted: from=b9652896-2ff0-2b6e-5aae-0641ea666b97 term=2 tally=1 +36:55.178 [INFO] TestAgent.server.raft: election won: tally=1 +36:55.178 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:21265 [Leader]" +36:55.178 [INFO] TestAgent.server: cluster leadership acquired +36:55.178 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:21265 +36:55.178 [INFO] TestAgent.server: New leader elected: payload=Node-b9652896-2ff0-2b6e-5aae-0641ea666b97 +36:55.181 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +36:55.184 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +36:55.184 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +36:55.184 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +36:55.184 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +36:55.184 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +36:55.184 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b9652896-2ff0-2b6e-5aae-0641ea666b97 +36:55.184 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b9652896-2ff0-2b6e-5aae-0641ea666b97 +36:55.185 [INFO] TestAgent.server: federation state anti-entropy synced +36:55.282 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +36:55.283 [INFO] TestAgent: Synced node info +36:55.283 [DEBUG] TestAgent: Node info in sync +36:55.359 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:44302 latency=1.137342ms +36:55.361 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/_rexec/a1f3f77e-3af2-6e04-12c0-19ae8c003957/job?acquire=a1f3f77e-3af2-6e04-12c0-19ae8c003957 from=127.0.0.1:44302 latency=1.265203ms +36:55.363 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/a1f3f77e-3af2-6e04-12c0-19ae8c003957/job from=127.0.0.1:44304 latency=383.861µs +36:55.364 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/_rexec/a1f3f77e-3af2-6e04-12c0-19ae8c003957?recurse= from=127.0.0.1:44302 latency=465.661µs +36:55.365 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/a1f3f77e-3af2-6e04-12c0-19ae8c003957/job from=127.0.0.1:44306 latency=61.301µs +36:55.365 [INFO] TestAgent: Requesting shutdown +36:55.365 [INFO] TestAgent.server: shutting down server +36:55.365 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +36:55.365 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +36:55.365 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +36:55.365 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +36:55.365 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +36:55.365 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +36:55.365 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +36:55.365 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +36:55.365 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +36:55.365 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +36:55.366 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:36:55.366Z [INFO] agent.router.manager: shutting down +2021-08-13T15:36:55.366Z [INFO] agent.router.manager: shutting down +36:55.366 [INFO] TestAgent: consul server down +36:55.366 [INFO] TestAgent: shutdown complete +36:55.366 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21260 network=tcp +36:55.366 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21260 network=udp +36:55.366 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:21261 network=tcp +36:55.866 [INFO] TestAgent: Waiting for endpoints to shut down +36:55.866 [INFO] TestAgent: Endpoints down +--- PASS: TestExecCommand_UploadDestroy (0.88s) +2021-08-13T15:36:55.098Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:36:55.133Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +36:55.270 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1980ae24-911e-78f4-9fe0-12347905912a Address:127.0.0.1:21279}]" +36:55.272 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:21279 [Follower]" leader= +36:55.279 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1980ae24-911e-78f4-9fe0-12347905912a.dc1 127.0.0.1 +36:55.280 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1980ae24-911e-78f4-9fe0-12347905912a 127.0.0.1 +2021-08-13T15:36:55.280Z [INFO] agent.router: Initializing LAN area manager +36:55.281 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21274 network=udp +36:55.281 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1980ae24-911e-78f4-9fe0-12347905912a.dc1 area=wan +36:55.282 [INFO] TestAgent.server: Adding LAN server: server="Node-1980ae24-911e-78f4-9fe0-12347905912a (Addr: tcp/127.0.0.1:21279) (DC: dc1)" +36:55.282 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21274 network=tcp +36:55.286 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:21275 network=tcp +36:55.286 [INFO] TestAgent: started state syncer +36:55.287 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:21280 network=tcp +36:55.324 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +36:55.324 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:21279 [Candidate]" term=2 +36:55.324 [DEBUG] TestAgent.server.raft: votes: needed=1 +36:55.324 [DEBUG] TestAgent.server.raft: vote granted: from=1980ae24-911e-78f4-9fe0-12347905912a term=2 tally=1 +36:55.324 [INFO] TestAgent.server.raft: election won: tally=1 +36:55.324 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:21279 [Leader]" +36:55.324 [INFO] TestAgent.server: cluster leadership acquired +36:55.325 [INFO] TestAgent.server: New leader elected: payload=Node-1980ae24-911e-78f4-9fe0-12347905912a +36:55.325 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:21279 +36:55.326 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +36:55.328 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +36:55.328 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +36:55.328 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +36:55.328 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +36:55.328 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +36:55.328 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1980ae24-911e-78f4-9fe0-12347905912a +36:55.328 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1980ae24-911e-78f4-9fe0-12347905912a +36:55.330 [INFO] TestAgent.server: federation state anti-entropy synced +36:55.420 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/health/service/consul?passing=1 from=127.0.0.1:35100 latency=1.375284ms +36:55.423 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:35100 latency=872.342µs +36:55.424 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/session/info/6d220fc0-9521-3c3e-1988-0470b79d684c from=127.0.0.1:35102 latency=231.24µs +36:55.426 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/6d220fc0-9521-3c3e-1988-0470b79d684c from=127.0.0.1:35100 latency=528.542µs +36:55.426 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/session/info/6d220fc0-9521-3c3e-1988-0470b79d684c from=127.0.0.1:35104 latency=104.64µs +36:55.427 [INFO] TestAgent: Requesting shutdown +36:55.427 [INFO] TestAgent.server: shutting down server +36:55.427 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +36:55.427 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +36:55.427 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +36:55.427 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +36:55.427 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +36:55.427 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +36:55.427 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +36:55.427 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +36:55.427 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +36:55.427 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +36:55.427 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +36:55.427 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:36:55.427Z [INFO] agent.router.manager: shutting down +2021-08-13T15:36:55.427Z [INFO] agent.router.manager: shutting down +36:55.427 [INFO] TestAgent: consul server down +36:55.427 [INFO] TestAgent: shutdown complete +36:55.427 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21274 network=tcp +36:55.427 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21274 network=udp +36:55.427 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:21275 network=tcp +36:55.928 [INFO] TestAgent: Waiting for endpoints to shut down +36:55.928 [INFO] TestAgent: Endpoints down +--- PASS: TestExecCommand_Sessions_Foreign (0.95s) +2021-08-13T15:36:55.072Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:36:55.129Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +36:55.133 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d3bce615-f94a-5ee6-d6be-3192a8f60033 Address:127.0.0.1:21272}]" +36:55.133 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:21272 [Follower]" leader= +36:55.134 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d3bce615-f94a-5ee6-d6be-3192a8f60033.dc1 127.0.0.1 +36:55.201 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +36:55.201 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:21272 [Candidate]" term=2 +36:55.202 [WARN] TestAgent.server.raft: unable to get address for sever, using fallback address: id=d3bce615-f94a-5ee6-d6be-3192a8f60033 fallback=127.0.0.1:21272 error="Could not find address for server id d3bce615-f94a-5ee6-d6be-3192a8f60033" +36:55.202 [DEBUG] TestAgent.server.raft: votes: needed=1 +36:55.202 [DEBUG] TestAgent.server.raft: vote granted: from=d3bce615-f94a-5ee6-d6be-3192a8f60033 term=2 tally=1 +36:55.202 [INFO] TestAgent.server.raft: election won: tally=1 +36:55.202 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:21272 [Leader]" +36:55.269 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d3bce615-f94a-5ee6-d6be-3192a8f60033 127.0.0.1 +2021-08-13T15:36:55.269Z [INFO] agent.router: Initializing LAN area manager +36:55.270 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21267 network=udp +36:55.272 [INFO] TestAgent.server: Adding LAN server: server="Node-d3bce615-f94a-5ee6-d6be-3192a8f60033 (Addr: tcp/127.0.0.1:21272) (DC: dc1)" +36:55.272 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d3bce615-f94a-5ee6-d6be-3192a8f60033.dc1 area=wan +36:55.272 [INFO] TestAgent.server: cluster leadership acquired +36:55.273 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:21272 +36:55.273 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21267 network=tcp +36:55.274 [INFO] TestAgent.server: New leader elected: payload=Node-d3bce615-f94a-5ee6-d6be-3192a8f60033 +36:55.279 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:21268 network=tcp +36:55.279 [INFO] TestAgent: started state syncer +36:55.280 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:21273 network=tcp +36:55.281 [INFO] TestAgent: Synced node info +36:55.281 [DEBUG] TestAgent: Node info in sync +36:55.284 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +36:55.287 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +36:55.287 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +36:55.287 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +36:55.287 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +36:55.287 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +36:55.287 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d3bce615-f94a-5ee6-d6be-3192a8f60033 +36:55.287 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d3bce615-f94a-5ee6-d6be-3192a8f60033 +36:55.288 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d3bce615-f94a-5ee6-d6be-3192a8f60033 +36:55.288 [INFO] TestAgent.server: federation state anti-entropy synced +36:55.308 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:53352 latency=1.827045ms +36:55.312 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:53352 latency=701.162µs +36:55.312 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/_rexec/44d97007-b023-680e-8510-e6bd354c172b/job?acquire=44d97007-b023-680e-8510-e6bd354c172b from=127.0.0.1:53352 latency=434.101µs +36:55.514 [DEBUG] TestAgent.server: User event: event=_rexec +36:55.514 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/event/fire/_rexec from=127.0.0.1:53352 latency=377.641µs +36:55.514 [DEBUG] TestAgent: received remote exec event: id=afeed7d3-06e0-dd7e-cdea-ab2f429f88f0 +36:55.514 [INFO] TestAgent: remote exec script: script= +36:55.515 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/44d97007-b023-680e-8510-e6bd354c172b/?keys=&wait=1000ms from=127.0.0.1:53352 latency=161.72µs +36:55.519 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/44d97007-b023-680e-8510-e6bd354c172b/?index=15&keys=&wait=1000ms from=127.0.0.1:53352 latency=3.877389ms +36:55.520 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/44d97007-b023-680e-8510-e6bd354c172b/Node-d3bce615-f94a-5ee6-d6be-3192a8f60033/out/00000 from=127.0.0.1:53352 latency=117.78µs +36:55.521 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/44d97007-b023-680e-8510-e6bd354c172b/?index=16&keys=&wait=1000ms from=127.0.0.1:53352 latency=159.84µs +36:55.521 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/44d97007-b023-680e-8510-e6bd354c172b/Node-d3bce615-f94a-5ee6-d6be-3192a8f60033/exit from=127.0.0.1:53352 latency=117.781µs +36:56.233 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +36:56.233 [DEBUG] TestAgent: Node info in sync +36:56.524 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/44d97007-b023-680e-8510-e6bd354c172b from=127.0.0.1:53376 latency=1.015743ms +36:56.524 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/44d97007-b023-680e-8510-e6bd354c172b/?index=17&keys=&wait=1000ms from=127.0.0.1:53352 latency=1.001876016s +36:56.525 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/_rexec/44d97007-b023-680e-8510-e6bd354c172b?recurse= from=127.0.0.1:53376 latency=453.601µs +36:56.525 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/44d97007-b023-680e-8510-e6bd354c172b from=127.0.0.1:53376 latency=325.821µs +36:56.526 [INFO] TestAgent: Requesting shutdown +36:56.526 [INFO] TestAgent.server: shutting down server +36:56.526 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +36:56.526 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +36:56.526 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +36:56.526 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +36:56.526 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +36:56.526 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +36:56.526 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +36:56.526 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +36:56.526 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +36:56.526 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +36:56.526 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:36:56.526Z [INFO] agent.router.manager: shutting down +2021-08-13T15:36:56.526Z [INFO] agent.router.manager: shutting down +36:56.526 [INFO] TestAgent: consul server down +36:56.526 [INFO] TestAgent: shutdown complete +36:56.526 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21267 network=tcp +36:56.526 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21267 network=udp +36:56.526 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:21268 network=tcp +36:57.026 [INFO] TestAgent: Waiting for endpoints to shut down +36:57.026 [INFO] TestAgent: Endpoints down +--- PASS: TestExecCommand_NoShell (2.05s) +2021-08-13T15:36:55.040Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:36:55.084Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +36:55.088 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8ff9ce4c-8d48-fdcd-8cd1-e728d7402e06 Address:127.0.0.1:21258}]" +36:55.089 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8ff9ce4c-8d48-fdcd-8cd1-e728d7402e06.dc1 127.0.0.1 +36:55.089 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8ff9ce4c-8d48-fdcd-8cd1-e728d7402e06 127.0.0.1 +2021-08-13T15:36:55.089Z [INFO] agent.router: Initializing LAN area manager +36:55.090 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21253 network=udp +36:55.090 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:21258 [Follower]" leader= +36:55.096 [INFO] TestAgent.server: Adding LAN server: server="Node-8ff9ce4c-8d48-fdcd-8cd1-e728d7402e06 (Addr: tcp/127.0.0.1:21258) (DC: dc1)" +36:55.098 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21253 network=tcp +36:55.099 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:21254 network=tcp +36:55.099 [INFO] TestAgent: started state syncer +36:55.100 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8ff9ce4c-8d48-fdcd-8cd1-e728d7402e06.dc1 area=wan +36:55.103 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:21259 network=tcp +36:55.153 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +36:55.153 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:21258 [Candidate]" term=2 +36:55.153 [DEBUG] TestAgent.server.raft: votes: needed=1 +36:55.153 [DEBUG] TestAgent.server.raft: vote granted: from=8ff9ce4c-8d48-fdcd-8cd1-e728d7402e06 term=2 tally=1 +36:55.153 [INFO] TestAgent.server.raft: election won: tally=1 +36:55.153 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:21258 [Leader]" +36:55.153 [INFO] TestAgent.server: cluster leadership acquired +36:55.153 [INFO] TestAgent.server: New leader elected: payload=Node-8ff9ce4c-8d48-fdcd-8cd1-e728d7402e06 +36:55.153 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:21258 +36:55.155 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +36:55.179 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +36:55.179 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +36:55.179 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +36:55.179 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +36:55.179 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +36:55.179 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8ff9ce4c-8d48-fdcd-8cd1-e728d7402e06 +36:55.179 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8ff9ce4c-8d48-fdcd-8cd1-e728d7402e06 +36:55.181 [INFO] TestAgent.server: federation state anti-entropy synced +36:55.370 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:49412 latency=1.673464ms +36:55.374 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:49412 latency=936.723µs +36:55.374 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +36:55.375 [INFO] TestAgent: Synced node info +36:55.375 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/_rexec/ddb84156-6675-c64c-52ea-905cc19b5960/job?acquire=ddb84156-6675-c64c-52ea-905cc19b5960 from=127.0.0.1:49412 latency=506.681µs +36:55.576 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/event/fire/_rexec from=127.0.0.1:49412 latency=302.88µs +36:55.577 [DEBUG] TestAgent.server: User event: event=_rexec +36:55.577 [DEBUG] TestAgent: received remote exec event: id=8d24604f-8fcc-04b5-2c6c-f653b370fa00 +36:55.578 [INFO] TestAgent: remote exec script: script=uptime +36:55.579 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/ddb84156-6675-c64c-52ea-905cc19b5960/?keys=&wait=1000ms from=127.0.0.1:49412 latency=979.922µs +36:55.586 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/ddb84156-6675-c64c-52ea-905cc19b5960/?index=15&keys=&wait=1000ms from=127.0.0.1:49412 latency=6.140315ms +36:55.586 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/ddb84156-6675-c64c-52ea-905cc19b5960/Node-8ff9ce4c-8d48-fdcd-8cd1-e728d7402e06/out/00000 from=127.0.0.1:49412 latency=130.881µs +36:55.587 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/ddb84156-6675-c64c-52ea-905cc19b5960/?index=16&keys=&wait=1000ms from=127.0.0.1:49412 latency=312.841µs +36:55.588 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/ddb84156-6675-c64c-52ea-905cc19b5960/Node-8ff9ce4c-8d48-fdcd-8cd1-e728d7402e06/exit from=127.0.0.1:49412 latency=232.481µs +36:56.590 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/ddb84156-6675-c64c-52ea-905cc19b5960 from=127.0.0.1:49430 latency=932.222µs +36:56.590 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/ddb84156-6675-c64c-52ea-905cc19b5960/?index=17&keys=&wait=1000ms from=127.0.0.1:49412 latency=1.001709895s +36:56.591 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/_rexec/ddb84156-6675-c64c-52ea-905cc19b5960?recurse= from=127.0.0.1:49430 latency=458.941µs +36:56.592 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/ddb84156-6675-c64c-52ea-905cc19b5960 from=127.0.0.1:49430 latency=315µs +36:56.592 [INFO] TestAgent: Requesting shutdown +36:56.592 [INFO] TestAgent.server: shutting down server +36:56.592 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +36:56.592 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +36:56.592 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +36:56.592 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +36:56.592 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +36:56.593 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +36:56.593 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +36:56.593 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +36:56.593 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +36:56.593 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +36:56.593 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:36:56.593Z [INFO] agent.router.manager: shutting down +2021-08-13T15:36:56.593Z [INFO] agent.router.manager: shutting down +36:56.593 [INFO] TestAgent: consul server down +36:56.593 [INFO] TestAgent: shutdown complete +36:56.593 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21253 network=tcp +36:56.593 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21253 network=udp +36:56.593 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:21254 network=tcp +36:57.093 [INFO] TestAgent: Waiting for endpoints to shut down +36:57.093 [INFO] TestAgent: Endpoints down +--- PASS: TestExecCommand (2.11s) +2021-08-13T15:36:55.544Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:36:55.560Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +36:55.563 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:61bef0b8-19b0-977e-0043-0de07d8d5191 Address:127.0.0.1:21286}]" +36:55.563 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:21286 [Follower]" leader= +36:55.564 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-61bef0b8-19b0-977e-0043-0de07d8d5191.dc2 127.0.0.1 +36:55.565 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-61bef0b8-19b0-977e-0043-0de07d8d5191 127.0.0.1 +2021-08-13T15:36:55.565Z [INFO] agent.router: Initializing LAN area manager +36:55.565 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-61bef0b8-19b0-977e-0043-0de07d8d5191.dc2 area=wan +36:55.565 [INFO] TestAgent.server: Adding LAN server: server="Node-61bef0b8-19b0-977e-0043-0de07d8d5191 (Addr: tcp/127.0.0.1:21286) (DC: dc2)" +36:55.565 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21281 network=tcp +36:55.565 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21281 network=udp +36:55.571 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:21282 network=tcp +36:55.571 [INFO] TestAgent: started state syncer +36:55.571 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:21287 network=tcp +36:55.627 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +36:55.627 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:21286 [Candidate]" term=2 +36:55.628 [DEBUG] TestAgent.server.raft: votes: needed=1 +36:55.628 [DEBUG] TestAgent.server.raft: vote granted: from=61bef0b8-19b0-977e-0043-0de07d8d5191 term=2 tally=1 +36:55.628 [INFO] TestAgent.server.raft: election won: tally=1 +36:55.628 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:21286 [Leader]" +36:55.628 [INFO] TestAgent.server: cluster leadership acquired +36:55.628 [INFO] TestAgent.server: New leader elected: payload=Node-61bef0b8-19b0-977e-0043-0de07d8d5191 +36:55.628 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:21286 +36:55.629 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +36:55.632 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +36:55.632 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +36:55.632 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +36:55.632 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +36:55.632 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +36:55.632 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-61bef0b8-19b0-977e-0043-0de07d8d5191 +36:55.632 [INFO] TestAgent.server: member joined, marking health alive: member=Node-61bef0b8-19b0-977e-0043-0de07d8d5191 +36:55.633 [INFO] TestAgent.server: federation state anti-entropy synced +36:55.951 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +36:55.952 [INFO] TestAgent: Synced node info +36:56.013 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:21243] +36:56.014 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:21243 +36:56.015 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-35b29f4f-62eb-7e42-8125-929abbdfb68c.dc1 127.0.0.1 +36:56.015 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 +36:56.015 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-35b29f4f-62eb-7e42-8125-929abbdfb68c.dc1 area=wan +36:56.226 [DEBUG] TestAgent.server: User event: event=_rexec +36:56.227 [DEBUG] TestAgent: received remote exec event: id=8dc63add-f5af-f989-3508-0bca0053aeb2 +36:56.227 [INFO] TestAgent: remote exec script: script=uptime +36:56.280 [DEBUG] TestAgent.server.serf.wan: serf: messageJoinType: Node-61bef0b8-19b0-977e-0043-0de07d8d5191.dc2 +36:56.743 [INFO] TestAgent: Requesting shutdown +36:56.743 [INFO] TestAgent.server: shutting down server +36:56.743 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +36:56.743 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +36:56.743 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +36:56.743 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +36:56.743 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +36:56.743 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +36:56.743 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +36:56.743 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +36:56.743 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +36:56.743 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +36:56.743 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:36:56.744Z [INFO] agent.router.manager: shutting down +2021-08-13T15:36:56.744Z [INFO] agent.router.manager: shutting down +2021-08-13T15:36:56.744Z [INFO] agent.router.manager: shutting down +36:56.744 [INFO] TestAgent: consul server down +36:56.744 [INFO] TestAgent: shutdown complete +36:56.744 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21281 network=tcp +36:56.744 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21281 network=udp +36:56.744 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:21282 network=tcp +36:57.244 [INFO] TestAgent: Waiting for endpoints to shut down +36:57.244 [INFO] TestAgent: Endpoints down +2021-08-13T15:36:55.117Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:36:55.270Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +36:55.275 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:35b29f4f-62eb-7e42-8125-929abbdfb68c Address:127.0.0.1:21244}]" +36:55.278 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:21244 [Follower]" leader= +36:55.279 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-35b29f4f-62eb-7e42-8125-929abbdfb68c.dc1 127.0.0.1 +36:55.280 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-35b29f4f-62eb-7e42-8125-929abbdfb68c 127.0.0.1 +2021-08-13T15:36:55.280Z [INFO] agent.router: Initializing LAN area manager +36:55.282 [INFO] TestAgent.server: Adding LAN server: server="Node-35b29f4f-62eb-7e42-8125-929abbdfb68c (Addr: tcp/127.0.0.1:21244) (DC: dc1)" +36:55.282 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21239 network=udp +36:55.282 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-35b29f4f-62eb-7e42-8125-929abbdfb68c.dc1 area=wan +36:55.282 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21239 network=tcp +36:55.285 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:21240 network=tcp +36:55.285 [INFO] TestAgent: started state syncer +36:55.286 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:21245 network=tcp +36:55.347 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +36:55.347 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:21244 [Candidate]" term=2 +36:55.348 [DEBUG] TestAgent.server.raft: votes: needed=1 +36:55.348 [DEBUG] TestAgent.server.raft: vote granted: from=35b29f4f-62eb-7e42-8125-929abbdfb68c term=2 tally=1 +36:55.348 [INFO] TestAgent.server.raft: election won: tally=1 +36:55.348 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:21244 [Leader]" +36:55.348 [INFO] TestAgent.server: cluster leadership acquired +36:55.348 [INFO] TestAgent.server: New leader elected: payload=Node-35b29f4f-62eb-7e42-8125-929abbdfb68c +36:55.348 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:21244 +36:55.349 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +36:55.354 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +36:55.354 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +36:55.354 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +36:55.354 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +36:55.354 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +36:55.354 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-35b29f4f-62eb-7e42-8125-929abbdfb68c +36:55.354 [INFO] TestAgent.server: member joined, marking health alive: member=Node-35b29f4f-62eb-7e42-8125-929abbdfb68c +36:55.355 [INFO] TestAgent.server: federation state anti-entropy synced +36:55.651 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +36:55.651 [INFO] TestAgent: Synced node info +36:55.651 [DEBUG] TestAgent: Node info in sync +36:56.014 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:45098 +36:56.015 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-61bef0b8-19b0-977e-0043-0de07d8d5191.dc2 127.0.0.1 +36:56.015 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-61bef0b8-19b0-977e-0043-0de07d8d5191.dc2 area=wan +36:56.018 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self?dc=dc2 from=127.0.0.1:38068 latency=1.455544ms +36:56.022 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/health/service/consul?dc=dc2&passing=1 from=127.0.0.1:38068 latency=1.636124ms +36:56.024 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create?dc=dc2 from=127.0.0.1:38068 latency=962.703µs +36:56.025 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/_rexec/27b5f200-6ca8-c97a-6982-24d678d2d303/job?acquire=27b5f200-6ca8-c97a-6982-24d678d2d303&dc=dc2 from=127.0.0.1:38068 latency=704.641µs +36:56.065 [DEBUG] TestAgent.server.serf.wan: serf: messageJoinType: Node-61bef0b8-19b0-977e-0043-0de07d8d5191.dc2 +36:56.101 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +36:56.101 [DEBUG] TestAgent: Node info in sync +36:56.227 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/event/fire/_rexec?dc=dc2 from=127.0.0.1:38068 latency=795.782µs +36:56.229 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/27b5f200-6ca8-c97a-6982-24d678d2d303/?dc=dc2&keys=&wait=500ms from=127.0.0.1:38068 latency=1.060083ms +36:56.234 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/27b5f200-6ca8-c97a-6982-24d678d2d303/?dc=dc2&index=15&keys=&wait=500ms from=127.0.0.1:38068 latency=5.092972ms +36:56.235 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/27b5f200-6ca8-c97a-6982-24d678d2d303/Node-61bef0b8-19b0-977e-0043-0de07d8d5191/out/00000?dc=dc2 from=127.0.0.1:38068 latency=600.682µs +36:56.237 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/27b5f200-6ca8-c97a-6982-24d678d2d303/?dc=dc2&index=16&keys=&wait=500ms from=127.0.0.1:38068 latency=486.481µs +36:56.238 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/27b5f200-6ca8-c97a-6982-24d678d2d303/Node-61bef0b8-19b0-977e-0043-0de07d8d5191/exit?dc=dc2 from=127.0.0.1:38068 latency=445.321µs +36:56.565 [DEBUG] TestAgent.server.serf.wan: serf: messageJoinType: Node-61bef0b8-19b0-977e-0043-0de07d8d5191.dc2 +36:56.741 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/27b5f200-6ca8-c97a-6982-24d678d2d303/?dc=dc2&index=17&keys=&wait=500ms from=127.0.0.1:38068 latency=502.461771ms +36:56.741 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/27b5f200-6ca8-c97a-6982-24d678d2d303?dc=dc2 from=127.0.0.1:38074 latency=1.644704ms +36:56.742 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/_rexec/27b5f200-6ca8-c97a-6982-24d678d2d303?dc=dc2&recurse= from=127.0.0.1:38074 latency=642.741µs +36:56.743 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/27b5f200-6ca8-c97a-6982-24d678d2d303?dc=dc2 from=127.0.0.1:38074 latency=600.482µs +36:57.244 [INFO] TestAgent: Requesting shutdown +36:57.244 [INFO] TestAgent.server: shutting down server +36:57.244 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +36:57.244 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +36:57.244 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +36:57.244 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +36:57.244 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +36:57.244 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +36:57.244 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +36:57.244 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +36:57.244 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +36:57.244 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +36:57.245 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:36:57.245Z [INFO] agent.router.manager: shutting down +2021-08-13T15:36:57.245Z [INFO] agent.router.manager: shutting down +2021-08-13T15:36:57.245Z [INFO] agent.router.manager: shutting down +36:57.245 [INFO] TestAgent: consul server down +36:57.245 [INFO] TestAgent: shutdown complete +36:57.245 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21239 network=tcp +36:57.245 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21239 network=udp +36:57.245 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:21240 network=tcp +36:57.745 [INFO] TestAgent: Waiting for endpoints to shut down +36:57.745 [INFO] TestAgent: Endpoints down +--- PASS: TestExecCommand_CrossDC (2.76s) PASS -ok github.com/hashicorp/consul/command/exec 3.590s +ok github.com/hashicorp/consul/command/exec 2.837s === RUN TestForceLeaveCommand_noTabs === PAUSE TestForceLeaveCommand_noTabs === RUN TestForceLeaveCommand @@ -84789,375 +84699,372 @@ === RUN TestForceLeaveCommand_noAddrs === PAUSE TestForceLeaveCommand_noAddrs === CONT TestForceLeaveCommand_noTabs +=== CONT TestForceLeaveCommand_prune +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... --- PASS: TestForceLeaveCommand_noTabs (0.00s) -=== CONT TestForceLeaveCommand_noAddrs ---- PASS: TestForceLeaveCommand_noAddrs (0.00s) === CONT TestForceLeaveCommand_NoNodeWithName === CONT TestForceLeaveCommand -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... -=== CONT TestForceLeaveCommand_prune +=== CONT TestForceLeaveCommand_noAddrs +--- PASS: TestForceLeaveCommand_noAddrs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:44:36.828Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:44:36.956Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -44:37.007 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:80040b29-307f-6fe2-371a-8b9b4e0592dc Address:127.0.0.1:23293}]" -44:37.008 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-80040b29-307f-6fe2-371a-8b9b4e0592dc.dc1 127.0.0.1 -44:37.009 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-80040b29-307f-6fe2-371a-8b9b4e0592dc 127.0.0.1 -2022-09-15T21:44:37.009Z [INFO] agent.router: Initializing LAN area manager -44:37.010 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23293 [Follower]" leader= -44:37.010 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23288 network=udp -44:37.010 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-80040b29-307f-6fe2-371a-8b9b4e0592dc.dc1 area=wan -44:37.010 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23288 network=tcp -44:37.010 [INFO] TestAgent.server: Adding LAN server: server="Node-80040b29-307f-6fe2-371a-8b9b4e0592dc (Addr: tcp/127.0.0.1:23293) (DC: dc1)" -44:37.018 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23289 network=tcp -44:37.019 [INFO] TestAgent: started state syncer -44:37.019 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23294 network=tcp -44:37.053 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -44:37.053 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23293 [Candidate]" term=2 -44:37.053 [DEBUG] TestAgent.server.raft: votes: needed=1 -44:37.053 [DEBUG] TestAgent.server.raft: vote granted: from=80040b29-307f-6fe2-371a-8b9b4e0592dc term=2 tally=1 -44:37.053 [INFO] TestAgent.server.raft: election won: tally=1 -44:37.053 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23293 [Leader]" -44:37.054 [INFO] TestAgent.server: cluster leadership acquired -44:37.054 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23293 -44:37.055 [INFO] TestAgent.server: New leader elected: payload=Node-80040b29-307f-6fe2-371a-8b9b4e0592dc -44:37.055 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -44:37.124 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -44:37.124 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -44:37.124 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -44:37.124 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -44:37.124 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -44:37.124 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-80040b29-307f-6fe2-371a-8b9b4e0592dc -44:37.124 [INFO] TestAgent.server: member joined, marking health alive: member=Node-80040b29-307f-6fe2-371a-8b9b4e0592dc -44:37.125 [INFO] TestAgent: Synced node info -44:37.125 [DEBUG] TestAgent: Node info in sync -44:37.125 [INFO] TestAgent.server: federation state anti-entropy synced -44:37.144 [INFO] TestAgent: Force leaving node: node=garbage-name -44:37.144 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/force-leave/garbage-name from=127.0.0.1:44410 error="agent: No node found with name 'garbage-name'" -44:37.144 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/force-leave/garbage-name from=127.0.0.1:44410 latency=301.62µs -44:37.144 [INFO] TestAgent: Requesting shutdown -44:37.144 [INFO] TestAgent.server: shutting down server -44:37.144 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -44:37.145 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -44:37.145 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -44:37.145 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -44:37.145 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -44:37.145 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -44:37.145 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -44:37.145 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -44:37.145 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -44:37.145 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -44:37.145 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:44:37.145Z [INFO] agent.router.manager: shutting down -2022-09-15T21:44:37.145Z [INFO] agent.router.manager: shutting down -44:37.145 [INFO] TestAgent: consul server down -44:37.145 [INFO] TestAgent: shutdown complete -44:37.145 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23288 network=tcp -44:37.145 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23288 network=udp -44:37.145 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23289 network=tcp -44:37.646 [INFO] TestAgent: Waiting for endpoints to shut down -44:37.646 [INFO] TestAgent: Endpoints down ---- PASS: TestForceLeaveCommand_NoNodeWithName (1.14s) -2022-09-15T21:44:37.325Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:44:37.413Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -44:37.424 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:df5404c7-4317-dfce-1f96-08946374b76b Address:127.0.0.1:23300}]" -44:37.425 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-df5404c7-4317-dfce-1f96-08946374b76b.dc1 127.0.0.1 -44:37.425 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23300 [Follower]" leader= -44:37.433 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-df5404c7-4317-dfce-1f96-08946374b76b 127.0.0.1 -2022-09-15T21:44:37.433Z [INFO] agent.router: Initializing LAN area manager -44:37.434 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23295 network=udp -44:37.434 [INFO] TestAgent.server: Adding LAN server: server="Node-df5404c7-4317-dfce-1f96-08946374b76b (Addr: tcp/127.0.0.1:23300) (DC: dc1)" -44:37.434 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-df5404c7-4317-dfce-1f96-08946374b76b.dc1 area=wan -44:37.434 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23295 network=tcp -44:37.435 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23296 network=tcp -44:37.435 [INFO] TestAgent: started state syncer -44:37.436 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23301 network=tcp -44:37.493 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -44:37.493 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23300 [Candidate]" term=2 -44:37.493 [DEBUG] TestAgent.server.raft: votes: needed=1 -44:37.493 [DEBUG] TestAgent.server.raft: vote granted: from=df5404c7-4317-dfce-1f96-08946374b76b term=2 tally=1 -44:37.493 [INFO] TestAgent.server.raft: election won: tally=1 -44:37.493 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23300 [Leader]" -44:37.494 [INFO] TestAgent.server: cluster leadership acquired -44:37.494 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23300 -44:37.494 [INFO] TestAgent.server: New leader elected: payload=Node-df5404c7-4317-dfce-1f96-08946374b76b -44:37.495 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -44:37.498 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -44:37.498 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -44:37.498 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -44:37.498 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -44:37.498 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -44:37.498 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-df5404c7-4317-dfce-1f96-08946374b76b -44:37.498 [INFO] TestAgent.server: member joined, marking health alive: member=Node-df5404c7-4317-dfce-1f96-08946374b76b -44:37.500 [INFO] TestAgent.server: federation state anti-entropy synced -44:37.814 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:23277] -44:37.814 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:23277 -44:37.818 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:32950 -44:37.818 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-aaa020c4-dbdd-4468-ab18-52c6020de278 127.0.0.1 -44:37.818 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 -44:37.818 [DEBUG] TestAgent: systemd notify failed: error="No socket" -44:37.818 [INFO] TestAgent: Requesting shutdown -44:37.818 [INFO] TestAgent.server: shutting down server -44:37.818 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -44:37.818 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -44:37.818 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -44:37.818 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -44:37.818 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -44:37.819 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -44:37.819 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -44:37.819 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -44:37.819 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -44:37.819 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -44:37.819 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -44:37.819 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-aaa020c4-dbdd-4468-ab18-52c6020de278.dc1 127.0.0.1 -44:37.819 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:44:37.819Z [INFO] agent.router.manager: shutting down -2022-09-15T21:44:37.819Z [INFO] agent.router.manager: shutting down -44:37.819 [INFO] TestAgent: consul server down -44:37.819 [INFO] TestAgent: shutdown complete -44:37.819 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23295 network=tcp -44:37.819 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23295 network=udp -44:37.819 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23296 network=tcp -44:38.320 [INFO] TestAgent: Waiting for endpoints to shut down -44:38.320 [INFO] TestAgent: Endpoints down -2022-09-15T21:44:36.824Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:44:36.924Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -44:36.928 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:aaa020c4-dbdd-4468-ab18-52c6020de278 Address:127.0.0.1:23279}]" -44:36.957 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23279 [Follower]" leader= -44:36.998 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-aaa020c4-dbdd-4468-ab18-52c6020de278.dc1 127.0.0.1 -44:36.999 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-aaa020c4-dbdd-4468-ab18-52c6020de278 127.0.0.1 -2022-09-15T21:44:36.999Z [INFO] agent.router: Initializing LAN area manager -44:36.999 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23274 network=udp -44:37.001 [INFO] TestAgent.server: Adding LAN server: server="Node-aaa020c4-dbdd-4468-ab18-52c6020de278 (Addr: tcp/127.0.0.1:23279) (DC: dc1)" -44:37.001 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-aaa020c4-dbdd-4468-ab18-52c6020de278.dc1 area=wan -44:37.005 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23274 network=tcp -44:37.006 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23275 network=tcp -44:37.006 [INFO] TestAgent: started state syncer -44:37.006 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23280 network=tcp -44:37.043 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -44:37.043 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23279 [Candidate]" term=2 -44:37.043 [DEBUG] TestAgent.server.raft: votes: needed=1 -44:37.043 [DEBUG] TestAgent.server.raft: vote granted: from=aaa020c4-dbdd-4468-ab18-52c6020de278 term=2 tally=1 -44:37.043 [INFO] TestAgent.server.raft: election won: tally=1 -44:37.043 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23279 [Leader]" -44:37.044 [INFO] TestAgent.server: cluster leadership acquired -44:37.044 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23279 -44:37.045 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -44:37.046 [INFO] TestAgent.server: New leader elected: payload=Node-aaa020c4-dbdd-4468-ab18-52c6020de278 -44:37.105 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -44:37.105 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -44:37.105 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -44:37.105 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -44:37.105 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -44:37.105 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-aaa020c4-dbdd-4468-ab18-52c6020de278 -44:37.105 [INFO] TestAgent.server: member joined, marking health alive: member=Node-aaa020c4-dbdd-4468-ab18-52c6020de278 -44:37.117 [INFO] TestAgent.server: federation state anti-entropy synced -44:37.306 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -44:37.307 [INFO] TestAgent: Synced node info -44:37.815 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:49264 -44:37.816 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-df5404c7-4317-dfce-1f96-08946374b76b 127.0.0.1 -44:37.816 [INFO] TestAgent.server: Adding LAN server: server="Node-df5404c7-4317-dfce-1f96-08946374b76b (Addr: tcp/127.0.0.1:23300) (DC: dc1)" -44:37.816 [INFO] TestAgent.server: New leader elected: payload=Node-df5404c7-4317-dfce-1f96-08946374b76b -44:37.816 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-df5404c7-4317-dfce-1f96-08946374b76b other=Node-aaa020c4-dbdd-4468-ab18-52c6020de278 -44:37.816 [INFO] TestAgent.server: member joined, marking health alive: member=Node-df5404c7-4317-dfce-1f96-08946374b76b -44:37.818 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: Node-df5404c7-4317-dfce-1f96-08946374b76b.dc1 127.0.0.1:23299 -44:37.819 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-df5404c7-4317-dfce-1f96-08946374b76b.dc1 127.0.0.1 -44:37.819 [DEBUG] TestAgent.server: Successfully performed flood-join for server at address: server=Node-df5404c7-4317-dfce-1f96-08946374b76b.dc1 address=127.0.0.1:23299 -44:37.820 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-df5404c7-4317-dfce-1f96-08946374b76b.dc1 area=wan -44:38.321 [INFO] TestAgent: Force leaving node: node=Node-df5404c7-4317-dfce-1f96-08946374b76b -44:38.321 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/force-leave/Node-df5404c7-4317-dfce-1f96-08946374b76b from=127.0.0.1:37818 latency=44.84µs -44:38.499 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Failed ping: Node-df5404c7-4317-dfce-1f96-08946374b76b (timeout reached) -44:39.002 [INFO] TestAgent.server.memberlist.lan: memberlist: Suspect Node-df5404c7-4317-dfce-1f96-08946374b76b has failed, no acks received -44:39.052 [WARN] TestAgent: error getting server health from server: server=Node-df5404c7-4317-dfce-1f96-08946374b76b error="rpc error getting client: failed to get conn: dial tcp 127.0.0.1:0->127.0.0.1:23300: connect: connection refused" -44:39.296 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -44:39.296 [DEBUG] TestAgent: Node info in sync -44:39.296 [DEBUG] TestAgent: Node info in sync -44:40.051 [WARN] TestAgent: error getting server health from server: server=Node-df5404c7-4317-dfce-1f96-08946374b76b error="context deadline exceeded" -44:40.503 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Failed ping: Node-df5404c7-4317-dfce-1f96-08946374b76b (timeout reached) -44:41.051 [WARN] TestAgent: error getting server health from server: server=Node-df5404c7-4317-dfce-1f96-08946374b76b error="rpc error getting client: failed to get conn: dial tcp 127.0.0.1:0->127.0.0.1:23300: connect: connection refused" -44:41.855 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -44:41.855 [DEBUG] TestAgent: Node info in sync -44:42.006 [INFO] TestAgent.server.memberlist.lan: memberlist: Suspect Node-df5404c7-4317-dfce-1f96-08946374b76b has failed, no acks received -44:42.050 [WARN] TestAgent: error getting server health from server: server=Node-df5404c7-4317-dfce-1f96-08946374b76b error="context deadline exceeded" -44:43.006 [INFO] TestAgent.server.memberlist.lan: memberlist: Marking Node-df5404c7-4317-dfce-1f96-08946374b76b as failed, suspect timeout reached (0 peer confirmations) -44:43.006 [INFO] TestAgent.server.serf.lan: serf: EventMemberLeave: Node-df5404c7-4317-dfce-1f96-08946374b76b 127.0.0.1 -44:43.006 [INFO] TestAgent.server: Removing LAN server: server="Node-df5404c7-4317-dfce-1f96-08946374b76b (Addr: tcp/127.0.0.1:23300) (DC: dc1)" -44:43.006 [INFO] TestAgent.server: deregistering member: member=Node-df5404c7-4317-dfce-1f96-08946374b76b reason=left -44:43.008 [INFO] TestAgent: Requesting shutdown -44:43.008 [INFO] TestAgent.server: shutting down server -44:43.008 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -44:43.008 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -44:43.008 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -44:43.008 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -44:43.008 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -44:43.008 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -44:43.008 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -44:43.008 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -44:43.008 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -44:43.008 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -44:43.008 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:44:43.008Z [INFO] agent.router.manager: shutting down -2022-09-15T21:44:43.008Z [INFO] agent.router.manager: shutting down -44:43.009 [INFO] TestAgent: consul server down -44:43.009 [INFO] TestAgent: shutdown complete -44:43.009 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23274 network=tcp -44:43.009 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23274 network=udp -44:43.009 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23275 network=tcp -44:43.510 [INFO] TestAgent: Waiting for endpoints to shut down -44:43.510 [INFO] TestAgent: Endpoints down ---- PASS: TestForceLeaveCommand (7.00s) -2022-09-15T21:44:37.367Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:44:37.437Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -44:37.441 [INFO] Agent2.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c331bf70-4961-ff50-a4f8-3c0fc1b8dba3 Address:127.0.0.1:23307}]" -44:37.442 [INFO] Agent2.server.raft: entering follower state: follower="Node at 127.0.0.1:23307 [Follower]" leader= -44:37.442 [INFO] Agent2.server.serf.wan: serf: EventMemberJoin: Node-c331bf70-4961-ff50-a4f8-3c0fc1b8dba3.dc1 127.0.0.1 -44:37.444 [INFO] Agent2.server.serf.lan: serf: EventMemberJoin: Node-c331bf70-4961-ff50-a4f8-3c0fc1b8dba3 127.0.0.1 -2022-09-15T21:44:37.444Z [INFO] agent.router: Initializing LAN area manager -44:37.444 [INFO] Agent2: Started DNS server: address=127.0.0.1:23302 network=udp -44:37.444 [INFO] Agent2.server: Adding LAN server: server="Node-c331bf70-4961-ff50-a4f8-3c0fc1b8dba3 (Addr: tcp/127.0.0.1:23307) (DC: dc1)" -44:37.445 [INFO] Agent2.server: Handled event for server in area: event=member-join server=Node-c331bf70-4961-ff50-a4f8-3c0fc1b8dba3.dc1 area=wan -44:37.445 [INFO] Agent2: Started DNS server: address=127.0.0.1:23302 network=tcp -44:37.446 [INFO] Agent2: Started HTTP server: address=127.0.0.1:23303 network=tcp -44:37.446 [INFO] Agent2: started state syncer -44:37.446 [INFO] Agent2: Started gRPC server: address=127.0.0.1:23308 network=tcp -44:37.499 [WARN] Agent2.server.raft: heartbeat timeout reached, starting election: last-leader= -44:37.499 [INFO] Agent2.server.raft: entering candidate state: node="Node at 127.0.0.1:23307 [Candidate]" term=2 -44:37.499 [DEBUG] Agent2.server.raft: votes: needed=1 -44:37.499 [DEBUG] Agent2.server.raft: vote granted: from=c331bf70-4961-ff50-a4f8-3c0fc1b8dba3 term=2 tally=1 -44:37.499 [INFO] Agent2.server.raft: election won: tally=1 -44:37.499 [INFO] Agent2.server.raft: entering leader state: leader="Node at 127.0.0.1:23307 [Leader]" -44:37.499 [INFO] Agent2.server: cluster leadership acquired -44:37.500 [DEBUG] Agent2.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23307 -44:37.500 [INFO] Agent2.server: New leader elected: payload=Node-c331bf70-4961-ff50-a4f8-3c0fc1b8dba3 -44:37.503 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -44:37.505 [INFO] Agent2.server.connect: initialized primary datacenter CA with provider: provider=consul -44:37.505 [INFO] Agent2.leader: started routine: routine="federation state anti-entropy" -44:37.505 [INFO] Agent2.leader: started routine: routine="federation state pruning" -44:37.505 [INFO] Agent2.leader: started routine: routine="intermediate cert renew watch" -44:37.505 [INFO] Agent2.leader: started routine: routine="CA root pruning" -44:37.505 [DEBUG] Agent2.server: Skipping self join check for node since the cluster is too small: node=Node-c331bf70-4961-ff50-a4f8-3c0fc1b8dba3 -44:37.505 [INFO] Agent2.server: member joined, marking health alive: member=Node-c331bf70-4961-ff50-a4f8-3c0fc1b8dba3 -44:37.508 [INFO] Agent2.server: federation state anti-entropy synced -44:37.518 [DEBUG] Agent2: Skipping remote check since it is managed automatically: check=serfHealth -44:37.518 [INFO] Agent2: Synced node info -44:37.518 [DEBUG] Agent2: Node info in sync -44:37.662 [INFO] Agent2: (LAN) joining: lan_addresses=[127.0.0.1:23284] -44:37.666 [DEBUG] Agent2.server.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:23284 -44:37.682 [INFO] Agent2.server.serf.lan: serf: EventMemberJoin: Node-d84e3b85-2527-94b8-a837-66c1834e1f4d 127.0.0.1 -44:37.682 [INFO] Agent2: (LAN) joined: number_of_nodes=1 -44:37.682 [DEBUG] Agent2: systemd notify failed: error="No socket" -44:37.682 [INFO] Agent2: Requesting shutdown -44:37.682 [INFO] Agent2.server: Adding LAN server: server="Node-d84e3b85-2527-94b8-a837-66c1834e1f4d (Addr: tcp/127.0.0.1:23286) (DC: dc1)" -44:37.682 [ERROR] Agent2.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-d84e3b85-2527-94b8-a837-66c1834e1f4d other=Node-c331bf70-4961-ff50-a4f8-3c0fc1b8dba3 -44:37.682 [INFO] Agent2.server: member joined, marking health alive: member=Node-d84e3b85-2527-94b8-a837-66c1834e1f4d -44:37.683 [INFO] Agent2.server: shutting down server -44:37.683 [DEBUG] Agent2.leader: stopping routine: routine="federation state anti-entropy" -44:37.683 [DEBUG] Agent2.leader: stopping routine: routine="federation state pruning" -44:37.683 [DEBUG] Agent2.leader: stopping routine: routine="intermediate cert renew watch" -44:37.683 [DEBUG] Agent2.leader: stopping routine: routine="CA root pruning" -44:37.683 [WARN] Agent2.server.serf.lan: serf: Shutdown without a Leave -44:37.683 [WARN] Agent2.server.serf.wan: serf: Shutdown without a Leave -44:37.684 [DEBUG] Agent2.leader: stopped routine: routine="intermediate cert renew watch" -44:37.684 [DEBUG] Agent2.leader: stopped routine: routine="CA root pruning" -44:37.684 [ERROR] Agent2.server: error performing anti-entropy sync of federation state: error="context canceled" -44:37.684 [DEBUG] Agent2.leader: stopped routine: routine="federation state anti-entropy" -44:37.684 [DEBUG] Agent2.leader: stopped routine: routine="federation state pruning" -44:37.684 [DEBUG] Agent2.server: Failed to flood-join server at address: server=Node-d84e3b85-2527-94b8-a837-66c1834e1f4d.dc1 address=127.0.0.1:23285 error="Serf can't Join after Leave or Shutdown" -2022-09-15T21:44:37.684Z [INFO] agent.router.manager: shutting down -2022-09-15T21:44:37.684Z [INFO] agent.router.manager: shutting down -44:37.691 [INFO] Agent2: consul server down -44:37.691 [INFO] Agent2: shutdown complete -44:37.691 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:23302 network=tcp -44:37.691 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:23302 network=udp -44:37.691 [INFO] Agent2: Stopping server: protocol=HTTP address=127.0.0.1:23303 network=tcp -44:38.193 [INFO] Agent2: Waiting for endpoints to shut down -44:38.193 [INFO] Agent2: Endpoints down -2022-09-15T21:44:36.827Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:44:36.909Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -44:36.958 [INFO] Agent1.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d84e3b85-2527-94b8-a837-66c1834e1f4d Address:127.0.0.1:23286}]" -44:36.962 [INFO] Agent1.server.raft: entering follower state: follower="Node at 127.0.0.1:23286 [Follower]" leader= -44:36.997 [INFO] Agent1.server.serf.wan: serf: EventMemberJoin: Node-d84e3b85-2527-94b8-a837-66c1834e1f4d.dc1 127.0.0.1 -44:36.998 [INFO] Agent1.server.serf.lan: serf: EventMemberJoin: Node-d84e3b85-2527-94b8-a837-66c1834e1f4d 127.0.0.1 -2022-09-15T21:44:36.998Z [INFO] agent.router: Initializing LAN area manager -44:36.999 [INFO] Agent1: Started DNS server: address=127.0.0.1:23281 network=udp -44:36.999 [INFO] Agent1.server: Adding LAN server: server="Node-d84e3b85-2527-94b8-a837-66c1834e1f4d (Addr: tcp/127.0.0.1:23286) (DC: dc1)" -44:36.999 [INFO] Agent1.server: Handled event for server in area: event=member-join server=Node-d84e3b85-2527-94b8-a837-66c1834e1f4d.dc1 area=wan -44:36.999 [INFO] Agent1: Started DNS server: address=127.0.0.1:23281 network=tcp -44:37.000 [INFO] Agent1: Started HTTP server: address=127.0.0.1:23282 network=tcp -44:37.000 [INFO] Agent1: started state syncer -44:37.001 [INFO] Agent1: Started gRPC server: address=127.0.0.1:23287 network=tcp -44:37.042 [WARN] Agent1.server.raft: heartbeat timeout reached, starting election: last-leader= -44:37.042 [INFO] Agent1.server.raft: entering candidate state: node="Node at 127.0.0.1:23286 [Candidate]" term=2 -44:37.053 [DEBUG] Agent1.server.raft: votes: needed=1 -44:37.053 [DEBUG] Agent1.server.raft: vote granted: from=d84e3b85-2527-94b8-a837-66c1834e1f4d term=2 tally=1 -44:37.053 [INFO] Agent1.server.raft: election won: tally=1 -44:37.053 [INFO] Agent1.server.raft: entering leader state: leader="Node at 127.0.0.1:23286 [Leader]" -44:37.053 [INFO] Agent1.server: cluster leadership acquired -44:37.054 [INFO] Agent1.server: New leader elected: payload=Node-d84e3b85-2527-94b8-a837-66c1834e1f4d -44:37.054 [DEBUG] Agent1.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23286 -44:37.055 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -44:37.122 [INFO] Agent1.server.connect: initialized primary datacenter CA with provider: provider=consul -44:37.123 [INFO] Agent1.leader: started routine: routine="federation state anti-entropy" -44:37.123 [INFO] Agent1.leader: started routine: routine="federation state pruning" -44:37.123 [INFO] Agent1.leader: started routine: routine="intermediate cert renew watch" -44:37.123 [INFO] Agent1.leader: started routine: routine="CA root pruning" -44:37.123 [DEBUG] Agent1.server: Skipping self join check for node since the cluster is too small: node=Node-d84e3b85-2527-94b8-a837-66c1834e1f4d -44:37.123 [INFO] Agent1.server: member joined, marking health alive: member=Node-d84e3b85-2527-94b8-a837-66c1834e1f4d -44:37.124 [INFO] Agent1.server: federation state anti-entropy synced -44:37.346 [DEBUG] Agent1: Skipping remote check since it is managed automatically: check=serfHealth -44:37.347 [INFO] Agent1: Synced node info -44:37.666 [DEBUG] Agent1.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:60594 -44:37.685 [INFO] Agent1.server.serf.lan: serf: EventMemberJoin: Node-c331bf70-4961-ff50-a4f8-3c0fc1b8dba3 127.0.0.1 -44:37.685 [INFO] Agent1.server: Adding LAN server: server="Node-c331bf70-4961-ff50-a4f8-3c0fc1b8dba3 (Addr: tcp/127.0.0.1:23307) (DC: dc1)" -44:37.685 [INFO] Agent1.server: New leader elected: payload=Node-c331bf70-4961-ff50-a4f8-3c0fc1b8dba3 -44:37.686 [ERROR] Agent1.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-c331bf70-4961-ff50-a4f8-3c0fc1b8dba3 other=Node-d84e3b85-2527-94b8-a837-66c1834e1f4d -44:37.686 [INFO] Agent1.server: member joined, marking health alive: member=Node-c331bf70-4961-ff50-a4f8-3c0fc1b8dba3 -44:37.686 [DEBUG] Agent1.server.memberlist.wan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:23306: connect: connection refused -44:37.686 [DEBUG] Agent1.server: Failed to flood-join server at address: server=Node-c331bf70-4961-ff50-a4f8-3c0fc1b8dba3.dc1 address=127.0.0.1:23306 error="1 error occurred: - * Failed to join 127.0.0.1: dial tcp 127.0.0.1:23306: connect: connection refused +2021-08-13T15:36:52.405Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:36:52.439Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +36:52.442 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d7d430cd-9b25-85cb-433a-d552b139de03 Address:127.0.0.1:19202}]" +36:52.442 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19202 [Follower]" leader= +36:52.443 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d7d430cd-9b25-85cb-433a-d552b139de03.dc1 127.0.0.1 +36:52.444 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d7d430cd-9b25-85cb-433a-d552b139de03 127.0.0.1 +2021-08-13T15:36:52.444Z [INFO] agent.router: Initializing LAN area manager +36:52.444 [INFO] TestAgent.server: Adding LAN server: server="Node-d7d430cd-9b25-85cb-433a-d552b139de03 (Addr: tcp/127.0.0.1:19202) (DC: dc1)" +36:52.444 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d7d430cd-9b25-85cb-433a-d552b139de03.dc1 area=wan +36:52.444 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19197 network=udp +36:52.444 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19197 network=tcp +36:52.445 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19198 network=tcp +36:52.445 [INFO] TestAgent: started state syncer +36:52.445 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19203 network=tcp +36:52.503 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +36:52.503 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19202 [Candidate]" term=2 +36:52.503 [DEBUG] TestAgent.server.raft: votes: needed=1 +36:52.503 [DEBUG] TestAgent.server.raft: vote granted: from=d7d430cd-9b25-85cb-433a-d552b139de03 term=2 tally=1 +36:52.503 [INFO] TestAgent.server.raft: election won: tally=1 +36:52.503 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19202 [Leader]" +36:52.503 [INFO] TestAgent.server: cluster leadership acquired +36:52.504 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:19202 +36:52.505 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +36:52.507 [INFO] TestAgent.server: New leader elected: payload=Node-d7d430cd-9b25-85cb-433a-d552b139de03 +36:52.508 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +36:52.508 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +36:52.508 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +36:52.508 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +36:52.508 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +36:52.509 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d7d430cd-9b25-85cb-433a-d552b139de03 +36:52.509 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d7d430cd-9b25-85cb-433a-d552b139de03 +36:52.510 [INFO] TestAgent.server: federation state anti-entropy synced +36:52.514 [INFO] TestAgent: Force leaving node: node=garbage-name +36:52.514 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/force-leave/garbage-name from=127.0.0.1:48956 error="agent: No node found with name 'garbage-name'" +36:52.515 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/force-leave/garbage-name from=127.0.0.1:48956 latency=941.582µs +36:52.515 [INFO] TestAgent: Requesting shutdown +36:52.516 [INFO] TestAgent.server: shutting down server +36:52.516 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +36:52.516 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +36:52.516 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +36:52.516 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +36:52.516 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +36:52.516 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +36:52.516 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +36:52.516 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +36:52.516 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +36:52.516 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +36:52.516 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +36:52.516 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:36:52.516Z [INFO] agent.router.manager: shutting down +2021-08-13T15:36:52.516Z [INFO] agent.router.manager: shutting down +36:52.516 [INFO] TestAgent: consul server down +36:52.516 [INFO] TestAgent: shutdown complete +36:52.516 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19197 network=tcp +36:52.516 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19197 network=udp +36:52.516 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19198 network=tcp +36:53.017 [INFO] TestAgent: Waiting for endpoints to shut down +36:53.017 [INFO] TestAgent: Endpoints down +--- PASS: TestForceLeaveCommand_NoNodeWithName (0.72s) +2021-08-13T15:36:52.607Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:36:52.622Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +36:52.625 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a9b3e40b-0ac4-0cae-ed48-82842b4a2734 Address:127.0.0.1:19216}]" +36:52.625 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19216 [Follower]" leader= +36:52.626 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a9b3e40b-0ac4-0cae-ed48-82842b4a2734.dc1 127.0.0.1 +36:52.627 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a9b3e40b-0ac4-0cae-ed48-82842b4a2734 127.0.0.1 +2021-08-13T15:36:52.627Z [INFO] agent.router: Initializing LAN area manager +36:52.627 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19211 network=udp +36:52.627 [INFO] TestAgent.server: Adding LAN server: server="Node-a9b3e40b-0ac4-0cae-ed48-82842b4a2734 (Addr: tcp/127.0.0.1:19216) (DC: dc1)" +36:52.627 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a9b3e40b-0ac4-0cae-ed48-82842b4a2734.dc1 area=wan +36:52.628 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19211 network=tcp +36:52.628 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19212 network=tcp +36:52.629 [INFO] TestAgent: started state syncer +36:52.629 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19217 network=tcp +36:52.688 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +36:52.688 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19216 [Candidate]" term=2 +36:52.688 [DEBUG] TestAgent.server.raft: votes: needed=1 +36:52.688 [DEBUG] TestAgent.server.raft: vote granted: from=a9b3e40b-0ac4-0cae-ed48-82842b4a2734 term=2 tally=1 +36:52.688 [INFO] TestAgent.server.raft: election won: tally=1 +36:52.688 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19216 [Leader]" +36:52.689 [INFO] TestAgent.server: cluster leadership acquired +36:52.689 [INFO] TestAgent.server: New leader elected: payload=Node-a9b3e40b-0ac4-0cae-ed48-82842b4a2734 +36:52.689 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:19216 +36:52.690 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +36:52.693 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +36:52.693 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +36:52.693 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +36:52.693 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +36:52.693 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +36:52.693 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a9b3e40b-0ac4-0cae-ed48-82842b4a2734 +36:52.693 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a9b3e40b-0ac4-0cae-ed48-82842b4a2734 +36:52.694 [INFO] TestAgent.server: federation state anti-entropy synced +36:52.981 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +36:52.982 [INFO] TestAgent: Synced node info +36:52.982 [DEBUG] TestAgent: Node info in sync +36:53.057 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:19207] +36:53.057 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:19207 +36:53.058 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c3eb616b-7630-30df-d108-8cccb63b9ac0 127.0.0.1 +36:53.059 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 +36:53.059 [DEBUG] TestAgent: systemd notify failed: error="No socket" +36:53.059 [INFO] TestAgent: Requesting shutdown +36:53.059 [INFO] TestAgent.server: shutting down server +36:53.059 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +36:53.059 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +36:53.059 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +36:53.059 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +36:53.059 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +36:53.059 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +36:53.059 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +36:53.059 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +36:53.059 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +36:53.059 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +36:53.059 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:36:53.059Z [INFO] agent.router.manager: shutting down +2021-08-13T15:36:53.059Z [INFO] agent.router.manager: shutting down +36:53.059 [INFO] TestAgent: consul server down +36:53.059 [INFO] TestAgent: shutdown complete +36:53.059 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19211 network=tcp +36:53.059 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19211 network=udp +36:53.059 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19212 network=tcp +36:53.560 [INFO] TestAgent: Waiting for endpoints to shut down +36:53.560 [INFO] TestAgent: Endpoints down +2021-08-13T15:36:52.374Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:36:52.403Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +36:52.407 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c3eb616b-7630-30df-d108-8cccb63b9ac0 Address:127.0.0.1:19209}]" +36:52.408 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c3eb616b-7630-30df-d108-8cccb63b9ac0.dc1 127.0.0.1 +36:52.409 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c3eb616b-7630-30df-d108-8cccb63b9ac0 127.0.0.1 +2021-08-13T15:36:52.409Z [INFO] agent.router: Initializing LAN area manager +36:52.410 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19204 network=udp +36:52.410 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19209 [Follower]" leader= +36:52.410 [INFO] TestAgent.server: Adding LAN server: server="Node-c3eb616b-7630-30df-d108-8cccb63b9ac0 (Addr: tcp/127.0.0.1:19209) (DC: dc1)" +36:52.410 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c3eb616b-7630-30df-d108-8cccb63b9ac0.dc1 area=wan +36:52.411 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19204 network=tcp +36:52.412 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19205 network=tcp +36:52.412 [INFO] TestAgent: started state syncer +36:52.412 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19210 network=tcp +36:52.464 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +36:52.464 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19209 [Candidate]" term=2 +36:52.464 [DEBUG] TestAgent.server.raft: votes: needed=1 +36:52.464 [DEBUG] TestAgent.server.raft: vote granted: from=c3eb616b-7630-30df-d108-8cccb63b9ac0 term=2 tally=1 +36:52.464 [INFO] TestAgent.server.raft: election won: tally=1 +36:52.464 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19209 [Leader]" +36:52.464 [INFO] TestAgent.server: cluster leadership acquired +36:52.464 [INFO] TestAgent.server: New leader elected: payload=Node-c3eb616b-7630-30df-d108-8cccb63b9ac0 +36:52.464 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:19209 +36:52.466 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +36:52.489 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +36:52.489 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +36:52.489 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +36:52.489 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +36:52.489 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +36:52.490 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c3eb616b-7630-30df-d108-8cccb63b9ac0 +36:52.490 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c3eb616b-7630-30df-d108-8cccb63b9ac0 +36:52.490 [INFO] TestAgent.server: federation state anti-entropy synced +36:52.775 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +36:52.775 [INFO] TestAgent: Synced node info +36:53.057 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:48270 +36:53.058 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a9b3e40b-0ac4-0cae-ed48-82842b4a2734 127.0.0.1 +36:53.059 [INFO] TestAgent.server: Adding LAN server: server="Node-a9b3e40b-0ac4-0cae-ed48-82842b4a2734 (Addr: tcp/127.0.0.1:19216) (DC: dc1)" +36:53.059 [INFO] TestAgent.server: New leader elected: payload=Node-a9b3e40b-0ac4-0cae-ed48-82842b4a2734 +36:53.059 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-a9b3e40b-0ac4-0cae-ed48-82842b4a2734 other=Node-c3eb616b-7630-30df-d108-8cccb63b9ac0 +36:53.059 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a9b3e40b-0ac4-0cae-ed48-82842b4a2734 +36:53.060 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:19215: connect: connection refused +36:53.060 [DEBUG] TestAgent.server: Failed to flood-join server at address: server=Node-a9b3e40b-0ac4-0cae-ed48-82842b4a2734.dc1 address=127.0.0.1:19215 error="1 error occurred: + * Failed to join 127.0.0.1: dial tcp 127.0.0.1:19215: connect: connection refused + +" +36:53.561 [INFO] TestAgent: Force leaving node: node=Node-a9b3e40b-0ac4-0cae-ed48-82842b4a2734 +36:53.561 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/force-leave/Node-a9b3e40b-0ac4-0cae-ed48-82842b4a2734 from=127.0.0.1:37726 latency=46.52µs +36:53.910 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Failed ping: Node-a9b3e40b-0ac4-0cae-ed48-82842b4a2734 (timeout reached) +36:54.409 [INFO] TestAgent.server.memberlist.lan: memberlist: Suspect Node-a9b3e40b-0ac4-0cae-ed48-82842b4a2734 has failed, no acks received +36:54.466 [WARN] TestAgent: error getting server health from server: server=Node-a9b3e40b-0ac4-0cae-ed48-82842b4a2734 error="rpc error getting client: failed to get conn: dial tcp 127.0.0.1:0->127.0.0.1:19216: connect: connection refused" +36:55.465 [WARN] TestAgent: error getting server health from server: server=Node-a9b3e40b-0ac4-0cae-ed48-82842b4a2734 error="context deadline exceeded" +36:55.694 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +36:55.694 [DEBUG] TestAgent: Node info in sync +36:55.910 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Failed ping: Node-a9b3e40b-0ac4-0cae-ed48-82842b4a2734 (timeout reached) +36:56.466 [WARN] TestAgent: error getting server health from server: server=Node-a9b3e40b-0ac4-0cae-ed48-82842b4a2734 error="rpc error getting client: failed to get conn: dial tcp 127.0.0.1:0->127.0.0.1:19216: connect: connection refused" +36:57.409 [INFO] TestAgent.server.memberlist.lan: memberlist: Suspect Node-a9b3e40b-0ac4-0cae-ed48-82842b4a2734 has failed, no acks received +36:57.465 [WARN] TestAgent: error getting server health from server: server=Node-a9b3e40b-0ac4-0cae-ed48-82842b4a2734 error="context deadline exceeded" +36:57.605 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +36:57.605 [DEBUG] TestAgent: Node info in sync +36:57.605 [DEBUG] TestAgent: Node info in sync +36:57.910 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Failed ping: Node-a9b3e40b-0ac4-0cae-ed48-82842b4a2734 (timeout reached) +36:58.410 [INFO] TestAgent.server.memberlist.lan: memberlist: Marking Node-a9b3e40b-0ac4-0cae-ed48-82842b4a2734 as failed, suspect timeout reached (0 peer confirmations) +36:58.410 [INFO] TestAgent.server.serf.lan: serf: EventMemberLeave: Node-a9b3e40b-0ac4-0cae-ed48-82842b4a2734 127.0.0.1 +36:58.410 [INFO] TestAgent.server: Removing LAN server: server="Node-a9b3e40b-0ac4-0cae-ed48-82842b4a2734 (Addr: tcp/127.0.0.1:19216) (DC: dc1)" +36:58.410 [INFO] TestAgent.server: deregistering member: member=Node-a9b3e40b-0ac4-0cae-ed48-82842b4a2734 reason=left +36:58.430 [INFO] TestAgent: Requesting shutdown +36:58.430 [INFO] TestAgent.server: shutting down server +36:58.430 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +36:58.430 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +36:58.430 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +36:58.430 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +36:58.431 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +36:58.431 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +36:58.431 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +36:58.431 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +36:58.431 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +36:58.431 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +36:58.431 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:36:58.431Z [INFO] agent.router.manager: shutting down +2021-08-13T15:36:58.431Z [INFO] agent.router.manager: shutting down +36:58.431 [INFO] TestAgent: consul server down +36:58.431 [INFO] TestAgent: shutdown complete +36:58.431 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19204 network=tcp +36:58.431 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19204 network=udp +36:58.431 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19205 network=tcp +36:58.931 [INFO] TestAgent: Waiting for endpoints to shut down +36:58.931 [INFO] TestAgent: Endpoints down +--- PASS: TestForceLeaveCommand (6.63s) +2021-08-13T15:36:52.856Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:36:52.874Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +36:52.876 [INFO] Agent2.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:08a26fdc-15e6-861c-0931-b91dd8c0843f Address:127.0.0.1:19223}]" +36:52.876 [INFO] Agent2.server.raft: entering follower state: follower="Node at 127.0.0.1:19223 [Follower]" leader= +36:52.877 [INFO] Agent2.server.serf.wan: serf: EventMemberJoin: Node-08a26fdc-15e6-861c-0931-b91dd8c0843f.dc1 127.0.0.1 +36:52.878 [INFO] Agent2.server.serf.lan: serf: EventMemberJoin: Node-08a26fdc-15e6-861c-0931-b91dd8c0843f 127.0.0.1 +2021-08-13T15:36:52.878Z [INFO] agent.router: Initializing LAN area manager +36:52.878 [INFO] Agent2.server: Handled event for server in area: event=member-join server=Node-08a26fdc-15e6-861c-0931-b91dd8c0843f.dc1 area=wan +36:52.879 [INFO] Agent2.server: Adding LAN server: server="Node-08a26fdc-15e6-861c-0931-b91dd8c0843f (Addr: tcp/127.0.0.1:19223) (DC: dc1)" +36:52.879 [INFO] Agent2: Started DNS server: address=127.0.0.1:19218 network=udp +36:52.879 [INFO] Agent2: Started DNS server: address=127.0.0.1:19218 network=tcp +36:52.880 [INFO] Agent2: Started HTTP server: address=127.0.0.1:19219 network=tcp +36:52.880 [INFO] Agent2: started state syncer +36:52.880 [INFO] Agent2: Started gRPC server: address=127.0.0.1:19224 network=tcp +36:52.938 [WARN] Agent2.server.raft: heartbeat timeout reached, starting election: last-leader= +36:52.939 [INFO] Agent2.server.raft: entering candidate state: node="Node at 127.0.0.1:19223 [Candidate]" term=2 +36:52.939 [DEBUG] Agent2.server.raft: votes: needed=1 +36:52.939 [DEBUG] Agent2.server.raft: vote granted: from=08a26fdc-15e6-861c-0931-b91dd8c0843f term=2 tally=1 +36:52.939 [INFO] Agent2.server.raft: election won: tally=1 +36:52.939 [INFO] Agent2.server.raft: entering leader state: leader="Node at 127.0.0.1:19223 [Leader]" +36:52.939 [INFO] Agent2.server: cluster leadership acquired +36:52.939 [INFO] Agent2.server: New leader elected: payload=Node-08a26fdc-15e6-861c-0931-b91dd8c0843f +36:52.939 [DEBUG] Agent2.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:19223 +36:52.941 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +36:52.943 [INFO] Agent2.server.connect: initialized primary datacenter CA with provider: provider=consul +36:52.943 [INFO] Agent2.leader: started routine: routine="federation state anti-entropy" +36:52.943 [INFO] Agent2.leader: started routine: routine="federation state pruning" +36:52.943 [INFO] Agent2.leader: started routine: routine="intermediate cert renew watch" +36:52.943 [INFO] Agent2.leader: started routine: routine="CA root pruning" +36:52.943 [DEBUG] Agent2.server: Skipping self join check for node since the cluster is too small: node=Node-08a26fdc-15e6-861c-0931-b91dd8c0843f +36:52.943 [INFO] Agent2.server: member joined, marking health alive: member=Node-08a26fdc-15e6-861c-0931-b91dd8c0843f +36:52.944 [INFO] Agent2.server: federation state anti-entropy synced +36:53.065 [DEBUG] Agent2: Skipping remote check since it is managed automatically: check=serfHealth +36:53.066 [INFO] Agent2: Synced node info +36:53.286 [INFO] Agent2: (LAN) joining: lan_addresses=[127.0.0.1:19193] +36:53.287 [DEBUG] Agent2.server.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:19193 +36:53.288 [INFO] Agent2.server.serf.lan: serf: EventMemberJoin: Node-ad30b2da-d62f-f58d-1af9-93ae2dea7328 127.0.0.1 +36:53.288 [INFO] Agent2: (LAN) joined: number_of_nodes=1 +36:53.288 [DEBUG] Agent2: systemd notify failed: error="No socket" +36:53.288 [INFO] Agent2: Requesting shutdown +36:53.288 [INFO] Agent2.server: shutting down server +36:53.288 [DEBUG] Agent2.leader: stopping routine: routine="federation state anti-entropy" +36:53.288 [DEBUG] Agent2.leader: stopping routine: routine="federation state pruning" +36:53.288 [DEBUG] Agent2.leader: stopping routine: routine="intermediate cert renew watch" +36:53.288 [DEBUG] Agent2.leader: stopping routine: routine="CA root pruning" +36:53.288 [WARN] Agent2.server.serf.lan: serf: Shutdown without a Leave +36:53.288 [ERROR] Agent2.server: error performing anti-entropy sync of federation state: error="context canceled" +36:53.288 [DEBUG] Agent2.leader: stopped routine: routine="federation state anti-entropy" +36:53.288 [DEBUG] Agent2.leader: stopped routine: routine="federation state pruning" +36:53.288 [DEBUG] Agent2.leader: stopped routine: routine="intermediate cert renew watch" +36:53.288 [DEBUG] Agent2.leader: stopped routine: routine="CA root pruning" +36:53.288 [WARN] Agent2.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:36:53.289Z [INFO] agent.router.manager: shutting down +2021-08-13T15:36:53.289Z [INFO] agent.router.manager: shutting down +36:53.289 [INFO] Agent2: consul server down +36:53.289 [INFO] Agent2: shutdown complete +36:53.289 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:19218 network=tcp +36:53.289 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:19218 network=udp +36:53.289 [INFO] Agent2: Stopping server: protocol=HTTP address=127.0.0.1:19219 network=tcp +36:53.789 [INFO] Agent2: Waiting for endpoints to shut down +36:53.789 [INFO] Agent2: Endpoints down +2021-08-13T15:36:52.384Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:36:52.425Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +36:52.430 [INFO] Agent1.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ad30b2da-d62f-f58d-1af9-93ae2dea7328 Address:127.0.0.1:19195}]" +36:52.431 [INFO] Agent1.server.raft: entering follower state: follower="Node at 127.0.0.1:19195 [Follower]" leader= +36:52.431 [INFO] Agent1.server.serf.wan: serf: EventMemberJoin: Node-ad30b2da-d62f-f58d-1af9-93ae2dea7328.dc1 127.0.0.1 +36:52.432 [INFO] Agent1.server.serf.lan: serf: EventMemberJoin: Node-ad30b2da-d62f-f58d-1af9-93ae2dea7328 127.0.0.1 +2021-08-13T15:36:52.432Z [INFO] agent.router: Initializing LAN area manager +36:52.432 [INFO] Agent1: Started DNS server: address=127.0.0.1:19190 network=udp +36:52.432 [INFO] Agent1.server: Handled event for server in area: event=member-join server=Node-ad30b2da-d62f-f58d-1af9-93ae2dea7328.dc1 area=wan +36:52.433 [INFO] Agent1.server: Adding LAN server: server="Node-ad30b2da-d62f-f58d-1af9-93ae2dea7328 (Addr: tcp/127.0.0.1:19195) (DC: dc1)" +36:52.433 [INFO] Agent1: Started DNS server: address=127.0.0.1:19190 network=tcp +36:52.433 [INFO] Agent1: Started HTTP server: address=127.0.0.1:19191 network=tcp +36:52.434 [INFO] Agent1: started state syncer +36:52.434 [INFO] Agent1: Started gRPC server: address=127.0.0.1:19196 network=tcp +36:52.495 [WARN] Agent1.server.raft: heartbeat timeout reached, starting election: last-leader= +36:52.495 [INFO] Agent1.server.raft: entering candidate state: node="Node at 127.0.0.1:19195 [Candidate]" term=2 +36:52.495 [DEBUG] Agent1.server.raft: votes: needed=1 +36:52.495 [DEBUG] Agent1.server.raft: vote granted: from=ad30b2da-d62f-f58d-1af9-93ae2dea7328 term=2 tally=1 +36:52.495 [INFO] Agent1.server.raft: election won: tally=1 +36:52.495 [INFO] Agent1.server.raft: entering leader state: leader="Node at 127.0.0.1:19195 [Leader]" +36:52.495 [INFO] Agent1.server: cluster leadership acquired +36:52.495 [INFO] Agent1.server: New leader elected: payload=Node-ad30b2da-d62f-f58d-1af9-93ae2dea7328 +36:52.496 [DEBUG] Agent1.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:19195 +36:52.497 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +36:52.499 [INFO] Agent1.server.connect: initialized primary datacenter CA with provider: provider=consul +36:52.499 [INFO] Agent1.leader: started routine: routine="federation state anti-entropy" +36:52.499 [INFO] Agent1.leader: started routine: routine="federation state pruning" +36:52.499 [INFO] Agent1.leader: started routine: routine="intermediate cert renew watch" +36:52.499 [INFO] Agent1.leader: started routine: routine="CA root pruning" +36:52.499 [DEBUG] Agent1.server: Skipping self join check for node since the cluster is too small: node=Node-ad30b2da-d62f-f58d-1af9-93ae2dea7328 +36:52.499 [INFO] Agent1.server: member joined, marking health alive: member=Node-ad30b2da-d62f-f58d-1af9-93ae2dea7328 +36:52.508 [INFO] Agent1.server: federation state anti-entropy synced +36:52.696 [DEBUG] Agent1: Skipping remote check since it is managed automatically: check=serfHealth +36:52.696 [INFO] Agent1: Synced node info +36:52.696 [DEBUG] Agent1: Node info in sync +36:53.180 [DEBUG] Agent1: Skipping remote check since it is managed automatically: check=serfHealth +36:53.180 [DEBUG] Agent1: Node info in sync +36:53.287 [DEBUG] Agent1.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:58660 +36:53.288 [INFO] Agent1.server.serf.lan: serf: EventMemberJoin: Node-08a26fdc-15e6-861c-0931-b91dd8c0843f 127.0.0.1 +36:53.288 [INFO] Agent1.server: Adding LAN server: server="Node-08a26fdc-15e6-861c-0931-b91dd8c0843f (Addr: tcp/127.0.0.1:19223) (DC: dc1)" +36:53.288 [INFO] Agent1.server: New leader elected: payload=Node-08a26fdc-15e6-861c-0931-b91dd8c0843f +36:53.288 [ERROR] Agent1.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-08a26fdc-15e6-861c-0931-b91dd8c0843f other=Node-ad30b2da-d62f-f58d-1af9-93ae2dea7328 +36:53.288 [INFO] Agent1.server: member joined, marking health alive: member=Node-08a26fdc-15e6-861c-0931-b91dd8c0843f +36:53.289 [DEBUG] Agent1.server.memberlist.wan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:19222: connect: connection reset by peer +36:53.289 [DEBUG] Agent1.server: Failed to flood-join server at address: server=Node-08a26fdc-15e6-861c-0931-b91dd8c0843f.dc1 address=127.0.0.1:19222 error="1 error occurred: + * Failed to join 127.0.0.1: dial tcp 127.0.0.1:19222: connect: connection reset by peer " -44:38.194 [INFO] Agent1: Force leaving node: node=Node-c331bf70-4961-ff50-a4f8-3c0fc1b8dba3 -44:38.499 [DEBUG] Agent1.server.memberlist.lan: memberlist: Failed ping: Node-c331bf70-4961-ff50-a4f8-3c0fc1b8dba3 (timeout reached) -44:39.002 [INFO] Agent1.server.memberlist.lan: memberlist: Suspect Node-c331bf70-4961-ff50-a4f8-3c0fc1b8dba3 has failed, no acks received -44:40.503 [DEBUG] Agent1.server.memberlist.lan: memberlist: Failed ping: Node-c331bf70-4961-ff50-a4f8-3c0fc1b8dba3 (timeout reached) -44:42.006 [INFO] Agent1.server.memberlist.lan: memberlist: Suspect Node-c331bf70-4961-ff50-a4f8-3c0fc1b8dba3 has failed, no acks received -44:42.510 [DEBUG] Agent1.server.memberlist.lan: memberlist: Failed ping: Node-c331bf70-4961-ff50-a4f8-3c0fc1b8dba3 (timeout reached) -44:43.008 [INFO] Agent1.server.memberlist.lan: memberlist: Marking Node-c331bf70-4961-ff50-a4f8-3c0fc1b8dba3 as failed, suspect timeout reached (0 peer confirmations) -44:45.007 [INFO] Agent1.server.memberlist.lan: memberlist: Suspect Node-c331bf70-4961-ff50-a4f8-3c0fc1b8dba3 has failed, no acks received -44:46.198 [INFO] Agent1.server.serf.lan: serf: EventMemberReap (forced): Node-c331bf70-4961-ff50-a4f8-3c0fc1b8dba3 127.0.0.1 -44:46.198 [DEBUG] Agent1.http: Request finished: method=PUT url=/v1/agent/force-leave/Node-c331bf70-4961-ff50-a4f8-3c0fc1b8dba3?prune=1 from=127.0.0.1:34938 latency=8.00340188s -44:46.198 [INFO] Agent1.server: Removing LAN server: server="Node-c331bf70-4961-ff50-a4f8-3c0fc1b8dba3 (Addr: tcp/127.0.0.1:23307) (DC: dc1)" -44:46.198 [INFO] Agent1.server: deregistering member: member=Node-c331bf70-4961-ff50-a4f8-3c0fc1b8dba3 reason=reaped -44:46.200 [DEBUG] Agent1: Skipping remote check since it is managed automatically: check=serfHealth -44:46.200 [DEBUG] Agent1: Node info in sync -44:46.200 [DEBUG] Agent1: Node info in sync -44:46.201 [INFO] Agent1: Requesting shutdown -44:46.201 [INFO] Agent1.server: shutting down server -44:46.201 [DEBUG] Agent1.leader: stopping routine: routine="intermediate cert renew watch" -44:46.201 [DEBUG] Agent1.leader: stopping routine: routine="CA root pruning" -44:46.201 [DEBUG] Agent1.leader: stopping routine: routine="federation state anti-entropy" -44:46.201 [DEBUG] Agent1.leader: stopping routine: routine="federation state pruning" -44:46.201 [WARN] Agent1.server.serf.lan: serf: Shutdown without a Leave -44:46.202 [ERROR] Agent1.server: error performing anti-entropy sync of federation state: error="context canceled" -44:46.202 [DEBUG] Agent1.leader: stopped routine: routine="federation state anti-entropy" -44:46.202 [DEBUG] Agent1.leader: stopped routine: routine="intermediate cert renew watch" -44:46.202 [DEBUG] Agent1.leader: stopped routine: routine="CA root pruning" -44:46.202 [DEBUG] Agent1.leader: stopped routine: routine="federation state pruning" -44:46.202 [WARN] Agent1.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:44:46.202Z [INFO] agent.router.manager: shutting down -44:46.202 [WARN] Agent1: error getting server health from server: server=Node-d84e3b85-2527-94b8-a837-66c1834e1f4d error="rpc error making call: EOF" -2022-09-15T21:44:46.202Z [INFO] agent.router.manager: shutting down -44:46.202 [INFO] Agent1: consul server down -44:46.202 [INFO] Agent1: shutdown complete -44:46.202 [INFO] Agent1: Stopping server: protocol=DNS address=127.0.0.1:23281 network=tcp -44:46.202 [INFO] Agent1: Stopping server: protocol=DNS address=127.0.0.1:23281 network=udp -44:46.202 [INFO] Agent1: Stopping server: protocol=HTTP address=127.0.0.1:23282 network=tcp -44:46.703 [INFO] Agent1: Waiting for endpoints to shut down -44:46.703 [INFO] Agent1: Endpoints down ---- PASS: TestForceLeaveCommand_prune (10.19s) +36:53.790 [INFO] Agent1: Force leaving node: node=Node-08a26fdc-15e6-861c-0931-b91dd8c0843f +36:53.933 [DEBUG] Agent1.server.memberlist.lan: memberlist: Failed ping: Node-08a26fdc-15e6-861c-0931-b91dd8c0843f (timeout reached) +36:54.432 [INFO] Agent1.server.memberlist.lan: memberlist: Suspect Node-08a26fdc-15e6-861c-0931-b91dd8c0843f has failed, no acks received +36:55.932 [DEBUG] Agent1.server.memberlist.lan: memberlist: Failed ping: Node-08a26fdc-15e6-861c-0931-b91dd8c0843f (timeout reached) +36:57.432 [INFO] Agent1.server.memberlist.lan: memberlist: Suspect Node-08a26fdc-15e6-861c-0931-b91dd8c0843f has failed, no acks received +36:57.933 [DEBUG] Agent1.server.memberlist.lan: memberlist: Failed ping: Node-08a26fdc-15e6-861c-0931-b91dd8c0843f (timeout reached) +36:58.432 [INFO] Agent1.server.memberlist.lan: memberlist: Marking Node-08a26fdc-15e6-861c-0931-b91dd8c0843f as failed, suspect timeout reached (0 peer confirmations) +37:00.432 [INFO] Agent1.server.memberlist.lan: memberlist: Suspect Node-08a26fdc-15e6-861c-0931-b91dd8c0843f has failed, no acks received +37:01.791 [INFO] Agent1.server.serf.lan: serf: EventMemberReap (forced): Node-08a26fdc-15e6-861c-0931-b91dd8c0843f 127.0.0.1 +37:01.791 [DEBUG] Agent1.http: Request finished: method=PUT url=/v1/agent/force-leave/Node-08a26fdc-15e6-861c-0931-b91dd8c0843f?prune=1 from=127.0.0.1:35650 latency=8.000224753s +37:01.791 [INFO] Agent1: Requesting shutdown +37:01.791 [INFO] Agent1.server: shutting down server +37:01.791 [DEBUG] Agent1.leader: stopping routine: routine="federation state anti-entropy" +37:01.791 [DEBUG] Agent1.leader: stopping routine: routine="federation state pruning" +37:01.791 [DEBUG] Agent1.leader: stopping routine: routine="intermediate cert renew watch" +37:01.791 [DEBUG] Agent1.leader: stopping routine: routine="CA root pruning" +37:01.791 [WARN] Agent1.server.serf.lan: serf: Shutdown without a Leave +37:01.791 [ERROR] Agent1.server: error performing anti-entropy sync of federation state: error="context canceled" +37:01.791 [DEBUG] Agent1.leader: stopped routine: routine="federation state anti-entropy" +37:01.791 [DEBUG] Agent1.leader: stopped routine: routine="federation state pruning" +37:01.791 [DEBUG] Agent1.leader: stopped routine: routine="intermediate cert renew watch" +37:01.791 [DEBUG] Agent1.leader: stopped routine: routine="CA root pruning" +37:01.791 [WARN] Agent1.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:37:01.792Z [INFO] agent.router.manager: shutting down +2021-08-13T15:37:01.792Z [INFO] agent.router.manager: shutting down +37:01.792 [INFO] Agent1: consul server down +37:01.792 [INFO] Agent1: shutdown complete +37:01.792 [INFO] Agent1: Stopping server: protocol=DNS address=127.0.0.1:19190 network=tcp +37:01.792 [INFO] Agent1: Stopping server: protocol=DNS address=127.0.0.1:19190 network=udp +37:01.792 [INFO] Agent1: Stopping server: protocol=HTTP address=127.0.0.1:19191 network=tcp +37:01.792 [ERROR] Agent1.server.memberlist.lan: memberlist: Failed to send gossip to 127.0.0.1:19221: write udp 127.0.0.1:19193->127.0.0.1:19221: use of closed network connection +37:01.792 [DEBUG] Agent1: Skipping remote check since it is managed automatically: check=serfHealth +37:01.792 [DEBUG] Agent1: Node info in sync +37:01.793 [WARN] Agent1: error getting server health from server: server=Node-ad30b2da-d62f-f58d-1af9-93ae2dea7328 error="rpc error getting client: failed to get conn: dial tcp 127.0.0.1:0->127.0.0.1:19195: connect: connection refused" +37:02.292 [INFO] Agent1: Waiting for endpoints to shut down +37:02.292 [INFO] Agent1: Endpoints down +--- PASS: TestForceLeaveCommand_prune (10.00s) PASS -ok github.com/hashicorp/consul/command/forceleave 10.343s +ok github.com/hashicorp/consul/command/forceleave 10.065s ? github.com/hashicorp/consul/command/helpers [no test files] === RUN TestInfoCommand_noTabs === PAUSE TestInfoCommand_noTabs @@ -85169,73 +85076,72 @@ [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:44:41.899Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:44:41.934Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -44:41.954 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4ac03690-ddee-f0a7-f6b2-d95ac4c62dbe Address:127.0.0.1:21237}]" -44:41.955 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4ac03690-ddee-f0a7-f6b2-d95ac4c62dbe.dc1 127.0.0.1 -44:41.956 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4ac03690-ddee-f0a7-f6b2-d95ac4c62dbe 127.0.0.1 -2022-09-15T21:44:41.957Z [INFO] agent.router: Initializing LAN area manager -44:41.957 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:21237 [Follower]" leader= -44:41.957 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21232 network=udp -44:41.957 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4ac03690-ddee-f0a7-f6b2-d95ac4c62dbe.dc1 area=wan -44:41.957 [INFO] TestAgent.server: Adding LAN server: server="Node-4ac03690-ddee-f0a7-f6b2-d95ac4c62dbe (Addr: tcp/127.0.0.1:21237) (DC: dc1)" -44:41.958 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21232 network=tcp -44:41.959 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:21233 network=tcp -44:41.959 [INFO] TestAgent: started state syncer -44:41.959 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:21238 network=tcp -44:42.017 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -44:42.017 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:21237 [Candidate]" term=2 -44:42.018 [DEBUG] TestAgent.server.raft: votes: needed=1 -44:42.018 [DEBUG] TestAgent.server.raft: vote granted: from=4ac03690-ddee-f0a7-f6b2-d95ac4c62dbe term=2 tally=1 -44:42.018 [INFO] TestAgent.server.raft: election won: tally=1 -44:42.018 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:21237 [Leader]" -44:42.018 [INFO] TestAgent.server: cluster leadership acquired -44:42.018 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:21237 -44:42.020 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -44:42.022 [INFO] TestAgent.server: New leader elected: payload=Node-4ac03690-ddee-f0a7-f6b2-d95ac4c62dbe -44:42.048 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -44:42.048 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -44:42.048 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -44:42.048 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -44:42.048 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -44:42.048 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4ac03690-ddee-f0a7-f6b2-d95ac4c62dbe -44:42.048 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4ac03690-ddee-f0a7-f6b2-d95ac4c62dbe -44:42.050 [INFO] TestAgent.server: federation state anti-entropy synced -44:42.146 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -44:42.147 [INFO] TestAgent: Synced node info -44:42.147 [DEBUG] TestAgent: Node info in sync -44:42.149 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:57336 latency=2.94668ms -44:42.155 [INFO] TestAgent: Requesting shutdown -44:42.155 [INFO] TestAgent.server: shutting down server -44:42.155 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -44:42.155 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -44:42.155 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -44:42.155 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -44:42.155 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -44:42.156 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -44:42.156 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -44:42.156 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -44:42.156 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -44:42.156 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -44:42.156 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:44:42.156Z [INFO] agent.router.manager: shutting down -2022-09-15T21:44:42.156Z [INFO] agent.router.manager: shutting down -44:42.156 [INFO] TestAgent: consul server down -44:42.156 [INFO] TestAgent: shutdown complete -44:42.156 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21232 network=tcp -44:42.156 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21232 network=udp -44:42.156 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:21233 network=tcp -44:42.657 [INFO] TestAgent: Waiting for endpoints to shut down -44:42.657 [INFO] TestAgent: Endpoints down ---- PASS: TestInfoCommand (0.83s) +2021-08-13T15:36:58.699Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:36:58.714Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +36:58.717 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7886829b-250e-09c6-3436-e390329c1e6d Address:127.0.0.1:30426}]" +36:58.718 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30426 [Follower]" leader= +36:58.721 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7886829b-250e-09c6-3436-e390329c1e6d.dc1 127.0.0.1 +36:58.722 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7886829b-250e-09c6-3436-e390329c1e6d 127.0.0.1 +2021-08-13T15:36:58.722Z [INFO] agent.router: Initializing LAN area manager +36:58.723 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=udp +36:58.723 [INFO] TestAgent.server: Adding LAN server: server="Node-7886829b-250e-09c6-3436-e390329c1e6d (Addr: tcp/127.0.0.1:30426) (DC: dc1)" +36:58.723 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7886829b-250e-09c6-3436-e390329c1e6d.dc1 area=wan +36:58.723 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=tcp +36:58.725 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30422 network=tcp +36:58.725 [INFO] TestAgent: started state syncer +36:58.725 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30427 network=tcp +36:58.777 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +36:58.777 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30426 [Candidate]" term=2 +36:58.777 [DEBUG] TestAgent.server.raft: votes: needed=1 +36:58.777 [DEBUG] TestAgent.server.raft: vote granted: from=7886829b-250e-09c6-3436-e390329c1e6d term=2 tally=1 +36:58.777 [INFO] TestAgent.server.raft: election won: tally=1 +36:58.777 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30426 [Leader]" +36:58.777 [INFO] TestAgent.server: cluster leadership acquired +36:58.777 [INFO] TestAgent.server: New leader elected: payload=Node-7886829b-250e-09c6-3436-e390329c1e6d +36:58.778 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:30426 +36:58.779 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +36:58.802 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +36:58.802 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +36:58.802 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +36:58.802 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +36:58.802 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +36:58.802 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7886829b-250e-09c6-3436-e390329c1e6d +36:58.802 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7886829b-250e-09c6-3436-e390329c1e6d +36:58.803 [INFO] TestAgent.server: federation state anti-entropy synced +36:59.025 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +36:59.026 [INFO] TestAgent: Synced node info +36:59.166 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:49086 latency=2.117825ms +36:59.169 [INFO] TestAgent: Requesting shutdown +36:59.170 [INFO] TestAgent.server: shutting down server +36:59.170 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +36:59.170 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +36:59.170 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +36:59.170 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +36:59.170 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +36:59.170 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +36:59.170 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +36:59.170 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +36:59.170 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +36:59.170 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +36:59.170 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:36:59.170Z [INFO] agent.router.manager: shutting down +2021-08-13T15:36:59.170Z [INFO] agent.router.manager: shutting down +36:59.170 [INFO] TestAgent: consul server down +36:59.170 [INFO] TestAgent: shutdown complete +36:59.170 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=tcp +36:59.170 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=udp +36:59.170 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30422 network=tcp +36:59.670 [INFO] TestAgent: Waiting for endpoints to shut down +36:59.670 [INFO] TestAgent: Endpoints down +--- PASS: TestInfoCommand (1.02s) PASS -ok github.com/hashicorp/consul/command/info 0.979s +ok github.com/hashicorp/consul/command/info 1.083s === RUN TestCommand_noTabs === PAUSE TestCommand_noTabs === CONT TestCommand_noTabs --- PASS: TestCommand_noTabs (0.00s) PASS -ok github.com/hashicorp/consul/command/intention 0.035s +ok github.com/hashicorp/consul/command/intention 0.006s === RUN TestCommand_noTabs === PAUSE TestCommand_noTabs === RUN TestCommand_Validation @@ -85250,74 +85156,57 @@ === RUN TestCommand_Validation/0_args === RUN TestCommand_Validation/1_args === RUN TestCommand_Validation/3_args +[INFO] freeport: detected ephemeral port range of [32768, 60999] +[INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range --- PASS: TestCommand_Validation (0.00s) --- PASS: TestCommand_Validation/0_args (0.00s) --- PASS: TestCommand_Validation/1_args (0.00s) --- PASS: TestCommand_Validation/3_args (0.00s) -[INFO] freeport: detected ephemeral port range of [32768, 60999] -[INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:44:48.451Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:44:48.503Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -44:48.508 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:24465db1-4568-811c-b97c-851e5094a42e Address:127.0.0.1:31447}]" -44:48.510 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31447 [Follower]" leader= -44:48.510 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-24465db1-4568-811c-b97c-851e5094a42e.dc1 127.0.0.1 -44:48.511 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-24465db1-4568-811c-b97c-851e5094a42e 127.0.0.1 -2022-09-15T21:44:48.511Z [INFO] agent.router: Initializing LAN area manager -44:48.512 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31442 network=udp -44:48.512 [INFO] TestAgent.server: Adding LAN server: server="Node-24465db1-4568-811c-b97c-851e5094a42e (Addr: tcp/127.0.0.1:31447) (DC: dc1)" -44:48.512 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-24465db1-4568-811c-b97c-851e5094a42e.dc1 area=wan -44:48.513 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31442 network=tcp -44:48.513 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31443 network=tcp -44:48.514 [INFO] TestAgent: started state syncer -44:48.514 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31448 network=tcp -44:48.549 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -44:48.550 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31447 [Candidate]" term=2 -44:48.550 [DEBUG] TestAgent.server.raft: votes: needed=1 -44:48.550 [DEBUG] TestAgent.server.raft: vote granted: from=24465db1-4568-811c-b97c-851e5094a42e term=2 tally=1 -44:48.550 [INFO] TestAgent.server.raft: election won: tally=1 -44:48.550 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31447 [Leader]" -44:48.550 [INFO] TestAgent.server: cluster leadership acquired -44:48.550 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:31447 -44:48.551 [INFO] TestAgent.server: New leader elected: payload=Node-24465db1-4568-811c-b97c-851e5094a42e -44:48.552 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -44:48.579 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -44:48.579 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -44:48.579 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -44:48.579 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -44:48.579 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -44:48.579 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-24465db1-4568-811c-b97c-851e5094a42e -44:48.579 [INFO] TestAgent.server: member joined, marking health alive: member=Node-24465db1-4568-811c-b97c-851e5094a42e -44:48.581 [INFO] TestAgent.server: federation state anti-entropy synced -44:48.794 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -44:48.795 [INFO] TestAgent: Synced node info -44:48.938 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:59620 latency=1.41072ms -44:48.940 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions/check?destination=db&source=foo&source-type=consul from=127.0.0.1:59622 latency=430.1µs -44:48.947 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions/check?destination=db&source=web&source-type=consul from=127.0.0.1:59624 latency=204.44µs -44:48.947 [INFO] TestAgent: Requesting shutdown -44:48.948 [INFO] TestAgent.server: shutting down server -44:48.948 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -44:48.948 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -44:48.948 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -44:48.948 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -44:48.948 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -44:48.948 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -44:48.948 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -44:48.948 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -44:48.948 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -44:48.948 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -44:48.948 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:44:48.948Z [INFO] agent.router.manager: shutting down -2022-09-15T21:44:48.948Z [INFO] agent.router.manager: shutting down -44:48.948 [INFO] TestAgent: consul server down -44:48.948 [INFO] TestAgent: shutdown complete -44:48.948 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31442 network=tcp -44:48.949 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31442 network=udp -44:48.949 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31443 network=tcp -44:49.450 [INFO] TestAgent: Waiting for endpoints to shut down -44:49.450 [INFO] TestAgent: Endpoints down ---- PASS: TestCommand (1.15s) +2021-08-13T15:37:11.882Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:37:11.904Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +37:11.908 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1cbb300a-5965-53a1-9da2-9d964fee99b7 Address:127.0.0.1:19195}]" +37:11.908 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19195 [Follower]" leader= +37:11.908 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1cbb300a-5965-53a1-9da2-9d964fee99b7.dc1 127.0.0.1 +37:11.909 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1cbb300a-5965-53a1-9da2-9d964fee99b7 127.0.0.1 +2021-08-13T15:37:11.909Z [INFO] agent.router: Initializing LAN area manager +37:11.910 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1cbb300a-5965-53a1-9da2-9d964fee99b7.dc1 area=wan +37:11.910 [INFO] TestAgent.server: Adding LAN server: server="Node-1cbb300a-5965-53a1-9da2-9d964fee99b7 (Addr: tcp/127.0.0.1:19195) (DC: dc1)" +37:11.910 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=udp +37:11.910 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=tcp +37:11.911 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19191 network=tcp +37:11.911 [INFO] TestAgent: started state syncer +37:11.911 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19196 network=tcp +37:11.971 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +37:11.971 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19195 [Candidate]" term=2 +37:11.971 [DEBUG] TestAgent.server.raft: votes: needed=1 +37:11.971 [DEBUG] TestAgent.server.raft: vote granted: from=1cbb300a-5965-53a1-9da2-9d964fee99b7 term=2 tally=1 +37:11.971 [INFO] TestAgent.server.raft: election won: tally=1 +37:11.971 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19195 [Leader]" +37:11.971 [INFO] TestAgent.server: cluster leadership acquired +37:11.972 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:19195 +37:11.973 [INFO] TestAgent.server: New leader elected: payload=Node-1cbb300a-5965-53a1-9da2-9d964fee99b7 +37:11.973 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +37:11.987 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:35710 latency=7.078217ms +37:11.990 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions/check?destination=db&source=foo&source-type=consul from=127.0.0.1:35712 latency=454.601µs +37:11.996 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions/check?destination=db&source=web&source-type=consul from=127.0.0.1:35714 latency=304.221µs +37:11.999 [INFO] TestAgent: Requesting shutdown +37:12.000 [INFO] TestAgent.server: shutting down server +37:12.000 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +37:12.000 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +37:12.000 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:37:12.000Z [INFO] agent.router.manager: shutting down +2021-08-13T15:37:12.000Z [INFO] agent.router.manager: shutting down +37:12.000 [INFO] TestAgent: consul server down +37:12.000 [INFO] TestAgent: shutdown complete +37:12.000 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=tcp +37:12.000 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=udp +37:12.000 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19191 network=tcp +37:12.002 [ERROR] TestAgent.server: failed to establish leadership: error="error generating CA root certificate: error computing next serial number: raft is already shutdown" +37:12.500 [INFO] TestAgent: Waiting for endpoints to shut down +37:12.500 [INFO] TestAgent: Endpoints down +--- PASS: TestCommand (0.68s) PASS -ok github.com/hashicorp/consul/command/intention/check 1.350s +ok github.com/hashicorp/consul/command/intention/check 0.743s === RUN TestCommand_noTabs === PAUSE TestCommand_noTabs === RUN TestCommand_Validation @@ -85339,361 +85228,340 @@ === CONT TestCommand_replace [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestCommand_meta +=== CONT TestCommand_FileNoExist === CONT TestCommand +=== CONT TestCommand_File === CONT TestCommand_deny === CONT TestCommand_Validation -=== CONT TestCommand_FileNoExist -=== CONT TestCommand_File === RUN TestCommand_Validation/-allow_and_-deny --- PASS: TestCommand_Validation (0.00s) --- PASS: TestCommand_Validation/-allow_and_-deny (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:44:49.112Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:44:49.196Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -44:49.207 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3250b81c-e0e2-86aa-7b3d-d1292d384f95 Address:127.0.0.1:20244}]" -44:49.208 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:20244 [Follower]" leader= -44:49.208 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3250b81c-e0e2-86aa-7b3d-d1292d384f95.dc1 127.0.0.1 -44:49.209 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3250b81c-e0e2-86aa-7b3d-d1292d384f95 127.0.0.1 -2022-09-15T21:44:49.209Z [INFO] agent.router: Initializing LAN area manager -44:49.209 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3250b81c-e0e2-86aa-7b3d-d1292d384f95.dc1 area=wan -44:49.209 [INFO] TestAgent.server: Adding LAN server: server="Node-3250b81c-e0e2-86aa-7b3d-d1292d384f95 (Addr: tcp/127.0.0.1:20244) (DC: dc1)" -44:49.277 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -44:49.277 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:20244 [Candidate]" term=2 -44:49.277 [DEBUG] TestAgent.server.raft: votes: needed=1 -44:49.277 [DEBUG] TestAgent.server.raft: vote granted: from=3250b81c-e0e2-86aa-7b3d-d1292d384f95 term=2 tally=1 -44:49.277 [INFO] TestAgent.server.raft: election won: tally=1 -44:49.277 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:20244 [Leader]" -44:49.278 [INFO] TestAgent.server: cluster leadership acquired -44:49.279 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20239 network=udp -44:49.279 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20239 network=tcp -44:49.281 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:20240 network=tcp -44:49.281 [INFO] TestAgent: started state syncer -44:49.287 [INFO] TestAgent.server: New leader elected: payload=Node-3250b81c-e0e2-86aa-7b3d-d1292d384f95 -44:49.287 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:20244 -44:49.288 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:20245 network=tcp -44:49.289 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -44:49.305 [INFO] TestAgent: Requesting shutdown -44:49.305 [INFO] TestAgent.server: shutting down server -44:49.305 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -44:49.305 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:44:49.306Z [INFO] agent.router.manager: shutting down -2022-09-15T21:44:49.306Z [INFO] agent.router.manager: shutting down -44:49.308 [ERROR] TestAgent.server: failed to establish leadership: error="error generating CA root certificate: error computing next serial number: raft is already shutdown" -44:49.308 [ERROR] TestAgent.server: failed to transfer leadership attempt, will retry: attempt=0 retry_limit=3 error="raft is already shutdown" -44:49.318 [INFO] TestAgent: Synced node info -44:49.342 [INFO] TestAgent: consul server down -44:49.342 [INFO] TestAgent: shutdown complete -44:49.342 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20239 network=tcp -44:49.342 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20239 network=udp -44:49.342 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:20240 network=tcp -44:49.843 [INFO] TestAgent: Waiting for endpoints to shut down -44:49.843 [INFO] TestAgent: Endpoints down ---- PASS: TestCommand_FileNoExist (0.99s) -2022-09-15T21:44:49.008Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:44:49.080Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -44:49.116 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6f118073-e4e0-685f-f073-298582feeae6 Address:127.0.0.1:20223}]" -44:49.117 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6f118073-e4e0-685f-f073-298582feeae6.dc1 127.0.0.1 -44:49.118 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6f118073-e4e0-685f-f073-298582feeae6 127.0.0.1 -2022-09-15T21:44:49.118Z [INFO] agent.router: Initializing LAN area manager -44:49.119 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20218 network=udp -44:49.119 [INFO] TestAgent.server: Adding LAN server: server="Node-6f118073-e4e0-685f-f073-298582feeae6 (Addr: tcp/127.0.0.1:20223) (DC: dc1)" -44:49.119 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20218 network=tcp -44:49.120 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:20219 network=tcp -44:49.120 [INFO] TestAgent: started state syncer -44:49.121 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:20224 network=tcp -44:49.121 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:20223 [Follower]" leader= -44:49.121 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6f118073-e4e0-685f-f073-298582feeae6.dc1 area=wan -44:49.186 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -44:49.186 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:20223 [Candidate]" term=2 -44:49.186 [DEBUG] TestAgent.server.raft: votes: needed=1 -44:49.186 [DEBUG] TestAgent.server.raft: vote granted: from=6f118073-e4e0-685f-f073-298582feeae6 term=2 tally=1 -44:49.186 [INFO] TestAgent.server.raft: election won: tally=1 -44:49.186 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:20223 [Leader]" -44:49.187 [INFO] TestAgent.server: cluster leadership acquired -44:49.187 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:20223 -44:49.189 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -44:49.189 [INFO] TestAgent.server: New leader elected: payload=Node-6f118073-e4e0-685f-f073-298582feeae6 -44:49.312 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -44:49.312 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -44:49.312 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -44:49.312 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -44:49.312 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -44:49.312 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6f118073-e4e0-685f-f073-298582feeae6 -44:49.312 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6f118073-e4e0-685f-f073-298582feeae6 -44:49.343 [INFO] TestAgent.server: federation state anti-entropy synced -44:49.348 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -44:49.348 [INFO] TestAgent: Synced node info -44:49.354 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:58300 latency=1.50344ms -44:49.356 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:58304 latency=603.1µs -44:49.357 [INFO] TestAgent: Requesting shutdown -44:49.357 [INFO] TestAgent.server: shutting down server -44:49.357 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -44:49.357 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -44:49.357 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -44:49.357 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -44:49.357 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -44:49.357 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -44:49.357 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -44:49.357 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -44:49.357 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -44:49.357 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -44:49.357 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:44:49.358Z [INFO] agent.router.manager: shutting down -2022-09-15T21:44:49.358Z [INFO] agent.router.manager: shutting down -44:49.358 [INFO] TestAgent: consul server down -44:49.358 [INFO] TestAgent: shutdown complete -44:49.358 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20218 network=tcp -44:49.358 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20218 network=udp -44:49.358 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:20219 network=tcp -44:49.870 [INFO] TestAgent: Waiting for endpoints to shut down -44:49.870 [INFO] TestAgent: Endpoints down ---- PASS: TestCommand_meta (1.03s) -2022-09-15T21:44:49.000Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:44:49.120Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -44:49.127 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0d40b875-1952-5acd-5f3a-cc0f1fc5ce2a Address:127.0.0.1:20237}]" -44:49.146 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:20237 [Follower]" leader= -44:49.154 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0d40b875-1952-5acd-5f3a-cc0f1fc5ce2a.dc1 127.0.0.1 -44:49.156 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0d40b875-1952-5acd-5f3a-cc0f1fc5ce2a 127.0.0.1 -2022-09-15T21:44:49.156Z [INFO] agent.router: Initializing LAN area manager -44:49.182 [INFO] TestAgent.server: Adding LAN server: server="Node-0d40b875-1952-5acd-5f3a-cc0f1fc5ce2a (Addr: tcp/127.0.0.1:20237) (DC: dc1)" -44:49.184 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0d40b875-1952-5acd-5f3a-cc0f1fc5ce2a.dc1 area=wan -44:49.185 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20232 network=udp -44:49.186 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20232 network=tcp -44:49.187 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:20233 network=tcp -44:49.187 [INFO] TestAgent: started state syncer -44:49.187 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:20238 network=tcp -44:49.205 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -44:49.205 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:20237 [Candidate]" term=2 -44:49.205 [DEBUG] TestAgent.server.raft: votes: needed=1 -44:49.205 [DEBUG] TestAgent.server.raft: vote granted: from=0d40b875-1952-5acd-5f3a-cc0f1fc5ce2a term=2 tally=1 -44:49.205 [INFO] TestAgent.server.raft: election won: tally=1 -44:49.205 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:20237 [Leader]" -44:49.205 [INFO] TestAgent.server: cluster leadership acquired -44:49.205 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:20237 -44:49.209 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -44:49.206 [INFO] TestAgent.server: New leader elected: payload=Node-0d40b875-1952-5acd-5f3a-cc0f1fc5ce2a -44:49.316 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -44:49.316 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -44:49.316 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -44:49.316 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -44:49.316 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -44:49.316 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0d40b875-1952-5acd-5f3a-cc0f1fc5ce2a -44:49.316 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0d40b875-1952-5acd-5f3a-cc0f1fc5ce2a -44:49.322 [INFO] TestAgent.server: federation state anti-entropy synced -44:49.364 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:59580 latency=1.17042ms -44:49.371 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:59584 latency=453.18µs -44:49.373 [INFO] TestAgent: Requesting shutdown -44:49.373 [INFO] TestAgent.server: shutting down server -44:49.373 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -44:49.373 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -44:49.373 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -44:49.373 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -44:49.373 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -44:49.373 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -44:49.373 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -44:49.373 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -44:49.373 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -44:49.373 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -44:49.373 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -44:49.373 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:44:49.373Z [INFO] agent.router.manager: shutting down -2022-09-15T21:44:49.373Z [INFO] agent.router.manager: shutting down -44:49.374 [INFO] TestAgent: consul server down -44:49.374 [INFO] TestAgent: shutdown complete -44:49.374 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20232 network=tcp -44:49.374 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20232 network=udp -44:49.374 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:20233 network=tcp -44:49.878 [INFO] TestAgent: Waiting for endpoints to shut down -44:49.878 [INFO] TestAgent: Endpoints down ---- PASS: TestCommand_deny (1.02s) -2022-09-15T21:44:48.999Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:44:49.095Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -44:49.137 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fbb39b47-dba7-19ad-b6b6-3db3e018e07d Address:127.0.0.1:20230}]" -44:49.138 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:20230 [Follower]" leader= -44:49.143 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fbb39b47-dba7-19ad-b6b6-3db3e018e07d.dc1 127.0.0.1 -44:49.144 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fbb39b47-dba7-19ad-b6b6-3db3e018e07d 127.0.0.1 -2022-09-15T21:44:49.144Z [INFO] agent.router: Initializing LAN area manager -44:49.145 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20225 network=udp -44:49.147 [INFO] TestAgent.server: Adding LAN server: server="Node-fbb39b47-dba7-19ad-b6b6-3db3e018e07d (Addr: tcp/127.0.0.1:20230) (DC: dc1)" -44:49.147 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fbb39b47-dba7-19ad-b6b6-3db3e018e07d.dc1 area=wan -44:49.148 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20225 network=tcp -44:49.154 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:20226 network=tcp -44:49.154 [INFO] TestAgent: started state syncer -44:49.155 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:20231 network=tcp -44:49.206 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -44:49.206 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:20230 [Candidate]" term=2 -44:49.206 [DEBUG] TestAgent.server.raft: votes: needed=1 -44:49.206 [DEBUG] TestAgent.server.raft: vote granted: from=fbb39b47-dba7-19ad-b6b6-3db3e018e07d term=2 tally=1 -44:49.206 [INFO] TestAgent.server.raft: election won: tally=1 -44:49.206 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:20230 [Leader]" -44:49.206 [INFO] TestAgent.server: cluster leadership acquired -44:49.206 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:20230 -44:49.208 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -44:49.208 [INFO] TestAgent.server: New leader elected: payload=Node-fbb39b47-dba7-19ad-b6b6-3db3e018e07d -44:49.324 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -44:49.324 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -44:49.324 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -44:49.324 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -44:49.324 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -44:49.324 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fbb39b47-dba7-19ad-b6b6-3db3e018e07d -44:49.324 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fbb39b47-dba7-19ad-b6b6-3db3e018e07d -44:49.345 [INFO] TestAgent.server: federation state anti-entropy synced -44:49.386 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -44:49.387 [INFO] TestAgent: Synced node info -44:49.435 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -44:49.436 [DEBUG] TestAgent: Node info in sync -44:49.436 [DEBUG] TestAgent: Node info in sync -44:49.439 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:44462 latency=1.07954ms -44:49.441 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:44464 latency=201.98µs -44:49.442 [INFO] TestAgent: Requesting shutdown -44:49.442 [INFO] TestAgent.server: shutting down server -44:49.442 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -44:49.442 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -44:49.442 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -44:49.442 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -44:49.442 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -44:49.442 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -44:49.442 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -44:49.442 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -44:49.442 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -44:49.442 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -44:49.442 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:44:49.442Z [INFO] agent.router.manager: shutting down -2022-09-15T21:44:49.442Z [INFO] agent.router.manager: shutting down -44:49.443 [INFO] TestAgent: consul server down -44:49.443 [INFO] TestAgent: shutdown complete -44:49.443 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20225 network=tcp -44:49.443 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20225 network=udp -44:49.443 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:20226 network=tcp -44:49.946 [INFO] TestAgent: Waiting for endpoints to shut down -44:49.946 [INFO] TestAgent: Endpoints down ---- PASS: TestCommand (1.10s) -2022-09-15T21:44:48.981Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:44:49.046Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -44:49.068 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:08d484ee-bdce-1cce-477e-96b35f5e9c20 Address:127.0.0.1:20216}]" -44:49.071 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-08d484ee-bdce-1cce-477e-96b35f5e9c20.dc1 127.0.0.1 -44:49.073 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-08d484ee-bdce-1cce-477e-96b35f5e9c20 127.0.0.1 -2022-09-15T21:44:49.073Z [INFO] agent.router: Initializing LAN area manager -44:49.079 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20211 network=udp -44:49.086 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-08d484ee-bdce-1cce-477e-96b35f5e9c20.dc1 area=wan -44:49.086 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:20216 [Follower]" leader= -44:49.095 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20211 network=tcp -44:49.112 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:20212 network=tcp -44:49.112 [INFO] TestAgent: started state syncer -44:49.112 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:20217 network=tcp -44:49.121 [INFO] TestAgent.server: Adding LAN server: server="Node-08d484ee-bdce-1cce-477e-96b35f5e9c20 (Addr: tcp/127.0.0.1:20216) (DC: dc1)" -44:49.156 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -44:49.156 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:20216 [Candidate]" term=2 -44:49.182 [DEBUG] TestAgent.server.raft: votes: needed=1 -44:49.182 [DEBUG] TestAgent.server.raft: vote granted: from=08d484ee-bdce-1cce-477e-96b35f5e9c20 term=2 tally=1 -44:49.182 [INFO] TestAgent.server.raft: election won: tally=1 -44:49.182 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:20216 [Leader]" -44:49.183 [INFO] TestAgent.server: cluster leadership acquired -44:49.183 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:20216 -44:49.185 [INFO] TestAgent.server: New leader elected: payload=Node-08d484ee-bdce-1cce-477e-96b35f5e9c20 -44:49.185 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -44:49.306 [INFO] TestAgent: Synced node info -44:49.306 [DEBUG] TestAgent: Node info in sync -44:49.345 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -44:49.345 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -44:49.345 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -44:49.345 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -44:49.345 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -44:49.345 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-08d484ee-bdce-1cce-477e-96b35f5e9c20 -44:49.345 [INFO] TestAgent.server: member joined, marking health alive: member=Node-08d484ee-bdce-1cce-477e-96b35f5e9c20 -44:49.348 [INFO] TestAgent.server: federation state anti-entropy synced -44:49.521 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:56450 latency=962.02µs -44:49.527 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:56452 latency=4.99066ms -44:49.548 [ERROR] TestAgent.http: Request error: method=POST url=/v1/connect/intentions from=127.0.0.1:56454 error="duplicate intention found: ALLOW default/foo => default/bar (ID: 63069a74-ea88-ee44-f441-e5c6d017da23, Precedence: 9)" -44:49.548 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:56454 latency=1.90424ms -44:49.561 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:56456 latency=418.64µs -44:49.564 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/connect/intentions/63069a74-ea88-ee44-f441-e5c6d017da23 from=127.0.0.1:56456 latency=1.27902ms -44:49.565 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:56452 latency=234.34µs -44:49.567 [INFO] TestAgent: Requesting shutdown -44:49.567 [INFO] TestAgent.server: shutting down server -44:49.567 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -44:49.567 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -44:49.567 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -44:49.567 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -44:49.567 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -44:49.567 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -44:49.567 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -44:49.567 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -44:49.567 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -44:49.567 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -44:49.568 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:44:49.568Z [INFO] agent.router.manager: shutting down -2022-09-15T21:44:49.568Z [INFO] agent.router.manager: shutting down -44:49.568 [INFO] TestAgent: consul server down -44:49.568 [INFO] TestAgent: shutdown complete -44:49.568 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20211 network=tcp -44:49.568 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20211 network=udp -44:49.568 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:20212 network=tcp -44:50.068 [INFO] TestAgent: Waiting for endpoints to shut down -44:50.068 [INFO] TestAgent: Endpoints down ---- PASS: TestCommand_replace (1.22s) -2022-09-15T21:44:49.061Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:44:49.216Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -44:49.273 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6337c8b1-b44b-d56f-2b11-1cf08c599afd Address:127.0.0.1:20251}]" -44:49.287 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:20251 [Follower]" leader= -44:49.292 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6337c8b1-b44b-d56f-2b11-1cf08c599afd.dc1 127.0.0.1 -44:49.302 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6337c8b1-b44b-d56f-2b11-1cf08c599afd 127.0.0.1 -2022-09-15T21:44:49.302Z [INFO] agent.router: Initializing LAN area manager -44:49.303 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20246 network=udp -44:49.303 [INFO] TestAgent.server: Adding LAN server: server="Node-6337c8b1-b44b-d56f-2b11-1cf08c599afd (Addr: tcp/127.0.0.1:20251) (DC: dc1)" -44:49.303 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6337c8b1-b44b-d56f-2b11-1cf08c599afd.dc1 area=wan -44:49.303 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20246 network=tcp -44:49.307 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:20247 network=tcp -44:49.307 [INFO] TestAgent: started state syncer -44:49.307 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:20252 network=tcp -44:49.343 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -44:49.343 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:20251 [Candidate]" term=2 -44:49.343 [DEBUG] TestAgent.server.raft: votes: needed=1 -44:49.343 [DEBUG] TestAgent.server.raft: vote granted: from=6337c8b1-b44b-d56f-2b11-1cf08c599afd term=2 tally=1 -44:49.343 [INFO] TestAgent.server.raft: election won: tally=1 -44:49.343 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:20251 [Leader]" -44:49.343 [INFO] TestAgent.server: cluster leadership acquired -44:49.344 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:20251 -44:49.345 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -44:49.347 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -44:49.347 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -44:49.347 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -44:49.347 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -44:49.347 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -44:49.348 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6337c8b1-b44b-d56f-2b11-1cf08c599afd -44:49.348 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6337c8b1-b44b-d56f-2b11-1cf08c599afd -44:49.348 [INFO] TestAgent.server: New leader elected: payload=Node-6337c8b1-b44b-d56f-2b11-1cf08c599afd -44:49.349 [INFO] TestAgent.server: federation state anti-entropy synced -44:49.702 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -44:49.703 [INFO] TestAgent: Synced node info -44:49.714 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:40762 latency=8.73798ms -44:49.717 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:40764 latency=390.56µs -44:49.718 [INFO] TestAgent: Requesting shutdown -44:49.718 [INFO] TestAgent.server: shutting down server -44:49.718 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -44:49.718 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -44:49.718 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -44:49.718 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -44:49.718 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -44:49.718 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -44:49.718 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -44:49.718 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -44:49.718 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -44:49.718 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -44:49.718 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:44:49.718Z [INFO] agent.router.manager: shutting down -2022-09-15T21:44:49.718Z [INFO] agent.router.manager: shutting down -44:49.718 [INFO] TestAgent: consul server down -44:49.719 [INFO] TestAgent: shutdown complete -44:49.719 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20246 network=tcp -44:49.719 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20246 network=udp -44:49.719 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:20247 network=tcp -44:50.226 [INFO] TestAgent: Waiting for endpoints to shut down -44:50.226 [INFO] TestAgent: Endpoints down ---- PASS: TestCommand_File (1.37s) +2021-08-13T15:37:14.071Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:37:14.098Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +37:14.108 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0567057f-27b1-adbd-d9e6-15ac92f8ea1e Address:127.0.0.1:31461}]" +37:14.109 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31461 [Follower]" leader= +37:14.111 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0567057f-27b1-adbd-d9e6-15ac92f8ea1e.dc1 127.0.0.1 +37:14.111 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0567057f-27b1-adbd-d9e6-15ac92f8ea1e 127.0.0.1 +2021-08-13T15:37:14.112Z [INFO] agent.router: Initializing LAN area manager +37:14.112 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31456 network=udp +37:14.112 [INFO] TestAgent.server: Adding LAN server: server="Node-0567057f-27b1-adbd-d9e6-15ac92f8ea1e (Addr: tcp/127.0.0.1:31461) (DC: dc1)" +37:14.112 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31456 network=tcp +37:14.113 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31457 network=tcp +37:14.113 [INFO] TestAgent: started state syncer +37:14.114 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31462 network=tcp +37:14.114 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0567057f-27b1-adbd-d9e6-15ac92f8ea1e.dc1 area=wan +37:14.171 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +37:14.171 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31461 [Candidate]" term=2 +37:14.171 [DEBUG] TestAgent.server.raft: votes: needed=1 +37:14.171 [DEBUG] TestAgent.server.raft: vote granted: from=0567057f-27b1-adbd-d9e6-15ac92f8ea1e term=2 tally=1 +37:14.171 [INFO] TestAgent.server.raft: election won: tally=1 +37:14.171 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31461 [Leader]" +37:14.172 [INFO] TestAgent.server: cluster leadership acquired +37:14.172 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:31461 +37:14.173 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +37:14.175 [INFO] TestAgent.server: New leader elected: payload=Node-0567057f-27b1-adbd-d9e6-15ac92f8ea1e +37:14.201 [INFO] TestAgent: Requesting shutdown +37:14.201 [INFO] TestAgent.server: shutting down server +37:14.201 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +37:14.201 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +37:14.201 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:37:14.201Z [INFO] agent.router.manager: shutting down +2021-08-13T15:37:14.202Z [INFO] agent.router.manager: shutting down +37:14.202 [INFO] TestAgent: consul server down +37:14.202 [INFO] TestAgent: shutdown complete +37:14.202 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31456 network=tcp +37:14.202 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31456 network=udp +37:14.202 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31457 network=tcp +37:14.246 [ERROR] TestAgent.server: failed to establish leadership: error="error generating CA root certificate: error computing next serial number: raft is already shutdown" +37:14.250 [ERROR] TestAgent.server: failed to transfer leadership attempt, will retry: attempt=0 retry_limit=3 error="raft is already shutdown" +37:14.703 [INFO] TestAgent: Waiting for endpoints to shut down +37:14.703 [INFO] TestAgent: Endpoints down +--- PASS: TestCommand_FileNoExist (0.74s) +2021-08-13T15:37:14.070Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:37:14.133Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +37:14.137 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:324e22d4-71e0-0ba3-b6b8-534d70cf3872 Address:127.0.0.1:31482}]" +37:14.138 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-324e22d4-71e0-0ba3-b6b8-534d70cf3872.dc1 127.0.0.1 +37:14.138 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-324e22d4-71e0-0ba3-b6b8-534d70cf3872 127.0.0.1 +2021-08-13T15:37:14.138Z [INFO] agent.router: Initializing LAN area manager +37:14.139 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31477 network=udp +37:14.139 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31482 [Follower]" leader= +37:14.139 [INFO] TestAgent.server: Adding LAN server: server="Node-324e22d4-71e0-0ba3-b6b8-534d70cf3872 (Addr: tcp/127.0.0.1:31482) (DC: dc1)" +37:14.139 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31477 network=tcp +37:14.139 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-324e22d4-71e0-0ba3-b6b8-534d70cf3872.dc1 area=wan +37:14.140 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31478 network=tcp +37:14.140 [INFO] TestAgent: started state syncer +37:14.140 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31483 network=tcp +37:14.195 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +37:14.195 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31482 [Candidate]" term=2 +37:14.195 [DEBUG] TestAgent.server.raft: votes: needed=1 +37:14.195 [DEBUG] TestAgent.server.raft: vote granted: from=324e22d4-71e0-0ba3-b6b8-534d70cf3872 term=2 tally=1 +37:14.195 [INFO] TestAgent.server.raft: election won: tally=1 +37:14.195 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31482 [Leader]" +37:14.198 [INFO] TestAgent.server: cluster leadership acquired +37:14.198 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:31482 +37:14.199 [INFO] TestAgent.server: New leader elected: payload=Node-324e22d4-71e0-0ba3-b6b8-534d70cf3872 +37:14.205 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +37:14.248 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:34882 latency=924.822µs +37:14.250 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:34884 latency=371.941µs +37:14.250 [INFO] TestAgent: Requesting shutdown +37:14.250 [INFO] TestAgent.server: shutting down server +37:14.250 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +37:14.251 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +37:14.251 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:37:14.251Z [INFO] agent.router.manager: shutting down +2021-08-13T15:37:14.251Z [INFO] agent.router.manager: shutting down +37:14.251 [INFO] TestAgent: consul server down +37:14.251 [INFO] TestAgent: shutdown complete +37:14.251 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31477 network=tcp +37:14.251 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31477 network=udp +37:14.251 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31478 network=tcp +37:14.251 [ERROR] TestAgent.server: failed to establish leadership: error="error generating CA root certificate: raft is already shutdown" +37:14.755 [INFO] TestAgent: Waiting for endpoints to shut down +37:14.755 [INFO] TestAgent: Endpoints down +--- PASS: TestCommand_deny (0.79s) +2021-08-13T15:37:14.048Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:37:14.093Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +37:14.106 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:aba8955c-0fd7-b745-075b-1f2b652fde1c Address:127.0.0.1:31475}]" +37:14.110 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31475 [Follower]" leader= +37:14.116 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-aba8955c-0fd7-b745-075b-1f2b652fde1c.dc1 127.0.0.1 +37:14.117 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-aba8955c-0fd7-b745-075b-1f2b652fde1c 127.0.0.1 +2021-08-13T15:37:14.117Z [INFO] agent.router: Initializing LAN area manager +37:14.117 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31470 network=udp +37:14.118 [INFO] TestAgent.server: Adding LAN server: server="Node-aba8955c-0fd7-b745-075b-1f2b652fde1c (Addr: tcp/127.0.0.1:31475) (DC: dc1)" +37:14.118 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-aba8955c-0fd7-b745-075b-1f2b652fde1c.dc1 area=wan +37:14.118 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31470 network=tcp +37:14.122 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31471 network=tcp +37:14.122 [INFO] TestAgent: started state syncer +37:14.122 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31476 network=tcp +37:14.175 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +37:14.175 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31475 [Candidate]" term=2 +37:14.175 [DEBUG] TestAgent.server.raft: votes: needed=1 +37:14.175 [DEBUG] TestAgent.server.raft: vote granted: from=aba8955c-0fd7-b745-075b-1f2b652fde1c term=2 tally=1 +37:14.175 [INFO] TestAgent.server.raft: election won: tally=1 +37:14.175 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31475 [Leader]" +37:14.175 [INFO] TestAgent.server: cluster leadership acquired +37:14.175 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:31475 +37:14.177 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +37:14.177 [INFO] TestAgent.server: New leader elected: payload=Node-aba8955c-0fd7-b745-075b-1f2b652fde1c +37:14.206 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:52892 latency=4.100989ms +37:14.249 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +37:14.249 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +37:14.249 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +37:14.249 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +37:14.249 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +37:14.249 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-aba8955c-0fd7-b745-075b-1f2b652fde1c +37:14.249 [INFO] TestAgent.server: member joined, marking health alive: member=Node-aba8955c-0fd7-b745-075b-1f2b652fde1c +37:14.251 [INFO] TestAgent.server: federation state anti-entropy synced +37:14.255 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:52894 latency=12.367808ms +37:14.256 [INFO] TestAgent: Requesting shutdown +37:14.256 [INFO] TestAgent.server: shutting down server +37:14.256 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +37:14.256 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +37:14.256 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +37:14.256 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +37:14.256 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +37:14.257 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +37:14.257 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +37:14.257 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +37:14.257 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +37:14.257 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +37:14.257 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +37:14.257 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:37:14.257Z [INFO] agent.router.manager: shutting down +2021-08-13T15:37:14.257Z [INFO] agent.router.manager: shutting down +37:14.257 [INFO] TestAgent: consul server down +37:14.257 [INFO] TestAgent: shutdown complete +37:14.257 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31470 network=tcp +37:14.257 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31470 network=udp +37:14.257 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31471 network=tcp +37:14.759 [INFO] TestAgent: Waiting for endpoints to shut down +37:14.759 [INFO] TestAgent: Endpoints down +--- PASS: TestCommand_File (0.80s) +2021-08-13T15:37:14.077Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:37:14.141Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +37:14.180 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f7a60b80-9ea3-981e-14e0-3d5b51086b04 Address:127.0.0.1:31447}]" +37:14.181 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f7a60b80-9ea3-981e-14e0-3d5b51086b04.dc1 127.0.0.1 +37:14.182 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f7a60b80-9ea3-981e-14e0-3d5b51086b04 127.0.0.1 +2021-08-13T15:37:14.182Z [INFO] agent.router: Initializing LAN area manager +37:14.183 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31442 network=udp +37:14.183 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31447 [Follower]" leader= +37:14.192 [INFO] TestAgent.server: Adding LAN server: server="Node-f7a60b80-9ea3-981e-14e0-3d5b51086b04 (Addr: tcp/127.0.0.1:31447) (DC: dc1)" +37:14.195 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f7a60b80-9ea3-981e-14e0-3d5b51086b04.dc1 area=wan +37:14.196 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31442 network=tcp +37:14.197 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31443 network=tcp +37:14.197 [INFO] TestAgent: started state syncer +37:14.198 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31448 network=tcp +37:14.242 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +37:14.242 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31447 [Candidate]" term=2 +37:14.242 [DEBUG] TestAgent.server.raft: votes: needed=1 +37:14.242 [DEBUG] TestAgent.server.raft: vote granted: from=f7a60b80-9ea3-981e-14e0-3d5b51086b04 term=2 tally=1 +37:14.242 [INFO] TestAgent.server.raft: election won: tally=1 +37:14.242 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31447 [Leader]" +37:14.242 [INFO] TestAgent.server: cluster leadership acquired +37:14.243 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:31447 +37:14.243 [INFO] TestAgent.server: New leader elected: payload=Node-f7a60b80-9ea3-981e-14e0-3d5b51086b04 +37:14.246 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +37:14.253 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +37:14.253 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +37:14.253 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +37:14.253 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +37:14.253 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +37:14.253 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f7a60b80-9ea3-981e-14e0-3d5b51086b04 +37:14.253 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f7a60b80-9ea3-981e-14e0-3d5b51086b04 +37:14.254 [INFO] TestAgent.server: federation state anti-entropy synced +37:14.309 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:60640 latency=1.263743ms +37:14.310 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:60642 latency=332.001µs +37:14.318 [ERROR] TestAgent.http: Request error: method=POST url=/v1/connect/intentions from=127.0.0.1:60644 error="duplicate intention found: ALLOW default/foo => default/bar (ID: ad7c7b6c-6dff-06b1-0f22-3d8a48c40b2f, Precedence: 9)" +37:14.318 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:60644 latency=1.047423ms +37:14.330 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:60646 latency=348.201µs +37:14.336 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/connect/intentions/ad7c7b6c-6dff-06b1-0f22-3d8a48c40b2f from=127.0.0.1:60646 latency=5.107032ms +37:14.338 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:60642 latency=553.222µs +37:14.339 [INFO] TestAgent: Requesting shutdown +37:14.340 [INFO] TestAgent.server: shutting down server +37:14.340 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +37:14.340 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +37:14.340 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +37:14.340 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +37:14.340 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +37:14.340 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +37:14.340 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +37:14.340 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +37:14.340 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +37:14.340 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +37:14.340 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +37:14.340 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:37:14.340Z [INFO] agent.router.manager: shutting down +2021-08-13T15:37:14.340Z [INFO] agent.router.manager: shutting down +37:14.340 [INFO] TestAgent: consul server down +37:14.340 [INFO] TestAgent: shutdown complete +37:14.341 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31442 network=tcp +37:14.341 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31442 network=udp +37:14.341 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31443 network=tcp +37:14.843 [INFO] TestAgent: Waiting for endpoints to shut down +37:14.843 [INFO] TestAgent: Endpoints down +--- PASS: TestCommand_replace (0.88s) +2021-08-13T15:37:14.098Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:37:14.136Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +37:14.139 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8875459b-35b9-e9d1-7a00-800370a1bae9 Address:127.0.0.1:31454}]" +37:14.140 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8875459b-35b9-e9d1-7a00-800370a1bae9.dc1 127.0.0.1 +37:14.141 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8875459b-35b9-e9d1-7a00-800370a1bae9 127.0.0.1 +2021-08-13T15:37:14.141Z [INFO] agent.router: Initializing LAN area manager +37:14.141 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31454 [Follower]" leader= +37:14.142 [INFO] TestAgent.server: Adding LAN server: server="Node-8875459b-35b9-e9d1-7a00-800370a1bae9 (Addr: tcp/127.0.0.1:31454) (DC: dc1)" +37:14.142 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8875459b-35b9-e9d1-7a00-800370a1bae9.dc1 area=wan +37:14.175 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31449 network=tcp +37:14.202 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31449 network=udp +37:14.204 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31450 network=tcp +37:14.204 [INFO] TestAgent: started state syncer +37:14.204 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31455 network=tcp +37:14.211 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +37:14.243 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31454 [Candidate]" term=2 +37:14.245 [DEBUG] TestAgent.server.raft: votes: needed=1 +37:14.245 [DEBUG] TestAgent.server.raft: vote granted: from=8875459b-35b9-e9d1-7a00-800370a1bae9 term=2 tally=1 +37:14.245 [INFO] TestAgent.server.raft: election won: tally=1 +37:14.245 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31454 [Leader]" +37:14.246 [INFO] TestAgent.server: cluster leadership acquired +37:14.246 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:31454 +37:14.247 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +37:14.249 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +37:14.249 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +37:14.249 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +37:14.249 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +37:14.249 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +37:14.249 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8875459b-35b9-e9d1-7a00-800370a1bae9 +37:14.249 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8875459b-35b9-e9d1-7a00-800370a1bae9 +37:14.247 [INFO] TestAgent.server: New leader elected: payload=Node-8875459b-35b9-e9d1-7a00-800370a1bae9 +37:14.251 [INFO] TestAgent.server: federation state anti-entropy synced +37:14.390 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:55104 latency=959.722µs +37:14.395 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:55106 latency=285.401µs +37:14.396 [INFO] TestAgent: Requesting shutdown +37:14.396 [INFO] TestAgent.server: shutting down server +37:14.396 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +37:14.396 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +37:14.396 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +37:14.396 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +37:14.396 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +37:14.397 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +37:14.397 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +37:14.397 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +37:14.397 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +37:14.397 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +37:14.397 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +37:14.397 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:37:14.397Z [INFO] agent.router.manager: shutting down +2021-08-13T15:37:14.397Z [INFO] agent.router.manager: shutting down +37:14.397 [INFO] TestAgent: consul server down +37:14.397 [INFO] TestAgent: shutdown complete +37:14.397 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31449 network=tcp +37:14.397 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31449 network=udp +37:14.397 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31450 network=tcp +37:14.897 [INFO] TestAgent: Waiting for endpoints to shut down +37:14.897 [INFO] TestAgent: Endpoints down +--- PASS: TestCommand_meta (0.94s) +2021-08-13T15:37:14.044Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:37:14.085Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +37:14.097 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ee841646-8000-590d-213a-44345130bf28 Address:127.0.0.1:31468}]" +37:14.098 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ee841646-8000-590d-213a-44345130bf28.dc1 127.0.0.1 +37:14.098 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ee841646-8000-590d-213a-44345130bf28 127.0.0.1 +2021-08-13T15:37:14.098Z [INFO] agent.router: Initializing LAN area manager +37:14.101 [INFO] TestAgent.server: Adding LAN server: server="Node-ee841646-8000-590d-213a-44345130bf28 (Addr: tcp/127.0.0.1:31468) (DC: dc1)" +37:14.108 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31468 [Follower]" leader= +37:14.109 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31463 network=udp +37:14.118 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ee841646-8000-590d-213a-44345130bf28.dc1 area=wan +37:14.118 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31463 network=tcp +37:14.121 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31464 network=tcp +37:14.121 [INFO] TestAgent: started state syncer +37:14.121 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31469 network=tcp +37:14.199 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +37:14.199 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31468 [Candidate]" term=2 +37:14.199 [DEBUG] TestAgent.server.raft: votes: needed=1 +37:14.199 [DEBUG] TestAgent.server.raft: vote granted: from=ee841646-8000-590d-213a-44345130bf28 term=2 tally=1 +37:14.200 [INFO] TestAgent.server.raft: election won: tally=1 +37:14.200 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31468 [Leader]" +37:14.200 [INFO] TestAgent.server: cluster leadership acquired +37:14.200 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:31468 +37:14.202 [INFO] TestAgent.server: New leader elected: payload=Node-ee841646-8000-590d-213a-44345130bf28 +37:14.202 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +37:14.241 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +37:14.241 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +37:14.241 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +37:14.241 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +37:14.241 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +37:14.241 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ee841646-8000-590d-213a-44345130bf28 +37:14.241 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ee841646-8000-590d-213a-44345130bf28 +37:14.243 [INFO] TestAgent.server: federation state anti-entropy synced +37:14.355 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +37:14.356 [INFO] TestAgent: Synced node info +37:14.356 [DEBUG] TestAgent: Node info in sync +37:14.452 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:48214 latency=1.283603ms +37:14.453 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:48216 latency=268.241µs +37:14.454 [INFO] TestAgent: Requesting shutdown +37:14.454 [INFO] TestAgent.server: shutting down server +37:14.454 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +37:14.454 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +37:14.454 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +37:14.454 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +37:14.454 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +37:14.454 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +37:14.454 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +37:14.454 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +37:14.454 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +37:14.454 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +37:14.454 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:37:14.454Z [INFO] agent.router.manager: shutting down +2021-08-13T15:37:14.455Z [INFO] agent.router.manager: shutting down +37:14.455 [INFO] TestAgent: consul server down +37:14.455 [INFO] TestAgent: shutdown complete +37:14.455 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31463 network=tcp +37:14.455 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31463 network=udp +37:14.455 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31464 network=tcp +37:14.955 [INFO] TestAgent: Waiting for endpoints to shut down +37:14.955 [INFO] TestAgent: Endpoints down +--- PASS: TestCommand (0.99s) PASS -ok github.com/hashicorp/consul/command/intention/create 1.566s +ok github.com/hashicorp/consul/command/intention/create 1.113s === RUN TestCommand_noTabs === PAUSE TestCommand_noTabs === RUN TestCommand_Validation @@ -85701,148 +85569,148 @@ === RUN TestCommand === PAUSE TestCommand === CONT TestCommand_noTabs ---- PASS: TestCommand_noTabs (0.00s) === CONT TestCommand === CONT TestCommand_Validation -=== RUN TestCommand_Validation/0_args [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestCommand_noTabs (0.00s) +=== RUN TestCommand_Validation/0_args === RUN TestCommand_Validation/3_args ---- PASS: TestCommand_Validation (0.01s) +--- PASS: TestCommand_Validation (0.00s) --- PASS: TestCommand_Validation/0_args (0.00s) --- PASS: TestCommand_Validation/3_args (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:44:58.539Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:44:58.585Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -44:58.604 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4dc30e7c-887a-e8fd-0771-585e5f69f7ea Address:127.0.0.1:20216}]" -44:58.605 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4dc30e7c-887a-e8fd-0771-585e5f69f7ea.dc1 127.0.0.1 -44:58.609 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:20216 [Follower]" leader= -44:58.612 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4dc30e7c-887a-e8fd-0771-585e5f69f7ea 127.0.0.1 -2022-09-15T21:44:58.612Z [INFO] agent.router: Initializing LAN area manager -44:58.612 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20211 network=udp -44:58.612 [INFO] TestAgent.server: Adding LAN server: server="Node-4dc30e7c-887a-e8fd-0771-585e5f69f7ea (Addr: tcp/127.0.0.1:20216) (DC: dc1)" -44:58.612 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4dc30e7c-887a-e8fd-0771-585e5f69f7ea.dc1 area=wan -44:58.613 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20211 network=tcp -44:58.615 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:20212 network=tcp -44:58.615 [INFO] TestAgent: started state syncer -44:58.615 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:20217 network=tcp -44:58.664 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -44:58.664 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:20216 [Candidate]" term=2 -44:58.665 [DEBUG] TestAgent.server.raft: votes: needed=1 -44:58.665 [DEBUG] TestAgent.server.raft: vote granted: from=4dc30e7c-887a-e8fd-0771-585e5f69f7ea term=2 tally=1 -44:58.665 [INFO] TestAgent.server.raft: election won: tally=1 -44:58.665 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:20216 [Leader]" -44:58.665 [INFO] TestAgent.server: cluster leadership acquired -44:58.665 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:20216 -44:58.667 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -44:58.670 [INFO] TestAgent.server: New leader elected: payload=Node-4dc30e7c-887a-e8fd-0771-585e5f69f7ea -44:58.714 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -44:58.714 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -44:58.714 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -44:58.714 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -44:58.714 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -44:58.714 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4dc30e7c-887a-e8fd-0771-585e5f69f7ea -44:58.714 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4dc30e7c-887a-e8fd-0771-585e5f69f7ea -44:58.716 [INFO] TestAgent.server: federation state anti-entropy synced -44:58.896 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -44:58.896 [INFO] TestAgent: Synced node info -44:58.896 [DEBUG] TestAgent: Node info in sync -44:58.918 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:56462 latency=1.76358ms -44:58.937 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:56464 latency=6.79726ms -44:58.945 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/connect/intentions/9dfd2f70-807a-de26-274a-cd74feded378 from=127.0.0.1:56464 latency=907.8µs -44:58.947 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:56462 latency=410.98µs -44:58.947 [INFO] TestAgent: Requesting shutdown -44:58.947 [INFO] TestAgent.server: shutting down server -44:58.947 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -44:58.947 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -44:58.947 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -44:58.947 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -44:58.947 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -44:58.947 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -44:58.947 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -44:58.947 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -44:58.947 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -44:58.947 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -44:58.947 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:44:58.947Z [INFO] agent.router.manager: shutting down -2022-09-15T21:44:58.948Z [INFO] agent.router.manager: shutting down -44:58.948 [INFO] TestAgent: consul server down -44:58.948 [INFO] TestAgent: shutdown complete -44:58.948 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20211 network=tcp -44:58.948 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20211 network=udp -44:58.948 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:20212 network=tcp -44:59.450 [INFO] TestAgent: Waiting for endpoints to shut down -44:59.450 [INFO] TestAgent: Endpoints down ---- PASS: TestCommand (1.00s) +2021-08-13T15:37:18.820Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:37:18.836Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +37:18.842 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:99a15dad-f424-4fea-eeb4-18a400a83d0f Address:127.0.0.1:24300}]" +37:18.842 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24300 [Follower]" leader= +37:18.843 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-99a15dad-f424-4fea-eeb4-18a400a83d0f.dc1 127.0.0.1 +37:18.844 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-99a15dad-f424-4fea-eeb4-18a400a83d0f 127.0.0.1 +2021-08-13T15:37:18.844Z [INFO] agent.router: Initializing LAN area manager +37:18.844 [INFO] TestAgent.server: Adding LAN server: server="Node-99a15dad-f424-4fea-eeb4-18a400a83d0f (Addr: tcp/127.0.0.1:24300) (DC: dc1)" +37:18.844 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24295 network=udp +37:18.844 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-99a15dad-f424-4fea-eeb4-18a400a83d0f.dc1 area=wan +37:18.844 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24295 network=tcp +37:18.845 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24296 network=tcp +37:18.846 [INFO] TestAgent: started state syncer +37:18.846 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24301 network=tcp +37:18.894 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +37:18.894 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24300 [Candidate]" term=2 +37:18.894 [DEBUG] TestAgent.server.raft: votes: needed=1 +37:18.894 [DEBUG] TestAgent.server.raft: vote granted: from=99a15dad-f424-4fea-eeb4-18a400a83d0f term=2 tally=1 +37:18.894 [INFO] TestAgent.server.raft: election won: tally=1 +37:18.894 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24300 [Leader]" +37:18.894 [INFO] TestAgent.server: cluster leadership acquired +37:18.894 [INFO] TestAgent.server: New leader elected: payload=Node-99a15dad-f424-4fea-eeb4-18a400a83d0f +37:18.895 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24300 +37:18.896 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +37:18.919 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +37:18.919 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +37:18.919 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +37:18.919 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +37:18.919 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +37:18.919 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-99a15dad-f424-4fea-eeb4-18a400a83d0f +37:18.919 [INFO] TestAgent.server: member joined, marking health alive: member=Node-99a15dad-f424-4fea-eeb4-18a400a83d0f +37:18.920 [INFO] TestAgent.server: federation state anti-entropy synced +37:19.144 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +37:19.144 [INFO] TestAgent: Synced node info +37:19.144 [DEBUG] TestAgent: Node info in sync +37:19.241 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:35094 latency=1.519623ms +37:19.248 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:35096 latency=673.481µs +37:19.250 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/connect/intentions/08ffb591-bb93-73f9-01ea-9ce826b824a0 from=127.0.0.1:35096 latency=807.882µs +37:19.251 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:35094 latency=222.28µs +37:19.251 [INFO] TestAgent: Requesting shutdown +37:19.251 [INFO] TestAgent.server: shutting down server +37:19.251 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +37:19.251 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +37:19.251 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +37:19.251 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +37:19.251 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +37:19.251 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +37:19.251 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +37:19.251 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +37:19.251 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +37:19.252 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +37:19.252 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:37:19.252Z [INFO] agent.router.manager: shutting down +2021-08-13T15:37:19.252Z [INFO] agent.router.manager: shutting down +37:19.252 [INFO] TestAgent: consul server down +37:19.252 [INFO] TestAgent: shutdown complete +37:19.252 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24295 network=tcp +37:19.252 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24295 network=udp +37:19.252 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24296 network=tcp +37:19.752 [INFO] TestAgent: Waiting for endpoints to shut down +37:19.752 [INFO] TestAgent: Endpoints down +--- PASS: TestCommand (0.99s) PASS -ok github.com/hashicorp/consul/command/intention/delete 1.129s +ok github.com/hashicorp/consul/command/intention/delete 1.048s === RUN TestFinder === PAUSE TestFinder === CONT TestFinder [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:45:03.673Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:03.727Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:03.735 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c853c46e-81e2-837b-bd0c-d0c2eb16c6c8 Address:127.0.0.1:19195}]" -45:03.736 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c853c46e-81e2-837b-bd0c-d0c2eb16c6c8.dc1 127.0.0.1 -45:03.737 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c853c46e-81e2-837b-bd0c-d0c2eb16c6c8 127.0.0.1 -2022-09-15T21:45:03.737Z [INFO] agent.router: Initializing LAN area manager -45:03.738 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19195 [Follower]" leader= -45:03.746 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=udp -45:03.746 [INFO] TestAgent.server: Adding LAN server: server="Node-c853c46e-81e2-837b-bd0c-d0c2eb16c6c8 (Addr: tcp/127.0.0.1:19195) (DC: dc1)" -45:03.746 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c853c46e-81e2-837b-bd0c-d0c2eb16c6c8.dc1 area=wan -45:03.747 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=tcp -45:03.748 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19191 network=tcp -45:03.748 [INFO] TestAgent: started state syncer -45:03.748 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19196 network=tcp -45:03.790 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:03.790 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19195 [Candidate]" term=2 -45:03.790 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:03.790 [DEBUG] TestAgent.server.raft: vote granted: from=c853c46e-81e2-837b-bd0c-d0c2eb16c6c8 term=2 tally=1 -45:03.790 [INFO] TestAgent.server.raft: election won: tally=1 -45:03.790 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19195 [Leader]" -45:03.790 [INFO] TestAgent.server: cluster leadership acquired -45:03.791 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:19195 -45:03.792 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:03.794 [INFO] TestAgent.server: New leader elected: payload=Node-c853c46e-81e2-837b-bd0c-d0c2eb16c6c8 -45:03.823 [INFO] TestAgent: Synced node info -45:03.823 [DEBUG] TestAgent: Node info in sync -45:03.847 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:03.847 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:03.847 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:03.847 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:03.847 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:03.848 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c853c46e-81e2-837b-bd0c-d0c2eb16c6c8 -45:03.848 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c853c46e-81e2-837b-bd0c-d0c2eb16c6c8 -45:03.851 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:40978 latency=11.7626ms -45:03.856 [INFO] TestAgent.server: federation state anti-entropy synced -45:03.866 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:40978 latency=4.10434ms -45:03.867 [INFO] TestAgent: Requesting shutdown -45:03.868 [INFO] TestAgent.server: shutting down server -45:03.868 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:03.868 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:03.868 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:03.868 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:03.868 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:03.868 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:03.868 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:03.868 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:03.868 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:03.868 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:03.868 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:03.868Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:03.868Z [INFO] agent.router.manager: shutting down -45:03.869 [INFO] TestAgent: consul server down -45:03.869 [INFO] TestAgent: shutdown complete -45:03.869 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=tcp -45:03.869 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=udp -45:03.869 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19191 network=tcp -45:04.372 [INFO] TestAgent: Waiting for endpoints to shut down -45:04.372 [INFO] TestAgent: Endpoints down ---- PASS: TestFinder (0.84s) +2021-08-13T15:37:18.444Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:37:18.459Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +37:18.463 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5a14b424-cbe9-c569-3f48-3bea886479d4 Address:127.0.0.1:14090}]" +37:18.463 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14090 [Follower]" leader= +37:18.464 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5a14b424-cbe9-c569-3f48-3bea886479d4.dc1 127.0.0.1 +37:18.465 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5a14b424-cbe9-c569-3f48-3bea886479d4 127.0.0.1 +2021-08-13T15:37:18.465Z [INFO] agent.router: Initializing LAN area manager +37:18.465 [INFO] TestAgent.server: Adding LAN server: server="Node-5a14b424-cbe9-c569-3f48-3bea886479d4 (Addr: tcp/127.0.0.1:14090) (DC: dc1)" +37:18.465 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5a14b424-cbe9-c569-3f48-3bea886479d4.dc1 area=wan +37:18.467 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14085 network=udp +37:18.467 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14085 network=tcp +37:18.469 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14086 network=tcp +37:18.469 [INFO] TestAgent: started state syncer +37:18.469 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14091 network=tcp +37:18.502 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +37:18.502 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14090 [Candidate]" term=2 +37:18.502 [DEBUG] TestAgent.server.raft: votes: needed=1 +37:18.502 [DEBUG] TestAgent.server.raft: vote granted: from=5a14b424-cbe9-c569-3f48-3bea886479d4 term=2 tally=1 +37:18.502 [INFO] TestAgent.server.raft: election won: tally=1 +37:18.502 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14090 [Leader]" +37:18.503 [INFO] TestAgent.server: cluster leadership acquired +37:18.503 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14090 +37:18.504 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +37:18.507 [INFO] TestAgent.server: New leader elected: payload=Node-5a14b424-cbe9-c569-3f48-3bea886479d4 +37:18.528 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +37:18.528 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +37:18.528 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +37:18.528 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +37:18.528 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +37:18.528 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5a14b424-cbe9-c569-3f48-3bea886479d4 +37:18.528 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5a14b424-cbe9-c569-3f48-3bea886479d4 +37:18.530 [INFO] TestAgent.server: federation state anti-entropy synced +37:18.568 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +37:18.569 [INFO] TestAgent: Synced node info +37:18.886 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:49498 latency=1.311763ms +37:18.887 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:49498 latency=400.101µs +37:18.888 [INFO] TestAgent: Requesting shutdown +37:18.888 [INFO] TestAgent.server: shutting down server +37:18.888 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +37:18.888 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +37:18.888 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +37:18.888 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +37:18.888 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +37:18.888 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +37:18.888 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +37:18.888 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +37:18.888 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +37:18.888 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +37:18.888 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:37:18.888Z [INFO] agent.router.manager: shutting down +2021-08-13T15:37:18.888Z [INFO] agent.router.manager: shutting down +37:18.888 [INFO] TestAgent: consul server down +37:18.888 [INFO] TestAgent: shutdown complete +37:18.888 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14085 network=tcp +37:18.889 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14085 network=udp +37:18.889 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14086 network=tcp +37:19.389 [INFO] TestAgent: Waiting for endpoints to shut down +37:19.389 [INFO] TestAgent: Endpoints down +--- PASS: TestFinder (1.00s) PASS -ok github.com/hashicorp/consul/command/intention/finder 1.031s +ok github.com/hashicorp/consul/command/intention/finder 1.076s === RUN TestCommand_noTabs === PAUSE TestCommand_noTabs === RUN TestCommand_Validation @@ -85854,12 +85722,12 @@ === RUN TestCommand_verticalBar === PAUSE TestCommand_verticalBar === CONT TestCommand_noTabs +=== CONT TestCommand_srcDst === CONT TestCommand_verticalBar +=== CONT TestCommand_Validation +=== CONT TestCommand_id [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... --- PASS: TestCommand_noTabs (0.00s) -=== CONT TestCommand_srcDst -=== CONT TestCommand_id -=== CONT TestCommand_Validation === RUN TestCommand_Validation/0_args === RUN TestCommand_Validation/3_args --- PASS: TestCommand_Validation (0.00s) @@ -85867,183 +85735,183 @@ --- PASS: TestCommand_Validation/3_args (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:45:04.823Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:04.936Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:04.940 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3adb9aa5-8cf7-ad8b-49b8-47f1d145bfa1 Address:127.0.0.1:17153}]" -45:04.941 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17153 [Follower]" leader= -45:04.942 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3adb9aa5-8cf7-ad8b-49b8-47f1d145bfa1.dc1 127.0.0.1 -45:04.955 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3adb9aa5-8cf7-ad8b-49b8-47f1d145bfa1 127.0.0.1 -2022-09-15T21:45:04.955Z [INFO] agent.router: Initializing LAN area manager -45:04.956 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17148 network=udp -45:04.956 [INFO] TestAgent.server: Adding LAN server: server="Node-3adb9aa5-8cf7-ad8b-49b8-47f1d145bfa1 (Addr: tcp/127.0.0.1:17153) (DC: dc1)" -45:04.956 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3adb9aa5-8cf7-ad8b-49b8-47f1d145bfa1.dc1 area=wan -45:04.956 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17148 network=tcp -45:04.957 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17149 network=tcp -45:04.957 [INFO] TestAgent: started state syncer -45:04.957 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17154 network=tcp -45:05.010 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:05.010 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17153 [Candidate]" term=2 -45:05.010 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:05.010 [DEBUG] TestAgent.server.raft: vote granted: from=3adb9aa5-8cf7-ad8b-49b8-47f1d145bfa1 term=2 tally=1 -45:05.010 [INFO] TestAgent.server.raft: election won: tally=1 -45:05.010 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17153 [Leader]" -45:05.011 [INFO] TestAgent.server: cluster leadership acquired -45:05.011 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17153 -45:05.013 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:05.014 [INFO] TestAgent.server: New leader elected: payload=Node-3adb9aa5-8cf7-ad8b-49b8-47f1d145bfa1 -45:05.029 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:05.030 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:05.030 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:05.030 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:05.030 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:05.030 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3adb9aa5-8cf7-ad8b-49b8-47f1d145bfa1 -45:05.030 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3adb9aa5-8cf7-ad8b-49b8-47f1d145bfa1 -45:05.032 [INFO] TestAgent.server: federation state anti-entropy synced -45:05.035 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:57840 latency=1.17794ms -45:05.046 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions/27516104-5c0b-5509-9d38-ea79fe8e2423 from=127.0.0.1:57842 latency=599.92µs -45:05.048 [INFO] TestAgent: Requesting shutdown -45:05.048 [INFO] TestAgent.server: shutting down server -45:05.048 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:05.048 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:05.048 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:05.048 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:05.048 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:05.048 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:05.048 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:05.048 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -45:05.048 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:05.048 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:05.048 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:05.048 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:05.048Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:05.048Z [INFO] agent.router.manager: shutting down -45:05.049 [INFO] TestAgent: consul server down -45:05.049 [INFO] TestAgent: shutdown complete -45:05.049 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17148 network=tcp -45:05.049 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17148 network=udp -45:05.049 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17149 network=tcp -45:05.549 [INFO] TestAgent: Waiting for endpoints to shut down -45:05.549 [INFO] TestAgent: Endpoints down ---- PASS: TestCommand_verticalBar (0.90s) -2022-09-15T21:45:04.776Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:04.867Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:04.880 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4dc4e933-dddb-a813-7fa1-d32886f7e5e8 Address:127.0.0.1:17160}]" -45:04.881 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4dc4e933-dddb-a813-7fa1-d32886f7e5e8.dc1 127.0.0.1 -45:04.882 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17160 [Follower]" leader= -45:04.906 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4dc4e933-dddb-a813-7fa1-d32886f7e5e8 127.0.0.1 -2022-09-15T21:45:04.906Z [INFO] agent.router: Initializing LAN area manager -45:04.918 [INFO] TestAgent.server: Adding LAN server: server="Node-4dc4e933-dddb-a813-7fa1-d32886f7e5e8 (Addr: tcp/127.0.0.1:17160) (DC: dc1)" -45:04.918 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4dc4e933-dddb-a813-7fa1-d32886f7e5e8.dc1 area=wan -45:04.919 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17155 network=udp -45:04.919 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17155 network=tcp -45:04.935 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17156 network=tcp -45:04.935 [INFO] TestAgent: started state syncer -45:04.935 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17161 network=tcp -45:04.944 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:04.944 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17160 [Candidate]" term=2 -45:04.944 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:04.944 [DEBUG] TestAgent.server.raft: vote granted: from=4dc4e933-dddb-a813-7fa1-d32886f7e5e8 term=2 tally=1 -45:04.944 [INFO] TestAgent.server.raft: election won: tally=1 -45:04.944 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17160 [Leader]" -45:04.946 [INFO] TestAgent.server: cluster leadership acquired -45:04.946 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17160 -45:04.948 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:04.949 [INFO] TestAgent.server: New leader elected: payload=Node-4dc4e933-dddb-a813-7fa1-d32886f7e5e8 -45:04.989 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:04.989 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:04.989 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:04.989 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:04.989 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:04.989 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4dc4e933-dddb-a813-7fa1-d32886f7e5e8 -45:04.989 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4dc4e933-dddb-a813-7fa1-d32886f7e5e8 -45:04.990 [INFO] TestAgent.server: federation state anti-entropy synced -45:05.136 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:46460 latency=1.2033ms -45:05.142 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:46462 latency=454.6µs -45:05.147 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions/75461ee3-b29a-0e42-19d4-7dbb9706b4da from=127.0.0.1:46462 latency=396.06µs -45:05.147 [INFO] TestAgent: Requesting shutdown -45:05.148 [INFO] TestAgent.server: shutting down server -45:05.148 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:05.148 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:05.148 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:05.148 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:05.148 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:05.148 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:05.148 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:05.148 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -45:05.148 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:05.148 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:05.148 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:05.148 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:05.148Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:05.148Z [INFO] agent.router.manager: shutting down -45:05.148 [INFO] TestAgent: consul server down -45:05.148 [INFO] TestAgent: shutdown complete -45:05.148 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17155 network=tcp -45:05.149 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17155 network=udp -45:05.149 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17156 network=tcp -45:05.649 [INFO] TestAgent: Waiting for endpoints to shut down -45:05.649 [INFO] TestAgent: Endpoints down ---- PASS: TestCommand_srcDst (1.00s) -2022-09-15T21:45:04.839Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:04.932Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:04.939 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:53810df2-c934-111c-99a7-d40ce7191a4e Address:127.0.0.1:17167}]" -45:04.939 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17167 [Follower]" leader= -45:04.942 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-53810df2-c934-111c-99a7-d40ce7191a4e.dc1 127.0.0.1 -45:04.943 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-53810df2-c934-111c-99a7-d40ce7191a4e 127.0.0.1 -2022-09-15T21:45:04.943Z [INFO] agent.router: Initializing LAN area manager -45:04.945 [INFO] TestAgent.server: Adding LAN server: server="Node-53810df2-c934-111c-99a7-d40ce7191a4e (Addr: tcp/127.0.0.1:17167) (DC: dc1)" -45:04.945 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-53810df2-c934-111c-99a7-d40ce7191a4e.dc1 area=wan -45:04.945 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17162 network=tcp -45:04.945 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17162 network=udp -45:04.958 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17163 network=tcp -45:04.958 [INFO] TestAgent: started state syncer -45:04.958 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17168 network=tcp -45:05.003 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:05.003 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17167 [Candidate]" term=2 -45:05.003 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:05.003 [DEBUG] TestAgent.server.raft: vote granted: from=53810df2-c934-111c-99a7-d40ce7191a4e term=2 tally=1 -45:05.003 [INFO] TestAgent.server.raft: election won: tally=1 -45:05.003 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17167 [Leader]" -45:05.003 [INFO] TestAgent.server: cluster leadership acquired -45:05.004 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17167 -45:05.010 [INFO] TestAgent.server: New leader elected: payload=Node-53810df2-c934-111c-99a7-d40ce7191a4e -45:05.014 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:05.017 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:05.017 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:05.017 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:05.017 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:05.017 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:05.020 [INFO] TestAgent.server: federation state anti-entropy synced -45:05.026 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-53810df2-c934-111c-99a7-d40ce7191a4e -45:05.026 [INFO] TestAgent.server: member joined, marking health alive: member=Node-53810df2-c934-111c-99a7-d40ce7191a4e -45:05.280 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:50342 latency=1.07236ms -45:05.282 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions/d9896798-67d7-3902-9ca8-16bc2d03de18 from=127.0.0.1:50344 latency=197.2µs -45:05.283 [INFO] TestAgent: Requesting shutdown -45:05.283 [INFO] TestAgent.server: shutting down server -45:05.283 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:05.283 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:05.283 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:05.283 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:05.283 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:05.283 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:05.283 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:05.283 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -45:05.283 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:05.283 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:05.283 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:05.284 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:05.284Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:05.284Z [INFO] agent.router.manager: shutting down -45:05.284 [INFO] TestAgent: consul server down -45:05.284 [INFO] TestAgent: shutdown complete -45:05.284 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17162 network=tcp -45:05.284 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17162 network=udp -45:05.284 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17163 network=tcp -45:05.784 [INFO] TestAgent: Waiting for endpoints to shut down -45:05.784 [INFO] TestAgent: Endpoints down ---- PASS: TestCommand_id (1.15s) +2021-08-13T15:37:30.886Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:37:30.911Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +37:30.915 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9ab8aabd-feab-f763-7be8-2884fcb388c5 Address:127.0.0.1:27377}]" +37:30.916 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9ab8aabd-feab-f763-7be8-2884fcb388c5.dc1 127.0.0.1 +37:30.918 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9ab8aabd-feab-f763-7be8-2884fcb388c5 127.0.0.1 +2021-08-13T15:37:30.918Z [INFO] agent.router: Initializing LAN area manager +37:30.918 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:27377 [Follower]" leader= +37:30.918 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27372 network=udp +37:30.919 [INFO] TestAgent.server: Adding LAN server: server="Node-9ab8aabd-feab-f763-7be8-2884fcb388c5 (Addr: tcp/127.0.0.1:27377) (DC: dc1)" +37:30.919 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27372 network=tcp +37:30.920 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9ab8aabd-feab-f763-7be8-2884fcb388c5.dc1 area=wan +37:30.920 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:27373 network=tcp +37:30.920 [INFO] TestAgent: started state syncer +37:30.920 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:27378 network=tcp +37:30.961 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +37:30.961 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:27377 [Candidate]" term=2 +37:30.961 [DEBUG] TestAgent.server.raft: votes: needed=1 +37:30.961 [DEBUG] TestAgent.server.raft: vote granted: from=9ab8aabd-feab-f763-7be8-2884fcb388c5 term=2 tally=1 +37:30.961 [INFO] TestAgent.server.raft: election won: tally=1 +37:30.961 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:27377 [Leader]" +37:30.961 [INFO] TestAgent.server: cluster leadership acquired +37:30.961 [INFO] TestAgent.server: New leader elected: payload=Node-9ab8aabd-feab-f763-7be8-2884fcb388c5 +37:30.961 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:27377 +37:30.962 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +37:30.983 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +37:30.983 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +37:30.983 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +37:30.983 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +37:30.983 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +37:30.983 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9ab8aabd-feab-f763-7be8-2884fcb388c5 +37:30.983 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9ab8aabd-feab-f763-7be8-2884fcb388c5 +37:30.984 [INFO] TestAgent.server: federation state anti-entropy synced +37:31.070 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:46296 latency=3.358728ms +37:31.076 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions/3d1c9605-cde5-287c-9985-4886e04baa54 from=127.0.0.1:46298 latency=499.241µs +37:31.077 [INFO] TestAgent: Requesting shutdown +37:31.077 [INFO] TestAgent.server: shutting down server +37:31.077 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +37:31.077 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +37:31.077 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +37:31.077 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +37:31.077 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +37:31.077 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +37:31.077 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +37:31.077 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +37:31.077 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +37:31.077 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +37:31.077 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +37:31.077 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:37:31.077Z [INFO] agent.router.manager: shutting down +2021-08-13T15:37:31.077Z [INFO] agent.router.manager: shutting down +37:31.077 [INFO] TestAgent: consul server down +37:31.077 [INFO] TestAgent: shutdown complete +37:31.077 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27372 network=tcp +37:31.077 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27372 network=udp +37:31.078 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:27373 network=tcp +37:31.578 [INFO] TestAgent: Waiting for endpoints to shut down +37:31.578 [INFO] TestAgent: Endpoints down +--- PASS: TestCommand_id (0.75s) +2021-08-13T15:37:30.884Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:37:30.904Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +37:30.908 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:750feb02-8aca-93eb-348e-538fdc3e00bc Address:127.0.0.1:27363}]" +37:30.908 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:27363 [Follower]" leader= +37:30.909 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-750feb02-8aca-93eb-348e-538fdc3e00bc.dc1 127.0.0.1 +37:30.910 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-750feb02-8aca-93eb-348e-538fdc3e00bc 127.0.0.1 +2021-08-13T15:37:30.910Z [INFO] agent.router: Initializing LAN area manager +37:30.910 [INFO] TestAgent.server: Adding LAN server: server="Node-750feb02-8aca-93eb-348e-538fdc3e00bc (Addr: tcp/127.0.0.1:27363) (DC: dc1)" +37:30.911 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-750feb02-8aca-93eb-348e-538fdc3e00bc.dc1 area=wan +37:30.911 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27358 network=udp +37:30.911 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27358 network=tcp +37:30.912 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:27359 network=tcp +37:30.912 [INFO] TestAgent: started state syncer +37:30.912 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:27364 network=tcp +37:30.957 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +37:30.958 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:27363 [Candidate]" term=2 +37:30.958 [DEBUG] TestAgent.server.raft: votes: needed=1 +37:30.958 [DEBUG] TestAgent.server.raft: vote granted: from=750feb02-8aca-93eb-348e-538fdc3e00bc term=2 tally=1 +37:30.958 [INFO] TestAgent.server.raft: election won: tally=1 +37:30.958 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:27363 [Leader]" +37:30.958 [INFO] TestAgent.server: cluster leadership acquired +37:30.958 [INFO] TestAgent.server: New leader elected: payload=Node-750feb02-8aca-93eb-348e-538fdc3e00bc +37:30.958 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:27363 +37:30.960 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +37:30.983 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +37:30.983 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +37:30.983 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +37:30.983 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +37:30.983 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +37:30.983 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-750feb02-8aca-93eb-348e-538fdc3e00bc +37:30.983 [INFO] TestAgent.server: member joined, marking health alive: member=Node-750feb02-8aca-93eb-348e-538fdc3e00bc +37:30.985 [INFO] TestAgent.server: federation state anti-entropy synced +37:31.235 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:34384 latency=1.110063ms +37:31.237 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:34386 latency=225.9µs +37:31.238 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions/087f5ee3-d58d-a2f1-5c5f-13a3826909a4 from=127.0.0.1:34386 latency=144.001µs +37:31.238 [INFO] TestAgent: Requesting shutdown +37:31.238 [INFO] TestAgent.server: shutting down server +37:31.238 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +37:31.238 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +37:31.238 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +37:31.238 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +37:31.238 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +37:31.238 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +37:31.238 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +37:31.238 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +37:31.238 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +37:31.238 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +37:31.238 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +37:31.239 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:37:31.239Z [INFO] agent.router.manager: shutting down +2021-08-13T15:37:31.239Z [INFO] agent.router.manager: shutting down +37:31.239 [INFO] TestAgent: consul server down +37:31.239 [INFO] TestAgent: shutdown complete +37:31.239 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27358 network=tcp +37:31.239 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27358 network=udp +37:31.239 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:27359 network=tcp +37:31.739 [INFO] TestAgent: Waiting for endpoints to shut down +37:31.739 [INFO] TestAgent: Endpoints down +--- PASS: TestCommand_srcDst (0.92s) +2021-08-13T15:37:30.893Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:37:30.916Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +37:30.921 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c3a82df1-abde-6caf-a29d-40153fde826a Address:127.0.0.1:27370}]" +37:30.921 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:27370 [Follower]" leader= +37:30.921 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c3a82df1-abde-6caf-a29d-40153fde826a.dc1 127.0.0.1 +37:30.922 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c3a82df1-abde-6caf-a29d-40153fde826a 127.0.0.1 +2021-08-13T15:37:30.922Z [INFO] agent.router: Initializing LAN area manager +37:30.922 [INFO] TestAgent.server: Adding LAN server: server="Node-c3a82df1-abde-6caf-a29d-40153fde826a (Addr: tcp/127.0.0.1:27370) (DC: dc1)" +37:30.923 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c3a82df1-abde-6caf-a29d-40153fde826a.dc1 area=wan +37:30.923 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27365 network=udp +37:30.923 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27365 network=tcp +37:30.924 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:27366 network=tcp +37:30.924 [INFO] TestAgent: started state syncer +37:30.924 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:27371 network=tcp +37:30.988 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +37:30.988 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:27370 [Candidate]" term=2 +37:30.988 [DEBUG] TestAgent.server.raft: votes: needed=1 +37:30.988 [DEBUG] TestAgent.server.raft: vote granted: from=c3a82df1-abde-6caf-a29d-40153fde826a term=2 tally=1 +37:30.988 [INFO] TestAgent.server.raft: election won: tally=1 +37:30.988 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:27370 [Leader]" +37:30.988 [INFO] TestAgent.server: cluster leadership acquired +37:30.988 [INFO] TestAgent.server: New leader elected: payload=Node-c3a82df1-abde-6caf-a29d-40153fde826a +37:30.988 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:27370 +37:30.990 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +37:30.992 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +37:30.992 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +37:30.992 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +37:30.992 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +37:30.992 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +37:30.992 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c3a82df1-abde-6caf-a29d-40153fde826a +37:30.992 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c3a82df1-abde-6caf-a29d-40153fde826a +37:30.993 [INFO] TestAgent.server: federation state anti-entropy synced +37:31.327 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:40824 latency=1.189983ms +37:31.329 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions/705aca2a-ff07-6d11-68ef-99abd4a61655 from=127.0.0.1:40826 latency=179.44µs +37:31.329 [INFO] TestAgent: Requesting shutdown +37:31.329 [INFO] TestAgent.server: shutting down server +37:31.329 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +37:31.329 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +37:31.329 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +37:31.329 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +37:31.329 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +37:31.330 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +37:31.330 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +37:31.330 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +37:31.330 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +37:31.330 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +37:31.330 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +37:31.330 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:37:31.330Z [INFO] agent.router.manager: shutting down +2021-08-13T15:37:31.330Z [INFO] agent.router.manager: shutting down +37:31.330 [INFO] TestAgent: consul server down +37:31.330 [INFO] TestAgent: shutdown complete +37:31.330 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27365 network=tcp +37:31.330 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27365 network=udp +37:31.330 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:27366 network=tcp +37:31.830 [INFO] TestAgent: Waiting for endpoints to shut down +37:31.830 [INFO] TestAgent: Endpoints down +--- PASS: TestCommand_verticalBar (1.01s) PASS -ok github.com/hashicorp/consul/command/intention/get 1.326s +ok github.com/hashicorp/consul/command/intention/get 1.068s === RUN TestCommand_noTabs === PAUSE TestCommand_noTabs === RUN TestCommand_Validation @@ -86058,137 +85926,139 @@ [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestCommand_matchDst === CONT TestCommand_Validation -=== RUN TestCommand_Validation/both_source_and_dest === RUN TestCommand_Validation/0_args -=== RUN TestCommand_Validation/3_args [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range ---- PASS: TestCommand_Validation (0.00s) - --- PASS: TestCommand_Validation/both_source_and_dest (0.00s) +=== RUN TestCommand_Validation/3_args +=== RUN TestCommand_Validation/both_source_and_dest +--- PASS: TestCommand_Validation (0.02s) --- PASS: TestCommand_Validation/0_args (0.00s) --- PASS: TestCommand_Validation/3_args (0.00s) -2022-09-15T21:45:05.689Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:05.713Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:05.718 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b10664d4-cb31-31af-cc80-ce7b8c09384f Address:127.0.0.1:24307}]" -45:05.719 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b10664d4-cb31-31af-cc80-ce7b8c09384f.dc1 127.0.0.1 -45:05.719 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24307 [Follower]" leader= -45:05.720 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b10664d4-cb31-31af-cc80-ce7b8c09384f 127.0.0.1 -2022-09-15T21:45:05.720Z [INFO] agent.router: Initializing LAN area manager -45:05.721 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24302 network=udp -45:05.722 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b10664d4-cb31-31af-cc80-ce7b8c09384f.dc1 area=wan -45:05.722 [INFO] TestAgent.server: Adding LAN server: server="Node-b10664d4-cb31-31af-cc80-ce7b8c09384f (Addr: tcp/127.0.0.1:24307) (DC: dc1)" -45:05.723 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24302 network=tcp -45:05.725 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24303 network=tcp -45:05.726 [INFO] TestAgent: started state syncer -45:05.734 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24308 network=tcp -45:05.767 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:05.767 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24307 [Candidate]" term=2 -45:05.767 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:05.767 [DEBUG] TestAgent.server.raft: vote granted: from=b10664d4-cb31-31af-cc80-ce7b8c09384f term=2 tally=1 -45:05.767 [INFO] TestAgent.server.raft: election won: tally=1 -45:05.767 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24307 [Leader]" -45:05.768 [INFO] TestAgent.server: cluster leadership acquired -45:05.768 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24307 -45:05.769 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:05.770 [INFO] TestAgent.server: New leader elected: payload=Node-b10664d4-cb31-31af-cc80-ce7b8c09384f -45:05.817 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:05.817 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:05.817 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:05.817 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:05.817 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:05.817 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b10664d4-cb31-31af-cc80-ce7b8c09384f -45:05.817 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b10664d4-cb31-31af-cc80-ce7b8c09384f -45:05.819 [INFO] TestAgent.server: federation state anti-entropy synced -45:05.858 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:53970 latency=1.40082ms -45:05.860 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:53970 latency=952.26µs -45:05.861 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:53970 latency=797.16µs -45:05.866 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions/match?by=destination&name=db from=127.0.0.1:53972 latency=2.80276ms -45:05.870 [INFO] TestAgent: Requesting shutdown -45:05.871 [INFO] TestAgent.server: shutting down server -45:05.871 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:05.871 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:05.871 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:05.871 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:05.871 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:05.872 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:05.872 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:05.872 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -45:05.872 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:05.872 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:05.872 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:05.872 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:05.872Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:05.872Z [INFO] agent.router.manager: shutting down -45:05.872 [INFO] TestAgent: consul server down -45:05.872 [INFO] TestAgent: shutdown complete -45:05.872 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24302 network=tcp -45:05.872 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24302 network=udp -45:05.873 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24303 network=tcp -45:06.374 [INFO] TestAgent: Waiting for endpoints to shut down -45:06.374 [INFO] TestAgent: Endpoints down ---- PASS: TestCommand_matchDst (0.75s) -2022-09-15T21:45:05.690Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:05.716Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:05.719 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5e8f1c5a-f2d7-6fd9-7973-259d512266c0 Address:127.0.0.1:24300}]" -45:05.720 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24300 [Follower]" leader= -45:05.722 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5e8f1c5a-f2d7-6fd9-7973-259d512266c0.dc1 127.0.0.1 -45:05.725 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5e8f1c5a-f2d7-6fd9-7973-259d512266c0 127.0.0.1 -2022-09-15T21:45:05.725Z [INFO] agent.router: Initializing LAN area manager -45:05.725 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24295 network=udp -45:05.734 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5e8f1c5a-f2d7-6fd9-7973-259d512266c0.dc1 area=wan -45:05.734 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24295 network=tcp -45:05.734 [INFO] TestAgent.server: Adding LAN server: server="Node-5e8f1c5a-f2d7-6fd9-7973-259d512266c0 (Addr: tcp/127.0.0.1:24300) (DC: dc1)" -45:05.735 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24296 network=tcp -45:05.735 [INFO] TestAgent: started state syncer -45:05.735 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24301 network=tcp -45:05.770 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:05.770 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24300 [Candidate]" term=2 -45:05.770 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:05.770 [DEBUG] TestAgent.server.raft: vote granted: from=5e8f1c5a-f2d7-6fd9-7973-259d512266c0 term=2 tally=1 -45:05.770 [INFO] TestAgent.server.raft: election won: tally=1 -45:05.770 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24300 [Leader]" -45:05.770 [INFO] TestAgent.server: cluster leadership acquired -45:05.770 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24300 -45:05.772 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:05.772 [INFO] TestAgent.server: New leader elected: payload=Node-5e8f1c5a-f2d7-6fd9-7973-259d512266c0 -45:05.815 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:05.815 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:05.815 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:05.815 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:05.815 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:05.815 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5e8f1c5a-f2d7-6fd9-7973-259d512266c0 -45:05.815 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5e8f1c5a-f2d7-6fd9-7973-259d512266c0 -45:05.817 [INFO] TestAgent.server: federation state anti-entropy synced -45:05.896 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:55440 latency=1.50966ms -45:05.903 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:55440 latency=965.98µs -45:05.905 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:55440 latency=914.66µs -45:05.915 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions/match?by=source&name=foo from=127.0.0.1:55442 latency=492.18µs -45:05.916 [INFO] TestAgent: Requesting shutdown -45:05.916 [INFO] TestAgent.server: shutting down server -45:05.916 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:05.916 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:05.916 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:05.916 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:05.916 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:05.916 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:05.916 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:05.916 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -45:05.916 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:05.916 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:05.916 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:05.916 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:05.917Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:05.917Z [INFO] agent.router.manager: shutting down -45:05.917 [INFO] TestAgent: consul server down -45:05.917 [INFO] TestAgent: shutdown complete -45:05.917 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24295 network=tcp -45:05.917 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24295 network=udp -45:05.917 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24296 network=tcp -45:06.417 [INFO] TestAgent: Waiting for endpoints to shut down -45:06.417 [INFO] TestAgent: Endpoints down ---- PASS: TestCommand_matchSource (0.80s) + --- PASS: TestCommand_Validation/both_source_and_dest (0.00s) +2021-08-13T15:37:36.611Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:37:36.626Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +37:36.633 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0c5912fb-0fd7-cbfb-769d-e96893bd3bd2 Address:127.0.0.1:26349}]" +37:36.636 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0c5912fb-0fd7-cbfb-769d-e96893bd3bd2.dc1 127.0.0.1 +37:36.637 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:26349 [Follower]" leader= +37:36.637 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0c5912fb-0fd7-cbfb-769d-e96893bd3bd2 127.0.0.1 +2021-08-13T15:37:36.637Z [INFO] agent.router: Initializing LAN area manager +37:36.637 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0c5912fb-0fd7-cbfb-769d-e96893bd3bd2.dc1 area=wan +37:36.637 [INFO] TestAgent.server: Adding LAN server: server="Node-0c5912fb-0fd7-cbfb-769d-e96893bd3bd2 (Addr: tcp/127.0.0.1:26349) (DC: dc1)" +37:36.637 [INFO] TestAgent: Started DNS server: address=127.0.0.1:26344 network=tcp +37:36.637 [INFO] TestAgent: Started DNS server: address=127.0.0.1:26344 network=udp +37:36.638 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:26345 network=tcp +37:36.638 [INFO] TestAgent: started state syncer +37:36.638 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:26350 network=tcp +37:36.694 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +37:36.695 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:26349 [Candidate]" term=2 +37:36.695 [DEBUG] TestAgent.server.raft: votes: needed=1 +37:36.695 [DEBUG] TestAgent.server.raft: vote granted: from=0c5912fb-0fd7-cbfb-769d-e96893bd3bd2 term=2 tally=1 +37:36.695 [INFO] TestAgent.server.raft: election won: tally=1 +37:36.695 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:26349 [Leader]" +37:36.695 [INFO] TestAgent.server: cluster leadership acquired +37:36.695 [INFO] TestAgent.server: New leader elected: payload=Node-0c5912fb-0fd7-cbfb-769d-e96893bd3bd2 +37:36.695 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:26349 +37:36.697 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +37:36.715 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:45576 latency=2.941827ms +37:36.717 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:45576 latency=1.069603ms +37:36.722 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:45576 latency=3.758369ms +37:36.722 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +37:36.722 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +37:36.722 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +37:36.722 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +37:36.722 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +37:36.723 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0c5912fb-0fd7-cbfb-769d-e96893bd3bd2 +37:36.723 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0c5912fb-0fd7-cbfb-769d-e96893bd3bd2 +37:36.723 [INFO] TestAgent.server: federation state anti-entropy synced +37:36.724 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions/match?by=destination&name=db from=127.0.0.1:45578 latency=558.501µs +37:36.725 [INFO] TestAgent: Requesting shutdown +37:36.725 [INFO] TestAgent.server: shutting down server +37:36.725 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +37:36.725 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +37:36.725 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +37:36.725 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +37:36.725 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +37:36.725 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +37:36.725 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +37:36.725 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +37:36.725 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +37:36.725 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +37:36.725 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +37:36.725 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:37:36.726Z [INFO] agent.router.manager: shutting down +2021-08-13T15:37:36.726Z [INFO] agent.router.manager: shutting down +37:36.726 [INFO] TestAgent: consul server down +37:36.726 [INFO] TestAgent: shutdown complete +37:36.726 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:26344 network=tcp +37:36.726 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:26344 network=udp +37:36.726 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:26345 network=tcp +37:37.226 [INFO] TestAgent: Waiting for endpoints to shut down +37:37.226 [INFO] TestAgent: Endpoints down +--- PASS: TestCommand_matchDst (0.69s) +2021-08-13T15:37:36.608Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:37:36.624Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +37:36.632 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f425c7b2-4a15-7e12-3c95-38202456cdba Address:127.0.0.1:26342}]" +37:36.635 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f425c7b2-4a15-7e12-3c95-38202456cdba.dc1 127.0.0.1 +37:36.636 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f425c7b2-4a15-7e12-3c95-38202456cdba 127.0.0.1 +2021-08-13T15:37:36.636Z [INFO] agent.router: Initializing LAN area manager +37:36.637 [INFO] TestAgent: Started DNS server: address=127.0.0.1:26337 network=udp +37:36.637 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f425c7b2-4a15-7e12-3c95-38202456cdba.dc1 area=wan +37:36.637 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:26342 [Follower]" leader= +37:36.637 [INFO] TestAgent: Started DNS server: address=127.0.0.1:26337 network=tcp +37:36.638 [INFO] TestAgent.server: Adding LAN server: server="Node-f425c7b2-4a15-7e12-3c95-38202456cdba (Addr: tcp/127.0.0.1:26342) (DC: dc1)" +37:36.638 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:26338 network=tcp +37:36.638 [INFO] TestAgent: started state syncer +37:36.638 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:26343 network=tcp +37:36.703 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +37:36.703 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:26342 [Candidate]" term=2 +37:36.704 [DEBUG] TestAgent.server.raft: votes: needed=1 +37:36.704 [DEBUG] TestAgent.server.raft: vote granted: from=f425c7b2-4a15-7e12-3c95-38202456cdba term=2 tally=1 +37:36.704 [INFO] TestAgent.server.raft: election won: tally=1 +37:36.704 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:26342 [Leader]" +37:36.704 [INFO] TestAgent.server: cluster leadership acquired +37:36.704 [INFO] TestAgent.server: New leader elected: payload=Node-f425c7b2-4a15-7e12-3c95-38202456cdba +37:36.704 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:26342 +37:36.705 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +37:36.723 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +37:36.723 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +37:36.723 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +37:36.723 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +37:36.723 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +37:36.724 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f425c7b2-4a15-7e12-3c95-38202456cdba +37:36.724 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f425c7b2-4a15-7e12-3c95-38202456cdba +37:36.724 [INFO] TestAgent.server: federation state anti-entropy synced +37:36.914 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +37:36.915 [INFO] TestAgent: Synced node info +37:36.915 [DEBUG] TestAgent: Node info in sync +37:37.008 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:34082 latency=1.074202ms +37:37.009 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:34082 latency=816.762µs +37:37.011 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:34082 latency=838.602µs +37:37.012 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions/match?by=source&name=foo from=127.0.0.1:34084 latency=223.741µs +37:37.013 [INFO] TestAgent: Requesting shutdown +37:37.013 [INFO] TestAgent.server: shutting down server +37:37.013 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +37:37.013 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +37:37.013 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +37:37.013 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +37:37.013 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +37:37.013 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +37:37.013 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +37:37.013 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +37:37.013 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +37:37.013 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +37:37.013 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:37:37.013Z [INFO] agent.router.manager: shutting down +2021-08-13T15:37:37.013Z [INFO] agent.router.manager: shutting down +37:37.013 [INFO] TestAgent: consul server down +37:37.013 [INFO] TestAgent: shutdown complete +37:37.013 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:26337 network=tcp +37:37.014 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:26337 network=udp +37:37.014 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:26338 network=tcp +37:37.514 [INFO] TestAgent: Waiting for endpoints to shut down +37:37.514 [INFO] TestAgent: Endpoints down +--- PASS: TestCommand_matchSource (0.98s) PASS -ok github.com/hashicorp/consul/command/intention/match 0.917s +ok github.com/hashicorp/consul/command/intention/match 1.060s === RUN TestJoinCommand_noTabs === PAUSE TestJoinCommand_noTabs === RUN TestJoinCommandJoin_lan @@ -86198,276 +86068,276 @@ === RUN TestJoinCommand_noAddrs === PAUSE TestJoinCommand_noAddrs === CONT TestJoinCommand_noTabs ---- PASS: TestJoinCommand_noTabs (0.00s) === CONT TestJoinCommand_noAddrs === CONT TestJoinCommand_wan ---- PASS: TestJoinCommand_noAddrs (0.00s) === CONT TestJoinCommandJoin_lan [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestJoinCommand_noTabs (0.00s) +--- PASS: TestJoinCommand_noAddrs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:45:14.131Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:14.149Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:14.152 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d72d5423-bdd3-dc8d-6534-c3237881ef08 Address:127.0.0.1:30440}]" -45:14.152 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30440 [Follower]" leader= -45:14.153 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d72d5423-bdd3-dc8d-6534-c3237881ef08.dc1 127.0.0.1 -45:14.155 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d72d5423-bdd3-dc8d-6534-c3237881ef08 127.0.0.1 -2022-09-15T21:45:14.155Z [INFO] agent.router: Initializing LAN area manager -45:14.155 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30435 network=udp -45:14.155 [INFO] TestAgent.server: Adding LAN server: server="Node-d72d5423-bdd3-dc8d-6534-c3237881ef08 (Addr: tcp/127.0.0.1:30440) (DC: dc1)" -45:14.156 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d72d5423-bdd3-dc8d-6534-c3237881ef08.dc1 area=wan -45:14.156 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30435 network=tcp -45:14.157 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30436 network=tcp -45:14.157 [INFO] TestAgent: started state syncer -45:14.157 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30441 network=tcp -45:14.203 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:14.203 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30440 [Candidate]" term=2 -45:14.203 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:14.203 [DEBUG] TestAgent.server.raft: vote granted: from=d72d5423-bdd3-dc8d-6534-c3237881ef08 term=2 tally=1 -45:14.203 [INFO] TestAgent.server.raft: election won: tally=1 -45:14.203 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30440 [Leader]" -45:14.203 [INFO] TestAgent.server: cluster leadership acquired -45:14.204 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:30440 -45:14.205 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:14.206 [INFO] TestAgent.server: New leader elected: payload=Node-d72d5423-bdd3-dc8d-6534-c3237881ef08 -45:14.208 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:14.208 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:14.208 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:14.208 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:14.208 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:14.208 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d72d5423-bdd3-dc8d-6534-c3237881ef08 -45:14.208 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d72d5423-bdd3-dc8d-6534-c3237881ef08 -45:14.210 [INFO] TestAgent.server: federation state anti-entropy synced -45:14.441 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:43270 -45:14.443 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d9ed875a-2d25-5fb2-4e71-6fe24a49380f 127.0.0.1 -45:14.443 [INFO] TestAgent.server: Adding LAN server: server="Node-d9ed875a-2d25-5fb2-4e71-6fe24a49380f (Addr: tcp/127.0.0.1:30433) (DC: dc1)" -45:14.443 [INFO] TestAgent.server: New leader elected: payload=Node-d9ed875a-2d25-5fb2-4e71-6fe24a49380f -45:14.443 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-d9ed875a-2d25-5fb2-4e71-6fe24a49380f other=Node-d72d5423-bdd3-dc8d-6534-c3237881ef08 -45:14.443 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d9ed875a-2d25-5fb2-4e71-6fe24a49380f -45:14.450 [INFO] TestAgent: Requesting shutdown -45:14.452 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: Node-d9ed875a-2d25-5fb2-4e71-6fe24a49380f.dc1 127.0.0.1:30432 -45:14.466 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:39298 -45:14.466 [INFO] TestAgent.server: shutting down server -45:14.466 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:14.466 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:14.466 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:14.466 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:14.466 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:14.467 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:14.467 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:14.467 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -45:14.467 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:14.467 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:14.467 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:14.467 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:14.467Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:14.467Z [INFO] agent.router.manager: shutting down -45:14.470 [INFO] TestAgent: consul server down -45:14.470 [INFO] TestAgent: shutdown complete -45:14.470 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30435 network=tcp -45:14.470 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30435 network=udp -45:14.470 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30436 network=tcp -45:14.475 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d9ed875a-2d25-5fb2-4e71-6fe24a49380f.dc1 127.0.0.1 -45:14.479 [DEBUG] TestAgent.server: Successfully performed flood-join for server at address: server=Node-d9ed875a-2d25-5fb2-4e71-6fe24a49380f.dc1 address=127.0.0.1:30432 -45:14.970 [INFO] TestAgent: Waiting for endpoints to shut down -45:14.970 [INFO] TestAgent: Endpoints down -2022-09-15T21:45:13.984Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:14.007Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:14.024 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d9ed875a-2d25-5fb2-4e71-6fe24a49380f Address:127.0.0.1:30433}]" -45:14.025 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30433 [Follower]" leader= -45:14.026 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d9ed875a-2d25-5fb2-4e71-6fe24a49380f.dc1 127.0.0.1 -45:14.027 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d9ed875a-2d25-5fb2-4e71-6fe24a49380f 127.0.0.1 -2022-09-15T21:45:14.027Z [INFO] agent.router: Initializing LAN area manager -45:14.027 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30428 network=udp -45:14.027 [INFO] TestAgent.server: Adding LAN server: server="Node-d9ed875a-2d25-5fb2-4e71-6fe24a49380f (Addr: tcp/127.0.0.1:30433) (DC: dc1)" -45:14.027 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d9ed875a-2d25-5fb2-4e71-6fe24a49380f.dc1 area=wan -45:14.028 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30428 network=tcp -45:14.029 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30429 network=tcp -45:14.029 [INFO] TestAgent: started state syncer -45:14.029 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30434 network=tcp -45:14.090 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:14.090 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30433 [Candidate]" term=2 -45:14.090 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:14.090 [DEBUG] TestAgent.server.raft: vote granted: from=d9ed875a-2d25-5fb2-4e71-6fe24a49380f term=2 tally=1 -45:14.090 [INFO] TestAgent.server.raft: election won: tally=1 -45:14.090 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30433 [Leader]" -45:14.090 [INFO] TestAgent.server: cluster leadership acquired -45:14.091 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:30433 -45:14.091 [INFO] TestAgent.server: New leader elected: payload=Node-d9ed875a-2d25-5fb2-4e71-6fe24a49380f -45:14.092 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:14.096 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:14.096 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:14.096 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:14.096 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:14.096 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:14.096 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d9ed875a-2d25-5fb2-4e71-6fe24a49380f -45:14.096 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d9ed875a-2d25-5fb2-4e71-6fe24a49380f -45:14.097 [INFO] TestAgent.server: federation state anti-entropy synced -45:14.196 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -45:14.197 [INFO] TestAgent: Synced node info -45:14.440 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:30438] -45:14.441 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:30438 -45:14.446 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:33618 -45:14.447 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d72d5423-bdd3-dc8d-6534-c3237881ef08 127.0.0.1 -45:14.447 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 -45:14.447 [DEBUG] TestAgent: systemd notify failed: error="No socket" -45:14.447 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/join/127.0.0.1:30438 from=127.0.0.1:48584 latency=6.4906ms -45:14.447 [INFO] TestAgent.server: Adding LAN server: server="Node-d72d5423-bdd3-dc8d-6534-c3237881ef08 (Addr: tcp/127.0.0.1:30440) (DC: dc1)" -45:14.447 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-d72d5423-bdd3-dc8d-6534-c3237881ef08 other=Node-d9ed875a-2d25-5fb2-4e71-6fe24a49380f -45:14.447 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d72d5423-bdd3-dc8d-6534-c3237881ef08 -45:14.467 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: Node-d72d5423-bdd3-dc8d-6534-c3237881ef08.dc1 127.0.0.1:30439 -45:14.477 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d72d5423-bdd3-dc8d-6534-c3237881ef08.dc1 127.0.0.1 -45:14.477 [DEBUG] TestAgent.server: Successfully performed flood-join for server at address: server=Node-d72d5423-bdd3-dc8d-6534-c3237881ef08.dc1 address=127.0.0.1:30439 -45:14.478 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d72d5423-bdd3-dc8d-6534-c3237881ef08.dc1 area=wan -45:14.970 [INFO] TestAgent: Requesting shutdown -45:14.970 [INFO] TestAgent.server: shutting down server -45:14.970 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:14.970 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:14.970 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:14.970 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:14.970 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:14.971 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:14.971 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:14.971 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:14.971 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:14.971 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:14.971 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:14.971Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:14.971Z [INFO] agent.router.manager: shutting down -45:14.971 [INFO] TestAgent: consul server down -45:14.971 [INFO] TestAgent: shutdown complete -45:14.971 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30428 network=tcp -45:14.971 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30428 network=udp -45:14.971 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30429 network=tcp -45:15.474 [INFO] TestAgent: Waiting for endpoints to shut down -45:15.474 [INFO] TestAgent: Endpoints down ---- PASS: TestJoinCommandJoin_lan (1.56s) -2022-09-15T21:45:14.489Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:14.507Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:14.511 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b3235805-9887-554c-abcf-c6b58853601c Address:127.0.0.1:30447}]" -45:14.512 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b3235805-9887-554c-abcf-c6b58853601c.dc1 127.0.0.1 -45:14.513 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b3235805-9887-554c-abcf-c6b58853601c 127.0.0.1 -2022-09-15T21:45:14.513Z [INFO] agent.router: Initializing LAN area manager -45:14.513 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30442 network=udp -45:14.513 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30447 [Follower]" leader= -45:14.514 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b3235805-9887-554c-abcf-c6b58853601c.dc1 area=wan -45:14.514 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30442 network=tcp -45:14.515 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30443 network=tcp -45:14.515 [INFO] TestAgent: started state syncer -45:14.515 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30448 network=tcp -45:14.515 [INFO] TestAgent.server: Adding LAN server: server="Node-b3235805-9887-554c-abcf-c6b58853601c (Addr: tcp/127.0.0.1:30447) (DC: dc1)" -45:14.562 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:14.562 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30447 [Candidate]" term=2 -45:14.563 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:14.563 [DEBUG] TestAgent.server.raft: vote granted: from=b3235805-9887-554c-abcf-c6b58853601c term=2 tally=1 -45:14.563 [INFO] TestAgent.server.raft: election won: tally=1 -45:14.563 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30447 [Leader]" -45:14.563 [INFO] TestAgent.server: cluster leadership acquired -45:14.563 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:30447 -45:14.565 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:14.566 [INFO] TestAgent.server: New leader elected: payload=Node-b3235805-9887-554c-abcf-c6b58853601c -45:14.567 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:14.567 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:14.567 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:14.567 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:14.567 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:14.568 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b3235805-9887-554c-abcf-c6b58853601c -45:14.568 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b3235805-9887-554c-abcf-c6b58853601c -45:14.569 [INFO] TestAgent.server: federation state anti-entropy synced -45:14.716 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:43868 -45:14.717 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-80aa4fcd-7ddc-ee38-3802-5fe889871ce8.dc1 127.0.0.1 -45:14.717 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-80aa4fcd-7ddc-ee38-3802-5fe889871ce8.dc1 area=wan -45:14.720 [INFO] TestAgent: Requesting shutdown -45:14.720 [INFO] TestAgent.server: shutting down server -45:14.720 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:14.720 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:14.720 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:14.720 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:14.721 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:14.721 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:14.721 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:14.721 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -45:14.721 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -45:14.721 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:14.721 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:14.721 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -2022-09-15T21:45:14.721Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:14.721Z [INFO] agent.router.manager: shutting down -45:14.721 [INFO] TestAgent: consul server down -45:14.721 [INFO] TestAgent: shutdown complete -45:14.721 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30442 network=tcp -45:14.721 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30442 network=udp -45:14.721 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30443 network=tcp -45:15.222 [INFO] TestAgent: Waiting for endpoints to shut down -45:15.222 [INFO] TestAgent: Endpoints down -2022-09-15T21:45:13.983Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:14.001Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:14.024 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:80aa4fcd-7ddc-ee38-3802-5fe889871ce8 Address:127.0.0.1:30426}]" -45:14.025 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30426 [Follower]" leader= -45:14.027 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-80aa4fcd-7ddc-ee38-3802-5fe889871ce8.dc1 127.0.0.1 -45:14.028 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-80aa4fcd-7ddc-ee38-3802-5fe889871ce8 127.0.0.1 -2022-09-15T21:45:14.028Z [INFO] agent.router: Initializing LAN area manager -45:14.028 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=udp -45:14.028 [INFO] TestAgent.server: Adding LAN server: server="Node-80aa4fcd-7ddc-ee38-3802-5fe889871ce8 (Addr: tcp/127.0.0.1:30426) (DC: dc1)" -45:14.028 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-80aa4fcd-7ddc-ee38-3802-5fe889871ce8.dc1 area=wan -45:14.029 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=tcp -45:14.030 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30422 network=tcp -45:14.030 [INFO] TestAgent: started state syncer -45:14.030 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30427 network=tcp -45:14.068 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:14.068 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30426 [Candidate]" term=2 -45:14.069 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:14.069 [DEBUG] TestAgent.server.raft: vote granted: from=80aa4fcd-7ddc-ee38-3802-5fe889871ce8 term=2 tally=1 -45:14.069 [INFO] TestAgent.server.raft: election won: tally=1 -45:14.069 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30426 [Leader]" -45:14.069 [INFO] TestAgent.server: cluster leadership acquired -45:14.069 [INFO] TestAgent.server: New leader elected: payload=Node-80aa4fcd-7ddc-ee38-3802-5fe889871ce8 -45:14.069 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:30426 -45:14.071 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:14.096 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:14.096 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:14.096 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:14.096 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:14.096 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:14.096 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-80aa4fcd-7ddc-ee38-3802-5fe889871ce8 -45:14.096 [INFO] TestAgent.server: member joined, marking health alive: member=Node-80aa4fcd-7ddc-ee38-3802-5fe889871ce8 -45:14.097 [INFO] TestAgent.server: federation state anti-entropy synced -45:14.215 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -45:14.216 [INFO] TestAgent: Synced node info -45:14.220 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -45:14.220 [DEBUG] TestAgent: Node info in sync -45:14.220 [DEBUG] TestAgent: Node info in sync -45:14.715 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:30446] -45:14.716 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:30446 -45:14.720 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b3235805-9887-554c-abcf-c6b58853601c.dc1 127.0.0.1 -45:14.720 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 -45:14.720 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/join/127.0.0.1:30446?wan=1 from=127.0.0.1:36348 latency=4.97228ms -45:14.720 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b3235805-9887-554c-abcf-c6b58853601c.dc1 area=wan -45:15.222 [INFO] TestAgent: Requesting shutdown -45:15.222 [INFO] TestAgent.server: shutting down server -45:15.222 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:15.222 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:15.222 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:15.222 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:15.222 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:15.222 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:15.222 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:15.222 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:15.222 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:15.222 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:15.222 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:15.222Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:15.222Z [INFO] agent.router.manager: shutting down -45:15.222 [INFO] TestAgent: consul server down -45:15.222 [INFO] TestAgent: shutdown complete -45:15.222 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=tcp -45:15.223 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=udp -45:15.223 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30422 network=tcp -45:15.723 [INFO] TestAgent: Waiting for endpoints to shut down -45:15.723 [INFO] TestAgent: Endpoints down ---- PASS: TestJoinCommand_wan (1.81s) +2021-08-13T15:37:31.503Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:37:31.522Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +37:31.525 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:53f8caf9-ba28-7587-6c5a-955bb351e422 Address:127.0.0.1:12062}]" +37:31.525 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:12062 [Follower]" leader= +37:31.526 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-53f8caf9-ba28-7587-6c5a-955bb351e422.dc1 127.0.0.1 +37:31.527 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-53f8caf9-ba28-7587-6c5a-955bb351e422 127.0.0.1 +2021-08-13T15:37:31.527Z [INFO] agent.router: Initializing LAN area manager +37:31.527 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-53f8caf9-ba28-7587-6c5a-955bb351e422.dc1 area=wan +37:31.527 [INFO] TestAgent.server: Adding LAN server: server="Node-53f8caf9-ba28-7587-6c5a-955bb351e422 (Addr: tcp/127.0.0.1:12062) (DC: dc1)" +37:31.527 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12057 network=tcp +37:31.527 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12057 network=udp +37:31.528 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:12058 network=tcp +37:31.528 [INFO] TestAgent: started state syncer +37:31.528 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:12063 network=tcp +37:31.566 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +37:31.566 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:12062 [Candidate]" term=2 +37:31.566 [DEBUG] TestAgent.server.raft: votes: needed=1 +37:31.566 [DEBUG] TestAgent.server.raft: vote granted: from=53f8caf9-ba28-7587-6c5a-955bb351e422 term=2 tally=1 +37:31.566 [INFO] TestAgent.server.raft: election won: tally=1 +37:31.566 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:12062 [Leader]" +37:31.566 [INFO] TestAgent.server: cluster leadership acquired +37:31.566 [INFO] TestAgent.server: New leader elected: payload=Node-53f8caf9-ba28-7587-6c5a-955bb351e422 +37:31.566 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:12062 +37:31.568 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +37:31.570 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +37:31.570 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +37:31.570 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +37:31.570 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +37:31.570 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +37:31.571 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-53f8caf9-ba28-7587-6c5a-955bb351e422 +37:31.571 [INFO] TestAgent.server: member joined, marking health alive: member=Node-53f8caf9-ba28-7587-6c5a-955bb351e422 +37:31.572 [INFO] TestAgent.server: federation state anti-entropy synced +37:31.636 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +37:31.636 [INFO] TestAgent: Synced node info +37:31.636 [DEBUG] TestAgent: Node info in sync +37:31.658 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:46810 +37:31.659 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2379b556-3585-2bf1-6ef6-64109c8105a7.dc1 127.0.0.1 +37:31.659 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2379b556-3585-2bf1-6ef6-64109c8105a7.dc1 area=wan +37:31.659 [INFO] TestAgent: Requesting shutdown +37:31.660 [INFO] TestAgent.server: shutting down server +37:31.660 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +37:31.660 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +37:31.660 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +37:31.660 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +37:31.660 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +37:31.660 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +37:31.660 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +37:31.660 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +37:31.660 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +37:31.660 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +37:31.660 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:37:31.660Z [INFO] agent.router.manager: shutting down +2021-08-13T15:37:31.660Z [INFO] agent.router.manager: shutting down +37:31.660 [INFO] TestAgent: consul server down +37:31.660 [INFO] TestAgent: shutdown complete +37:31.660 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12057 network=tcp +37:31.660 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12057 network=udp +37:31.660 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:12058 network=tcp +37:32.161 [INFO] TestAgent: Waiting for endpoints to shut down +37:32.161 [INFO] TestAgent: Endpoints down +2021-08-13T15:37:31.291Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:37:31.309Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +37:31.313 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2379b556-3585-2bf1-6ef6-64109c8105a7 Address:127.0.0.1:12055}]" +37:31.313 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:12055 [Follower]" leader= +37:31.314 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2379b556-3585-2bf1-6ef6-64109c8105a7.dc1 127.0.0.1 +37:31.315 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2379b556-3585-2bf1-6ef6-64109c8105a7 127.0.0.1 +2021-08-13T15:37:31.315Z [INFO] agent.router: Initializing LAN area manager +37:31.315 [INFO] TestAgent.server: Adding LAN server: server="Node-2379b556-3585-2bf1-6ef6-64109c8105a7 (Addr: tcp/127.0.0.1:12055) (DC: dc1)" +37:31.315 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2379b556-3585-2bf1-6ef6-64109c8105a7.dc1 area=wan +37:31.316 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12050 network=tcp +37:31.316 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12050 network=udp +37:31.319 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:12051 network=tcp +37:31.319 [INFO] TestAgent: started state syncer +37:31.319 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:12056 network=tcp +37:31.368 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +37:31.368 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:12055 [Candidate]" term=2 +37:31.368 [DEBUG] TestAgent.server.raft: votes: needed=1 +37:31.368 [DEBUG] TestAgent.server.raft: vote granted: from=2379b556-3585-2bf1-6ef6-64109c8105a7 term=2 tally=1 +37:31.368 [INFO] TestAgent.server.raft: election won: tally=1 +37:31.368 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:12055 [Leader]" +37:31.368 [INFO] TestAgent.server: cluster leadership acquired +37:31.369 [INFO] TestAgent.server: New leader elected: payload=Node-2379b556-3585-2bf1-6ef6-64109c8105a7 +37:31.369 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:12055 +37:31.370 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +37:31.394 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +37:31.394 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +37:31.394 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +37:31.394 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +37:31.394 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +37:31.394 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2379b556-3585-2bf1-6ef6-64109c8105a7 +37:31.394 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2379b556-3585-2bf1-6ef6-64109c8105a7 +37:31.395 [INFO] TestAgent.server: federation state anti-entropy synced +37:31.655 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +37:31.656 [INFO] TestAgent: Synced node info +37:31.658 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:12061] +37:31.658 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:12061 +37:31.659 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-53f8caf9-ba28-7587-6c5a-955bb351e422.dc1 127.0.0.1 +37:31.659 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 +37:31.659 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/join/127.0.0.1:12061?wan=1 from=127.0.0.1:59176 latency=1.691964ms +37:31.659 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-53f8caf9-ba28-7587-6c5a-955bb351e422.dc1 area=wan +37:32.161 [INFO] TestAgent: Requesting shutdown +37:32.161 [INFO] TestAgent.server: shutting down server +37:32.161 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +37:32.161 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +37:32.161 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +37:32.161 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +37:32.161 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +37:32.161 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +37:32.161 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +37:32.161 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +37:32.161 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +37:32.161 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +37:32.161 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:37:32.161Z [INFO] agent.router.manager: shutting down +2021-08-13T15:37:32.161Z [INFO] agent.router.manager: shutting down +37:32.161 [INFO] TestAgent: consul server down +37:32.161 [INFO] TestAgent: shutdown complete +37:32.161 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12050 network=tcp +37:32.161 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12050 network=udp +37:32.162 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:12051 network=tcp +37:32.662 [INFO] TestAgent: Waiting for endpoints to shut down +37:32.662 [INFO] TestAgent: Endpoints down +--- PASS: TestJoinCommand_wan (1.43s) +2021-08-13T15:37:31.577Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:37:31.597Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +37:31.600 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a261060c-4d78-147f-efe4-cc0671511dcf Address:127.0.0.1:12069}]" +37:31.600 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:12069 [Follower]" leader= +37:31.601 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a261060c-4d78-147f-efe4-cc0671511dcf.dc1 127.0.0.1 +37:31.601 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a261060c-4d78-147f-efe4-cc0671511dcf 127.0.0.1 +2021-08-13T15:37:31.602Z [INFO] agent.router: Initializing LAN area manager +37:31.602 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a261060c-4d78-147f-efe4-cc0671511dcf.dc1 area=wan +37:31.602 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12064 network=tcp +37:31.602 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12064 network=udp +37:31.602 [INFO] TestAgent.server: Adding LAN server: server="Node-a261060c-4d78-147f-efe4-cc0671511dcf (Addr: tcp/127.0.0.1:12069) (DC: dc1)" +37:31.603 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:12065 network=tcp +37:31.603 [INFO] TestAgent: started state syncer +37:31.603 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:12070 network=tcp +37:31.668 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +37:31.668 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:12069 [Candidate]" term=2 +37:31.668 [DEBUG] TestAgent.server.raft: votes: needed=1 +37:31.668 [DEBUG] TestAgent.server.raft: vote granted: from=a261060c-4d78-147f-efe4-cc0671511dcf term=2 tally=1 +37:31.668 [INFO] TestAgent.server.raft: election won: tally=1 +37:31.668 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:12069 [Leader]" +37:31.669 [INFO] TestAgent.server: cluster leadership acquired +37:31.669 [INFO] TestAgent.server: New leader elected: payload=Node-a261060c-4d78-147f-efe4-cc0671511dcf +37:31.669 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:12069 +37:31.670 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +37:31.673 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +37:31.673 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +37:31.673 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +37:31.673 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +37:31.673 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +37:31.673 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a261060c-4d78-147f-efe4-cc0671511dcf +37:31.673 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a261060c-4d78-147f-efe4-cc0671511dcf +37:31.673 [INFO] TestAgent.server: federation state anti-entropy synced +37:31.732 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +37:31.733 [INFO] TestAgent: Synced node info +37:31.801 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:58894 +37:31.803 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5f53cbc5-878f-95fa-8ec5-aac515dd47f3 127.0.0.1 +37:31.803 [INFO] TestAgent.server: Adding LAN server: server="Node-5f53cbc5-878f-95fa-8ec5-aac515dd47f3 (Addr: tcp/127.0.0.1:12048) (DC: dc1)" +37:31.803 [INFO] TestAgent.server: New leader elected: payload=Node-5f53cbc5-878f-95fa-8ec5-aac515dd47f3 +37:31.803 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-5f53cbc5-878f-95fa-8ec5-aac515dd47f3 other=Node-a261060c-4d78-147f-efe4-cc0671511dcf +37:31.803 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5f53cbc5-878f-95fa-8ec5-aac515dd47f3 +37:31.803 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: Node-5f53cbc5-878f-95fa-8ec5-aac515dd47f3.dc1 127.0.0.1:12047 +37:31.803 [INFO] TestAgent: Requesting shutdown +37:31.803 [INFO] TestAgent.server: shutting down server +37:31.803 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +37:31.803 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +37:31.803 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +37:31.803 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +37:31.803 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +37:31.804 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +37:31.804 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +37:31.804 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +37:31.804 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +37:31.804 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:45768 +37:31.804 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:37:31.804Z [INFO] agent.router.manager: shutting down +2021-08-13T15:37:31.804Z [INFO] agent.router.manager: shutting down +37:31.804 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +37:31.806 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5f53cbc5-878f-95fa-8ec5-aac515dd47f3.dc1 127.0.0.1 +37:31.806 [DEBUG] TestAgent.server: Successfully performed flood-join for server at address: server=Node-5f53cbc5-878f-95fa-8ec5-aac515dd47f3.dc1 address=127.0.0.1:12047 +37:31.806 [INFO] TestAgent: consul server down +37:31.806 [INFO] TestAgent: shutdown complete +37:31.806 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12064 network=tcp +37:31.806 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12064 network=udp +37:31.806 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:12065 network=tcp +37:32.306 [INFO] TestAgent: Waiting for endpoints to shut down +37:32.306 [INFO] TestAgent: Endpoints down +2021-08-13T15:37:31.290Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:37:31.309Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +37:31.313 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5f53cbc5-878f-95fa-8ec5-aac515dd47f3 Address:127.0.0.1:12048}]" +37:31.313 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:12048 [Follower]" leader= +37:31.314 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5f53cbc5-878f-95fa-8ec5-aac515dd47f3.dc1 127.0.0.1 +37:31.315 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5f53cbc5-878f-95fa-8ec5-aac515dd47f3 127.0.0.1 +2021-08-13T15:37:31.315Z [INFO] agent.router: Initializing LAN area manager +37:31.317 [INFO] TestAgent.server: Adding LAN server: server="Node-5f53cbc5-878f-95fa-8ec5-aac515dd47f3 (Addr: tcp/127.0.0.1:12048) (DC: dc1)" +37:31.317 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5f53cbc5-878f-95fa-8ec5-aac515dd47f3.dc1 area=wan +37:31.318 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12043 network=tcp +37:31.318 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12043 network=udp +37:31.319 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:12044 network=tcp +37:31.320 [INFO] TestAgent: started state syncer +37:31.320 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:12049 network=tcp +37:31.383 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +37:31.383 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:12048 [Candidate]" term=2 +37:31.383 [DEBUG] TestAgent.server.raft: votes: needed=1 +37:31.383 [DEBUG] TestAgent.server.raft: vote granted: from=5f53cbc5-878f-95fa-8ec5-aac515dd47f3 term=2 tally=1 +37:31.383 [INFO] TestAgent.server.raft: election won: tally=1 +37:31.383 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:12048 [Leader]" +37:31.383 [INFO] TestAgent.server: cluster leadership acquired +37:31.384 [INFO] TestAgent.server: New leader elected: payload=Node-5f53cbc5-878f-95fa-8ec5-aac515dd47f3 +37:31.384 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:12048 +37:31.385 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +37:31.394 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +37:31.394 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +37:31.394 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +37:31.394 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +37:31.394 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +37:31.394 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5f53cbc5-878f-95fa-8ec5-aac515dd47f3 +37:31.394 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5f53cbc5-878f-95fa-8ec5-aac515dd47f3 +37:31.395 [INFO] TestAgent.server: federation state anti-entropy synced +37:31.613 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +37:31.613 [INFO] TestAgent: Synced node info +37:31.801 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:12067] +37:31.801 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:12067 +37:31.803 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a261060c-4d78-147f-efe4-cc0671511dcf 127.0.0.1 +37:31.803 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 +37:31.803 [DEBUG] TestAgent: systemd notify failed: error="No socket" +37:31.803 [INFO] TestAgent.server: Adding LAN server: server="Node-a261060c-4d78-147f-efe4-cc0671511dcf (Addr: tcp/127.0.0.1:12069) (DC: dc1)" +37:31.803 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/join/127.0.0.1:12067 from=127.0.0.1:48238 latency=1.803624ms +37:31.803 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-a261060c-4d78-147f-efe4-cc0671511dcf other=Node-5f53cbc5-878f-95fa-8ec5-aac515dd47f3 +37:31.803 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a261060c-4d78-147f-efe4-cc0671511dcf +37:31.803 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:41598 +37:31.803 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: Node-a261060c-4d78-147f-efe4-cc0671511dcf.dc1 127.0.0.1:12068 +37:31.805 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a261060c-4d78-147f-efe4-cc0671511dcf.dc1 127.0.0.1 +37:31.806 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a261060c-4d78-147f-efe4-cc0671511dcf.dc1 area=wan +37:31.806 [DEBUG] TestAgent.server: Successfully performed flood-join for server at address: server=Node-a261060c-4d78-147f-efe4-cc0671511dcf.dc1 address=127.0.0.1:12068 +37:32.306 [INFO] TestAgent: Requesting shutdown +37:32.307 [INFO] TestAgent.server: shutting down server +37:32.307 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +37:32.307 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +37:32.307 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +37:32.307 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +37:32.307 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +37:32.307 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +37:32.307 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +37:32.307 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +37:32.307 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +37:32.307 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +37:32.307 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:37:32.307Z [INFO] agent.router.manager: shutting down +2021-08-13T15:37:32.307Z [INFO] agent.router.manager: shutting down +37:32.307 [INFO] TestAgent: consul server down +37:32.307 [INFO] TestAgent: shutdown complete +37:32.307 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12043 network=tcp +37:32.307 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12043 network=udp +37:32.307 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:12044 network=tcp +37:32.807 [INFO] TestAgent: Waiting for endpoints to shut down +37:32.807 [INFO] TestAgent: Endpoints down +--- PASS: TestJoinCommandJoin_lan (1.58s) PASS -ok github.com/hashicorp/consul/command/join 1.918s +ok github.com/hashicorp/consul/command/join 1.639s === RUN TestKeygenCommand_noTabs === PAUSE TestKeygenCommand_noTabs === RUN TestKeygenCommand @@ -86501,118 +86371,118 @@ === RUN TestKeyringCommand_numActions --- PASS: TestKeyringCommand_numActions (0.00s) === CONT TestKeyringCommand_noTabs ---- PASS: TestKeyringCommand_noTabs (0.00s) -=== CONT TestKeyringCommand_invalidRelayFactor -=== CONT TestKeyringCommand_failedConnection === CONT TestKeyringCommand_invalidLocalOnly === CONT TestKeyringCommand_help +--- PASS: TestKeyringCommand_noTabs (0.00s) === CONT TestKeyringCommand ---- PASS: TestKeyringCommand_invalidLocalOnly (0.00s) [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestKeyringCommand_help (0.00s) +=== CONT TestKeyringCommand_failedConnection +=== CONT TestKeyringCommand_invalidRelayFactor +--- PASS: TestKeyringCommand_invalidRelayFactor (0.00s) --- PASS: TestKeyringCommand_failedConnection (0.00s) ---- PASS: TestKeyringCommand_invalidRelayFactor (0.01s) ---- PASS: TestKeyringCommand_help (0.01s) +--- PASS: TestKeyringCommand_invalidLocalOnly (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:45:20.799Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:20.880Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:20.893 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e8fc4e53-7ae6-1170-fa54-db1102ed22d8 Address:127.0.0.1:28384}]" -45:20.898 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:28384 [Follower]" leader= -45:20.899 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e8fc4e53-7ae6-1170-fa54-db1102ed22d8.dc1 127.0.0.1 -45:20.900 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e8fc4e53-7ae6-1170-fa54-db1102ed22d8 127.0.0.1 -2022-09-15T21:45:20.900Z [INFO] agent.router: Initializing LAN area manager -45:20.900 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28379 network=udp -45:20.901 [INFO] TestAgent.server: Adding LAN server: server="Node-e8fc4e53-7ae6-1170-fa54-db1102ed22d8 (Addr: tcp/127.0.0.1:28384) (DC: dc1)" -45:20.901 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e8fc4e53-7ae6-1170-fa54-db1102ed22d8.dc1 area=wan -45:20.901 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28379 network=tcp -45:20.902 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:28380 network=tcp -45:20.902 [INFO] TestAgent: started state syncer -45:20.903 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:28385 network=tcp -45:20.962 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:20.962 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:28384 [Candidate]" term=2 -45:20.962 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:20.962 [DEBUG] TestAgent.server.raft: vote granted: from=e8fc4e53-7ae6-1170-fa54-db1102ed22d8 term=2 tally=1 -45:20.962 [INFO] TestAgent.server.raft: election won: tally=1 -45:20.962 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:28384 [Leader]" -45:20.962 [INFO] TestAgent.server: cluster leadership acquired -45:20.963 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:28384 -45:20.978 [INFO] TestAgent.server: New leader elected: payload=Node-e8fc4e53-7ae6-1170-fa54-db1102ed22d8 -45:20.990 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:21.053 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:21.053 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:21.053 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:21.053 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:21.053 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:21.053 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e8fc4e53-7ae6-1170-fa54-db1102ed22d8 -45:21.053 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e8fc4e53-7ae6-1170-fa54-db1102ed22d8 -45:21.070 [INFO] TestAgent.server: federation state anti-entropy synced -45:21.175 [INFO] TestAgent.server.serf.wan: serf: Received list-keys query -45:21.176 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-e8fc4e53-7ae6-1170-fa54-db1102ed22d8.dc1 -45:21.176 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query -45:21.176 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-e8fc4e53-7ae6-1170-fa54-db1102ed22d8 -45:21.177 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/keyring from=127.0.0.1:57186 latency=1.84688ms -45:21.181 [INFO] TestAgent.server.serf.wan: serf: Received install-key query -45:21.183 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-e8fc4e53-7ae6-1170-fa54-db1102ed22d8.dc1 -45:21.183 [INFO] TestAgent.server.serf.lan: serf: Received install-key query -45:21.184 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-e8fc4e53-7ae6-1170-fa54-db1102ed22d8 -45:21.184 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/operator/keyring from=127.0.0.1:57190 latency=3.89808ms -45:21.186 [INFO] TestAgent.server.serf.wan: serf: Received list-keys query -45:21.187 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-e8fc4e53-7ae6-1170-fa54-db1102ed22d8.dc1 -45:21.188 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query -45:21.188 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-e8fc4e53-7ae6-1170-fa54-db1102ed22d8 -45:21.198 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/keyring from=127.0.0.1:57192 latency=13.02706ms -45:21.205 [INFO] TestAgent.server.serf.wan: serf: Received use-key query -45:21.205 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-e8fc4e53-7ae6-1170-fa54-db1102ed22d8.dc1 -45:21.206 [INFO] TestAgent.server.serf.lan: serf: Received use-key query -45:21.210 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-e8fc4e53-7ae6-1170-fa54-db1102ed22d8 -45:21.210 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/operator/keyring from=127.0.0.1:57196 latency=5.67884ms -45:21.213 [INFO] TestAgent.server.serf.wan: serf: Received list-keys query -45:21.213 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-e8fc4e53-7ae6-1170-fa54-db1102ed22d8.dc1 -45:21.213 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query -45:21.215 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-e8fc4e53-7ae6-1170-fa54-db1102ed22d8 -45:21.215 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/keyring from=127.0.0.1:57198 latency=2.46108ms -45:21.217 [INFO] TestAgent.server.serf.wan: serf: Received remove-key query -45:21.218 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-e8fc4e53-7ae6-1170-fa54-db1102ed22d8.dc1 -45:21.218 [INFO] TestAgent.server.serf.lan: serf: Received remove-key query -45:21.218 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-e8fc4e53-7ae6-1170-fa54-db1102ed22d8 -45:21.218 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/operator/keyring from=127.0.0.1:57200 latency=1.404ms -45:21.220 [INFO] TestAgent.server.serf.wan: serf: Received list-keys query -45:21.220 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-e8fc4e53-7ae6-1170-fa54-db1102ed22d8.dc1 -45:21.220 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query -45:21.220 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-e8fc4e53-7ae6-1170-fa54-db1102ed22d8 -45:21.221 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/keyring from=127.0.0.1:57202 latency=1.13466ms -45:21.221 [INFO] TestAgent: Requesting shutdown -45:21.221 [INFO] TestAgent.server: shutting down server -45:21.221 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:21.221 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:21.221 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:21.221 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:21.221 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:21.221 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:21.221 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:21.222 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -45:21.222 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:21.222 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:21.222 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:21.222 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:21.222Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:21.222Z [INFO] agent.router.manager: shutting down -45:21.222 [INFO] TestAgent: consul server down -45:21.222 [INFO] TestAgent: shutdown complete -45:21.222 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28379 network=tcp -45:21.222 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28379 network=udp -45:21.222 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:28380 network=tcp -45:21.726 [INFO] TestAgent: Waiting for endpoints to shut down -45:21.726 [INFO] TestAgent: Endpoints down ---- PASS: TestKeyringCommand (1.11s) +2021-08-13T15:37:36.546Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:37:36.573Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +37:36.580 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:916b760e-0d1f-27df-ee47-77bac021acf2 Address:127.0.0.1:11027}]" +37:36.580 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:11027 [Follower]" leader= +37:36.581 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-916b760e-0d1f-27df-ee47-77bac021acf2.dc1 127.0.0.1 +37:36.582 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-916b760e-0d1f-27df-ee47-77bac021acf2 127.0.0.1 +2021-08-13T15:37:36.582Z [INFO] agent.router: Initializing LAN area manager +37:36.584 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11022 network=udp +37:36.584 [INFO] TestAgent.server: Adding LAN server: server="Node-916b760e-0d1f-27df-ee47-77bac021acf2 (Addr: tcp/127.0.0.1:11027) (DC: dc1)" +37:36.584 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-916b760e-0d1f-27df-ee47-77bac021acf2.dc1 area=wan +37:36.584 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11022 network=tcp +37:36.585 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:11023 network=tcp +37:36.585 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:11028 network=tcp +37:36.585 [INFO] TestAgent: started state syncer +37:36.650 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +37:36.650 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:11027 [Candidate]" term=2 +37:36.650 [DEBUG] TestAgent.server.raft: votes: needed=1 +37:36.650 [DEBUG] TestAgent.server.raft: vote granted: from=916b760e-0d1f-27df-ee47-77bac021acf2 term=2 tally=1 +37:36.650 [INFO] TestAgent.server.raft: election won: tally=1 +37:36.650 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:11027 [Leader]" +37:36.650 [INFO] TestAgent.server: cluster leadership acquired +37:36.651 [INFO] TestAgent.server: New leader elected: payload=Node-916b760e-0d1f-27df-ee47-77bac021acf2 +37:36.651 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:11027 +37:36.652 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +37:36.676 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +37:36.676 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +37:36.676 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +37:36.676 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +37:36.676 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +37:36.676 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-916b760e-0d1f-27df-ee47-77bac021acf2 +37:36.676 [INFO] TestAgent.server: member joined, marking health alive: member=Node-916b760e-0d1f-27df-ee47-77bac021acf2 +37:36.677 [INFO] TestAgent.server: federation state anti-entropy synced +37:36.680 [INFO] TestAgent.server.serf.wan: serf: Received list-keys query +37:36.680 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-916b760e-0d1f-27df-ee47-77bac021acf2.dc1 +37:36.680 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query +37:36.681 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-916b760e-0d1f-27df-ee47-77bac021acf2 +37:36.681 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/keyring from=127.0.0.1:49710 latency=1.632564ms +37:36.683 [INFO] TestAgent.server.serf.wan: serf: Received install-key query +37:36.684 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-916b760e-0d1f-27df-ee47-77bac021acf2.dc1 +37:36.684 [INFO] TestAgent.server.serf.lan: serf: Received install-key query +37:36.685 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-916b760e-0d1f-27df-ee47-77bac021acf2 +37:36.685 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/operator/keyring from=127.0.0.1:49712 latency=1.449244ms +37:36.687 [INFO] TestAgent.server.serf.wan: serf: Received list-keys query +37:36.690 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-916b760e-0d1f-27df-ee47-77bac021acf2.dc1 +37:36.690 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query +37:36.691 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-916b760e-0d1f-27df-ee47-77bac021acf2 +37:36.691 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/keyring from=127.0.0.1:49714 latency=4.651351ms +37:36.694 [INFO] TestAgent.server.serf.wan: serf: Received use-key query +37:36.694 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-916b760e-0d1f-27df-ee47-77bac021acf2.dc1 +37:36.694 [INFO] TestAgent.server.serf.lan: serf: Received use-key query +37:36.695 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-916b760e-0d1f-27df-ee47-77bac021acf2 +37:36.695 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/operator/keyring from=127.0.0.1:49716 latency=1.412323ms +37:36.696 [INFO] TestAgent.server.serf.wan: serf: Received list-keys query +37:36.697 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-916b760e-0d1f-27df-ee47-77bac021acf2.dc1 +37:36.697 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query +37:36.697 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-916b760e-0d1f-27df-ee47-77bac021acf2 +37:36.697 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/keyring from=127.0.0.1:49718 latency=1.290464ms +37:36.699 [INFO] TestAgent.server.serf.wan: serf: Received remove-key query +37:36.700 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-916b760e-0d1f-27df-ee47-77bac021acf2.dc1 +37:36.700 [INFO] TestAgent.server.serf.lan: serf: Received remove-key query +37:36.700 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-916b760e-0d1f-27df-ee47-77bac021acf2 +37:36.700 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/operator/keyring from=127.0.0.1:49720 latency=1.304683ms +37:36.701 [INFO] TestAgent.server.serf.wan: serf: Received list-keys query +37:36.702 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-916b760e-0d1f-27df-ee47-77bac021acf2.dc1 +37:36.702 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query +37:36.702 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-916b760e-0d1f-27df-ee47-77bac021acf2 +37:36.702 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/keyring from=127.0.0.1:49722 latency=1.053962ms +37:36.703 [INFO] TestAgent: Requesting shutdown +37:36.703 [INFO] TestAgent.server: shutting down server +37:36.703 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +37:36.703 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +37:36.703 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +37:36.703 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +37:36.703 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +37:36.703 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +37:36.703 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +37:36.703 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +37:36.703 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +37:36.703 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +37:36.703 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +37:36.703 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:37:36.703Z [INFO] agent.router.manager: shutting down +2021-08-13T15:37:36.703Z [INFO] agent.router.manager: shutting down +37:36.703 [INFO] TestAgent: consul server down +37:36.703 [INFO] TestAgent: shutdown complete +37:36.703 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11022 network=tcp +37:36.704 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11022 network=udp +37:36.704 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:11023 network=tcp +37:37.204 [INFO] TestAgent: Waiting for endpoints to shut down +37:37.204 [INFO] TestAgent: Endpoints down +--- PASS: TestKeyringCommand (0.72s) PASS -ok github.com/hashicorp/consul/command/keyring 1.306s +ok github.com/hashicorp/consul/command/keyring 0.788s === RUN TestKVCommand_noTabs === PAUSE TestKVCommand_noTabs === CONT TestKVCommand_noTabs --- PASS: TestKVCommand_noTabs (0.00s) PASS -ok github.com/hashicorp/consul/command/kv 0.016s +ok github.com/hashicorp/consul/command/kv 0.009s === RUN TestKVDeleteCommand_noTabs === PAUSE TestKVDeleteCommand_noTabs === RUN TestKVDeleteCommand_Validation @@ -86624,276 +86494,277 @@ === RUN TestKVDeleteCommand_CAS === PAUSE TestKVDeleteCommand_CAS === CONT TestKVDeleteCommand_noTabs ---- PASS: TestKVDeleteCommand_noTabs (0.00s) -=== CONT TestKVDeleteCommand_CAS -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... -=== CONT TestKVDeleteCommand === CONT TestKVDeleteCommand_Recurse +=== CONT TestKVDeleteCommand +=== CONT TestKVDeleteCommand_CAS === CONT TestKVDeleteCommand_Validation +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestKVDeleteCommand_noTabs (0.00s) --- PASS: TestKVDeleteCommand_Validation (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:45:20.962Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:21.061Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:21.064 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e93e23bc-5ee5-d956-bd66-88de597ac133 Address:127.0.0.1:30433}]" -45:21.065 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e93e23bc-5ee5-d956-bd66-88de597ac133.dc1 127.0.0.1 -45:21.066 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30433 [Follower]" leader= -45:21.076 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e93e23bc-5ee5-d956-bd66-88de597ac133 127.0.0.1 -2022-09-15T21:45:21.076Z [INFO] agent.router: Initializing LAN area manager -45:21.076 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30428 network=udp -45:21.077 [INFO] TestAgent.server: Adding LAN server: server="Node-e93e23bc-5ee5-d956-bd66-88de597ac133 (Addr: tcp/127.0.0.1:30433) (DC: dc1)" -45:21.077 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e93e23bc-5ee5-d956-bd66-88de597ac133.dc1 area=wan -45:21.082 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30428 network=tcp -45:21.091 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30429 network=tcp -45:21.091 [INFO] TestAgent: started state syncer -45:21.091 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30434 network=tcp -45:21.134 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:21.134 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30433 [Candidate]" term=2 -45:21.134 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:21.134 [DEBUG] TestAgent.server.raft: vote granted: from=e93e23bc-5ee5-d956-bd66-88de597ac133 term=2 tally=1 -45:21.134 [INFO] TestAgent.server.raft: election won: tally=1 -45:21.134 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30433 [Leader]" -45:21.134 [INFO] TestAgent.server: cluster leadership acquired -45:21.134 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:30433 -45:21.136 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:21.137 [INFO] TestAgent.server: New leader elected: payload=Node-e93e23bc-5ee5-d956-bd66-88de597ac133 -45:21.138 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:21.138 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:21.138 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:21.138 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:21.138 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:21.138 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e93e23bc-5ee5-d956-bd66-88de597ac133 -45:21.138 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e93e23bc-5ee5-d956-bd66-88de597ac133 -45:21.140 [INFO] TestAgent.server: federation state anti-entropy synced -45:21.186 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:48600 latency=4.92856ms -45:21.189 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/foo from=127.0.0.1:48606 latency=820.68µs -45:21.190 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:48600 latency=103.48µs -45:21.191 [INFO] TestAgent: Requesting shutdown -45:21.191 [INFO] TestAgent.server: shutting down server -45:21.191 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:21.191 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:21.191 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:21.191 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:21.191 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:21.191 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:21.191 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:21.191 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -45:21.191 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:21.191 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:21.191 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:21.191 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:21.191Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:21.191Z [INFO] agent.router.manager: shutting down -45:21.191 [INFO] TestAgent: consul server down -45:21.191 [INFO] TestAgent: shutdown complete -45:21.191 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30428 network=tcp -45:21.192 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30428 network=udp -45:21.192 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30429 network=tcp -45:21.693 [INFO] TestAgent: Waiting for endpoints to shut down -45:21.693 [INFO] TestAgent: Endpoints down ---- PASS: TestKVDeleteCommand (0.95s) -2022-09-15T21:45:20.981Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:21.022Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:21.039 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d9e4edfc-241c-3ffc-d2c5-20c64e3ba339 Address:127.0.0.1:30440}]" -45:21.040 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30440 [Follower]" leader= -45:21.054 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d9e4edfc-241c-3ffc-d2c5-20c64e3ba339.dc1 127.0.0.1 -45:21.055 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d9e4edfc-241c-3ffc-d2c5-20c64e3ba339 127.0.0.1 -2022-09-15T21:45:21.055Z [INFO] agent.router: Initializing LAN area manager -45:21.056 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30435 network=udp -45:21.058 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d9e4edfc-241c-3ffc-d2c5-20c64e3ba339.dc1 area=wan -45:21.058 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30435 network=tcp -45:21.068 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30436 network=tcp -45:21.068 [INFO] TestAgent: started state syncer -45:21.058 [INFO] TestAgent.server: Adding LAN server: server="Node-d9e4edfc-241c-3ffc-d2c5-20c64e3ba339 (Addr: tcp/127.0.0.1:30440) (DC: dc1)" -45:21.069 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30441 network=tcp -45:21.086 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:21.086 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30440 [Candidate]" term=2 -45:21.090 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:21.090 [DEBUG] TestAgent.server.raft: vote granted: from=d9e4edfc-241c-3ffc-d2c5-20c64e3ba339 term=2 tally=1 -45:21.090 [INFO] TestAgent.server.raft: election won: tally=1 -45:21.090 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30440 [Leader]" -45:21.090 [INFO] TestAgent.server: cluster leadership acquired -45:21.090 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:30440 -45:21.091 [INFO] TestAgent.server: New leader elected: payload=Node-d9e4edfc-241c-3ffc-d2c5-20c64e3ba339 -45:21.092 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:21.116 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:21.116 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:21.116 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:21.116 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:21.116 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:21.116 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d9e4edfc-241c-3ffc-d2c5-20c64e3ba339 -45:21.116 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d9e4edfc-241c-3ffc-d2c5-20c64e3ba339 -45:21.118 [INFO] TestAgent.server: federation state anti-entropy synced -45:21.239 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/a from=127.0.0.1:35516 latency=735.62µs -45:21.243 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/b from=127.0.0.1:35516 latency=2.70034ms -45:21.248 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/food from=127.0.0.1:35516 latency=898.46µs -45:21.250 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/foo?recurse= from=127.0.0.1:35518 latency=1.0388ms -45:21.254 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo/a from=127.0.0.1:35516 latency=103.14µs -45:21.255 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo/b from=127.0.0.1:35516 latency=62.78µs -45:21.257 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/food from=127.0.0.1:35516 latency=61.64µs -45:21.257 [INFO] TestAgent: Requesting shutdown -45:21.258 [INFO] TestAgent.server: shutting down server -45:21.258 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:21.258 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:21.258 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:21.258 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:21.258 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:21.258 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:21.258 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:21.258 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -45:21.258 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:21.258 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:21.258 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:21.258 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:21.258Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:21.258Z [INFO] agent.router.manager: shutting down -45:21.258 [INFO] TestAgent: consul server down -45:21.258 [INFO] TestAgent: shutdown complete -45:21.258 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30435 network=tcp -45:21.258 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30435 network=udp -45:21.258 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30436 network=tcp -45:21.760 [INFO] TestAgent: Waiting for endpoints to shut down -45:21.760 [INFO] TestAgent: Endpoints down ---- PASS: TestKVDeleteCommand_Recurse (1.02s) -2022-09-15T21:45:20.979Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:21.061Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:21.068 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d41a373c-397f-fb64-c1e1-cc9912b762fa Address:127.0.0.1:30426}]" -45:21.069 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d41a373c-397f-fb64-c1e1-cc9912b762fa.dc1 127.0.0.1 -45:21.076 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d41a373c-397f-fb64-c1e1-cc9912b762fa 127.0.0.1 -2022-09-15T21:45:21.076Z [INFO] agent.router: Initializing LAN area manager -45:21.078 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30426 [Follower]" leader= -45:21.080 [INFO] TestAgent.server: Adding LAN server: server="Node-d41a373c-397f-fb64-c1e1-cc9912b762fa (Addr: tcp/127.0.0.1:30426) (DC: dc1)" -45:21.080 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d41a373c-397f-fb64-c1e1-cc9912b762fa.dc1 area=wan -45:21.081 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=tcp -45:21.083 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=udp -45:21.090 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30422 network=tcp -45:21.091 [INFO] TestAgent: started state syncer -45:21.091 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30427 network=tcp -45:21.118 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:21.118 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30426 [Candidate]" term=2 -45:21.118 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:21.118 [DEBUG] TestAgent.server.raft: vote granted: from=d41a373c-397f-fb64-c1e1-cc9912b762fa term=2 tally=1 -45:21.118 [INFO] TestAgent.server.raft: election won: tally=1 -45:21.118 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30426 [Leader]" -45:21.118 [INFO] TestAgent.server: cluster leadership acquired -45:21.118 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:30426 -45:21.119 [INFO] TestAgent.server: New leader elected: payload=Node-d41a373c-397f-fb64-c1e1-cc9912b762fa -45:21.120 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:21.122 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:21.122 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:21.122 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:21.122 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:21.122 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:21.122 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d41a373c-397f-fb64-c1e1-cc9912b762fa -45:21.122 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d41a373c-397f-fb64-c1e1-cc9912b762fa -45:21.124 [INFO] TestAgent.server: federation state anti-entropy synced -45:21.488 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:36376 latency=893.18µs -45:21.489 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -45:21.499 [INFO] TestAgent: Synced node info -45:21.499 [DEBUG] TestAgent: Node info in sync -45:21.500 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/foo?cas=1 from=127.0.0.1:36378 latency=10.13658ms -45:21.501 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:36376 latency=509.38µs -45:21.503 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/foo?cas=12 from=127.0.0.1:36380 latency=607.44µs -45:21.503 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:36376 latency=65.92µs -45:21.504 [INFO] TestAgent: Requesting shutdown -45:21.504 [INFO] TestAgent.server: shutting down server -45:21.504 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:21.504 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:21.504 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:21.504 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:21.504 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:21.504 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:21.504 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:21.504 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:21.504 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:21.504 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:21.504 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:21.504Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:21.504Z [INFO] agent.router.manager: shutting down -45:21.504 [INFO] TestAgent: consul server down -45:21.504 [INFO] TestAgent: shutdown complete -45:21.504 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=tcp -45:21.504 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=udp -45:21.505 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30422 network=tcp -45:22.007 [INFO] TestAgent: Waiting for endpoints to shut down -45:22.007 [INFO] TestAgent: Endpoints down ---- PASS: TestKVDeleteCommand_CAS (1.27s) +2021-08-13T15:37:47.822Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:37:47.846Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +37:47.855 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:80b8d591-8971-6625-cbc1-584b2dbbc607 Address:127.0.0.1:24300}]" +37:47.855 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24300 [Follower]" leader= +37:47.856 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-80b8d591-8971-6625-cbc1-584b2dbbc607.dc1 127.0.0.1 +37:47.859 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-80b8d591-8971-6625-cbc1-584b2dbbc607 127.0.0.1 +2021-08-13T15:37:47.859Z [INFO] agent.router: Initializing LAN area manager +37:47.859 [INFO] TestAgent.server: Adding LAN server: server="Node-80b8d591-8971-6625-cbc1-584b2dbbc607 (Addr: tcp/127.0.0.1:24300) (DC: dc1)" +37:47.859 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24295 network=udp +37:47.859 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-80b8d591-8971-6625-cbc1-584b2dbbc607.dc1 area=wan +37:47.859 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24295 network=tcp +37:47.860 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24296 network=tcp +37:47.860 [INFO] TestAgent: started state syncer +37:47.860 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24301 network=tcp +37:47.907 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +37:47.907 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24300 [Candidate]" term=2 +37:47.907 [DEBUG] TestAgent.server.raft: votes: needed=1 +37:47.907 [DEBUG] TestAgent.server.raft: vote granted: from=80b8d591-8971-6625-cbc1-584b2dbbc607 term=2 tally=1 +37:47.907 [INFO] TestAgent.server.raft: election won: tally=1 +37:47.907 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24300 [Leader]" +37:47.907 [INFO] TestAgent.server: cluster leadership acquired +37:47.907 [INFO] TestAgent.server: New leader elected: payload=Node-80b8d591-8971-6625-cbc1-584b2dbbc607 +37:47.907 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24300 +37:47.909 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +37:47.925 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +37:47.925 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +37:47.925 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +37:47.925 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +37:47.925 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +37:47.926 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-80b8d591-8971-6625-cbc1-584b2dbbc607 +37:47.926 [INFO] TestAgent.server: member joined, marking health alive: member=Node-80b8d591-8971-6625-cbc1-584b2dbbc607 +37:47.926 [INFO] TestAgent.server: federation state anti-entropy synced +37:48.087 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/a from=127.0.0.1:35146 latency=954.802µs +37:48.088 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/b from=127.0.0.1:35146 latency=560.321µs +37:48.090 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/food from=127.0.0.1:35146 latency=617.582µs +37:48.091 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/foo?recurse= from=127.0.0.1:35148 latency=665.421µs +37:48.092 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo/a from=127.0.0.1:35146 latency=71.941µs +37:48.092 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo/b from=127.0.0.1:35146 latency=50.6µs +37:48.092 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/food from=127.0.0.1:35146 latency=51.26µs +37:48.092 [INFO] TestAgent: Requesting shutdown +37:48.093 [INFO] TestAgent.server: shutting down server +37:48.093 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +37:48.093 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +37:48.093 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +37:48.093 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +37:48.093 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +37:48.093 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +37:48.093 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +37:48.093 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +37:48.093 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +37:48.093 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +37:48.093 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +37:48.093 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:37:48.093Z [INFO] agent.router.manager: shutting down +2021-08-13T15:37:48.093Z [INFO] agent.router.manager: shutting down +37:48.093 [INFO] TestAgent: consul server down +37:48.093 [INFO] TestAgent: shutdown complete +37:48.093 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24295 network=tcp +37:48.093 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24295 network=udp +37:48.093 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24296 network=tcp +37:48.593 [INFO] TestAgent: Waiting for endpoints to shut down +37:48.593 [INFO] TestAgent: Endpoints down +--- PASS: TestKVDeleteCommand_Recurse (0.83s) +2021-08-13T15:37:47.825Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:37:47.846Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +37:47.855 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:49e55b54-387d-5c33-7167-bcac49e1d0a6 Address:127.0.0.1:24307}]" +37:47.855 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24307 [Follower]" leader= +37:47.857 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-49e55b54-387d-5c33-7167-bcac49e1d0a6.dc1 127.0.0.1 +37:47.858 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-49e55b54-387d-5c33-7167-bcac49e1d0a6 127.0.0.1 +2021-08-13T15:37:47.858Z [INFO] agent.router: Initializing LAN area manager +37:47.858 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24302 network=udp +37:47.858 [INFO] TestAgent.server: Adding LAN server: server="Node-49e55b54-387d-5c33-7167-bcac49e1d0a6 (Addr: tcp/127.0.0.1:24307) (DC: dc1)" +37:47.858 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-49e55b54-387d-5c33-7167-bcac49e1d0a6.dc1 area=wan +37:47.858 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24302 network=tcp +37:47.859 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24303 network=tcp +37:47.859 [INFO] TestAgent: started state syncer +37:47.859 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24308 network=tcp +37:47.894 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +37:47.894 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24307 [Candidate]" term=2 +37:47.895 [DEBUG] TestAgent.server.raft: votes: needed=1 +37:47.895 [DEBUG] TestAgent.server.raft: vote granted: from=49e55b54-387d-5c33-7167-bcac49e1d0a6 term=2 tally=1 +37:47.895 [INFO] TestAgent.server.raft: election won: tally=1 +37:47.895 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24307 [Leader]" +37:47.895 [INFO] TestAgent.server: cluster leadership acquired +37:47.895 [INFO] TestAgent.server: New leader elected: payload=Node-49e55b54-387d-5c33-7167-bcac49e1d0a6 +37:47.895 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24307 +37:47.897 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +37:47.926 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +37:47.926 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +37:47.926 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +37:47.926 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +37:47.926 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +37:47.926 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-49e55b54-387d-5c33-7167-bcac49e1d0a6 +37:47.926 [INFO] TestAgent.server: member joined, marking health alive: member=Node-49e55b54-387d-5c33-7167-bcac49e1d0a6 +37:48.048 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +37:48.049 [INFO] TestAgent: Synced node info +37:48.049 [DEBUG] TestAgent: Node info in sync +37:48.254 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:41534 latency=661.101µs +37:48.255 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/foo?cas=1 from=127.0.0.1:41536 latency=422.641µs +37:48.256 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:41534 latency=212.881µs +37:48.257 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/foo?cas=12 from=127.0.0.1:41538 latency=534.801µs +37:48.258 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:41534 latency=56.24µs +37:48.258 [INFO] TestAgent: Requesting shutdown +37:48.258 [INFO] TestAgent.server: shutting down server +37:48.258 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +37:48.258 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +37:48.258 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +37:48.258 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +37:48.258 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +37:48.258 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="shutdown waiting for leader" +37:48.258 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +37:48.258 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +37:48.258 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +37:48.258 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +37:48.258 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:37:48.258Z [INFO] agent.router.manager: shutting down +2021-08-13T15:37:48.258Z [INFO] agent.router.manager: shutting down +37:48.258 [INFO] TestAgent: consul server down +37:48.258 [INFO] TestAgent: shutdown complete +37:48.258 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24302 network=tcp +37:48.259 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24302 network=udp +37:48.259 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24303 network=tcp +37:48.759 [INFO] TestAgent: Waiting for endpoints to shut down +37:48.759 [INFO] TestAgent: Endpoints down +--- PASS: TestKVDeleteCommand_CAS (1.00s) +2021-08-13T15:37:47.824Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:37:47.845Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +37:47.852 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fe720239-3299-de0b-e561-5c2269d4b922 Address:127.0.0.1:24314}]" +37:47.853 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24314 [Follower]" leader= +37:47.854 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fe720239-3299-de0b-e561-5c2269d4b922.dc1 127.0.0.1 +37:47.855 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fe720239-3299-de0b-e561-5c2269d4b922 127.0.0.1 +2021-08-13T15:37:47.855Z [INFO] agent.router: Initializing LAN area manager +37:47.855 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fe720239-3299-de0b-e561-5c2269d4b922.dc1 area=wan +37:47.855 [INFO] TestAgent.server: Adding LAN server: server="Node-fe720239-3299-de0b-e561-5c2269d4b922 (Addr: tcp/127.0.0.1:24314) (DC: dc1)" +37:47.856 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24309 network=tcp +37:47.856 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24309 network=udp +37:47.857 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24310 network=tcp +37:47.857 [INFO] TestAgent: started state syncer +37:47.857 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24315 network=tcp +37:47.907 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +37:47.907 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24314 [Candidate]" term=2 +37:47.907 [DEBUG] TestAgent.server.raft: votes: needed=1 +37:47.907 [DEBUG] TestAgent.server.raft: vote granted: from=fe720239-3299-de0b-e561-5c2269d4b922 term=2 tally=1 +37:47.907 [INFO] TestAgent.server.raft: election won: tally=1 +37:47.907 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24314 [Leader]" +37:47.908 [INFO] TestAgent.server: cluster leadership acquired +37:47.908 [INFO] TestAgent.server: New leader elected: payload=Node-fe720239-3299-de0b-e561-5c2269d4b922 +37:47.908 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24314 +37:47.909 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +37:47.922 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +37:47.922 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +37:47.922 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +37:47.922 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +37:47.922 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +37:47.922 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fe720239-3299-de0b-e561-5c2269d4b922 +37:47.922 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fe720239-3299-de0b-e561-5c2269d4b922 +37:47.925 [INFO] TestAgent.server: federation state anti-entropy synced +37:48.040 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +37:48.041 [INFO] TestAgent: Synced node info +37:48.041 [DEBUG] TestAgent: Node info in sync +37:48.262 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:39018 latency=668.002µs +37:48.264 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/foo from=127.0.0.1:39020 latency=706.821µs +37:48.264 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:39018 latency=59.781µs +37:48.264 [INFO] TestAgent: Requesting shutdown +37:48.264 [INFO] TestAgent.server: shutting down server +37:48.264 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +37:48.264 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +37:48.264 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +37:48.264 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +37:48.264 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +37:48.264 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +37:48.264 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +37:48.264 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +37:48.264 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +37:48.264 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +37:48.265 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:37:48.265Z [INFO] agent.router.manager: shutting down +2021-08-13T15:37:48.265Z [INFO] agent.router.manager: shutting down +37:48.265 [INFO] TestAgent: consul server down +37:48.265 [INFO] TestAgent: shutdown complete +37:48.265 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24309 network=tcp +37:48.265 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24309 network=udp +37:48.265 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24310 network=tcp +37:48.765 [INFO] TestAgent: Waiting for endpoints to shut down +37:48.765 [INFO] TestAgent: Endpoints down +--- PASS: TestKVDeleteCommand (1.00s) PASS -ok github.com/hashicorp/consul/command/kv/del 1.497s +ok github.com/hashicorp/consul/command/kv/del 1.066s === RUN TestKVExportCommand_noTabs === PAUSE TestKVExportCommand_noTabs === RUN TestKVExportCommand === PAUSE TestKVExportCommand === CONT TestKVExportCommand_noTabs ---- PASS: TestKVExportCommand_noTabs (0.00s) === CONT TestKVExportCommand [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestKVExportCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:45:21.696Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:21.767Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:21.784 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2fa0bb5e-fd8d-5729-3cbd-cac4144994d5 Address:127.0.0.1:23279}]" -45:21.785 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23279 [Follower]" leader= -45:21.791 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2fa0bb5e-fd8d-5729-3cbd-cac4144994d5.dc1 127.0.0.1 -45:21.792 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2fa0bb5e-fd8d-5729-3cbd-cac4144994d5 127.0.0.1 -2022-09-15T21:45:21.792Z [INFO] agent.router: Initializing LAN area manager -45:21.792 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23274 network=udp -45:21.792 [INFO] TestAgent.server: Adding LAN server: server="Node-2fa0bb5e-fd8d-5729-3cbd-cac4144994d5 (Addr: tcp/127.0.0.1:23279) (DC: dc1)" -45:21.792 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2fa0bb5e-fd8d-5729-3cbd-cac4144994d5.dc1 area=wan -45:21.795 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23274 network=tcp -45:21.796 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23275 network=tcp -45:21.797 [INFO] TestAgent: started state syncer -45:21.797 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23280 network=tcp -45:21.847 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:21.847 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23279 [Candidate]" term=2 -45:21.848 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:21.848 [DEBUG] TestAgent.server.raft: vote granted: from=2fa0bb5e-fd8d-5729-3cbd-cac4144994d5 term=2 tally=1 -45:21.848 [INFO] TestAgent.server.raft: election won: tally=1 -45:21.848 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23279 [Leader]" -45:21.848 [INFO] TestAgent.server: cluster leadership acquired -45:21.848 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23279 -45:21.850 [INFO] TestAgent.server: New leader elected: payload=Node-2fa0bb5e-fd8d-5729-3cbd-cac4144994d5 -45:21.854 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:21.880 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:21.880 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:21.880 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:21.880 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:21.880 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:21.881 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2fa0bb5e-fd8d-5729-3cbd-cac4144994d5 -45:21.881 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2fa0bb5e-fd8d-5729-3cbd-cac4144994d5 -45:21.891 [INFO] TestAgent.server: federation state anti-entropy synced -45:21.935 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -45:21.936 [INFO] TestAgent: Synced node info -45:22.003 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/a from=127.0.0.1:37932 latency=989.54µs -45:22.006 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/b from=127.0.0.1:37932 latency=720.72µs -45:22.007 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/c from=127.0.0.1:37932 latency=729.26µs -45:22.008 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/bar from=127.0.0.1:37932 latency=425.48µs -45:22.010 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo?recurse= from=127.0.0.1:37934 latency=400.28µs -45:22.011 [INFO] TestAgent: Requesting shutdown -45:22.011 [INFO] TestAgent.server: shutting down server -45:22.011 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:22.011 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:22.011 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:22.011 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:22.011 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:22.011 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:22.011 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:22.011 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:22.011 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:22.011 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:22.011 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:22.011Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:22.011Z [INFO] agent.router.manager: shutting down -45:22.012 [INFO] TestAgent: consul server down -45:22.012 [INFO] TestAgent: shutdown complete -45:22.012 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23274 network=tcp -45:22.012 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23274 network=udp -45:22.012 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23275 network=tcp -45:22.513 [INFO] TestAgent: Waiting for endpoints to shut down -45:22.513 [INFO] TestAgent: Endpoints down ---- PASS: TestKVExportCommand (0.91s) +2021-08-13T15:37:48.838Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:37:48.857Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +37:48.862 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7aabf024-73ea-ff6d-b273-78f0a298f287 Address:127.0.0.1:31447}]" +37:48.862 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31447 [Follower]" leader= +37:48.863 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7aabf024-73ea-ff6d-b273-78f0a298f287.dc1 127.0.0.1 +37:48.864 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7aabf024-73ea-ff6d-b273-78f0a298f287 127.0.0.1 +2021-08-13T15:37:48.864Z [INFO] agent.router: Initializing LAN area manager +37:48.864 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7aabf024-73ea-ff6d-b273-78f0a298f287.dc1 area=wan +37:48.864 [INFO] TestAgent.server: Adding LAN server: server="Node-7aabf024-73ea-ff6d-b273-78f0a298f287 (Addr: tcp/127.0.0.1:31447) (DC: dc1)" +37:48.865 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31442 network=tcp +37:48.865 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31442 network=udp +37:48.866 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31443 network=tcp +37:48.866 [INFO] TestAgent: started state syncer +37:48.866 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31448 network=tcp +37:48.910 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +37:48.910 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31447 [Candidate]" term=2 +37:48.910 [DEBUG] TestAgent.server.raft: votes: needed=1 +37:48.910 [DEBUG] TestAgent.server.raft: vote granted: from=7aabf024-73ea-ff6d-b273-78f0a298f287 term=2 tally=1 +37:48.910 [INFO] TestAgent.server.raft: election won: tally=1 +37:48.910 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31447 [Leader]" +37:48.910 [INFO] TestAgent.server: cluster leadership acquired +37:48.911 [INFO] TestAgent.server: New leader elected: payload=Node-7aabf024-73ea-ff6d-b273-78f0a298f287 +37:48.911 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:31447 +37:48.913 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +37:48.937 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +37:48.937 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +37:48.937 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +37:48.937 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +37:48.937 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +37:48.937 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7aabf024-73ea-ff6d-b273-78f0a298f287 +37:48.937 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7aabf024-73ea-ff6d-b273-78f0a298f287 +37:48.938 [INFO] TestAgent.server: federation state anti-entropy synced +37:49.010 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +37:49.010 [INFO] TestAgent: Synced node info +37:49.081 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/b from=127.0.0.1:60750 latency=1.921624ms +37:49.084 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/c from=127.0.0.1:60750 latency=691.281µs +37:49.085 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/bar from=127.0.0.1:60750 latency=638.462µs +37:49.086 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/a from=127.0.0.1:60750 latency=381.261µs +37:49.088 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo?recurse= from=127.0.0.1:60754 latency=481.701µs +37:49.089 [INFO] TestAgent: Requesting shutdown +37:49.089 [INFO] TestAgent.server: shutting down server +37:49.089 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +37:49.089 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +37:49.089 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +37:49.089 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +37:49.089 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +37:49.089 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +37:49.090 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +37:49.090 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +37:49.090 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +37:49.090 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +37:49.090 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:37:49.090Z [INFO] agent.router.manager: shutting down +2021-08-13T15:37:49.090Z [INFO] agent.router.manager: shutting down +37:49.090 [INFO] TestAgent: consul server down +37:49.090 [INFO] TestAgent: shutdown complete +37:49.090 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31442 network=tcp +37:49.090 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31442 network=udp +37:49.090 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31443 network=tcp +37:49.590 [INFO] TestAgent: Waiting for endpoints to shut down +37:49.590 [INFO] TestAgent: Endpoints down +--- PASS: TestKVExportCommand (0.81s) PASS -ok github.com/hashicorp/consul/command/kv/exp 1.063s +ok github.com/hashicorp/consul/command/kv/exp 0.880s === RUN TestKVGetCommand_noTabs === PAUSE TestKVGetCommand_noTabs === RUN TestKVGetCommand_Validation @@ -86917,615 +86788,627 @@ === RUN TestKVGetCommand_DetailedBase64 === PAUSE TestKVGetCommand_DetailedBase64 === CONT TestKVGetCommand_noTabs -=== CONT TestKVGetCommand_Base64 -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestKVGetCommand_RecurseBase64 -=== CONT TestKVGetCommand_DetailedBase64 === CONT TestKVGetCommand_Empty -=== CONT TestKVGetCommand_Validation -=== CONT TestKVGetCommand_Detailed +=== CONT TestKVGetCommand_DetailedBase64 +=== CONT TestKVGetCommand_Base64 +=== CONT TestKVGetCommand_Keys +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestKVGetCommand_Missing ---- PASS: TestKVGetCommand_noTabs (0.02s) === CONT TestKVGetCommand_Recurse ---- PASS: TestKVGetCommand_Validation (0.00s) -=== CONT TestKVGetCommand_Keys +--- PASS: TestKVGetCommand_noTabs (0.00s) +=== CONT TestKVGetCommand_Detailed [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:45:30.509Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:30.561Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:30.611 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:31e160fa-d3fc-7031-05a1-5353cc3372fe Address:127.0.0.1:25370}]" -45:30.612 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-31e160fa-d3fc-7031-05a1-5353cc3372fe.dc1 127.0.0.1 -45:30.613 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-31e160fa-d3fc-7031-05a1-5353cc3372fe 127.0.0.1 -2022-09-15T21:45:30.613Z [INFO] agent.router: Initializing LAN area manager -45:30.613 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25365 network=udp -45:30.614 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25370 [Follower]" leader= -45:30.614 [INFO] TestAgent.server: Adding LAN server: server="Node-31e160fa-d3fc-7031-05a1-5353cc3372fe (Addr: tcp/127.0.0.1:25370) (DC: dc1)" -45:30.614 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-31e160fa-d3fc-7031-05a1-5353cc3372fe.dc1 area=wan -45:30.614 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25365 network=tcp -45:30.641 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25366 network=tcp -45:30.641 [INFO] TestAgent: started state syncer -45:30.666 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:30.666 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25370 [Candidate]" term=2 -45:30.666 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:30.666 [DEBUG] TestAgent.server.raft: vote granted: from=31e160fa-d3fc-7031-05a1-5353cc3372fe term=2 tally=1 -45:30.666 [INFO] TestAgent.server.raft: election won: tally=1 -45:30.666 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25370 [Leader]" -45:30.666 [INFO] TestAgent.server: cluster leadership acquired -45:30.667 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25370 -45:30.667 [INFO] TestAgent.server: New leader elected: payload=Node-31e160fa-d3fc-7031-05a1-5353cc3372fe -45:30.671 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:30.683 [INFO] TestAgent: Synced node info -45:30.685 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25371 network=tcp -45:30.722 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:30.722 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:30.722 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:30.722 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:30.722 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:30.722 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-31e160fa-d3fc-7031-05a1-5353cc3372fe -45:30.722 [INFO] TestAgent.server: member joined, marking health alive: member=Node-31e160fa-d3fc-7031-05a1-5353cc3372fe -45:30.724 [INFO] TestAgent.server: federation state anti-entropy synced -45:30.830 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/not-a-real-key from=127.0.0.1:45322 latency=141.7µs -45:30.830 [INFO] TestAgent: Requesting shutdown -45:30.830 [INFO] TestAgent.server: shutting down server -45:30.830 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:30.830 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:30.830 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:30.830 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:30.830 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:30.830 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:30.830 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:30.830 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:30.830 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:30.830 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:30.830 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:30.830Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:30.830Z [INFO] agent.router.manager: shutting down -45:30.831 [INFO] TestAgent: consul server down -45:30.831 [INFO] TestAgent: shutdown complete -45:30.831 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25365 network=tcp -45:30.831 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25365 network=udp -45:30.831 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25366 network=tcp -45:31.331 [INFO] TestAgent: Waiting for endpoints to shut down -45:31.331 [INFO] TestAgent: Endpoints down ---- PASS: TestKVGetCommand_Missing (1.10s) +2021-08-13T15:37:48.620Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:37:48.661Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +37:48.664 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e73e67c2-e5f7-58aa-ecf7-8ab4ae77335b Address:127.0.0.1:18188}]" +37:48.665 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e73e67c2-e5f7-58aa-ecf7-8ab4ae77335b.dc1 127.0.0.1 +37:48.666 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e73e67c2-e5f7-58aa-ecf7-8ab4ae77335b 127.0.0.1 +2021-08-13T15:37:48.666Z [INFO] agent.router: Initializing LAN area manager +37:48.666 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18183 network=udp +37:48.666 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:18188 [Follower]" leader= +37:48.668 [INFO] TestAgent.server: Adding LAN server: server="Node-e73e67c2-e5f7-58aa-ecf7-8ab4ae77335b (Addr: tcp/127.0.0.1:18188) (DC: dc1)" +37:48.669 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e73e67c2-e5f7-58aa-ecf7-8ab4ae77335b.dc1 area=wan +37:48.669 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18183 network=tcp +37:48.673 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:18184 network=tcp +37:48.674 [INFO] TestAgent: started state syncer +37:48.674 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:18189 network=tcp +37:48.717 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +37:48.717 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:18188 [Candidate]" term=2 +37:48.717 [DEBUG] TestAgent.server.raft: votes: needed=1 +37:48.717 [DEBUG] TestAgent.server.raft: vote granted: from=e73e67c2-e5f7-58aa-ecf7-8ab4ae77335b term=2 tally=1 +37:48.717 [INFO] TestAgent.server.raft: election won: tally=1 +37:48.717 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:18188 [Leader]" +37:48.717 [INFO] TestAgent.server: cluster leadership acquired +37:48.717 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:18188 +37:48.718 [INFO] TestAgent.server: New leader elected: payload=Node-e73e67c2-e5f7-58aa-ecf7-8ab4ae77335b +37:48.719 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +37:48.736 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +37:48.736 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +37:48.736 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +37:48.736 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +37:48.736 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +37:48.736 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e73e67c2-e5f7-58aa-ecf7-8ab4ae77335b +37:48.736 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e73e67c2-e5f7-58aa-ecf7-8ab4ae77335b +37:48.738 [INFO] TestAgent.server: federation state anti-entropy synced +37:48.776 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:55110 latency=801.762µs +37:48.778 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:55112 latency=420.921µs +37:48.778 [INFO] TestAgent: Requesting shutdown +37:48.778 [INFO] TestAgent.server: shutting down server +37:48.778 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +37:48.778 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +37:48.778 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +37:48.778 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +37:48.778 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +37:48.778 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +37:48.778 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +37:48.778 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +37:48.778 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +37:48.778 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +37:48.778 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +37:48.779 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:37:48.779Z [INFO] agent.router.manager: shutting down +2021-08-13T15:37:48.779Z [INFO] agent.router.manager: shutting down +37:48.779 [INFO] TestAgent: consul server down +37:48.779 [INFO] TestAgent: shutdown complete +37:48.779 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18183 network=tcp +37:48.779 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18183 network=udp +37:48.779 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:18184 network=tcp +37:49.279 [INFO] TestAgent: Waiting for endpoints to shut down +37:49.279 [INFO] TestAgent: Endpoints down +--- PASS: TestKVGetCommand_DetailedBase64 (0.74s) === CONT TestKVGetCommand -2022-09-15T21:45:30.476Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:30.611Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:30.623 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8e36a87e-8b93-8b48-5aa1-2400e246aae3 Address:127.0.0.1:25335}]" -45:30.626 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25335 [Follower]" leader= -45:30.631 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8e36a87e-8b93-8b48-5aa1-2400e246aae3.dc1 127.0.0.1 -45:30.633 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8e36a87e-8b93-8b48-5aa1-2400e246aae3 127.0.0.1 -2022-09-15T21:45:30.633Z [INFO] agent.router: Initializing LAN area manager -45:30.636 [INFO] TestAgent.server: Adding LAN server: server="Node-8e36a87e-8b93-8b48-5aa1-2400e246aae3 (Addr: tcp/127.0.0.1:25335) (DC: dc1)" -45:30.637 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25330 network=udp -45:30.637 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8e36a87e-8b93-8b48-5aa1-2400e246aae3.dc1 area=wan -45:30.637 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25330 network=tcp -45:30.682 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:30.682 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25335 [Candidate]" term=2 -45:30.682 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:30.682 [DEBUG] TestAgent.server.raft: vote granted: from=8e36a87e-8b93-8b48-5aa1-2400e246aae3 term=2 tally=1 -45:30.682 [INFO] TestAgent.server.raft: election won: tally=1 -45:30.682 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25335 [Leader]" -45:30.683 [INFO] TestAgent.server: cluster leadership acquired -45:30.683 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25335 -45:30.685 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:30.687 [INFO] TestAgent.server: New leader elected: payload=Node-8e36a87e-8b93-8b48-5aa1-2400e246aae3 -45:30.687 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25331 network=tcp -45:30.688 [INFO] TestAgent: started state syncer -45:30.691 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25336 network=tcp -45:30.695 [INFO] TestAgent: Synced node info -45:30.755 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:30.755 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:30.755 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:30.755 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:30.755 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:30.756 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8e36a87e-8b93-8b48-5aa1-2400e246aae3 -45:30.756 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8e36a87e-8b93-8b48-5aa1-2400e246aae3 -45:30.757 [INFO] TestAgent.server: federation state anti-entropy synced -45:30.864 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:59040 latency=16.58038ms -45:30.910 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:59044 latency=19.80362ms -45:30.911 [INFO] TestAgent: Requesting shutdown -45:30.911 [INFO] TestAgent.server: shutting down server -45:30.911 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:30.911 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:30.911 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:30.911 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:30.911 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:30.912 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:30.912 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:30.912 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:30.912 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:30.912 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:30.912 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:30.912Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:30.912Z [INFO] agent.router.manager: shutting down -45:30.916 [INFO] TestAgent: consul server down -45:30.916 [INFO] TestAgent: shutdown complete -45:30.919 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25330 network=tcp -45:30.919 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25330 network=udp -45:30.919 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25331 network=tcp -45:31.420 [INFO] TestAgent: Waiting for endpoints to shut down -45:31.420 [INFO] TestAgent: Endpoints down ---- PASS: TestKVGetCommand_DetailedBase64 (1.19s) -2022-09-15T21:45:30.605Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:30.824Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:30.827 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:32d66be9-7552-5746-4927-bba016634af5 Address:127.0.0.1:25321}]" -45:30.828 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-32d66be9-7552-5746-4927-bba016634af5.dc1 127.0.0.1 -45:30.828 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-32d66be9-7552-5746-4927-bba016634af5 127.0.0.1 -2022-09-15T21:45:30.828Z [INFO] agent.router: Initializing LAN area manager -45:30.829 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25316 network=udp -45:30.829 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25321 [Follower]" leader= -45:30.831 [INFO] TestAgent.server: Adding LAN server: server="Node-32d66be9-7552-5746-4927-bba016634af5 (Addr: tcp/127.0.0.1:25321) (DC: dc1)" -45:30.831 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-32d66be9-7552-5746-4927-bba016634af5.dc1 area=wan -45:30.831 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25316 network=tcp -45:30.890 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:30.890 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25321 [Candidate]" term=2 -45:30.902 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:30.903 [DEBUG] TestAgent.server.raft: vote granted: from=32d66be9-7552-5746-4927-bba016634af5 term=2 tally=1 -45:30.903 [INFO] TestAgent.server.raft: election won: tally=1 -45:30.903 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25321 [Leader]" -45:30.903 [INFO] TestAgent.server: cluster leadership acquired -45:30.903 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25317 network=tcp -45:30.903 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25321 -45:30.903 [INFO] TestAgent: started state syncer -45:30.905 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25322 network=tcp -45:30.906 [INFO] TestAgent.server: New leader elected: payload=Node-32d66be9-7552-5746-4927-bba016634af5 -45:30.917 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:30.923 [INFO] TestAgent: Synced node info -45:30.923 [DEBUG] TestAgent: Node info in sync -45:30.929 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:30.929 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:30.929 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:30.929 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:30.929 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:30.929 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-32d66be9-7552-5746-4927-bba016634af5 -45:30.929 [INFO] TestAgent.server: member joined, marking health alive: member=Node-32d66be9-7552-5746-4927-bba016634af5 -45:30.931 [INFO] TestAgent.server: federation state anti-entropy synced -45:30.934 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:36752 latency=18.63458ms -45:30.936 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:36756 latency=426.74µs -45:30.937 [INFO] TestAgent: Requesting shutdown -45:30.937 [INFO] TestAgent.server: shutting down server -45:30.937 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:30.937 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:30.937 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:30.937 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:30.937 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:30.937 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:30.937 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:30.937 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:30.937 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:30.937 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:30.937 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:30.937Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:30.937Z [INFO] agent.router.manager: shutting down -45:30.937 [INFO] TestAgent: consul server down -45:30.937 [INFO] TestAgent: shutdown complete -45:30.937 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25316 network=tcp -45:30.938 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25316 network=udp -45:30.938 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25317 network=tcp -45:31.442 [INFO] TestAgent: Waiting for endpoints to shut down -45:31.442 [INFO] TestAgent: Endpoints down -2022-09-15T21:45:30.652Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:30.723Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:30.755 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e6a5b2a0-29a3-5ef6-1ade-bb2229d1cf45 Address:127.0.0.1:25356}]" -45:30.758 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25356 [Follower]" leader= -45:30.802 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:30.802 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25356 [Candidate]" term=2 -45:30.802 [WARN] TestAgent.server.raft: unable to get address for sever, using fallback address: id=e6a5b2a0-29a3-5ef6-1ade-bb2229d1cf45 fallback=127.0.0.1:25356 error="Could not find address for server id e6a5b2a0-29a3-5ef6-1ade-bb2229d1cf45" -45:30.802 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:30.802 [DEBUG] TestAgent.server.raft: vote granted: from=e6a5b2a0-29a3-5ef6-1ade-bb2229d1cf45 term=2 tally=1 -45:30.802 [INFO] TestAgent.server.raft: election won: tally=1 -45:30.802 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25356 [Leader]" -45:30.904 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e6a5b2a0-29a3-5ef6-1ade-bb2229d1cf45.dc1 127.0.0.1 -45:30.921 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e6a5b2a0-29a3-5ef6-1ade-bb2229d1cf45 127.0.0.1 -2022-09-15T21:45:30.921Z [INFO] agent.router: Initializing LAN area manager -45:30.921 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25351 network=udp -45:30.925 [INFO] TestAgent.server: cluster leadership acquired -45:30.930 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25351 network=tcp -45:30.933 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25352 network=tcp -45:30.933 [INFO] TestAgent: started state syncer -45:30.934 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e6a5b2a0-29a3-5ef6-1ade-bb2229d1cf45.dc1 area=wan -45:30.932 [INFO] TestAgent.server: Adding LAN server: server="Node-e6a5b2a0-29a3-5ef6-1ade-bb2229d1cf45 (Addr: tcp/127.0.0.1:25356) (DC: dc1)" -45:30.934 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25357 network=tcp -45:30.934 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25356 -45:30.935 [INFO] TestAgent.server: New leader elected: payload=Node-e6a5b2a0-29a3-5ef6-1ade-bb2229d1cf45 -45:30.936 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:40704 latency=1.1242ms -45:30.939 [INFO] TestAgent: Synced node info -45:30.939 [DEBUG] TestAgent: Node info in sync -45:30.939 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:30.939 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:40708 latency=161.98µs -45:30.939 [INFO] TestAgent: Requesting shutdown -45:30.940 [INFO] TestAgent.server: shutting down server -45:30.940 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:30.940 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:30.940Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:30.940Z [INFO] agent.router.manager: shutting down -45:30.940 [ERROR] TestAgent.server: failed to establish leadership: error="error generating CA root certificate: error computing next serial number: leadership lost while committing log" -45:30.940 [INFO] TestAgent: consul server down -45:30.940 [INFO] TestAgent: shutdown complete -45:30.940 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25351 network=tcp -45:30.940 [ERROR] TestAgent.server: failed to transfer leadership attempt, will retry: attempt=0 retry_limit=3 error="raft is already shutdown" -45:30.940 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25351 network=udp -45:30.940 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25352 network=tcp -45:31.442 [INFO] TestAgent: Waiting for endpoints to shut down -45:31.442 [INFO] TestAgent: Endpoints down ---- PASS: TestKVGetCommand_Detailed (1.21s) ---- PASS: TestKVGetCommand_Base64 (1.21s) -2022-09-15T21:45:30.480Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:30.558Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:30.650 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a36cf803-4392-228e-3378-94258d64f841 Address:127.0.0.1:25342}]" -45:30.654 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a36cf803-4392-228e-3378-94258d64f841.dc1 127.0.0.1 -45:30.655 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a36cf803-4392-228e-3378-94258d64f841 127.0.0.1 -2022-09-15T21:45:30.655Z [INFO] agent.router: Initializing LAN area manager -45:30.656 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25337 network=udp -45:30.656 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25342 [Follower]" leader= -45:30.657 [INFO] TestAgent.server: Adding LAN server: server="Node-a36cf803-4392-228e-3378-94258d64f841 (Addr: tcp/127.0.0.1:25342) (DC: dc1)" -45:30.657 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a36cf803-4392-228e-3378-94258d64f841.dc1 area=wan -45:30.657 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25337 network=tcp -45:30.659 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25338 network=tcp -45:30.659 [INFO] TestAgent: started state syncer -45:30.659 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25343 network=tcp -45:30.758 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:30.758 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25342 [Candidate]" term=2 -45:30.826 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:30.826 [DEBUG] TestAgent.server.raft: vote granted: from=a36cf803-4392-228e-3378-94258d64f841 term=2 tally=1 -45:30.826 [INFO] TestAgent.server.raft: election won: tally=1 -45:30.826 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25342 [Leader]" -45:30.829 [INFO] TestAgent.server: cluster leadership acquired -45:30.830 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25342 -45:30.831 [INFO] TestAgent.server: New leader elected: payload=Node-a36cf803-4392-228e-3378-94258d64f841 -45:30.866 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:30.908 [INFO] TestAgent: Synced node info -45:30.908 [DEBUG] TestAgent: Node info in sync -45:30.914 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:30.914 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:30.914 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:30.914 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:30.914 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:30.916 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a36cf803-4392-228e-3378-94258d64f841 -45:30.916 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a36cf803-4392-228e-3378-94258d64f841 -45:30.927 [INFO] TestAgent.server: federation state anti-entropy synced -45:31.032 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/empty from=127.0.0.1:34042 latency=847.34µs -45:31.034 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/empty from=127.0.0.1:34044 latency=441.52µs -45:31.034 [INFO] TestAgent: Requesting shutdown -45:31.034 [INFO] TestAgent.server: shutting down server -45:31.034 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:31.034 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:31.034 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:31.034 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:31.034 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:31.035 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:31.035 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:31.035 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:31.035 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:31.035 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:31.035 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:31.035Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:31.035Z [INFO] agent.router.manager: shutting down -45:31.035 [INFO] TestAgent: consul server down -45:31.035 [INFO] TestAgent: shutdown complete -45:31.035 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25337 network=tcp -45:31.035 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25337 network=udp -45:31.035 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25338 network=tcp -45:31.537 [INFO] TestAgent: Waiting for endpoints to shut down -45:31.537 [INFO] TestAgent: Endpoints down ---- PASS: TestKVGetCommand_Empty (1.31s) -2022-09-15T21:45:30.735Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:30.878Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:30.890 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ccd25cd7-7bf9-8240-9d8c-7670a6bc5591 Address:127.0.0.1:25349}]" -45:30.890 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25349 [Follower]" leader= -45:30.891 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ccd25cd7-7bf9-8240-9d8c-7670a6bc5591.dc1 127.0.0.1 -45:30.900 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ccd25cd7-7bf9-8240-9d8c-7670a6bc5591 127.0.0.1 -2022-09-15T21:45:30.900Z [INFO] agent.router: Initializing LAN area manager -45:30.915 [INFO] TestAgent.server: Adding LAN server: server="Node-ccd25cd7-7bf9-8240-9d8c-7670a6bc5591 (Addr: tcp/127.0.0.1:25349) (DC: dc1)" -45:30.915 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ccd25cd7-7bf9-8240-9d8c-7670a6bc5591.dc1 area=wan -45:30.917 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25344 network=tcp -45:30.918 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25344 network=udp -45:30.923 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25345 network=tcp -45:30.923 [INFO] TestAgent: started state syncer -45:30.932 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25350 network=tcp -45:30.937 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:30.937 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25349 [Candidate]" term=2 -45:30.942 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:30.942 [DEBUG] TestAgent.server.raft: vote granted: from=ccd25cd7-7bf9-8240-9d8c-7670a6bc5591 term=2 tally=1 -45:30.942 [INFO] TestAgent.server.raft: election won: tally=1 -45:30.942 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25349 [Leader]" -45:30.942 [INFO] TestAgent.server: cluster leadership acquired -45:30.942 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25349 -45:30.944 [INFO] TestAgent.server: New leader elected: payload=Node-ccd25cd7-7bf9-8240-9d8c-7670a6bc5591 -45:30.944 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:30.948 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:30.948 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:30.948 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:30.948 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:30.948 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:30.948 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ccd25cd7-7bf9-8240-9d8c-7670a6bc5591 -45:30.948 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ccd25cd7-7bf9-8240-9d8c-7670a6bc5591 -45:30.950 [INFO] TestAgent.server: federation state anti-entropy synced -45:30.979 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -45:30.980 [INFO] TestAgent: Synced node info -45:30.980 [DEBUG] TestAgent: Node info in sync -45:31.049 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/a from=127.0.0.1:48634 latency=675.74µs -45:31.050 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/b from=127.0.0.1:48634 latency=444.8µs -45:31.051 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/c from=127.0.0.1:48634 latency=400.54µs -45:31.053 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo?recurse= from=127.0.0.1:48636 latency=414.78µs -45:31.054 [INFO] TestAgent: Requesting shutdown -45:31.054 [INFO] TestAgent.server: shutting down server -45:31.054 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:31.054 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:31.054 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:31.054 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:31.054 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:31.054 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:31.054 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:31.054 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:31.054 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:31.054 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:31.054 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:31.055Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:31.055Z [INFO] agent.router.manager: shutting down -45:31.055 [INFO] TestAgent: consul server down -45:31.055 [INFO] TestAgent: shutdown complete -45:31.055 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25344 network=tcp -45:31.055 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25344 network=udp -45:31.055 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25345 network=tcp -45:31.555 [INFO] TestAgent: Waiting for endpoints to shut down -45:31.556 [INFO] TestAgent: Endpoints down ---- PASS: TestKVGetCommand_Recurse (1.32s) -2022-09-15T21:45:30.461Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:30.717Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:30.720 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:762dc705-5705-fc70-1a39-082e936130b1 Address:127.0.0.1:25328}]" -45:30.721 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25328 [Follower]" leader= -45:30.723 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-762dc705-5705-fc70-1a39-082e936130b1.dc1 127.0.0.1 -45:30.739 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-762dc705-5705-fc70-1a39-082e936130b1 127.0.0.1 -2022-09-15T21:45:30.739Z [INFO] agent.router: Initializing LAN area manager -45:30.740 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25323 network=udp -45:30.747 [INFO] TestAgent.server: Adding LAN server: server="Node-762dc705-5705-fc70-1a39-082e936130b1 (Addr: tcp/127.0.0.1:25328) (DC: dc1)" -45:30.747 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-762dc705-5705-fc70-1a39-082e936130b1.dc1 area=wan -45:30.748 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25323 network=tcp -45:30.749 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25324 network=tcp -45:30.749 [INFO] TestAgent: started state syncer -45:30.751 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25329 network=tcp -45:30.831 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:30.831 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25328 [Candidate]" term=2 -45:30.832 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:30.832 [DEBUG] TestAgent.server.raft: vote granted: from=762dc705-5705-fc70-1a39-082e936130b1 term=2 tally=1 -45:30.832 [INFO] TestAgent.server.raft: election won: tally=1 -45:30.832 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25328 [Leader]" -45:30.832 [INFO] TestAgent.server: cluster leadership acquired -45:30.834 [INFO] TestAgent.server: New leader elected: payload=Node-762dc705-5705-fc70-1a39-082e936130b1 -45:30.844 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25328 -45:30.863 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:30.935 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:30.935 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:30.935 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:30.935 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:30.935 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:30.936 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-762dc705-5705-fc70-1a39-082e936130b1 -45:30.936 [INFO] TestAgent.server: member joined, marking health alive: member=Node-762dc705-5705-fc70-1a39-082e936130b1 -45:30.937 [INFO] TestAgent.server: federation state anti-entropy synced -45:30.939 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -45:30.941 [INFO] TestAgent: Synced node info -45:31.073 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/a from=127.0.0.1:54918 latency=818.32µs -45:31.074 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/b from=127.0.0.1:54918 latency=856.98µs -45:31.079 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/c from=127.0.0.1:54918 latency=630.88µs -45:31.081 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo?recurse= from=127.0.0.1:54920 latency=210.74µs -45:31.081 [INFO] TestAgent: Requesting shutdown -45:31.081 [INFO] TestAgent.server: shutting down server -45:31.081 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:31.081 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:31.081 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:31.081 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:31.081 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:31.081 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:31.081 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:31.081 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:31.082 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -45:31.082 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2022-09-15T21:45:31.082Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:31.082Z [INFO] agent.router.manager: shutting down -45:31.082 [INFO] TestAgent: consul server down -45:31.082 [INFO] TestAgent: shutdown complete -45:31.082 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25323 network=tcp -45:31.082 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25323 network=udp -45:31.082 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25324 network=tcp -45:31.083 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:31.602 [INFO] TestAgent: Waiting for endpoints to shut down -45:31.602 [INFO] TestAgent: Endpoints down ---- PASS: TestKVGetCommand_RecurseBase64 (1.37s) -2022-09-15T21:45:30.629Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:30.713Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:30.716 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f8a8fd6c-7898-05a1-9db0-e0e98b0709a0 Address:127.0.0.1:25363}]" -45:30.718 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25363 [Follower]" leader= -45:30.751 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f8a8fd6c-7898-05a1-9db0-e0e98b0709a0.dc1 127.0.0.1 -45:30.827 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f8a8fd6c-7898-05a1-9db0-e0e98b0709a0 127.0.0.1 -2022-09-15T21:45:30.827Z [INFO] agent.router: Initializing LAN area manager -45:30.827 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25358 network=udp -45:30.827 [INFO] TestAgent.server: Adding LAN server: server="Node-f8a8fd6c-7898-05a1-9db0-e0e98b0709a0 (Addr: tcp/127.0.0.1:25363) (DC: dc1)" -45:30.827 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f8a8fd6c-7898-05a1-9db0-e0e98b0709a0.dc1 area=wan -45:30.828 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25358 network=tcp -45:30.829 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25359 network=tcp -45:30.829 [INFO] TestAgent: started state syncer -45:30.829 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25364 network=tcp -45:30.831 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:30.832 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25363 [Candidate]" term=2 -45:30.832 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:30.832 [DEBUG] TestAgent.server.raft: vote granted: from=f8a8fd6c-7898-05a1-9db0-e0e98b0709a0 term=2 tally=1 -45:30.832 [INFO] TestAgent.server.raft: election won: tally=1 -45:30.832 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25363 [Leader]" -45:30.832 [INFO] TestAgent.server: cluster leadership acquired -45:30.842 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25363 -45:30.846 [INFO] TestAgent.server: New leader elected: payload=Node-f8a8fd6c-7898-05a1-9db0-e0e98b0709a0 -45:30.864 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:30.880 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:30.880 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:30.880 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:30.880 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:30.880 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:30.880 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f8a8fd6c-7898-05a1-9db0-e0e98b0709a0 -45:30.880 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f8a8fd6c-7898-05a1-9db0-e0e98b0709a0 -45:30.883 [INFO] TestAgent.server: federation state anti-entropy synced -45:30.904 [INFO] TestAgent: Synced node info -45:31.143 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/bar from=127.0.0.1:48390 latency=773.64µs -45:31.144 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/baz from=127.0.0.1:48390 latency=460.56µs -45:31.145 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/zip from=127.0.0.1:48390 latency=440.32µs -45:31.149 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo/?keys=&separator=%2F from=127.0.0.1:48392 latency=241.96µs -45:31.149 [INFO] TestAgent: Requesting shutdown -45:31.149 [INFO] TestAgent.server: shutting down server -45:31.149 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:31.149 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:31.149 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:31.150 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:31.150 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:31.150 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:31.150 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:31.150 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:31.150 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:31.150 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:31.150 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:31.150Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:31.150Z [INFO] agent.router.manager: shutting down -45:31.150 [INFO] TestAgent: consul server down -45:31.150 [INFO] TestAgent: shutdown complete -45:31.150 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25358 network=tcp -45:31.150 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25358 network=udp -45:31.150 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25359 network=tcp -45:31.654 [INFO] TestAgent: Waiting for endpoints to shut down -45:31.654 [INFO] TestAgent: Endpoints down ---- PASS: TestKVGetCommand_Keys (1.42s) -2022-09-15T21:45:31.368Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:31.387Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:31.390 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:18e1486b-2d93-9da1-b409-1e822846e287 Address:127.0.0.1:25377}]" -45:31.392 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-18e1486b-2d93-9da1-b409-1e822846e287.dc1 127.0.0.1 -45:31.392 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-18e1486b-2d93-9da1-b409-1e822846e287 127.0.0.1 -2022-09-15T21:45:31.392Z [INFO] agent.router: Initializing LAN area manager -45:31.393 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25372 network=udp -45:31.393 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25377 [Follower]" leader= -45:31.394 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-18e1486b-2d93-9da1-b409-1e822846e287.dc1 area=wan -45:31.394 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25372 network=tcp -45:31.395 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25373 network=tcp -45:31.395 [INFO] TestAgent: started state syncer -45:31.395 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25378 network=tcp -45:31.396 [INFO] TestAgent.server: Adding LAN server: server="Node-18e1486b-2d93-9da1-b409-1e822846e287 (Addr: tcp/127.0.0.1:25377) (DC: dc1)" -45:31.450 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:31.450 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25377 [Candidate]" term=2 -45:31.450 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:31.450 [DEBUG] TestAgent.server.raft: vote granted: from=18e1486b-2d93-9da1-b409-1e822846e287 term=2 tally=1 -45:31.450 [INFO] TestAgent.server.raft: election won: tally=1 -45:31.450 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25377 [Leader]" -45:31.450 [INFO] TestAgent.server: cluster leadership acquired -45:31.451 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25377 -45:31.452 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:31.454 [INFO] TestAgent.server: New leader elected: payload=Node-18e1486b-2d93-9da1-b409-1e822846e287 -45:31.455 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:31.455 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:31.455 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:31.455 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:31.455 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:31.455 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-18e1486b-2d93-9da1-b409-1e822846e287 -45:31.455 [INFO] TestAgent.server: member joined, marking health alive: member=Node-18e1486b-2d93-9da1-b409-1e822846e287 -45:31.457 [INFO] TestAgent.server: federation state anti-entropy synced -45:31.705 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:41902 latency=934.5µs -45:31.707 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:41904 latency=202.14µs -45:31.707 [INFO] TestAgent: Requesting shutdown -45:31.707 [INFO] TestAgent.server: shutting down server -45:31.707 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:31.707 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:31.707 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:31.707 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:31.707 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:31.708 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:31.708 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:31.708 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -45:31.708 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:31.708 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:31.708 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:31.708 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:31.708Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:31.708Z [INFO] agent.router.manager: shutting down -45:31.708 [INFO] TestAgent: consul server down -45:31.708 [INFO] TestAgent: shutdown complete -45:31.708 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25372 network=tcp -45:31.708 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25372 network=udp -45:31.708 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25373 network=tcp -45:32.209 [INFO] TestAgent: Waiting for endpoints to shut down -45:32.209 [INFO] TestAgent: Endpoints down ---- PASS: TestKVGetCommand (0.88s) +2021-08-13T15:37:48.621Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:37:48.666Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +37:48.670 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f9f3209c-ac87-6c7a-bfaf-72ec27ae92fa Address:127.0.0.1:18202}]" +37:48.672 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:18202 [Follower]" leader= +37:48.672 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f9f3209c-ac87-6c7a-bfaf-72ec27ae92fa.dc1 127.0.0.1 +37:48.677 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f9f3209c-ac87-6c7a-bfaf-72ec27ae92fa 127.0.0.1 +2021-08-13T15:37:48.677Z [INFO] agent.router: Initializing LAN area manager +37:48.677 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18197 network=udp +37:48.679 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f9f3209c-ac87-6c7a-bfaf-72ec27ae92fa.dc1 area=wan +37:48.680 [INFO] TestAgent.server: Adding LAN server: server="Node-f9f3209c-ac87-6c7a-bfaf-72ec27ae92fa (Addr: tcp/127.0.0.1:18202) (DC: dc1)" +37:48.680 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18197 network=tcp +37:48.681 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:18198 network=tcp +37:48.681 [INFO] TestAgent: started state syncer +37:48.681 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:18203 network=tcp +37:48.708 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +37:48.708 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:18202 [Candidate]" term=2 +37:48.708 [DEBUG] TestAgent.server.raft: votes: needed=1 +37:48.708 [DEBUG] TestAgent.server.raft: vote granted: from=f9f3209c-ac87-6c7a-bfaf-72ec27ae92fa term=2 tally=1 +37:48.708 [INFO] TestAgent.server.raft: election won: tally=1 +37:48.708 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:18202 [Leader]" +37:48.709 [INFO] TestAgent.server: cluster leadership acquired +37:48.709 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:18202 +37:48.710 [INFO] TestAgent.server: New leader elected: payload=Node-f9f3209c-ac87-6c7a-bfaf-72ec27ae92fa +37:48.711 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +37:48.736 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +37:48.736 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +37:48.736 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +37:48.736 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +37:48.736 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +37:48.736 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f9f3209c-ac87-6c7a-bfaf-72ec27ae92fa +37:48.736 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f9f3209c-ac87-6c7a-bfaf-72ec27ae92fa +37:48.738 [INFO] TestAgent.server: federation state anti-entropy synced +37:48.786 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/bar from=127.0.0.1:50550 latency=721.522µs +37:48.787 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/baz from=127.0.0.1:50550 latency=434.421µs +37:48.788 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/zip from=127.0.0.1:50550 latency=486.861µs +37:48.790 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo/?keys=&separator=%2F from=127.0.0.1:50552 latency=435.141µs +37:48.790 [INFO] TestAgent: Requesting shutdown +37:48.790 [INFO] TestAgent.server: shutting down server +37:48.790 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +37:48.790 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +37:48.790 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +37:48.790 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +37:48.790 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +37:48.790 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +37:48.790 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +37:48.790 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +37:48.790 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +37:48.790 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +37:48.790 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +37:48.790 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +2021-08-13T15:37:48.790Z [INFO] agent.router.manager: shutting down +2021-08-13T15:37:48.790Z [INFO] agent.router.manager: shutting down +37:48.790 [INFO] TestAgent: consul server down +37:48.790 [INFO] TestAgent: shutdown complete +37:48.790 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18197 network=tcp +37:48.790 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18197 network=udp +37:48.790 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:18198 network=tcp +37:49.291 [INFO] TestAgent: Waiting for endpoints to shut down +37:49.291 [INFO] TestAgent: Endpoints down +--- PASS: TestKVGetCommand_Keys (0.75s) +=== CONT TestKVGetCommand_Validation +--- PASS: TestKVGetCommand_Validation (0.00s) +2021-08-13T15:37:48.636Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:37:48.680Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +37:48.684 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c193aa14-701e-36f7-7139-45b8bf0a6cd4 Address:127.0.0.1:18209}]" +37:48.685 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c193aa14-701e-36f7-7139-45b8bf0a6cd4.dc1 127.0.0.1 +37:48.685 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:18209 [Follower]" leader= +37:48.709 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c193aa14-701e-36f7-7139-45b8bf0a6cd4 127.0.0.1 +2021-08-13T15:37:48.709Z [INFO] agent.router: Initializing LAN area manager +37:48.709 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18204 network=udp +37:48.709 [INFO] TestAgent.server: Adding LAN server: server="Node-c193aa14-701e-36f7-7139-45b8bf0a6cd4 (Addr: tcp/127.0.0.1:18209) (DC: dc1)" +37:48.709 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c193aa14-701e-36f7-7139-45b8bf0a6cd4.dc1 area=wan +37:48.710 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18204 network=tcp +37:48.710 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:18205 network=tcp +37:48.710 [INFO] TestAgent: started state syncer +37:48.711 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:18210 network=tcp +37:48.750 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +37:48.750 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:18209 [Candidate]" term=2 +37:48.751 [DEBUG] TestAgent.server.raft: votes: needed=1 +37:48.751 [DEBUG] TestAgent.server.raft: vote granted: from=c193aa14-701e-36f7-7139-45b8bf0a6cd4 term=2 tally=1 +37:48.751 [INFO] TestAgent.server.raft: election won: tally=1 +37:48.751 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:18209 [Leader]" +37:48.751 [INFO] TestAgent.server: cluster leadership acquired +37:48.751 [INFO] TestAgent.server: New leader elected: payload=Node-c193aa14-701e-36f7-7139-45b8bf0a6cd4 +37:48.751 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:18209 +37:48.752 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +37:48.755 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +37:48.755 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +37:48.755 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +37:48.755 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +37:48.755 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +37:48.755 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c193aa14-701e-36f7-7139-45b8bf0a6cd4 +37:48.755 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c193aa14-701e-36f7-7139-45b8bf0a6cd4 +37:48.756 [INFO] TestAgent.server: federation state anti-entropy synced +37:48.868 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/not-a-real-key from=127.0.0.1:46138 latency=97.841µs +37:48.868 [INFO] TestAgent: Requesting shutdown +37:48.869 [INFO] TestAgent.server: shutting down server +37:48.869 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +37:48.869 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +37:48.869 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +37:48.869 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +37:48.869 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +37:48.869 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +37:48.869 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +37:48.869 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +37:48.869 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +37:48.869 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +37:48.869 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +37:48.869 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:37:48.869Z [INFO] agent.router.manager: shutting down +2021-08-13T15:37:48.869Z [INFO] agent.router.manager: shutting down +37:48.869 [INFO] TestAgent: consul server down +37:48.869 [INFO] TestAgent: shutdown complete +37:48.869 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18204 network=tcp +37:48.869 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18204 network=udp +37:48.869 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:18205 network=tcp +37:49.369 [INFO] TestAgent: Waiting for endpoints to shut down +37:49.369 [INFO] TestAgent: Endpoints down +--- PASS: TestKVGetCommand_Missing (0.83s) +2021-08-13T15:37:48.611Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:37:48.644Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +37:48.650 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7dd89f6c-52f7-9bb7-169a-4c4ee8632638 Address:127.0.0.1:18216}]" +37:48.651 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:18216 [Follower]" leader= +37:48.654 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7dd89f6c-52f7-9bb7-169a-4c4ee8632638.dc1 127.0.0.1 +37:48.675 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7dd89f6c-52f7-9bb7-169a-4c4ee8632638 127.0.0.1 +2021-08-13T15:37:48.675Z [INFO] agent.router: Initializing LAN area manager +37:48.675 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18211 network=udp +37:48.679 [INFO] TestAgent.server: Adding LAN server: server="Node-7dd89f6c-52f7-9bb7-169a-4c4ee8632638 (Addr: tcp/127.0.0.1:18216) (DC: dc1)" +37:48.679 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7dd89f6c-52f7-9bb7-169a-4c4ee8632638.dc1 area=wan +37:48.679 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18211 network=tcp +37:48.680 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:18212 network=tcp +37:48.680 [INFO] TestAgent: started state syncer +37:48.680 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:18217 network=tcp +37:48.718 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +37:48.718 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:18216 [Candidate]" term=2 +37:48.719 [DEBUG] TestAgent.server.raft: votes: needed=1 +37:48.719 [DEBUG] TestAgent.server.raft: vote granted: from=7dd89f6c-52f7-9bb7-169a-4c4ee8632638 term=2 tally=1 +37:48.719 [INFO] TestAgent.server.raft: election won: tally=1 +37:48.719 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:18216 [Leader]" +37:48.719 [INFO] TestAgent.server: cluster leadership acquired +37:48.719 [INFO] TestAgent.server: New leader elected: payload=Node-7dd89f6c-52f7-9bb7-169a-4c4ee8632638 +37:48.719 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:18216 +37:48.720 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +37:48.736 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +37:48.736 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +37:48.736 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +37:48.736 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +37:48.736 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +37:48.736 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7dd89f6c-52f7-9bb7-169a-4c4ee8632638 +37:48.736 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7dd89f6c-52f7-9bb7-169a-4c4ee8632638 +37:48.738 [INFO] TestAgent.server: federation state anti-entropy synced +37:48.826 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +37:48.827 [INFO] TestAgent: Synced node info +37:48.827 [DEBUG] TestAgent: Node info in sync +37:48.942 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/a from=127.0.0.1:47736 latency=3.289588ms +37:48.944 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/b from=127.0.0.1:47736 latency=920.602µs +37:48.945 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/c from=127.0.0.1:47736 latency=565.422µs +37:48.947 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo?recurse= from=127.0.0.1:47738 latency=227.901µs +37:48.947 [INFO] TestAgent: Requesting shutdown +37:48.947 [INFO] TestAgent.server: shutting down server +37:48.947 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +37:48.947 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +37:48.947 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +37:48.947 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +37:48.947 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +37:48.947 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +37:48.947 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +37:48.947 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +37:48.948 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +37:48.947 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +37:48.948 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:37:48.948Z [INFO] agent.router.manager: shutting down +2021-08-13T15:37:48.948Z [INFO] agent.router.manager: shutting down +37:48.948 [INFO] TestAgent: consul server down +37:48.948 [INFO] TestAgent: shutdown complete +37:48.948 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18211 network=tcp +37:48.948 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18211 network=udp +37:48.948 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:18212 network=tcp +37:49.449 [INFO] TestAgent: Waiting for endpoints to shut down +37:49.449 [INFO] TestAgent: Endpoints down +--- PASS: TestKVGetCommand_Recurse (0.91s) +2021-08-13T15:37:48.638Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:37:48.712Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +37:48.715 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:055094e1-dce2-18b0-66ae-b39e54cec68b Address:127.0.0.1:18195}]" +37:48.715 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:18195 [Follower]" leader= +37:48.716 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-055094e1-dce2-18b0-66ae-b39e54cec68b.dc1 127.0.0.1 +37:48.717 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-055094e1-dce2-18b0-66ae-b39e54cec68b 127.0.0.1 +2021-08-13T15:37:48.717Z [INFO] agent.router: Initializing LAN area manager +37:48.717 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18190 network=udp +37:48.717 [INFO] TestAgent.server: Adding LAN server: server="Node-055094e1-dce2-18b0-66ae-b39e54cec68b (Addr: tcp/127.0.0.1:18195) (DC: dc1)" +37:48.717 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-055094e1-dce2-18b0-66ae-b39e54cec68b.dc1 area=wan +37:48.718 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18190 network=tcp +37:48.719 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:18191 network=tcp +37:48.719 [INFO] TestAgent: started state syncer +37:48.719 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:18196 network=tcp +37:48.764 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +37:48.764 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:18195 [Candidate]" term=2 +37:48.764 [DEBUG] TestAgent.server.raft: votes: needed=1 +37:48.764 [DEBUG] TestAgent.server.raft: vote granted: from=055094e1-dce2-18b0-66ae-b39e54cec68b term=2 tally=1 +37:48.764 [INFO] TestAgent.server.raft: election won: tally=1 +37:48.764 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:18195 [Leader]" +37:48.765 [INFO] TestAgent.server: cluster leadership acquired +37:48.765 [INFO] TestAgent.server: New leader elected: payload=Node-055094e1-dce2-18b0-66ae-b39e54cec68b +37:48.765 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:18195 +37:48.766 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +37:48.769 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +37:48.769 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +37:48.769 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +37:48.769 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +37:48.769 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +37:48.769 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-055094e1-dce2-18b0-66ae-b39e54cec68b +37:48.769 [INFO] TestAgent.server: member joined, marking health alive: member=Node-055094e1-dce2-18b0-66ae-b39e54cec68b +37:48.770 [INFO] TestAgent.server: federation state anti-entropy synced +37:48.956 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/empty from=127.0.0.1:33894 latency=914.642µs +37:48.957 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/empty from=127.0.0.1:33896 latency=179.1µs +37:48.958 [INFO] TestAgent: Requesting shutdown +37:48.958 [INFO] TestAgent.server: shutting down server +37:48.958 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +37:48.958 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +37:48.958 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +37:48.958 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +37:48.958 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +37:48.958 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +37:48.958 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +37:48.958 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +37:48.958 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +37:48.958 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +37:48.958 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +37:48.958 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:37:48.958Z [INFO] agent.router.manager: shutting down +2021-08-13T15:37:48.958Z [INFO] agent.router.manager: shutting down +37:48.958 [INFO] TestAgent: consul server down +37:48.958 [INFO] TestAgent: shutdown complete +37:48.958 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18190 network=tcp +37:48.959 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18190 network=udp +37:48.959 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:18191 network=tcp +37:49.459 [INFO] TestAgent: Waiting for endpoints to shut down +37:49.459 [INFO] TestAgent: Endpoints down +--- PASS: TestKVGetCommand_Empty (0.92s) +2021-08-13T15:37:48.636Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:37:48.667Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +37:48.674 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:de49dc05-b398-9617-f286-c977b6d65803 Address:127.0.0.1:18223}]" +37:48.675 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-de49dc05-b398-9617-f286-c977b6d65803.dc1 127.0.0.1 +37:48.676 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-de49dc05-b398-9617-f286-c977b6d65803 127.0.0.1 +2021-08-13T15:37:48.676Z [INFO] agent.router: Initializing LAN area manager +37:48.676 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18218 network=udp +37:48.678 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:18223 [Follower]" leader= +37:48.678 [INFO] TestAgent.server: Adding LAN server: server="Node-de49dc05-b398-9617-f286-c977b6d65803 (Addr: tcp/127.0.0.1:18223) (DC: dc1)" +37:48.678 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-de49dc05-b398-9617-f286-c977b6d65803.dc1 area=wan +37:48.678 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18218 network=tcp +37:48.679 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:18219 network=tcp +37:48.679 [INFO] TestAgent: started state syncer +37:48.680 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:18224 network=tcp +37:48.734 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +37:48.734 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:18223 [Candidate]" term=2 +37:48.734 [DEBUG] TestAgent.server.raft: votes: needed=1 +37:48.734 [DEBUG] TestAgent.server.raft: vote granted: from=de49dc05-b398-9617-f286-c977b6d65803 term=2 tally=1 +37:48.734 [INFO] TestAgent.server.raft: election won: tally=1 +37:48.734 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:18223 [Leader]" +37:48.734 [INFO] TestAgent.server: cluster leadership acquired +37:48.735 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:18223 +37:48.735 [INFO] TestAgent.server: New leader elected: payload=Node-de49dc05-b398-9617-f286-c977b6d65803 +37:48.736 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +37:48.739 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +37:48.739 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +37:48.739 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +37:48.739 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +37:48.739 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +37:48.739 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-de49dc05-b398-9617-f286-c977b6d65803 +37:48.739 [INFO] TestAgent.server: member joined, marking health alive: member=Node-de49dc05-b398-9617-f286-c977b6d65803 +37:48.739 [INFO] TestAgent.server: federation state anti-entropy synced +37:49.024 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:46226 latency=855.262µs +37:49.026 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:46228 latency=248.16µs +37:49.026 [INFO] TestAgent: Requesting shutdown +37:49.026 [INFO] TestAgent.server: shutting down server +37:49.026 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +37:49.026 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +37:49.026 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +37:49.026 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +37:49.026 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +37:49.026 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +37:49.026 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +37:49.026 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +37:49.026 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +37:49.026 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +37:49.026 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +37:49.027 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:37:49.027Z [INFO] agent.router.manager: shutting down +2021-08-13T15:37:49.027Z [INFO] agent.router.manager: shutting down +37:49.027 [INFO] TestAgent: consul server down +37:49.027 [INFO] TestAgent: shutdown complete +37:49.027 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18218 network=tcp +37:49.027 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18218 network=udp +37:49.027 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:18219 network=tcp +37:49.527 [INFO] TestAgent: Waiting for endpoints to shut down +37:49.527 [INFO] TestAgent: Endpoints down +--- PASS: TestKVGetCommand_Detailed (0.99s) +2021-08-13T15:37:48.638Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:37:48.681Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +37:48.686 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f63593f3-6df6-0477-4c21-cdc5fe1560ba Address:127.0.0.1:18174}]" +37:48.687 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:18174 [Follower]" leader= +37:48.709 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f63593f3-6df6-0477-4c21-cdc5fe1560ba.dc1 127.0.0.1 +37:48.710 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f63593f3-6df6-0477-4c21-cdc5fe1560ba 127.0.0.1 +2021-08-13T15:37:48.710Z [INFO] agent.router: Initializing LAN area manager +37:48.710 [INFO] TestAgent.server: Adding LAN server: server="Node-f63593f3-6df6-0477-4c21-cdc5fe1560ba (Addr: tcp/127.0.0.1:18174) (DC: dc1)" +37:48.710 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f63593f3-6df6-0477-4c21-cdc5fe1560ba.dc1 area=wan +37:48.710 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18169 network=udp +37:48.710 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18169 network=tcp +37:48.711 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:18170 network=tcp +37:48.711 [INFO] TestAgent: started state syncer +37:48.711 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:18175 network=tcp +37:48.755 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +37:48.755 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:18174 [Candidate]" term=2 +37:48.755 [DEBUG] TestAgent.server.raft: votes: needed=1 +37:48.755 [DEBUG] TestAgent.server.raft: vote granted: from=f63593f3-6df6-0477-4c21-cdc5fe1560ba term=2 tally=1 +37:48.755 [INFO] TestAgent.server.raft: election won: tally=1 +37:48.755 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:18174 [Leader]" +37:48.755 [INFO] TestAgent.server: cluster leadership acquired +37:48.755 [INFO] TestAgent.server: New leader elected: payload=Node-f63593f3-6df6-0477-4c21-cdc5fe1560ba +37:48.755 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:18174 +37:48.757 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +37:48.759 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +37:48.759 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +37:48.759 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +37:48.759 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +37:48.759 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +37:48.759 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f63593f3-6df6-0477-4c21-cdc5fe1560ba +37:48.759 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f63593f3-6df6-0477-4c21-cdc5fe1560ba +37:48.760 [INFO] TestAgent.server: federation state anti-entropy synced +37:48.816 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +37:48.816 [INFO] TestAgent: Synced node info +37:49.054 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:37420 latency=970.082µs +37:49.055 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:37422 latency=143.9µs +37:49.056 [INFO] TestAgent: Requesting shutdown +37:49.056 [INFO] TestAgent.server: shutting down server +37:49.056 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +37:49.056 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +37:49.056 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +37:49.056 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +37:49.056 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +37:49.056 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +37:49.056 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +37:49.056 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +37:49.056 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +37:49.056 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +37:49.056 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:37:49.056Z [INFO] agent.router.manager: shutting down +2021-08-13T15:37:49.056Z [INFO] agent.router.manager: shutting down +37:49.057 [INFO] TestAgent: consul server down +37:49.057 [INFO] TestAgent: shutdown complete +37:49.057 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18169 network=tcp +37:49.057 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18169 network=udp +37:49.057 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:18170 network=tcp +37:49.557 [INFO] TestAgent: Waiting for endpoints to shut down +37:49.557 [INFO] TestAgent: Endpoints down +--- PASS: TestKVGetCommand_Base64 (1.02s) +2021-08-13T15:37:48.678Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:37:48.718Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +37:48.720 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7016fe5c-44d2-f07b-731c-9a5d91304b09 Address:127.0.0.1:18181}]" +37:48.720 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:18181 [Follower]" leader= +37:48.721 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7016fe5c-44d2-f07b-731c-9a5d91304b09.dc1 127.0.0.1 +37:48.722 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7016fe5c-44d2-f07b-731c-9a5d91304b09 127.0.0.1 +2021-08-13T15:37:48.722Z [INFO] agent.router: Initializing LAN area manager +37:48.722 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7016fe5c-44d2-f07b-731c-9a5d91304b09.dc1 area=wan +37:48.722 [INFO] TestAgent.server: Adding LAN server: server="Node-7016fe5c-44d2-f07b-731c-9a5d91304b09 (Addr: tcp/127.0.0.1:18181) (DC: dc1)" +37:48.722 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18176 network=udp +37:48.723 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18176 network=tcp +37:48.726 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:18177 network=tcp +37:48.726 [INFO] TestAgent: started state syncer +37:48.726 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:18182 network=tcp +37:48.765 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +37:48.765 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:18181 [Candidate]" term=2 +37:48.766 [DEBUG] TestAgent.server.raft: votes: needed=1 +37:48.766 [DEBUG] TestAgent.server.raft: vote granted: from=7016fe5c-44d2-f07b-731c-9a5d91304b09 term=2 tally=1 +37:48.766 [INFO] TestAgent.server.raft: election won: tally=1 +37:48.766 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:18181 [Leader]" +37:48.766 [INFO] TestAgent.server: cluster leadership acquired +37:48.766 [INFO] TestAgent.server: New leader elected: payload=Node-7016fe5c-44d2-f07b-731c-9a5d91304b09 +37:48.767 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:18181 +37:48.768 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +37:48.770 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +37:48.770 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +37:48.770 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +37:48.770 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +37:48.770 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +37:48.770 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7016fe5c-44d2-f07b-731c-9a5d91304b09 +37:48.770 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7016fe5c-44d2-f07b-731c-9a5d91304b09 +37:48.771 [INFO] TestAgent.server: federation state anti-entropy synced +37:48.851 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +37:48.852 [INFO] TestAgent: Synced node info +37:49.087 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/a from=127.0.0.1:36826 latency=828.302µs +37:49.089 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/b from=127.0.0.1:36826 latency=599.821µs +37:49.089 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/c from=127.0.0.1:36826 latency=401.021µs +37:49.091 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo?recurse= from=127.0.0.1:36830 latency=217.34µs +37:49.091 [INFO] TestAgent: Requesting shutdown +37:49.091 [INFO] TestAgent.server: shutting down server +37:49.091 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +37:49.091 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +37:49.091 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +37:49.091 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +37:49.091 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +37:49.091 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +37:49.091 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +37:49.091 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +37:49.091 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +37:49.091 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +37:49.091 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:37:49.092Z [INFO] agent.router.manager: shutting down +2021-08-13T15:37:49.092Z [INFO] agent.router.manager: shutting down +37:49.092 [INFO] TestAgent: consul server down +37:49.092 [INFO] TestAgent: shutdown complete +37:49.092 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18176 network=tcp +37:49.092 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18176 network=udp +37:49.092 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:18177 network=tcp +37:49.592 [INFO] TestAgent: Waiting for endpoints to shut down +37:49.592 [INFO] TestAgent: Endpoints down +--- PASS: TestKVGetCommand_RecurseBase64 (1.05s) +2021-08-13T15:37:49.296Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:37:49.312Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +37:49.315 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e2e45be1-4cb0-262c-fd71-655aa13e9b62 Address:127.0.0.1:18230}]" +37:49.315 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:18230 [Follower]" leader= +37:49.316 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e2e45be1-4cb0-262c-fd71-655aa13e9b62.dc1 127.0.0.1 +37:49.317 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e2e45be1-4cb0-262c-fd71-655aa13e9b62 127.0.0.1 +2021-08-13T15:37:49.317Z [INFO] agent.router: Initializing LAN area manager +37:49.317 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e2e45be1-4cb0-262c-fd71-655aa13e9b62.dc1 area=wan +37:49.317 [INFO] TestAgent.server: Adding LAN server: server="Node-e2e45be1-4cb0-262c-fd71-655aa13e9b62 (Addr: tcp/127.0.0.1:18230) (DC: dc1)" +37:49.317 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18225 network=tcp +37:49.317 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18225 network=udp +37:49.318 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:18226 network=tcp +37:49.318 [INFO] TestAgent: started state syncer +37:49.318 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:18231 network=tcp +37:49.367 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +37:49.367 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:18230 [Candidate]" term=2 +37:49.367 [DEBUG] TestAgent.server.raft: votes: needed=1 +37:49.367 [DEBUG] TestAgent.server.raft: vote granted: from=e2e45be1-4cb0-262c-fd71-655aa13e9b62 term=2 tally=1 +37:49.367 [INFO] TestAgent.server.raft: election won: tally=1 +37:49.367 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:18230 [Leader]" +37:49.367 [INFO] TestAgent.server: cluster leadership acquired +37:49.368 [INFO] TestAgent.server: New leader elected: payload=Node-e2e45be1-4cb0-262c-fd71-655aa13e9b62 +37:49.368 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:18230 +37:49.369 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +37:49.372 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +37:49.372 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +37:49.372 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +37:49.372 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +37:49.372 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +37:49.372 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e2e45be1-4cb0-262c-fd71-655aa13e9b62 +37:49.372 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e2e45be1-4cb0-262c-fd71-655aa13e9b62 +37:49.372 [INFO] TestAgent.server: federation state anti-entropy synced +37:49.514 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:35832 latency=619.581µs +37:49.515 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:35834 latency=130.841µs +37:49.516 [INFO] TestAgent: Requesting shutdown +37:49.516 [INFO] TestAgent.server: shutting down server +37:49.516 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +37:49.516 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +37:49.516 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +37:49.516 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +37:49.516 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +37:49.516 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +37:49.516 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +37:49.516 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +37:49.516 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +37:49.516 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +37:49.516 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +37:49.516 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:37:49.516Z [INFO] agent.router.manager: shutting down +2021-08-13T15:37:49.516Z [INFO] agent.router.manager: shutting down +37:49.517 [INFO] TestAgent: consul server down +37:49.517 [INFO] TestAgent: shutdown complete +37:49.517 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18225 network=tcp +37:49.517 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18225 network=udp +37:49.517 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:18226 network=tcp +37:50.017 [INFO] TestAgent: Waiting for endpoints to shut down +37:50.017 [INFO] TestAgent: Endpoints down +--- PASS: TestKVGetCommand (0.74s) PASS -ok github.com/hashicorp/consul/command/kv/get 2.220s +ok github.com/hashicorp/consul/command/kv/get 1.543s === RUN TestKVImportCommand_noTabs === PAUSE TestKVImportCommand_noTabs === RUN TestKVImportCommand === PAUSE TestKVImportCommand === CONT TestKVImportCommand_noTabs +--- PASS: TestKVImportCommand_noTabs (0.00s) === CONT TestKVImportCommand [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... ---- PASS: TestKVImportCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:45:36.357Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:36.387Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:36.391 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2df518b4-529e-fe2b-a599-79abf6e5e3d6 Address:127.0.0.1:29405}]" -45:36.392 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2df518b4-529e-fe2b-a599-79abf6e5e3d6.dc1 127.0.0.1 -45:36.393 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29405 [Follower]" leader= -45:36.395 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2df518b4-529e-fe2b-a599-79abf6e5e3d6 127.0.0.1 -2022-09-15T21:45:36.395Z [INFO] agent.router: Initializing LAN area manager -45:36.396 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29400 network=udp -45:36.396 [INFO] TestAgent.server: Adding LAN server: server="Node-2df518b4-529e-fe2b-a599-79abf6e5e3d6 (Addr: tcp/127.0.0.1:29405) (DC: dc1)" -45:36.396 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2df518b4-529e-fe2b-a599-79abf6e5e3d6.dc1 area=wan -45:36.396 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29400 network=tcp -45:36.397 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29401 network=tcp -45:36.397 [INFO] TestAgent: started state syncer -45:36.397 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29406 network=tcp -45:36.433 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:36.434 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29405 [Candidate]" term=2 -45:36.434 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:36.434 [DEBUG] TestAgent.server.raft: vote granted: from=2df518b4-529e-fe2b-a599-79abf6e5e3d6 term=2 tally=1 -45:36.434 [INFO] TestAgent.server.raft: election won: tally=1 -45:36.434 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29405 [Leader]" -45:36.434 [INFO] TestAgent.server: cluster leadership acquired -45:36.435 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29405 -45:36.436 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:36.438 [INFO] TestAgent.server: New leader elected: payload=Node-2df518b4-529e-fe2b-a599-79abf6e5e3d6 -45:36.481 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:36.481 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:36.481 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:36.481 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:36.481 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:36.481 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2df518b4-529e-fe2b-a599-79abf6e5e3d6 -45:36.481 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2df518b4-529e-fe2b-a599-79abf6e5e3d6 -45:36.500 [INFO] TestAgent.server: federation state anti-entropy synced -45:36.511 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -45:36.512 [INFO] TestAgent: Synced node info -45:36.625 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:35620 latency=877.62µs -45:36.626 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/a from=127.0.0.1:35620 latency=680.4µs -45:36.628 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:35622 latency=524.4µs -45:36.629 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo/a from=127.0.0.1:35622 latency=106.76µs -45:36.629 [INFO] TestAgent: Requesting shutdown -45:36.629 [INFO] TestAgent.server: shutting down server -45:36.629 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:36.629 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:36.629 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:36.629 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:36.629 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:36.629 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:36.629 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:36.629 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:36.629 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:36.629 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:36.630 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:36.630Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:36.630Z [INFO] agent.router.manager: shutting down -45:36.630 [INFO] TestAgent: consul server down -45:36.630 [INFO] TestAgent: shutdown complete -45:36.630 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29400 network=tcp -45:36.630 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29400 network=udp -45:36.630 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29401 network=tcp -45:37.134 [INFO] TestAgent: Waiting for endpoints to shut down -45:37.134 [INFO] TestAgent: Endpoints down ---- PASS: TestKVImportCommand (0.85s) +2021-08-13T15:37:55.968Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:37:55.983Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +37:55.987 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a1faf789-84f0-ce53-4cba-ad523a60f1cd Address:127.0.0.1:16132}]" +37:55.987 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16132 [Follower]" leader= +37:55.988 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a1faf789-84f0-ce53-4cba-ad523a60f1cd.dc1 127.0.0.1 +37:55.989 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a1faf789-84f0-ce53-4cba-ad523a60f1cd 127.0.0.1 +2021-08-13T15:37:55.989Z [INFO] agent.router: Initializing LAN area manager +37:55.989 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16127 network=udp +37:55.990 [INFO] TestAgent.server: Adding LAN server: server="Node-a1faf789-84f0-ce53-4cba-ad523a60f1cd (Addr: tcp/127.0.0.1:16132) (DC: dc1)" +37:55.990 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a1faf789-84f0-ce53-4cba-ad523a60f1cd.dc1 area=wan +37:55.990 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16127 network=tcp +37:55.991 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16128 network=tcp +37:55.991 [INFO] TestAgent: started state syncer +37:55.991 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16133 network=tcp +37:56.055 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +37:56.055 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16132 [Candidate]" term=2 +37:56.057 [DEBUG] TestAgent.server.raft: votes: needed=1 +37:56.057 [DEBUG] TestAgent.server.raft: vote granted: from=a1faf789-84f0-ce53-4cba-ad523a60f1cd term=2 tally=1 +37:56.057 [INFO] TestAgent.server.raft: election won: tally=1 +37:56.057 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16132 [Leader]" +37:56.059 [INFO] TestAgent.server: cluster leadership acquired +37:56.059 [INFO] TestAgent.server: New leader elected: payload=Node-a1faf789-84f0-ce53-4cba-ad523a60f1cd +37:56.062 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16132 +37:56.065 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +37:56.089 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +37:56.089 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +37:56.089 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +37:56.089 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +37:56.089 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +37:56.089 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a1faf789-84f0-ce53-4cba-ad523a60f1cd +37:56.089 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a1faf789-84f0-ce53-4cba-ad523a60f1cd +37:56.089 [INFO] TestAgent.server: federation state anti-entropy synced +37:56.104 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +37:56.104 [INFO] TestAgent: Synced node info +37:56.383 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:47490 latency=979.583µs +37:56.384 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/a from=127.0.0.1:47490 latency=622.101µs +37:56.385 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:47492 latency=450.501µs +37:56.386 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo/a from=127.0.0.1:47492 latency=268µs +37:56.387 [INFO] TestAgent: Requesting shutdown +37:56.387 [INFO] TestAgent.server: shutting down server +37:56.387 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +37:56.387 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +37:56.387 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +37:56.387 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +37:56.387 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +37:56.387 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +37:56.387 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +37:56.387 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +37:56.387 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +37:56.387 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +37:56.387 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:37:56.387Z [INFO] agent.router.manager: shutting down +2021-08-13T15:37:56.387Z [INFO] agent.router.manager: shutting down +37:56.387 [INFO] TestAgent: consul server down +37:56.387 [INFO] TestAgent: shutdown complete +37:56.387 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16127 network=tcp +37:56.388 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16127 network=udp +37:56.388 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16128 network=tcp +37:56.888 [INFO] TestAgent: Waiting for endpoints to shut down +37:56.888 [INFO] TestAgent: Endpoints down +--- PASS: TestKVImportCommand (0.98s) PASS -ok github.com/hashicorp/consul/command/kv/imp 0.954s +ok github.com/hashicorp/consul/command/kv/imp 1.041s ? github.com/hashicorp/consul/command/kv/impexp [no test files] === RUN TestKVPutCommand_noTabs === PAUSE TestKVPutCommand_noTabs @@ -87550,483 +87433,497 @@ === RUN TestKVPutCommand_CAS === PAUSE TestKVPutCommand_CAS === CONT TestKVPutCommand_noTabs ---- PASS: TestKVPutCommand_noTabs (0.00s) === CONT TestKVPutCommand_FileNoExist ---- PASS: TestKVPutCommand_FileNoExist (0.00s) +=== CONT TestKVPutCommand_NegativeVal +=== CONT TestKVPutCommand_Flags === CONT TestKVPutCommand_CAS [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... -=== CONT TestKVPutCommand_EmptyDataQuoted === CONT TestKVPutCommand_File === CONT TestKVPutCommand_Base64 -=== CONT TestKVPutCommand_Flags -=== CONT TestKVPutCommand_Validation -=== CONT TestKVPutCommand +=== CONT TestKVPutCommand_EmptyDataQuoted +--- PASS: TestKVPutCommand_noTabs (0.00s) === CONT TestKVPutCommand_Stdin +--- PASS: TestKVPutCommand_FileNoExist (0.00s) +=== CONT TestKVPutCommand [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range ---- PASS: TestKVPutCommand_Validation (0.03s) -=== CONT TestKVPutCommand_NegativeVal -2022-09-15T21:45:37.115Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:37.217Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:37.279 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2488292f-c931-ef2e-bb11-9bfc4a5a304c Address:127.0.0.1:28405}]" -45:37.280 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:28405 [Follower]" leader= -45:37.284 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2488292f-c931-ef2e-bb11-9bfc4a5a304c.dc1 127.0.0.1 -45:37.285 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2488292f-c931-ef2e-bb11-9bfc4a5a304c 127.0.0.1 -2022-09-15T21:45:37.285Z [INFO] agent.router: Initializing LAN area manager -45:37.285 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28400 network=udp -45:37.295 [INFO] TestAgent.server: Adding LAN server: server="Node-2488292f-c931-ef2e-bb11-9bfc4a5a304c (Addr: tcp/127.0.0.1:28405) (DC: dc1)" -45:37.295 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2488292f-c931-ef2e-bb11-9bfc4a5a304c.dc1 area=wan -45:37.322 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:37.328 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:28405 [Candidate]" term=2 -45:37.329 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:37.329 [DEBUG] TestAgent.server.raft: vote granted: from=2488292f-c931-ef2e-bb11-9bfc4a5a304c term=2 tally=1 -45:37.329 [INFO] TestAgent.server.raft: election won: tally=1 -45:37.329 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:28405 [Leader]" -45:37.330 [INFO] TestAgent.server: cluster leadership acquired -45:37.374 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28400 network=tcp -45:37.375 [INFO] TestAgent.server: New leader elected: payload=Node-2488292f-c931-ef2e-bb11-9bfc4a5a304c -45:37.376 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:28401 network=tcp -45:37.376 [INFO] TestAgent: started state syncer -45:37.376 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:28405 -45:37.379 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:28406 network=tcp -45:37.387 [INFO] TestAgent: Synced node info -45:37.387 [DEBUG] TestAgent: Node info in sync -45:37.410 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:38136 latency=30.17752ms -45:37.415 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:38140 latency=604.7µs -45:37.416 [INFO] TestAgent: Requesting shutdown -45:37.416 [INFO] TestAgent.server: shutting down server -45:37.416 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:37.416 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:37.417Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:37.417Z [INFO] agent.router.manager: shutting down -45:37.417 [INFO] TestAgent: consul server down -45:37.417 [INFO] TestAgent: shutdown complete -45:37.417 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28400 network=tcp -45:37.417 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28400 network=udp -45:37.417 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:28401 network=tcp -45:37.418 [ERROR] TestAgent.server: failed to establish leadership: error="error configuring provider: raft is already shutdown" -45:37.418 [ERROR] TestAgent.server: failed to transfer leadership attempt, will retry: attempt=0 retry_limit=3 error="raft is already shutdown" -45:37.918 [INFO] TestAgent: Waiting for endpoints to shut down -45:37.918 [INFO] TestAgent: Endpoints down ---- PASS: TestKVPutCommand_Base64 (1.05s) -2022-09-15T21:45:36.965Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:37.158Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:37.250 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fbc14560-6ef5-abf6-0eba-75a977880f59 Address:127.0.0.1:28384}]" -45:37.252 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:28384 [Follower]" leader= -45:37.279 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fbc14560-6ef5-abf6-0eba-75a977880f59.dc1 127.0.0.1 -45:37.280 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fbc14560-6ef5-abf6-0eba-75a977880f59 127.0.0.1 -2022-09-15T21:45:37.281Z [INFO] agent.router: Initializing LAN area manager -45:37.281 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28379 network=udp -45:37.282 [INFO] TestAgent.server: Adding LAN server: server="Node-fbc14560-6ef5-abf6-0eba-75a977880f59 (Addr: tcp/127.0.0.1:28384) (DC: dc1)" -45:37.282 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fbc14560-6ef5-abf6-0eba-75a977880f59.dc1 area=wan -45:37.282 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28379 network=tcp -45:37.283 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:28380 network=tcp -45:37.283 [INFO] TestAgent: started state syncer -45:37.283 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:28385 network=tcp -45:37.328 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:37.328 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:28384 [Candidate]" term=2 -45:37.329 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:37.329 [DEBUG] TestAgent.server.raft: vote granted: from=fbc14560-6ef5-abf6-0eba-75a977880f59 term=2 tally=1 -45:37.329 [INFO] TestAgent.server.raft: election won: tally=1 -45:37.329 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:28384 [Leader]" -45:37.330 [INFO] TestAgent.server: cluster leadership acquired -45:37.379 [INFO] TestAgent: Synced node info -45:37.379 [DEBUG] TestAgent: Node info in sync -45:37.418 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:28384 -45:37.419 [INFO] TestAgent.server: New leader elected: payload=Node-fbc14560-6ef5-abf6-0eba-75a977880f59 -45:37.420 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:37.422 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:37.422 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:37.422 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:37.422 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:37.422 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:37.422 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fbc14560-6ef5-abf6-0eba-75a977880f59 -45:37.423 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fbc14560-6ef5-abf6-0eba-75a977880f59 -45:37.423 [INFO] TestAgent.server: federation state anti-entropy synced -45:37.424 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:57260 latency=39.7356ms -45:37.445 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo?cas=123 from=127.0.0.1:57264 latency=911.82µs -45:37.446 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:57260 latency=474.8µs -45:37.463 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo?cas=4 from=127.0.0.1:57266 latency=987.08µs -45:37.465 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:57260 latency=294.36µs -45:37.465 [INFO] TestAgent: Requesting shutdown -45:37.465 [INFO] TestAgent.server: shutting down server -45:37.465 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:37.465 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:37.465 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:37.465 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:37.465 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:37.466 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:37.466 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:37.466 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:37.466 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:37.466 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -45:37.466 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2022-09-15T21:45:37.466Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:37.466Z [INFO] agent.router.manager: shutting down -45:37.466 [INFO] TestAgent: consul server down -45:37.466 [INFO] TestAgent: shutdown complete -45:37.466 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28379 network=tcp -45:37.466 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28379 network=udp -45:37.466 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:28380 network=tcp -45:37.967 [INFO] TestAgent: Waiting for endpoints to shut down -45:37.967 [INFO] TestAgent: Endpoints down ---- PASS: TestKVPutCommand_CAS (1.10s) -2022-09-15T21:45:37.003Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:37.168Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:37.194 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:90716f86-6cf6-28b3-242b-dc8dc848bd64 Address:127.0.0.1:28412}]" -45:37.202 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-90716f86-6cf6-28b3-242b-dc8dc848bd64.dc1 127.0.0.1 -45:37.202 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-90716f86-6cf6-28b3-242b-dc8dc848bd64 127.0.0.1 -2022-09-15T21:45:37.203Z [INFO] agent.router: Initializing LAN area manager -45:37.203 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28407 network=udp -45:37.203 [INFO] TestAgent.server: Adding LAN server: server="Node-90716f86-6cf6-28b3-242b-dc8dc848bd64 (Addr: tcp/127.0.0.1:28412) (DC: dc1)" -45:37.203 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-90716f86-6cf6-28b3-242b-dc8dc848bd64.dc1 area=wan -45:37.204 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28407 network=tcp -45:37.204 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:28408 network=tcp -45:37.205 [INFO] TestAgent: started state syncer -45:37.205 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:28413 network=tcp -45:37.231 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:28412 [Follower]" leader= -45:37.278 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:37.278 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:28412 [Candidate]" term=2 -45:37.278 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:37.278 [DEBUG] TestAgent.server.raft: vote granted: from=90716f86-6cf6-28b3-242b-dc8dc848bd64 term=2 tally=1 -45:37.278 [INFO] TestAgent.server.raft: election won: tally=1 -45:37.278 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:28412 [Leader]" -45:37.278 [INFO] TestAgent.server: cluster leadership acquired -45:37.279 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:28412 -45:37.279 [INFO] TestAgent.server: New leader elected: payload=Node-90716f86-6cf6-28b3-242b-dc8dc848bd64 -45:37.296 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:37.419 [INFO] TestAgent: Synced node info -45:37.419 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:37.419 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:37.419 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:37.419 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:37.419 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:37.419 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-90716f86-6cf6-28b3-242b-dc8dc848bd64 -45:37.423 [INFO] TestAgent.server: member joined, marking health alive: member=Node-90716f86-6cf6-28b3-242b-dc8dc848bd64 -45:37.438 [INFO] TestAgent.server: federation state anti-entropy synced -45:37.535 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo?flags=12345 from=127.0.0.1:59296 latency=679.24µs -45:37.536 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:59298 latency=341.58µs -45:37.542 [INFO] TestAgent: Requesting shutdown -45:37.542 [INFO] TestAgent.server: shutting down server -45:37.542 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:37.542 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:37.542 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:37.542 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:37.542 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:37.542 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:37.542 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:37.542 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:37.542 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:37.542 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:37.542 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:37.542Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:37.542Z [INFO] agent.router.manager: shutting down -45:37.543 [INFO] TestAgent: consul server down -45:37.543 [INFO] TestAgent: shutdown complete -45:37.543 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28407 network=tcp -45:37.543 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28407 network=udp -45:37.543 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:28408 network=tcp -45:38.043 [INFO] TestAgent: Waiting for endpoints to shut down -45:38.043 [INFO] TestAgent: Endpoints down ---- PASS: TestKVPutCommand_Flags (1.18s) -2022-09-15T21:45:37.155Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:37.389Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:37.393 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:03f816d1-59f3-0ef3-b688-fce4f44a3562 Address:127.0.0.1:28426}]" -45:37.397 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-03f816d1-59f3-0ef3-b688-fce4f44a3562.dc1 127.0.0.1 -45:37.399 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-03f816d1-59f3-0ef3-b688-fce4f44a3562 127.0.0.1 -2022-09-15T21:45:37.399Z [INFO] agent.router: Initializing LAN area manager -45:37.411 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:28426 [Follower]" leader= -45:37.419 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28421 network=udp -45:37.419 [INFO] TestAgent.server: Adding LAN server: server="Node-03f816d1-59f3-0ef3-b688-fce4f44a3562 (Addr: tcp/127.0.0.1:28426) (DC: dc1)" -45:37.419 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-03f816d1-59f3-0ef3-b688-fce4f44a3562.dc1 area=wan -45:37.420 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28421 network=tcp -45:37.421 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:28422 network=tcp -45:37.421 [INFO] TestAgent: started state syncer -45:37.423 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:28427 network=tcp -45:37.477 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:37.481 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:28426 [Candidate]" term=2 -45:37.482 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:37.482 [DEBUG] TestAgent.server.raft: vote granted: from=03f816d1-59f3-0ef3-b688-fce4f44a3562 term=2 tally=1 -45:37.482 [INFO] TestAgent.server.raft: election won: tally=1 -45:37.482 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:28426 [Leader]" -45:37.482 [INFO] TestAgent.server: cluster leadership acquired -45:37.482 [INFO] TestAgent.server: New leader elected: payload=Node-03f816d1-59f3-0ef3-b688-fce4f44a3562 -45:37.483 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:28426 -45:37.485 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:37.492 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:37.492 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:37.492 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:37.492 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:37.492 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:37.493 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-03f816d1-59f3-0ef3-b688-fce4f44a3562 -45:37.493 [INFO] TestAgent.server: member joined, marking health alive: member=Node-03f816d1-59f3-0ef3-b688-fce4f44a3562 -45:37.494 [INFO] TestAgent.server: federation state anti-entropy synced -45:37.575 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -45:37.575 [INFO] TestAgent: Synced node info -45:37.590 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:41374 latency=722.14µs -45:37.591 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:41376 latency=167.1µs -45:37.592 [INFO] TestAgent: Requesting shutdown -45:37.592 [INFO] TestAgent.server: shutting down server -45:37.592 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:37.592 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:37.592 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:37.592 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:37.592 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:37.592 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:37.592 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:37.592 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:37.592 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:37.592 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:37.592 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:37.593Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:37.593Z [INFO] agent.router.manager: shutting down -45:37.593 [INFO] TestAgent: consul server down -45:37.593 [INFO] TestAgent: shutdown complete -45:37.593 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28421 network=tcp -45:37.593 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28421 network=udp -45:37.593 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:28422 network=tcp -45:38.093 [INFO] TestAgent: Waiting for endpoints to shut down -45:38.093 [INFO] TestAgent: Endpoints down ---- PASS: TestKVPutCommand_Stdin (1.21s) -2022-09-15T21:45:37.049Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:37.214Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:37.217 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:129f0031-0197-4c1a-4775-37672fd27364 Address:127.0.0.1:28419}]" -45:37.218 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:28419 [Follower]" leader= -45:37.267 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-129f0031-0197-4c1a-4775-37672fd27364.dc1 127.0.0.1 -45:37.268 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-129f0031-0197-4c1a-4775-37672fd27364 127.0.0.1 -2022-09-15T21:45:37.268Z [INFO] agent.router: Initializing LAN area manager -45:37.268 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28414 network=udp -45:37.270 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-129f0031-0197-4c1a-4775-37672fd27364.dc1 area=wan -45:37.270 [INFO] TestAgent.server: Adding LAN server: server="Node-129f0031-0197-4c1a-4775-37672fd27364 (Addr: tcp/127.0.0.1:28419) (DC: dc1)" -45:37.270 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28414 network=tcp -45:37.271 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:28415 network=tcp -45:37.272 [INFO] TestAgent: started state syncer -45:37.272 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:28420 network=tcp -45:37.280 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:37.280 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:28419 [Candidate]" term=2 -45:37.280 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:37.280 [DEBUG] TestAgent.server.raft: vote granted: from=129f0031-0197-4c1a-4775-37672fd27364 term=2 tally=1 -45:37.280 [INFO] TestAgent.server.raft: election won: tally=1 -45:37.280 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:28419 [Leader]" -45:37.280 [INFO] TestAgent.server: cluster leadership acquired -45:37.298 [INFO] TestAgent.server: New leader elected: payload=Node-129f0031-0197-4c1a-4775-37672fd27364 -45:37.300 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:28419 -45:37.302 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:37.311 [INFO] TestAgent: Synced node info -45:37.422 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:37.422 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:37.422 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:37.422 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:37.422 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:37.422 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-129f0031-0197-4c1a-4775-37672fd27364 -45:37.422 [INFO] TestAgent.server: member joined, marking health alive: member=Node-129f0031-0197-4c1a-4775-37672fd27364 -45:37.425 [INFO] TestAgent.server: federation state anti-entropy synced -45:37.630 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:53432 latency=823.6µs -45:37.631 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:53434 latency=154.62µs -45:37.632 [INFO] TestAgent: Requesting shutdown -45:37.632 [INFO] TestAgent.server: shutting down server -45:37.632 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:37.632 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:37.632 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:37.632 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:37.632 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:37.632 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:37.632 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:37.632 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:37.632 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:37.632 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:37.632 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:37.633Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:37.633Z [INFO] agent.router.manager: shutting down -45:37.633 [INFO] TestAgent: consul server down -45:37.633 [INFO] TestAgent: shutdown complete -45:37.633 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28414 network=tcp -45:37.633 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28414 network=udp -45:37.633 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:28415 network=tcp -45:38.133 [INFO] TestAgent: Waiting for endpoints to shut down -45:38.133 [INFO] TestAgent: Endpoints down ---- PASS: TestKVPutCommand (1.25s) -2022-09-15T21:45:37.011Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:37.137Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:37.223 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:72fb0dc2-1963-4e38-bfa0-c606952d8ba5 Address:127.0.0.1:28398}]" -45:37.230 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:28398 [Follower]" leader= -45:37.244 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-72fb0dc2-1963-4e38-bfa0-c606952d8ba5.dc1 127.0.0.1 -45:37.245 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-72fb0dc2-1963-4e38-bfa0-c606952d8ba5 127.0.0.1 -2022-09-15T21:45:37.246Z [INFO] agent.router: Initializing LAN area manager -45:37.246 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28393 network=udp -45:37.247 [INFO] TestAgent.server: Adding LAN server: server="Node-72fb0dc2-1963-4e38-bfa0-c606952d8ba5 (Addr: tcp/127.0.0.1:28398) (DC: dc1)" -45:37.247 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-72fb0dc2-1963-4e38-bfa0-c606952d8ba5.dc1 area=wan -45:37.280 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28393 network=tcp -45:37.281 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:28394 network=tcp -45:37.281 [INFO] TestAgent: started state syncer -45:37.281 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:28399 network=tcp -45:37.304 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:37.304 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:28398 [Candidate]" term=2 -45:37.304 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:37.304 [DEBUG] TestAgent.server.raft: vote granted: from=72fb0dc2-1963-4e38-bfa0-c606952d8ba5 term=2 tally=1 -45:37.304 [INFO] TestAgent.server.raft: election won: tally=1 -45:37.304 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:28398 [Leader]" -45:37.304 [INFO] TestAgent.server: cluster leadership acquired -45:37.305 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:28398 -45:37.309 [INFO] TestAgent.server: New leader elected: payload=Node-72fb0dc2-1963-4e38-bfa0-c606952d8ba5 -45:37.317 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:37.430 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:37.430 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:37.430 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:37.430 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:37.430 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:37.430 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-72fb0dc2-1963-4e38-bfa0-c606952d8ba5 -45:37.430 [INFO] TestAgent.server: member joined, marking health alive: member=Node-72fb0dc2-1963-4e38-bfa0-c606952d8ba5 -45:37.443 [INFO] TestAgent.server: federation state anti-entropy synced -45:37.629 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -45:37.630 [INFO] TestAgent: Synced node info -45:37.671 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:55892 latency=1.17006ms -45:37.673 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:55894 latency=166.22µs -45:37.673 [INFO] TestAgent: Requesting shutdown -45:37.674 [INFO] TestAgent.server: shutting down server -45:37.674 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:37.674 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:37.674 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:37.674 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:37.674 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:37.674 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:37.674 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:37.674 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:37.674 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:37.674 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:37.674 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:37.674Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:37.674Z [INFO] agent.router.manager: shutting down -45:37.674 [INFO] TestAgent: consul server down -45:37.674 [INFO] TestAgent: shutdown complete -45:37.674 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28393 network=tcp -45:37.674 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28393 network=udp -45:37.675 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:28394 network=tcp -45:38.175 [INFO] TestAgent: Waiting for endpoints to shut down -45:38.175 [INFO] TestAgent: Endpoints down ---- PASS: TestKVPutCommand_File (1.31s) -2022-09-15T21:45:37.088Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:37.277Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:37.427 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:19805af2-1e0f-5b01-b387-1a61c36e7c37 Address:127.0.0.1:28391}]" -45:37.428 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-19805af2-1e0f-5b01-b387-1a61c36e7c37.dc1 127.0.0.1 -45:37.429 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-19805af2-1e0f-5b01-b387-1a61c36e7c37 127.0.0.1 -2022-09-15T21:45:37.429Z [INFO] agent.router: Initializing LAN area manager -45:37.429 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28386 network=udp -45:37.429 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:28391 [Follower]" leader= -45:37.430 [INFO] TestAgent.server: Adding LAN server: server="Node-19805af2-1e0f-5b01-b387-1a61c36e7c37 (Addr: tcp/127.0.0.1:28391) (DC: dc1)" -45:37.430 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-19805af2-1e0f-5b01-b387-1a61c36e7c37.dc1 area=wan -45:37.430 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28386 network=tcp -45:37.431 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:28387 network=tcp -45:37.431 [INFO] TestAgent: started state syncer -45:37.438 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:28392 network=tcp -45:37.467 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:37.467 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:28391 [Candidate]" term=2 -45:37.467 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:37.467 [DEBUG] TestAgent.server.raft: vote granted: from=19805af2-1e0f-5b01-b387-1a61c36e7c37 term=2 tally=1 -45:37.467 [INFO] TestAgent.server.raft: election won: tally=1 -45:37.467 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:28391 [Leader]" -45:37.467 [INFO] TestAgent.server: cluster leadership acquired -45:37.468 [INFO] TestAgent.server: New leader elected: payload=Node-19805af2-1e0f-5b01-b387-1a61c36e7c37 -45:37.468 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:28391 -45:37.470 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:37.474 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:37.474 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:37.474 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:37.474 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:37.474 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:37.474 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-19805af2-1e0f-5b01-b387-1a61c36e7c37 -45:37.474 [INFO] TestAgent.server: member joined, marking health alive: member=Node-19805af2-1e0f-5b01-b387-1a61c36e7c37 -45:37.479 [INFO] TestAgent.server: federation state anti-entropy synced -45:37.519 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -45:37.520 [INFO] TestAgent: Synced node info -45:37.520 [DEBUG] TestAgent: Node info in sync -45:37.784 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:48728 latency=654.32µs -45:37.790 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:48730 latency=206.2µs -45:37.791 [INFO] TestAgent: Requesting shutdown -45:37.791 [INFO] TestAgent.server: shutting down server -45:37.791 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:37.791 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:37.791 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:37.791 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:37.791 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:37.791 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:37.791 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:37.791 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:37.791 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:37.791 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:37.791 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:37.791Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:37.791Z [INFO] agent.router.manager: shutting down -45:37.791 [INFO] TestAgent: consul server down -45:37.791 [INFO] TestAgent: shutdown complete -45:37.791 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28386 network=tcp -45:37.792 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28386 network=udp -45:37.792 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:28387 network=tcp -45:38.292 [INFO] TestAgent: Waiting for endpoints to shut down -45:38.292 [INFO] TestAgent: Endpoints down ---- PASS: TestKVPutCommand_EmptyDataQuoted (1.42s) -2022-09-15T21:45:37.207Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:37.412Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:37.452 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2b7f6c4b-bf5f-5365-e9e7-a7027d519fa6 Address:127.0.0.1:28433}]" -45:37.454 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:28433 [Follower]" leader= -45:37.454 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2b7f6c4b-bf5f-5365-e9e7-a7027d519fa6.dc1 127.0.0.1 -45:37.455 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2b7f6c4b-bf5f-5365-e9e7-a7027d519fa6 127.0.0.1 -2022-09-15T21:45:37.455Z [INFO] agent.router: Initializing LAN area manager -45:37.456 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28428 network=udp -45:37.462 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28428 network=tcp -45:37.462 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2b7f6c4b-bf5f-5365-e9e7-a7027d519fa6.dc1 area=wan -45:37.463 [INFO] TestAgent.server: Adding LAN server: server="Node-2b7f6c4b-bf5f-5365-e9e7-a7027d519fa6 (Addr: tcp/127.0.0.1:28433) (DC: dc1)" -45:37.466 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:28429 network=tcp -45:37.467 [INFO] TestAgent: started state syncer -45:37.467 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:28434 network=tcp -45:37.519 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:37.519 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:28433 [Candidate]" term=2 -45:37.519 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:37.519 [DEBUG] TestAgent.server.raft: vote granted: from=2b7f6c4b-bf5f-5365-e9e7-a7027d519fa6 term=2 tally=1 -45:37.519 [INFO] TestAgent.server.raft: election won: tally=1 -45:37.519 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:28433 [Leader]" -45:37.519 [INFO] TestAgent.server: cluster leadership acquired -45:37.520 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:28433 -45:37.520 [INFO] TestAgent.server: New leader elected: payload=Node-2b7f6c4b-bf5f-5365-e9e7-a7027d519fa6 -45:37.521 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:37.523 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:37.523 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:37.523 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:37.523 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:37.523 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:37.523 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2b7f6c4b-bf5f-5365-e9e7-a7027d519fa6 -45:37.523 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2b7f6c4b-bf5f-5365-e9e7-a7027d519fa6 -45:37.524 [INFO] TestAgent.server: federation state anti-entropy synced -45:37.574 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -45:37.575 [INFO] TestAgent: Synced node info -45:37.575 [DEBUG] TestAgent: Node info in sync -45:37.863 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:35680 latency=695.68µs -45:37.864 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:35682 latency=258.72µs -45:37.865 [INFO] TestAgent: Requesting shutdown -45:37.865 [INFO] TestAgent.server: shutting down server -45:37.865 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:37.865 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:37.865 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:37.865 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:37.865 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:37.865 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:37.865 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:37.865 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:37.865 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:37.865 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:37.865 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -45:37.866 [INFO] TestAgent: consul server down -45:37.866 [INFO] TestAgent: shutdown complete -45:37.866 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28428 network=tcp -45:37.866 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28428 network=udp -45:37.866 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:28429 network=tcp -2022-09-15T21:45:37.866Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:37.866Z [INFO] agent.router.manager: shutting down -45:38.366 [INFO] TestAgent: Waiting for endpoints to shut down -45:38.366 [INFO] TestAgent: Endpoints down ---- PASS: TestKVPutCommand_NegativeVal (1.47s) +2021-08-13T15:38:02.944Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:38:02.996Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:03.029 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:76e95c0a-f51d-69fd-17dc-b4d69324db22 Address:127.0.0.1:21237}]" +38:03.029 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:21237 [Follower]" leader= +38:03.030 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-76e95c0a-f51d-69fd-17dc-b4d69324db22.dc1 127.0.0.1 +38:03.079 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +38:03.079 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:21237 [Candidate]" term=2 +38:03.079 [WARN] TestAgent.server.raft: unable to get address for sever, using fallback address: id=76e95c0a-f51d-69fd-17dc-b4d69324db22 fallback=127.0.0.1:21237 error="Could not find address for server id 76e95c0a-f51d-69fd-17dc-b4d69324db22" +38:03.079 [DEBUG] TestAgent.server.raft: votes: needed=1 +38:03.079 [DEBUG] TestAgent.server.raft: vote granted: from=76e95c0a-f51d-69fd-17dc-b4d69324db22 term=2 tally=1 +38:03.079 [INFO] TestAgent.server.raft: election won: tally=1 +38:03.079 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:21237 [Leader]" +38:03.096 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-76e95c0a-f51d-69fd-17dc-b4d69324db22 127.0.0.1 +2021-08-13T15:38:03.096Z [INFO] agent.router: Initializing LAN area manager +38:03.096 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21232 network=udp +38:03.096 [INFO] TestAgent.server: Adding LAN server: server="Node-76e95c0a-f51d-69fd-17dc-b4d69324db22 (Addr: tcp/127.0.0.1:21237) (DC: dc1)" +38:03.097 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-76e95c0a-f51d-69fd-17dc-b4d69324db22.dc1 area=wan +38:03.097 [INFO] TestAgent.server: cluster leadership acquired +38:03.097 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:21237 +38:03.115 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21232 network=tcp +38:03.122 [INFO] TestAgent.server: New leader elected: payload=Node-76e95c0a-f51d-69fd-17dc-b4d69324db22 +38:03.139 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +38:03.140 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:21233 network=tcp +38:03.141 [INFO] TestAgent: started state syncer +38:03.154 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:21238 network=tcp +38:03.155 [INFO] TestAgent: Synced node info +38:03.157 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +38:03.157 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +38:03.157 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +38:03.157 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +38:03.157 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +38:03.157 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-76e95c0a-f51d-69fd-17dc-b4d69324db22 +38:03.157 [INFO] TestAgent.server: member joined, marking health alive: member=Node-76e95c0a-f51d-69fd-17dc-b4d69324db22 +38:03.158 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-76e95c0a-f51d-69fd-17dc-b4d69324db22 +38:03.158 [INFO] TestAgent.server: federation state anti-entropy synced +38:03.160 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo?flags=12345 from=127.0.0.1:42846 latency=692.601µs +38:03.161 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:42856 latency=352.82µs +38:03.162 [INFO] TestAgent: Requesting shutdown +38:03.162 [INFO] TestAgent.server: shutting down server +38:03.162 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +38:03.162 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +38:03.162 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +38:03.162 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +38:03.162 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +38:03.162 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +38:03.162 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +38:03.162 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +38:03.162 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +38:03.162 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +38:03.162 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:38:03.162Z [INFO] agent.router.manager: shutting down +2021-08-13T15:38:03.162Z [INFO] agent.router.manager: shutting down +38:03.162 [INFO] TestAgent: consul server down +38:03.162 [INFO] TestAgent: shutdown complete +38:03.162 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21232 network=tcp +38:03.162 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21232 network=udp +38:03.163 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:21233 network=tcp +38:03.663 [INFO] TestAgent: Waiting for endpoints to shut down +38:03.663 [INFO] TestAgent: Endpoints down +2021-08-13T15:38:02.936Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:38:03.001Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:03.007 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5481e944-b376-9a4c-0596-8b744f39f667 Address:127.0.0.1:21258}]" +38:03.009 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5481e944-b376-9a4c-0596-8b744f39f667.dc1 127.0.0.1 +38:03.030 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:21258 [Follower]" leader= +38:03.089 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +38:03.089 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:21258 [Candidate]" term=2 +38:03.090 [WARN] TestAgent.server.raft: unable to get address for sever, using fallback address: id=5481e944-b376-9a4c-0596-8b744f39f667 fallback=127.0.0.1:21258 error="Could not find address for server id 5481e944-b376-9a4c-0596-8b744f39f667" +38:03.090 [DEBUG] TestAgent.server.raft: votes: needed=1 +38:03.090 [DEBUG] TestAgent.server.raft: vote granted: from=5481e944-b376-9a4c-0596-8b744f39f667 term=2 tally=1 +38:03.090 [INFO] TestAgent.server.raft: election won: tally=1 +38:03.090 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:21258 [Leader]" +38:03.092 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5481e944-b376-9a4c-0596-8b744f39f667 127.0.0.1 +2021-08-13T15:38:03.092Z [INFO] agent.router: Initializing LAN area manager +38:03.093 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21253 network=udp +38:03.093 [INFO] TestAgent.server: Adding LAN server: server="Node-5481e944-b376-9a4c-0596-8b744f39f667 (Addr: tcp/127.0.0.1:21258) (DC: dc1)" +38:03.093 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5481e944-b376-9a4c-0596-8b744f39f667.dc1 area=wan +38:03.093 [INFO] TestAgent.server: cluster leadership acquired +38:03.093 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:21258 +38:03.094 [INFO] TestAgent.server: New leader elected: payload=Node-5481e944-b376-9a4c-0596-8b744f39f667 +38:03.094 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21253 network=tcp +38:03.094 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +38:03.110 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:21254 network=tcp +38:03.110 [INFO] TestAgent: started state syncer +38:03.111 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +38:03.111 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +38:03.111 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +38:03.111 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +38:03.111 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +38:03.112 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5481e944-b376-9a4c-0596-8b744f39f667 +38:03.112 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5481e944-b376-9a4c-0596-8b744f39f667 +38:03.123 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:21259 network=tcp +38:03.148 [INFO] TestAgent: Synced node info +38:03.156 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:49582 latency=14.605254ms +38:03.156 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5481e944-b376-9a4c-0596-8b744f39f667 +38:03.158 [INFO] TestAgent.server: federation state anti-entropy synced +38:03.161 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:49590 latency=774.882µs +38:03.161 [INFO] TestAgent: Requesting shutdown +38:03.162 [INFO] TestAgent.server: shutting down server +38:03.162 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +38:03.162 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +38:03.162 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +38:03.162 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +38:03.162 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +38:03.162 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +38:03.162 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +38:03.162 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +38:03.162 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +38:03.162 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +38:03.162 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:38:03.163Z [INFO] agent.router.manager: shutting down +2021-08-13T15:38:03.163Z [INFO] agent.router.manager: shutting down +38:03.163 [INFO] TestAgent: consul server down +38:03.163 [INFO] TestAgent: shutdown complete +38:03.163 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21253 network=tcp +38:03.163 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21253 network=udp +38:03.163 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:21254 network=tcp +38:03.663 [INFO] TestAgent: Waiting for endpoints to shut down +38:03.663 [INFO] TestAgent: Endpoints down +2021-08-13T15:38:02.903Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:38:02.982Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:02.989 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5c6ba664-8ea0-dcf1-30a3-388ebc3c3c70 Address:127.0.0.1:21244}]" +38:02.990 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5c6ba664-8ea0-dcf1-30a3-388ebc3c3c70.dc1 127.0.0.1 +38:02.991 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5c6ba664-8ea0-dcf1-30a3-388ebc3c3c70 127.0.0.1 +38:02.992 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:21244 [Follower]" leader= +2021-08-13T15:38:02.995Z [INFO] agent.router: Initializing LAN area manager +38:02.995 [INFO] TestAgent.server: Adding LAN server: server="Node-5c6ba664-8ea0-dcf1-30a3-388ebc3c3c70 (Addr: tcp/127.0.0.1:21244) (DC: dc1)" +38:02.995 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5c6ba664-8ea0-dcf1-30a3-388ebc3c3c70.dc1 area=wan +38:02.996 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21239 network=udp +38:02.996 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21239 network=tcp +38:03.032 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +38:03.032 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:21244 [Candidate]" term=2 +38:03.032 [DEBUG] TestAgent.server.raft: votes: needed=1 +38:03.032 [DEBUG] TestAgent.server.raft: vote granted: from=5c6ba664-8ea0-dcf1-30a3-388ebc3c3c70 term=2 tally=1 +38:03.032 [INFO] TestAgent.server.raft: election won: tally=1 +38:03.032 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:21244 [Leader]" +38:03.033 [INFO] TestAgent.server: cluster leadership acquired +38:03.033 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:21244 +38:03.034 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +38:03.034 [INFO] TestAgent.server: New leader elected: payload=Node-5c6ba664-8ea0-dcf1-30a3-388ebc3c3c70 +38:03.035 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:21240 network=tcp +38:03.085 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +38:03.085 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +38:03.085 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +38:03.085 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +38:03.085 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +38:03.085 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5c6ba664-8ea0-dcf1-30a3-388ebc3c3c70 +38:03.085 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5c6ba664-8ea0-dcf1-30a3-388ebc3c3c70 +38:03.086 [INFO] TestAgent.server: federation state anti-entropy synced +38:03.091 [INFO] TestAgent: started state syncer +38:03.093 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:21245 network=tcp +38:03.093 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +38:03.093 [INFO] TestAgent: Synced node info +38:03.093 [DEBUG] TestAgent: Node info in sync +38:03.155 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:38222 latency=35.699423ms +38:03.161 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:38234 latency=558.561µs +38:03.162 [INFO] TestAgent: Requesting shutdown +38:03.162 [INFO] TestAgent.server: shutting down server +38:03.162 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +38:03.162 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +38:03.162 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +38:03.162 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +38:03.162 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +38:03.162 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +38:03.162 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +38:03.163 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +38:03.163 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +38:03.163 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +38:03.163 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:38:03.163Z [INFO] agent.router.manager: shutting down +2021-08-13T15:38:03.163Z [INFO] agent.router.manager: shutting down +38:03.163 [INFO] TestAgent: consul server down +38:03.163 [INFO] TestAgent: shutdown complete +38:03.163 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21239 network=tcp +38:03.163 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21239 network=udp +38:03.163 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:21240 network=tcp +38:03.663 [INFO] TestAgent: Waiting for endpoints to shut down +38:03.663 [INFO] TestAgent: Endpoints down +--- PASS: TestKVPutCommand_Flags (0.93s) +=== CONT TestKVPutCommand_Validation +--- PASS: TestKVPutCommand_File (0.93s) +--- PASS: TestKVPutCommand_NegativeVal (0.93s) +2021-08-13T15:38:02.911Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:38:02.960Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:02.965 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:456565c4-8aa3-0a02-3e77-66083d08efca Address:127.0.0.1:21251}]" +38:02.966 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-456565c4-8aa3-0a02-3e77-66083d08efca.dc1 127.0.0.1 +38:02.966 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-456565c4-8aa3-0a02-3e77-66083d08efca 127.0.0.1 +2021-08-13T15:38:02.967Z [INFO] agent.router: Initializing LAN area manager +38:02.967 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:21251 [Follower]" leader= +38:02.967 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21246 network=udp +38:02.967 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-456565c4-8aa3-0a02-3e77-66083d08efca.dc1 area=wan +38:02.967 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21246 network=tcp +38:02.967 [INFO] TestAgent.server: Adding LAN server: server="Node-456565c4-8aa3-0a02-3e77-66083d08efca (Addr: tcp/127.0.0.1:21251) (DC: dc1)" +38:02.968 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:21247 network=tcp +38:02.968 [INFO] TestAgent: started state syncer +38:02.968 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:21252 network=tcp +38:03.031 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +38:03.031 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:21251 [Candidate]" term=2 +38:03.031 [DEBUG] TestAgent.server.raft: votes: needed=1 +38:03.031 [DEBUG] TestAgent.server.raft: vote granted: from=456565c4-8aa3-0a02-3e77-66083d08efca term=2 tally=1 +38:03.031 [INFO] TestAgent.server.raft: election won: tally=1 +38:03.031 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:21251 [Leader]" +38:03.031 [INFO] TestAgent.server: cluster leadership acquired +38:03.032 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:21251 +38:03.032 [INFO] TestAgent.server: New leader elected: payload=Node-456565c4-8aa3-0a02-3e77-66083d08efca +38:03.035 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +38:03.082 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +38:03.082 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +38:03.082 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +38:03.082 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +38:03.082 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +38:03.082 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-456565c4-8aa3-0a02-3e77-66083d08efca +38:03.082 [INFO] TestAgent.server: member joined, marking health alive: member=Node-456565c4-8aa3-0a02-3e77-66083d08efca +38:03.087 [INFO] TestAgent.server: federation state anti-entropy synced +38:03.157 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:47090 latency=659.561µs +38:03.162 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo?cas=123 from=127.0.0.1:47100 latency=727.321µs +38:03.162 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:47090 latency=128.32µs +38:03.164 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo?cas=12 from=127.0.0.1:47104 latency=548.921µs +38:03.165 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:47090 latency=133.941µs +38:03.165 [INFO] TestAgent: Requesting shutdown +38:03.165 [INFO] TestAgent.server: shutting down server +38:03.165 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +38:03.165 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +38:03.165 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +38:03.165 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +38:03.165 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +38:03.165 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +38:03.165 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +38:03.165 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +38:03.165 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +38:03.165 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +38:03.165 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +38:03.165 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:38:03.166Z [INFO] agent.router.manager: shutting down +2021-08-13T15:38:03.166Z [INFO] agent.router.manager: shutting down +38:03.166 [INFO] TestAgent: consul server down +38:03.166 [INFO] TestAgent: shutdown complete +38:03.166 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21246 network=tcp +38:03.166 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21246 network=udp +38:03.166 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:21247 network=tcp +38:03.666 [INFO] TestAgent: Waiting for endpoints to shut down +38:03.666 [INFO] TestAgent: Endpoints down +2021-08-13T15:38:02.930Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:38:02.989Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:02.992 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5d791091-9fb0-ee1d-2911-4ab876e3200d Address:127.0.0.1:21286}]" +38:02.992 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:21286 [Follower]" leader= +38:02.998 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5d791091-9fb0-ee1d-2911-4ab876e3200d.dc1 127.0.0.1 +38:03.003 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5d791091-9fb0-ee1d-2911-4ab876e3200d 127.0.0.1 +2021-08-13T15:38:03.003Z [INFO] agent.router: Initializing LAN area manager +38:03.004 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5d791091-9fb0-ee1d-2911-4ab876e3200d.dc1 area=wan +38:03.004 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21281 network=tcp +38:03.005 [INFO] TestAgent.server: Adding LAN server: server="Node-5d791091-9fb0-ee1d-2911-4ab876e3200d (Addr: tcp/127.0.0.1:21286) (DC: dc1)" +38:03.005 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21281 network=udp +38:03.007 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:21282 network=tcp +38:03.007 [INFO] TestAgent: started state syncer +38:03.007 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:21287 network=tcp +38:03.051 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +38:03.051 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:21286 [Candidate]" term=2 +38:03.051 [DEBUG] TestAgent.server.raft: votes: needed=1 +38:03.051 [DEBUG] TestAgent.server.raft: vote granted: from=5d791091-9fb0-ee1d-2911-4ab876e3200d term=2 tally=1 +38:03.051 [INFO] TestAgent.server.raft: election won: tally=1 +38:03.051 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:21286 [Leader]" +38:03.052 [INFO] TestAgent.server: cluster leadership acquired +38:03.052 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:21286 +38:03.053 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +38:03.054 [INFO] TestAgent.server: New leader elected: payload=Node-5d791091-9fb0-ee1d-2911-4ab876e3200d +38:03.085 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +38:03.086 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +38:03.086 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +38:03.086 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +38:03.086 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +38:03.086 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5d791091-9fb0-ee1d-2911-4ab876e3200d +38:03.086 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5d791091-9fb0-ee1d-2911-4ab876e3200d +38:03.087 [INFO] TestAgent.server: federation state anti-entropy synced +38:03.164 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:59460 latency=5.853774ms +38:03.165 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:59472 latency=133.32µs +38:03.166 [INFO] TestAgent: Requesting shutdown +38:03.166 [INFO] TestAgent.server: shutting down server +38:03.166 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +38:03.166 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +38:03.166 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +38:03.166 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +38:03.166 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +38:03.166 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +38:03.166 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +38:03.166 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +38:03.166 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +38:03.166 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +38:03.166 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +38:03.166 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:38:03.166Z [INFO] agent.router.manager: shutting down +2021-08-13T15:38:03.166Z [INFO] agent.router.manager: shutting down +38:03.166 [INFO] TestAgent: consul server down +38:03.166 [INFO] TestAgent: shutdown complete +38:03.166 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21281 network=tcp +38:03.166 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21281 network=udp +38:03.166 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:21282 network=tcp +38:03.666 [INFO] TestAgent: Waiting for endpoints to shut down +38:03.666 [INFO] TestAgent: Endpoints down +--- PASS: TestKVPutCommand_CAS (0.93s) +--- PASS: TestKVPutCommand (0.93s) +--- PASS: TestKVPutCommand_Validation (0.00s) +2021-08-13T15:38:02.966Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:38:03.109Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:03.156 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:768884a5-909f-0d66-6a41-5d55f113924b Address:127.0.0.1:21279}]" +38:03.157 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-768884a5-909f-0d66-6a41-5d55f113924b.dc1 127.0.0.1 +38:03.158 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-768884a5-909f-0d66-6a41-5d55f113924b 127.0.0.1 +2021-08-13T15:38:03.158Z [INFO] agent.router: Initializing LAN area manager +38:03.158 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21274 network=udp +38:03.158 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:21279 [Follower]" leader= +38:03.159 [INFO] TestAgent.server: Adding LAN server: server="Node-768884a5-909f-0d66-6a41-5d55f113924b (Addr: tcp/127.0.0.1:21279) (DC: dc1)" +38:03.159 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-768884a5-909f-0d66-6a41-5d55f113924b.dc1 area=wan +38:03.159 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21274 network=tcp +38:03.160 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:21275 network=tcp +38:03.160 [INFO] TestAgent: started state syncer +38:03.160 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:21280 network=tcp +38:03.194 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +38:03.194 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:21279 [Candidate]" term=2 +38:03.195 [DEBUG] TestAgent.server.raft: votes: needed=1 +38:03.195 [DEBUG] TestAgent.server.raft: vote granted: from=768884a5-909f-0d66-6a41-5d55f113924b term=2 tally=1 +38:03.195 [INFO] TestAgent.server.raft: election won: tally=1 +38:03.195 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:21279 [Leader]" +38:03.195 [INFO] TestAgent.server: cluster leadership acquired +38:03.195 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:21279 +38:03.197 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +38:03.199 [INFO] TestAgent.server: New leader elected: payload=Node-768884a5-909f-0d66-6a41-5d55f113924b +38:03.206 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +38:03.206 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +38:03.206 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +38:03.206 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +38:03.206 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +38:03.206 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-768884a5-909f-0d66-6a41-5d55f113924b +38:03.206 [INFO] TestAgent.server: member joined, marking health alive: member=Node-768884a5-909f-0d66-6a41-5d55f113924b +38:03.209 [INFO] TestAgent.server: federation state anti-entropy synced +38:03.236 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:35288 latency=702.322µs +38:03.238 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:35290 latency=325.801µs +38:03.238 [INFO] TestAgent: Requesting shutdown +38:03.238 [INFO] TestAgent.server: shutting down server +38:03.238 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +38:03.238 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +38:03.238 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +38:03.238 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +38:03.238 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +38:03.238 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +38:03.238 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +38:03.238 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +38:03.238 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +38:03.238 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +38:03.238 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +38:03.239 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:38:03.239Z [INFO] agent.router.manager: shutting down +2021-08-13T15:38:03.239Z [INFO] agent.router.manager: shutting down +38:03.239 [INFO] TestAgent: consul server down +38:03.239 [INFO] TestAgent: shutdown complete +38:03.239 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21274 network=tcp +38:03.239 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21274 network=udp +38:03.239 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:21275 network=tcp +38:03.739 [INFO] TestAgent: Waiting for endpoints to shut down +38:03.739 [INFO] TestAgent: Endpoints down +--- PASS: TestKVPutCommand_Stdin (1.00s) +2021-08-13T15:38:02.893Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:38:02.944Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:02.948 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:930003e4-5142-e83f-de13-af3882344d85 Address:127.0.0.1:21265}]" +38:02.949 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:21265 [Follower]" leader= +38:02.956 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-930003e4-5142-e83f-de13-af3882344d85.dc1 127.0.0.1 +38:02.960 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-930003e4-5142-e83f-de13-af3882344d85 127.0.0.1 +2021-08-13T15:38:02.960Z [INFO] agent.router: Initializing LAN area manager +38:02.960 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21260 network=udp +38:02.960 [INFO] TestAgent.server: Adding LAN server: server="Node-930003e4-5142-e83f-de13-af3882344d85 (Addr: tcp/127.0.0.1:21265) (DC: dc1)" +38:02.960 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-930003e4-5142-e83f-de13-af3882344d85.dc1 area=wan +38:02.961 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21260 network=tcp +38:02.963 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:21261 network=tcp +38:02.963 [INFO] TestAgent: started state syncer +38:02.986 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:21266 network=tcp +38:03.030 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +38:03.030 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:21265 [Candidate]" term=2 +38:03.031 [DEBUG] TestAgent.server.raft: votes: needed=1 +38:03.031 [DEBUG] TestAgent.server.raft: vote granted: from=930003e4-5142-e83f-de13-af3882344d85 term=2 tally=1 +38:03.031 [INFO] TestAgent.server.raft: election won: tally=1 +38:03.031 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:21265 [Leader]" +38:03.031 [INFO] TestAgent.server: cluster leadership acquired +38:03.031 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:21265 +38:03.031 [INFO] TestAgent.server: New leader elected: payload=Node-930003e4-5142-e83f-de13-af3882344d85 +38:03.033 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +38:03.083 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +38:03.083 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +38:03.083 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +38:03.083 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +38:03.083 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +38:03.083 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-930003e4-5142-e83f-de13-af3882344d85 +38:03.083 [INFO] TestAgent.server: member joined, marking health alive: member=Node-930003e4-5142-e83f-de13-af3882344d85 +38:03.084 [INFO] TestAgent.server: federation state anti-entropy synced +38:03.245 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:44502 latency=909.562µs +38:03.246 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:44504 latency=148.441µs +38:03.246 [INFO] TestAgent: Requesting shutdown +38:03.246 [INFO] TestAgent.server: shutting down server +38:03.246 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +38:03.246 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +38:03.246 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +38:03.246 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +38:03.246 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +38:03.247 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +38:03.247 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +38:03.247 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +38:03.247 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +38:03.247 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +38:03.247 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +38:03.247 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:38:03.247Z [INFO] agent.router.manager: shutting down +2021-08-13T15:38:03.247Z [INFO] agent.router.manager: shutting down +38:03.247 [INFO] TestAgent: consul server down +38:03.247 [INFO] TestAgent: shutdown complete +38:03.247 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21260 network=tcp +38:03.247 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21260 network=udp +38:03.247 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:21261 network=tcp +38:03.747 [INFO] TestAgent: Waiting for endpoints to shut down +38:03.747 [INFO] TestAgent: Endpoints down +--- PASS: TestKVPutCommand_Base64 (1.01s) +2021-08-13T15:38:02.938Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:38:03.031Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:03.097 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:536cec16-2632-ea7d-bf06-b6e78cd34c54 Address:127.0.0.1:21272}]" +38:03.105 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-536cec16-2632-ea7d-bf06-b6e78cd34c54.dc1 127.0.0.1 +38:03.107 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:21272 [Follower]" leader= +38:03.111 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-536cec16-2632-ea7d-bf06-b6e78cd34c54 127.0.0.1 +2021-08-13T15:38:03.111Z [INFO] agent.router: Initializing LAN area manager +38:03.140 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21267 network=udp +38:03.141 [INFO] TestAgent.server: Adding LAN server: server="Node-536cec16-2632-ea7d-bf06-b6e78cd34c54 (Addr: tcp/127.0.0.1:21272) (DC: dc1)" +38:03.141 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-536cec16-2632-ea7d-bf06-b6e78cd34c54.dc1 area=wan +38:03.141 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21267 network=tcp +38:03.155 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:21268 network=tcp +38:03.156 [INFO] TestAgent: started state syncer +38:03.156 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:21273 network=tcp +38:03.173 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +38:03.173 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:21272 [Candidate]" term=2 +38:03.173 [DEBUG] TestAgent.server.raft: votes: needed=1 +38:03.173 [DEBUG] TestAgent.server.raft: vote granted: from=536cec16-2632-ea7d-bf06-b6e78cd34c54 term=2 tally=1 +38:03.173 [INFO] TestAgent.server.raft: election won: tally=1 +38:03.173 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:21272 [Leader]" +38:03.173 [INFO] TestAgent.server: cluster leadership acquired +38:03.174 [INFO] TestAgent.server: New leader elected: payload=Node-536cec16-2632-ea7d-bf06-b6e78cd34c54 +38:03.174 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:21272 +38:03.175 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +38:03.178 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +38:03.178 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +38:03.178 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +38:03.178 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +38:03.178 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +38:03.178 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-536cec16-2632-ea7d-bf06-b6e78cd34c54 +38:03.178 [INFO] TestAgent.server: member joined, marking health alive: member=Node-536cec16-2632-ea7d-bf06-b6e78cd34c54 +38:03.179 [INFO] TestAgent.server: federation state anti-entropy synced +38:03.251 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:53558 latency=853.542µs +38:03.252 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:53560 latency=369.761µs +38:03.253 [INFO] TestAgent: Requesting shutdown +38:03.253 [INFO] TestAgent.server: shutting down server +38:03.253 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +38:03.253 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +38:03.253 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +38:03.253 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +38:03.253 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +38:03.253 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +38:03.253 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +38:03.253 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +38:03.253 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +38:03.253 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +38:03.253 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +38:03.253 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:38:03.253Z [INFO] agent.router.manager: shutting down +2021-08-13T15:38:03.253Z [INFO] agent.router.manager: shutting down +38:03.254 [INFO] TestAgent: consul server down +38:03.254 [INFO] TestAgent: shutdown complete +38:03.254 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21267 network=tcp +38:03.254 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21267 network=udp +38:03.254 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:21268 network=tcp +38:03.754 [INFO] TestAgent: Waiting for endpoints to shut down +38:03.754 [INFO] TestAgent: Endpoints down +--- PASS: TestKVPutCommand_EmptyDataQuoted (1.02s) PASS -ok github.com/hashicorp/consul/command/kv/put 1.614s +ok github.com/hashicorp/consul/command/kv/put 1.081s === RUN TestLeaveCommand_noTabs === PAUSE TestLeaveCommand_noTabs === RUN TestLeaveCommand @@ -88034,142 +87931,140 @@ === RUN TestLeaveCommand_FailOnNonFlagArgs === PAUSE TestLeaveCommand_FailOnNonFlagArgs === CONT TestLeaveCommand_noTabs -=== CONT TestLeaveCommand +--- PASS: TestLeaveCommand_noTabs (0.00s) === CONT TestLeaveCommand_FailOnNonFlagArgs [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... ---- PASS: TestLeaveCommand_noTabs (0.00s) +=== CONT TestLeaveCommand [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:45:36.670Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:36.698Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:36.700 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ce9f932b-0567-cb4b-95fe-eaf123686e50 Address:127.0.0.1:13076}]" -45:36.701 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13076 [Follower]" leader= -45:36.703 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ce9f932b-0567-cb4b-95fe-eaf123686e50.dc1 127.0.0.1 -45:36.703 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ce9f932b-0567-cb4b-95fe-eaf123686e50 127.0.0.1 -2022-09-15T21:45:36.703Z [INFO] agent.router: Initializing LAN area manager -45:36.707 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13071 network=udp -45:36.707 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ce9f932b-0567-cb4b-95fe-eaf123686e50.dc1 area=wan -45:36.708 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13071 network=tcp -45:36.710 [INFO] TestAgent.server: Adding LAN server: server="Node-ce9f932b-0567-cb4b-95fe-eaf123686e50 (Addr: tcp/127.0.0.1:13076) (DC: dc1)" -45:36.710 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13072 network=tcp -45:36.711 [INFO] TestAgent: started state syncer -45:36.711 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13077 network=tcp -45:36.736 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:36.736 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13076 [Candidate]" term=2 -45:36.737 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:36.737 [DEBUG] TestAgent.server.raft: vote granted: from=ce9f932b-0567-cb4b-95fe-eaf123686e50 term=2 tally=1 -45:36.737 [INFO] TestAgent.server.raft: election won: tally=1 -45:36.737 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13076 [Leader]" -45:36.737 [INFO] TestAgent.server: cluster leadership acquired -45:36.737 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13076 -45:36.738 [INFO] TestAgent.server: New leader elected: payload=Node-ce9f932b-0567-cb4b-95fe-eaf123686e50 -45:36.739 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:36.763 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:36.764 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:36.764 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:36.764 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:36.764 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:36.764 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ce9f932b-0567-cb4b-95fe-eaf123686e50 -45:36.764 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ce9f932b-0567-cb4b-95fe-eaf123686e50 -45:36.764 [INFO] TestAgent.server: federation state anti-entropy synced -45:36.902 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -45:36.903 [INFO] TestAgent: Synced node info -45:36.934 [INFO] TestAgent: Requesting shutdown -45:36.934 [INFO] TestAgent.server: shutting down server -45:36.934 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:36.934 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:36.934 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:36.934 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:36.935 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:36.935 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:36.935 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:36.935 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:36.935 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:36.935 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:36.935 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:36.935Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:36.935Z [INFO] agent.router.manager: shutting down -45:36.935 [INFO] TestAgent: consul server down -45:36.935 [INFO] TestAgent: shutdown complete -45:36.935 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13071 network=tcp -45:36.935 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13071 network=udp -45:36.935 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13072 network=tcp -45:37.436 [INFO] TestAgent: Waiting for endpoints to shut down -45:37.436 [INFO] TestAgent: Endpoints down ---- PASS: TestLeaveCommand_FailOnNonFlagArgs (0.85s) -2022-09-15T21:45:36.667Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:36.697Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:36.700 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1730e5be-bb62-01c7-4a56-e8a83d7b294f Address:127.0.0.1:13069}]" -45:36.701 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13069 [Follower]" leader= -45:36.703 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1730e5be-bb62-01c7-4a56-e8a83d7b294f.dc1 127.0.0.1 -45:36.703 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1730e5be-bb62-01c7-4a56-e8a83d7b294f 127.0.0.1 -2022-09-15T21:45:36.703Z [INFO] agent.router: Initializing LAN area manager -45:36.708 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13064 network=udp -45:36.708 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13064 network=tcp -45:36.710 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1730e5be-bb62-01c7-4a56-e8a83d7b294f.dc1 area=wan -45:36.710 [INFO] TestAgent.server: Adding LAN server: server="Node-1730e5be-bb62-01c7-4a56-e8a83d7b294f (Addr: tcp/127.0.0.1:13069) (DC: dc1)" -45:36.710 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13065 network=tcp -45:36.711 [INFO] TestAgent: started state syncer -45:36.711 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13070 network=tcp -45:36.765 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:36.765 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13069 [Candidate]" term=2 -45:36.765 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:36.765 [DEBUG] TestAgent.server.raft: vote granted: from=1730e5be-bb62-01c7-4a56-e8a83d7b294f term=2 tally=1 -45:36.765 [INFO] TestAgent.server.raft: election won: tally=1 -45:36.765 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13069 [Leader]" -45:36.765 [INFO] TestAgent.server: cluster leadership acquired -45:36.765 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13069 -45:36.765 [INFO] TestAgent.server: New leader elected: payload=Node-1730e5be-bb62-01c7-4a56-e8a83d7b294f -45:36.767 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:36.769 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:36.769 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:36.769 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:36.769 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:36.769 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:36.769 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1730e5be-bb62-01c7-4a56-e8a83d7b294f -45:36.769 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1730e5be-bb62-01c7-4a56-e8a83d7b294f -45:36.770 [INFO] TestAgent.server: federation state anti-entropy synced -45:36.800 [INFO] TestAgent.server: server starting leave -45:36.800 [INFO] TestAgent.server.serf.wan: serf: EventMemberLeave: Node-1730e5be-bb62-01c7-4a56-e8a83d7b294f.dc1 127.0.0.1 -45:36.800 [INFO] TestAgent.server: Handled event for server in area: event=member-leave server=Node-1730e5be-bb62-01c7-4a56-e8a83d7b294f.dc1 area=wan -2022-09-15T21:45:36.800Z [INFO] agent.router.manager: shutting down -45:36.800 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -45:36.801 [INFO] TestAgent: Synced node info -45:36.801 [DEBUG] TestAgent: Node info in sync -45:37.710 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -45:37.710 [DEBUG] TestAgent: Node info in sync -45:39.800 [INFO] TestAgent.server.serf.lan: serf: EventMemberLeave: Node-1730e5be-bb62-01c7-4a56-e8a83d7b294f 127.0.0.1 -45:39.800 [INFO] TestAgent.server: Removing LAN server: server="Node-1730e5be-bb62-01c7-4a56-e8a83d7b294f (Addr: tcp/127.0.0.1:13069) (DC: dc1)" -45:39.800 [WARN] TestAgent.server: deregistering self should be done by follower: name=Node-1730e5be-bb62-01c7-4a56-e8a83d7b294f -45:40.770 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" -45:42.770 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" -45:42.802 [INFO] TestAgent.server: Waiting to drain RPC traffic: drain_time=5s -45:44.766 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" -45:46.768 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" -45:46.768 [ERROR] TestAgent.server.autopilot: Error promoting servers: error="error getting server raft protocol versions: No servers found" -45:47.803 [INFO] TestAgent: Requesting shutdown -45:47.803 [INFO] TestAgent.server: shutting down server -45:47.803 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:47.803 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:47.803 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:47.803 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:47.803 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:47.803 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:47.803 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:47.803 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:47.803 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2022-09-15T21:45:47.803Z [INFO] agent.router.manager: shutting down -45:47.804 [INFO] TestAgent: consul server down -45:47.804 [INFO] TestAgent: shutdown complete -45:47.804 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/leave from=127.0.0.1:46544 latency=11.0041148s -45:47.804 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13064 network=tcp -45:47.804 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13064 network=udp -45:47.804 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13065 network=tcp -45:48.304 [INFO] TestAgent: Waiting for endpoints to shut down -45:48.304 [INFO] TestAgent: Endpoints down ---- PASS: TestLeaveCommand (11.72s) +2021-08-13T15:38:07.189Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:38:07.213Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:07.219 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:71d2bac9-7e8a-0da9-59ed-b6369a7e3dfc Address:127.0.0.1:27363}]" +38:07.219 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:27363 [Follower]" leader= +38:07.220 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-71d2bac9-7e8a-0da9-59ed-b6369a7e3dfc.dc1 127.0.0.1 +38:07.221 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-71d2bac9-7e8a-0da9-59ed-b6369a7e3dfc 127.0.0.1 +2021-08-13T15:38:07.221Z [INFO] agent.router: Initializing LAN area manager +38:07.221 [INFO] TestAgent.server: Adding LAN server: server="Node-71d2bac9-7e8a-0da9-59ed-b6369a7e3dfc (Addr: tcp/127.0.0.1:27363) (DC: dc1)" +38:07.221 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-71d2bac9-7e8a-0da9-59ed-b6369a7e3dfc.dc1 area=wan +38:07.221 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27358 network=udp +38:07.221 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27358 network=tcp +38:07.223 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:27359 network=tcp +38:07.223 [INFO] TestAgent: started state syncer +38:07.223 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:27364 network=tcp +38:07.259 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +38:07.260 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:27363 [Candidate]" term=2 +38:07.260 [DEBUG] TestAgent.server.raft: votes: needed=1 +38:07.260 [DEBUG] TestAgent.server.raft: vote granted: from=71d2bac9-7e8a-0da9-59ed-b6369a7e3dfc term=2 tally=1 +38:07.260 [INFO] TestAgent.server.raft: election won: tally=1 +38:07.260 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:27363 [Leader]" +38:07.260 [INFO] TestAgent.server: cluster leadership acquired +38:07.260 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:27363 +38:07.261 [INFO] TestAgent.server: New leader elected: payload=Node-71d2bac9-7e8a-0da9-59ed-b6369a7e3dfc +38:07.262 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +38:07.304 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +38:07.304 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +38:07.304 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +38:07.304 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +38:07.304 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +38:07.304 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-71d2bac9-7e8a-0da9-59ed-b6369a7e3dfc +38:07.305 [INFO] TestAgent.server: member joined, marking health alive: member=Node-71d2bac9-7e8a-0da9-59ed-b6369a7e3dfc +38:07.306 [INFO] TestAgent.server: federation state anti-entropy synced +38:07.449 [INFO] TestAgent: Requesting shutdown +38:07.449 [INFO] TestAgent.server: shutting down server +38:07.449 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +38:07.449 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +38:07.449 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +38:07.449 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +38:07.449 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +38:07.449 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +38:07.449 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +38:07.449 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +38:07.449 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +38:07.449 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +38:07.449 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:38:07.449Z [INFO] agent.router.manager: shutting down +2021-08-13T15:38:07.449Z [INFO] agent.router.manager: shutting down +38:07.449 [INFO] TestAgent: consul server down +38:07.449 [INFO] TestAgent: shutdown complete +38:07.450 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27358 network=tcp +38:07.450 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27358 network=udp +38:07.450 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:27359 network=tcp +38:07.950 [INFO] TestAgent: Waiting for endpoints to shut down +38:07.950 [INFO] TestAgent: Endpoints down +--- PASS: TestLeaveCommand_FailOnNonFlagArgs (0.84s) +2021-08-13T15:38:07.191Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:38:07.217Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:07.220 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b2857adc-9220-38cb-6f47-b343aab0f36c Address:127.0.0.1:27370}]" +38:07.220 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:27370 [Follower]" leader= +38:07.220 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b2857adc-9220-38cb-6f47-b343aab0f36c.dc1 127.0.0.1 +38:07.221 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b2857adc-9220-38cb-6f47-b343aab0f36c 127.0.0.1 +2021-08-13T15:38:07.221Z [INFO] agent.router: Initializing LAN area manager +38:07.221 [INFO] TestAgent.server: Adding LAN server: server="Node-b2857adc-9220-38cb-6f47-b343aab0f36c (Addr: tcp/127.0.0.1:27370) (DC: dc1)" +38:07.221 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b2857adc-9220-38cb-6f47-b343aab0f36c.dc1 area=wan +38:07.222 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27365 network=udp +38:07.222 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27365 network=tcp +38:07.222 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:27366 network=tcp +38:07.223 [INFO] TestAgent: started state syncer +38:07.223 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:27371 network=tcp +38:07.261 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +38:07.261 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:27370 [Candidate]" term=2 +38:07.261 [DEBUG] TestAgent.server.raft: votes: needed=1 +38:07.261 [DEBUG] TestAgent.server.raft: vote granted: from=b2857adc-9220-38cb-6f47-b343aab0f36c term=2 tally=1 +38:07.261 [INFO] TestAgent.server.raft: election won: tally=1 +38:07.261 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:27370 [Leader]" +38:07.263 [INFO] TestAgent.server: cluster leadership acquired +38:07.263 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:27370 +38:07.264 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +38:07.264 [INFO] TestAgent.server: New leader elected: payload=Node-b2857adc-9220-38cb-6f47-b343aab0f36c +38:07.291 [INFO] TestAgent.server: server starting leave +38:07.291 [INFO] TestAgent.server.serf.wan: serf: EventMemberLeave: Node-b2857adc-9220-38cb-6f47-b343aab0f36c.dc1 127.0.0.1 +38:07.291 [INFO] TestAgent.server: Handled event for server in area: event=member-leave server=Node-b2857adc-9220-38cb-6f47-b343aab0f36c.dc1 area=wan +2021-08-13T15:38:07.291Z [INFO] agent.router.manager: shutting down +38:07.300 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +38:07.300 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +38:07.300 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +38:07.300 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +38:07.300 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +38:07.300 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b2857adc-9220-38cb-6f47-b343aab0f36c +38:07.300 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b2857adc-9220-38cb-6f47-b343aab0f36c +38:07.306 [INFO] TestAgent.server: federation state anti-entropy synced +38:07.327 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +38:07.328 [INFO] TestAgent: Synced node info +38:07.749 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +38:07.749 [DEBUG] TestAgent: Node info in sync +38:07.749 [DEBUG] TestAgent: Node info in sync +38:10.291 [INFO] TestAgent.server.serf.lan: serf: EventMemberLeave: Node-b2857adc-9220-38cb-6f47-b343aab0f36c 127.0.0.1 +38:10.292 [INFO] TestAgent.server: Removing LAN server: server="Node-b2857adc-9220-38cb-6f47-b343aab0f36c (Addr: tcp/127.0.0.1:27370) (DC: dc1)" +38:10.292 [WARN] TestAgent.server: deregistering self should be done by follower: name=Node-b2857adc-9220-38cb-6f47-b343aab0f36c +38:11.264 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" +38:13.267 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" +38:13.292 [INFO] TestAgent.server: Waiting to drain RPC traffic: drain_time=5s +38:15.264 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" +38:17.265 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" +38:17.265 [ERROR] TestAgent.server.autopilot: Error promoting servers: error="error getting server raft protocol versions: No servers found" +38:18.292 [INFO] TestAgent: Requesting shutdown +38:18.292 [INFO] TestAgent.server: shutting down server +38:18.292 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +38:18.292 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +38:18.292 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +38:18.292 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +38:18.292 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +38:18.292 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +38:18.292 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +38:18.292 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +38:18.292 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2021-08-13T15:38:18.292Z [INFO] agent.router.manager: shutting down +38:18.293 [INFO] TestAgent: consul server down +38:18.293 [INFO] TestAgent: shutdown complete +38:18.293 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/leave from=127.0.0.1:40966 latency=11.001723701s +38:18.293 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27365 network=tcp +38:18.293 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27365 network=udp +38:18.293 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:27366 network=tcp +38:18.795 [INFO] TestAgent: Waiting for endpoints to shut down +38:18.795 [INFO] TestAgent: Endpoints down +--- PASS: TestLeaveCommand (11.69s) PASS -ok github.com/hashicorp/consul/command/leave 11.838s +ok github.com/hashicorp/consul/command/leave 11.785s === RUN TestLockCommand_noTabs === PAUSE TestLockCommand_noTabs === RUN TestLockCommand_BadArgs @@ -88193,671 +88088,672 @@ === RUN TestLockCommand_ChildExitCode === PAUSE TestLockCommand_ChildExitCode === CONT TestLockCommand_noTabs ---- PASS: TestLockCommand_noTabs (0.00s) +=== CONT TestLockCommand_MonitorRetry_Lock_Default +=== CONT TestLockCommand_MonitorRetry_Semaphore_Arg === CONT TestLockCommand_ChildExitCode -=== CONT TestLockCommand_TrySemaphore [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... -=== CONT TestLockCommand_MonitorRetry_Semaphore_Arg -=== CONT TestLockCommand_MonitorRetry_Lock_Arg -=== CONT TestLockCommand_MonitorRetry_Semaphore_Default -=== CONT TestLockCommand_MonitorRetry_Lock_Default === CONT TestLockCommand_NoShell +=== CONT TestLockCommand_TrySemaphore === CONT TestLockCommand_TryLock +=== CONT TestLockCommand +--- PASS: TestLockCommand_noTabs (0.00s) +=== CONT TestLockCommand_BadArgs +--- PASS: TestLockCommand_BadArgs (0.00s) +=== CONT TestLockCommand_MonitorRetry_Lock_Arg [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range === RUN TestLockCommand_ChildExitCode/clean_exit === RUN TestLockCommand_ChildExitCode/error_exit === RUN TestLockCommand_ChildExitCode/not_propagated -2022-09-15T21:45:46.047Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:46.132Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:46.138 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bbc96e07-2a36-2e06-ba5d-bca63d2e3222 Address:127.0.0.1:11034}]" -45:46.139 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bbc96e07-2a36-2e06-ba5d-bca63d2e3222.dc1 127.0.0.1 -45:46.140 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bbc96e07-2a36-2e06-ba5d-bca63d2e3222 127.0.0.1 -2022-09-15T21:45:46.140Z [INFO] agent.router: Initializing LAN area manager -45:46.141 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11029 network=udp -45:46.141 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:11034 [Follower]" leader= -45:46.141 [INFO] TestAgent.server: Adding LAN server: server="Node-bbc96e07-2a36-2e06-ba5d-bca63d2e3222 (Addr: tcp/127.0.0.1:11034) (DC: dc1)" -45:46.141 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bbc96e07-2a36-2e06-ba5d-bca63d2e3222.dc1 area=wan -45:46.142 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11029 network=tcp -45:46.143 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:11030 network=tcp -45:46.143 [INFO] TestAgent: started state syncer -45:46.143 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:11035 network=tcp -45:46.182 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:46.182 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:11034 [Candidate]" term=2 -45:46.182 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:46.182 [DEBUG] TestAgent.server.raft: vote granted: from=bbc96e07-2a36-2e06-ba5d-bca63d2e3222 term=2 tally=1 -45:46.182 [INFO] TestAgent.server.raft: election won: tally=1 -45:46.182 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:11034 [Leader]" -45:46.182 [INFO] TestAgent.server: cluster leadership acquired -45:46.185 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:11034 -45:46.186 [INFO] TestAgent.server: New leader elected: payload=Node-bbc96e07-2a36-2e06-ba5d-bca63d2e3222 -45:46.203 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:46.303 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:46.303 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:46.303 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:46.303 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:46.303 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:46.303 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bbc96e07-2a36-2e06-ba5d-bca63d2e3222 -45:46.303 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bbc96e07-2a36-2e06-ba5d-bca63d2e3222 -45:46.364 [INFO] TestAgent.server: federation state anti-entropy synced -45:46.450 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:34612 latency=49.59364ms -45:46.457 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:34612 latency=2.86572ms -45:46.459 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/91131d8b-60d8-718c-b797-cfd57cf733aa?acquire=91131d8b-60d8-718c-b797-cfd57cf733aa&flags=16210313421097356768 from=127.0.0.1:34612 latency=795.82µs -45:46.460 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?recurse=&wait=10000ms from=127.0.0.1:34612 latency=285.18µs -45:46.461 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?cas=0&flags=16210313421097356768 from=127.0.0.1:34612 latency=528.4µs -45:46.462 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?consistent=&recurse= from=127.0.0.1:34612 latency=310.3µs -45:46.496 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:34614 latency=1.20398ms -45:46.500 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?cas=14&flags=16210313421097356768 from=127.0.0.1:34614 latency=989.5µs -45:46.500 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?consistent=&index=14&recurse= from=127.0.0.1:34612 latency=36.66618ms -45:46.511 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/91131d8b-60d8-718c-b797-cfd57cf733aa from=127.0.0.1:34614 latency=10.15056ms -45:46.513 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/91131d8b-60d8-718c-b797-cfd57cf733aa from=127.0.0.1:34612 latency=1.11634ms -45:46.513 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?recurse= from=127.0.0.1:34614 latency=201.96µs -45:46.519 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=15 from=127.0.0.1:34614 latency=720.24µs -45:46.554 [INFO] TestAgent: Requesting shutdown -45:46.554 [INFO] TestAgent.server: shutting down server -45:46.554 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:46.554 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:46.554 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:46.554 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:46.554 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:46.563 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -45:46.564 [INFO] TestAgent: Synced node info -45:46.564 [DEBUG] TestAgent: Node info in sync -45:46.564 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:46.564 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:46.564 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -45:46.564 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:46.564 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:46.564 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2022-09-15T21:45:46.567Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:46.567Z [INFO] agent.router.manager: shutting down -45:46.570 [INFO] TestAgent: consul server down -45:46.570 [INFO] TestAgent: shutdown complete -45:46.570 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11029 network=tcp -45:46.570 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11029 network=udp -45:46.570 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:11030 network=tcp -45:47.070 [INFO] TestAgent: Waiting for endpoints to shut down -45:47.070 [INFO] TestAgent: Endpoints down ---- PASS: TestLockCommand_TrySemaphore (1.16s) -=== CONT TestLockCommand -2022-09-15T21:45:46.260Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:46.355Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:46.399 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6e46fd3a-e3dd-767c-d1e7-81ea8f8c8840 Address:127.0.0.1:11048}]" -45:46.401 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6e46fd3a-e3dd-767c-d1e7-81ea8f8c8840.dc1 127.0.0.1 -45:46.406 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:11048 [Follower]" leader= -45:46.410 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6e46fd3a-e3dd-767c-d1e7-81ea8f8c8840 127.0.0.1 -2022-09-15T21:45:46.410Z [INFO] agent.router: Initializing LAN area manager -45:46.410 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11043 network=udp -45:46.466 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:46.466 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:11048 [Candidate]" term=2 -45:46.466 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6e46fd3a-e3dd-767c-d1e7-81ea8f8c8840.dc1 area=wan -45:46.466 [WARN] TestAgent.server.raft: unable to get address for sever, using fallback address: id=6e46fd3a-e3dd-767c-d1e7-81ea8f8c8840 fallback=127.0.0.1:11048 error="Could not find address for server id 6e46fd3a-e3dd-767c-d1e7-81ea8f8c8840" -45:46.466 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:46.466 [DEBUG] TestAgent.server.raft: vote granted: from=6e46fd3a-e3dd-767c-d1e7-81ea8f8c8840 term=2 tally=1 -45:46.466 [INFO] TestAgent.server.raft: election won: tally=1 -45:46.466 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:11048 [Leader]" -45:46.466 [INFO] TestAgent.server: cluster leadership acquired -45:46.466 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=1 found=false leader=127.0.0.1:11048 -45:46.468 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:46.470 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:46.470 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:46.470 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:46.470 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:46.470 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:46.470 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6e46fd3a-e3dd-767c-d1e7-81ea8f8c8840 -45:46.470 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6e46fd3a-e3dd-767c-d1e7-81ea8f8c8840 -45:46.470 [INFO] TestAgent.server: Adding LAN server: server="Node-6e46fd3a-e3dd-767c-d1e7-81ea8f8c8840 (Addr: tcp/127.0.0.1:11048) (DC: dc1)" -45:46.476 [INFO] TestAgent.server: New leader elected: payload=Node-6e46fd3a-e3dd-767c-d1e7-81ea8f8c8840 -45:46.502 [INFO] TestAgent.server: federation state anti-entropy synced -45:46.502 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6e46fd3a-e3dd-767c-d1e7-81ea8f8c8840 -45:46.510 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11043 network=tcp -45:46.568 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:11044 network=tcp -45:46.568 [INFO] TestAgent: started state syncer -45:46.570 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:11049 network=tcp -45:46.572 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -45:46.573 [INFO] TestAgent: Synced node info -45:46.578 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:54130 latency=1.7125ms -45:46.590 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:54130 latency=9.27896ms -45:46.594 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?wait=15000ms from=127.0.0.1:54130 latency=125.86µs -45:46.595 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?acquire=4f37e590-a93e-b2fe-a325-7e0c0ce66e71&flags=3304740253564472344 from=127.0.0.1:54130 latency=575.48µs -45:46.598 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent= from=127.0.0.1:54130 latency=337.86µs -45:46.629 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent=&index=14 from=127.0.0.1:54130 latency=10.6519ms -45:46.634 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?flags=3304740253564472344&release=4f37e590-a93e-b2fe-a325-7e0c0ce66e71 from=127.0.0.1:54132 latency=15.9805ms -45:46.636 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:54132 latency=289.38µs -45:46.636 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/4f37e590-a93e-b2fe-a325-7e0c0ce66e71 from=127.0.0.1:54130 latency=1.06348ms -45:46.638 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=15 from=127.0.0.1:54130 latency=783.28µs -45:46.638 [INFO] TestAgent: Requesting shutdown -45:46.638 [INFO] TestAgent.server: shutting down server -45:46.639 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:46.639 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:46.639 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:46.639 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:46.639 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:46.639 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:46.639 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:46.639 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:46.639 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:46.639 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:46.639 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:46.639Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:46.639Z [INFO] agent.router.manager: shutting down -45:46.646 [INFO] TestAgent: consul server down -45:46.646 [INFO] TestAgent: shutdown complete -45:46.646 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11043 network=tcp -45:46.646 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11043 network=udp -45:46.646 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:11044 network=tcp -45:47.147 [INFO] TestAgent: Waiting for endpoints to shut down -45:47.147 [INFO] TestAgent: Endpoints down ---- PASS: TestLockCommand_MonitorRetry_Lock_Arg (1.23s) -=== CONT TestLockCommand_BadArgs ---- PASS: TestLockCommand_BadArgs (0.00s) -2022-09-15T21:45:46.273Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:46.574Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:46.576 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8dd3efbd-be65-d2e1-16ef-0f9eb1716257 Address:127.0.0.1:11062}]" -45:46.577 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8dd3efbd-be65-d2e1-16ef-0f9eb1716257.dc1 127.0.0.1 -45:46.578 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:11062 [Follower]" leader= -45:46.583 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8dd3efbd-be65-d2e1-16ef-0f9eb1716257 127.0.0.1 -2022-09-15T21:45:46.583Z [INFO] agent.router: Initializing LAN area manager -45:46.583 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11057 network=udp -45:46.583 [INFO] TestAgent.server: Adding LAN server: server="Node-8dd3efbd-be65-d2e1-16ef-0f9eb1716257 (Addr: tcp/127.0.0.1:11062) (DC: dc1)" -45:46.584 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8dd3efbd-be65-d2e1-16ef-0f9eb1716257.dc1 area=wan -45:46.584 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11057 network=tcp -45:46.585 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:11058 network=tcp -45:46.585 [INFO] TestAgent: started state syncer -45:46.585 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:11063 network=tcp -45:46.618 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:46.618 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:11062 [Candidate]" term=2 -45:46.619 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:46.619 [DEBUG] TestAgent.server.raft: vote granted: from=8dd3efbd-be65-d2e1-16ef-0f9eb1716257 term=2 tally=1 -45:46.619 [INFO] TestAgent.server.raft: election won: tally=1 -45:46.619 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:11062 [Leader]" -45:46.619 [INFO] TestAgent.server: cluster leadership acquired -45:46.619 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:11062 -45:46.621 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:46.623 [INFO] TestAgent.server: New leader elected: payload=Node-8dd3efbd-be65-d2e1-16ef-0f9eb1716257 -45:46.637 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:46.637 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:46.637 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:46.637 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:46.637 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:46.637 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8dd3efbd-be65-d2e1-16ef-0f9eb1716257 -45:46.637 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8dd3efbd-be65-d2e1-16ef-0f9eb1716257 -45:46.646 [INFO] TestAgent.server: federation state anti-entropy synced -45:46.668 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:38370 latency=10.40008ms -45:46.670 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -45:46.671 [INFO] TestAgent: Synced node info -45:46.671 [DEBUG] TestAgent: Node info in sync -45:46.688 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:38370 latency=860.58µs -45:46.694 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?wait=15000ms from=127.0.0.1:38370 latency=135.88µs -45:46.695 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?acquire=3d59fb4b-7868-2e3d-d629-ee94af052961&flags=3304740253564472344 from=127.0.0.1:38370 latency=788.62µs -45:46.697 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent= from=127.0.0.1:38370 latency=320.2µs -45:46.723 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?flags=3304740253564472344&release=3d59fb4b-7868-2e3d-d629-ee94af052961 from=127.0.0.1:38374 latency=765.14µs -45:46.724 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent=&index=14 from=127.0.0.1:38370 latency=26.24288ms -45:46.725 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:38374 latency=160.46µs -45:46.726 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/3d59fb4b-7868-2e3d-d629-ee94af052961 from=127.0.0.1:38370 latency=690.78µs -45:46.731 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=15 from=127.0.0.1:38374 latency=922.76µs -45:46.731 [INFO] TestAgent: Requesting shutdown -45:46.731 [INFO] TestAgent.server: shutting down server -45:46.731 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:46.731 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:46.731 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:46.731 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:46.732 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:46.732 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:46.732 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:46.732 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:46.732 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:46.732 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:46.732 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:46.732Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:46.732Z [INFO] agent.router.manager: shutting down -45:46.732 [INFO] TestAgent: consul server down -45:46.732 [INFO] TestAgent: shutdown complete -45:46.732 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11057 network=tcp -45:46.732 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11057 network=udp -45:46.732 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:11058 network=tcp -45:47.233 [INFO] TestAgent: Waiting for endpoints to shut down -45:47.233 [INFO] TestAgent: Endpoints down ---- PASS: TestLockCommand_MonitorRetry_Lock_Default (1.32s) -2022-09-15T21:45:46.149Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:46.244Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:46.260 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:092bdab5-a421-e239-1834-2afefb125cfe Address:127.0.0.1:11027}]" -45:46.261 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-092bdab5-a421-e239-1834-2afefb125cfe.dc1 127.0.0.1 -45:46.261 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-092bdab5-a421-e239-1834-2afefb125cfe 127.0.0.1 -2022-09-15T21:45:46.261Z [INFO] agent.router: Initializing LAN area manager -45:46.270 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:11027 [Follower]" leader= -45:46.270 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11022 network=udp -45:46.270 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-092bdab5-a421-e239-1834-2afefb125cfe.dc1 area=wan -45:46.270 [INFO] TestAgent.server: Adding LAN server: server="Node-092bdab5-a421-e239-1834-2afefb125cfe (Addr: tcp/127.0.0.1:11027) (DC: dc1)" -45:46.270 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11022 network=tcp -45:46.271 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:11023 network=tcp -45:46.271 [INFO] TestAgent: started state syncer -45:46.271 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:11028 network=tcp -45:46.361 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:46.361 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:11027 [Candidate]" term=2 -45:46.361 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:46.361 [DEBUG] TestAgent.server.raft: vote granted: from=092bdab5-a421-e239-1834-2afefb125cfe term=2 tally=1 -45:46.361 [INFO] TestAgent.server.raft: election won: tally=1 -45:46.361 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:11027 [Leader]" -45:46.362 [INFO] TestAgent.server: cluster leadership acquired -45:46.362 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:11027 -45:46.363 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:46.364 [INFO] TestAgent.server: New leader elected: payload=Node-092bdab5-a421-e239-1834-2afefb125cfe -45:46.370 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:46.370 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:46.370 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:46.370 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:46.370 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:46.370 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-092bdab5-a421-e239-1834-2afefb125cfe -45:46.370 [INFO] TestAgent.server: member joined, marking health alive: member=Node-092bdab5-a421-e239-1834-2afefb125cfe -45:46.372 [INFO] TestAgent.server: federation state anti-entropy synced -45:46.674 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -45:46.675 [INFO] TestAgent: Synced node info -45:46.675 [DEBUG] TestAgent: Node info in sync -45:46.706 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:48452 latency=6.05432ms -45:46.711 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:48452 latency=802.82µs -45:46.712 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?wait=15000ms from=127.0.0.1:48452 latency=98.72µs -45:46.713 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?acquire=c4d83fca-c5e3-beb7-2a17-7516f25a782a&flags=3304740253564472344 from=127.0.0.1:48452 latency=669.94µs -45:46.714 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent= from=127.0.0.1:48452 latency=200.26µs -45:46.737 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?flags=3304740253564472344&release=c4d83fca-c5e3-beb7-2a17-7516f25a782a from=127.0.0.1:48456 latency=723.18µs -45:46.737 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent=&index=14 from=127.0.0.1:48452 latency=22.82338ms -45:46.738 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:48456 latency=149.3µs -45:46.739 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/c4d83fca-c5e3-beb7-2a17-7516f25a782a from=127.0.0.1:48452 latency=636.84µs -45:46.739 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=15 from=127.0.0.1:48456 latency=427.14µs -45:46.748 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:48458 latency=1.83486ms -45:46.752 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:48458 latency=887.48µs -45:46.753 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?wait=15000ms from=127.0.0.1:48458 latency=91.72µs -45:46.759 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?acquire=fe2aba59-2a46-e7aa-75cf-e14980b2e7e9&flags=3304740253564472344 from=127.0.0.1:48458 latency=666.72µs -45:46.766 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent= from=127.0.0.1:48458 latency=311.88µs -45:46.779 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?flags=3304740253564472344&release=fe2aba59-2a46-e7aa-75cf-e14980b2e7e9 from=127.0.0.1:48460 latency=754.84µs -45:46.779 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent=&index=19 from=127.0.0.1:48458 latency=11.939ms -45:46.781 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/fe2aba59-2a46-e7aa-75cf-e14980b2e7e9 from=127.0.0.1:48458 latency=702.9µs -45:46.781 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:48460 latency=147.22µs -45:46.783 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=20 from=127.0.0.1:48460 latency=580.7µs -45:46.795 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:48462 latency=9.9501ms -45:46.800 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:48462 latency=973.08µs -45:46.801 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?wait=15000ms from=127.0.0.1:48462 latency=99.34µs -45:46.802 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?acquire=d006af5f-81f9-0099-3808-c0ba12cde77e&flags=3304740253564472344 from=127.0.0.1:48462 latency=648.44µs -45:46.805 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent= from=127.0.0.1:48462 latency=1.24668ms -45:46.812 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?flags=3304740253564472344&release=d006af5f-81f9-0099-3808-c0ba12cde77e from=127.0.0.1:48464 latency=1.22986ms -45:46.812 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent=&index=24 from=127.0.0.1:48462 latency=4.61702ms -45:46.814 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/d006af5f-81f9-0099-3808-c0ba12cde77e from=127.0.0.1:48462 latency=747.66µs -45:46.814 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:48464 latency=141.96µs -45:46.816 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=25 from=127.0.0.1:48464 latency=553.14µs -45:46.816 [INFO] TestAgent: Requesting shutdown -45:46.816 [INFO] TestAgent.server: shutting down server -45:46.816 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:46.816 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:46.816 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:46.816 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:46.816 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:46.816 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:46.816 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:46.816 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:46.816 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:46.816 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:46.817 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:46.817Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:46.817Z [INFO] agent.router.manager: shutting down -45:46.817 [INFO] TestAgent: consul server down -45:46.817 [INFO] TestAgent: shutdown complete -45:46.817 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11022 network=tcp -45:46.817 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11022 network=udp -45:46.817 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:11023 network=tcp -45:47.317 [INFO] TestAgent: Waiting for endpoints to shut down -45:47.317 [INFO] TestAgent: Endpoints down ---- PASS: TestLockCommand_ChildExitCode (1.40s) +2021-08-13T15:38:06.997Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:38:07.096Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:07.099 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:eccf19b6-3fb5-a9ec-722d-e1999568fcf9 Address:127.0.0.1:28384}]" +38:07.100 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-eccf19b6-3fb5-a9ec-722d-e1999568fcf9.dc1 127.0.0.1 +38:07.100 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:28384 [Follower]" leader= +38:07.100 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-eccf19b6-3fb5-a9ec-722d-e1999568fcf9 127.0.0.1 +2021-08-13T15:38:07.100Z [INFO] agent.router: Initializing LAN area manager +38:07.170 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28379 network=udp +38:07.170 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28379 network=tcp +38:07.172 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:28380 network=tcp +38:07.173 [INFO] TestAgent: started state syncer +38:07.173 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:28385 network=tcp +38:07.175 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +38:07.175 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:28384 [Candidate]" term=2 +38:07.175 [WARN] TestAgent.server.raft: unable to get address for sever, using fallback address: id=eccf19b6-3fb5-a9ec-722d-e1999568fcf9 fallback=127.0.0.1:28384 error="Could not find address for server id eccf19b6-3fb5-a9ec-722d-e1999568fcf9" +38:07.176 [DEBUG] TestAgent.server.raft: votes: needed=1 +38:07.176 [DEBUG] TestAgent.server.raft: vote granted: from=eccf19b6-3fb5-a9ec-722d-e1999568fcf9 term=2 tally=1 +38:07.176 [INFO] TestAgent.server.raft: election won: tally=1 +38:07.176 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:28384 [Leader]" +38:07.180 [INFO] TestAgent.server: Adding LAN server: server="Node-eccf19b6-3fb5-a9ec-722d-e1999568fcf9 (Addr: tcp/127.0.0.1:28384) (DC: dc1)" +38:07.181 [INFO] TestAgent.server: cluster leadership acquired +38:07.185 [INFO] TestAgent.server: New leader elected: payload=Node-eccf19b6-3fb5-a9ec-722d-e1999568fcf9 +38:07.185 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:28384 +38:07.196 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-eccf19b6-3fb5-a9ec-722d-e1999568fcf9.dc1 area=wan +38:07.198 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +38:07.233 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +38:07.233 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +38:07.233 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +38:07.233 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +38:07.233 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +38:07.233 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-eccf19b6-3fb5-a9ec-722d-e1999568fcf9 +38:07.233 [INFO] TestAgent.server: member joined, marking health alive: member=Node-eccf19b6-3fb5-a9ec-722d-e1999568fcf9 +38:07.237 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-eccf19b6-3fb5-a9ec-722d-e1999568fcf9 +38:07.238 [INFO] TestAgent.server: federation state anti-entropy synced +38:07.307 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:48718 latency=21.37375ms +38:07.311 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:48718 latency=745.442µs +38:07.312 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +38:07.312 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?wait=15000ms from=127.0.0.1:48718 latency=62.62µs +38:07.312 [INFO] TestAgent: Synced node info +38:07.312 [DEBUG] TestAgent: Node info in sync +38:07.313 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?acquire=f11050ad-27e3-453d-8145-9be7ea686072&flags=3304740253564472344 from=127.0.0.1:48718 latency=405.861µs +38:07.314 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent= from=127.0.0.1:48718 latency=147.201µs +38:07.324 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?flags=3304740253564472344&release=f11050ad-27e3-453d-8145-9be7ea686072 from=127.0.0.1:48732 latency=639.221µs +38:07.324 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent=&index=14 from=127.0.0.1:48718 latency=9.129581ms +38:07.325 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:48732 latency=114.901µs +38:07.325 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/f11050ad-27e3-453d-8145-9be7ea686072 from=127.0.0.1:48718 latency=575.821µs +38:07.326 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=15 from=127.0.0.1:48732 latency=412.761µs +38:07.326 [INFO] TestAgent: Requesting shutdown +38:07.326 [INFO] TestAgent.server: shutting down server +38:07.326 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +38:07.326 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +38:07.326 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +38:07.326 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +38:07.326 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +38:07.326 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +38:07.326 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +38:07.326 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +38:07.326 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +38:07.326 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +38:07.326 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:38:07.327Z [INFO] agent.router.manager: shutting down +2021-08-13T15:38:07.327Z [INFO] agent.router.manager: shutting down +38:07.327 [INFO] TestAgent: consul server down +38:07.327 [INFO] TestAgent: shutdown complete +38:07.327 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28379 network=tcp +38:07.327 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28379 network=udp +38:07.327 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:28380 network=tcp +38:07.827 [INFO] TestAgent: Waiting for endpoints to shut down +38:07.827 [INFO] TestAgent: Endpoints down +--- PASS: TestLockCommand_MonitorRetry_Lock_Default (0.93s) +=== CONT TestLockCommand_MonitorRetry_Semaphore_Default +2021-08-13T15:38:07.007Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:38:07.053Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:07.059 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:77be7c51-a91a-d2e0-5ae5-eaf3f142e90c Address:127.0.0.1:28405}]" +38:07.061 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-77be7c51-a91a-d2e0-5ae5-eaf3f142e90c.dc1 127.0.0.1 +38:07.062 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-77be7c51-a91a-d2e0-5ae5-eaf3f142e90c 127.0.0.1 +2021-08-13T15:38:07.062Z [INFO] agent.router: Initializing LAN area manager +38:07.062 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28400 network=udp +38:07.065 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:28405 [Follower]" leader= +38:07.070 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-77be7c51-a91a-d2e0-5ae5-eaf3f142e90c.dc1 area=wan +38:07.072 [INFO] TestAgent.server: Adding LAN server: server="Node-77be7c51-a91a-d2e0-5ae5-eaf3f142e90c (Addr: tcp/127.0.0.1:28405) (DC: dc1)" +38:07.074 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28400 network=tcp +38:07.100 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:28401 network=tcp +38:07.100 [INFO] TestAgent: started state syncer +38:07.100 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:28406 network=tcp +38:07.173 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +38:07.173 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:28405 [Candidate]" term=2 +38:07.174 [DEBUG] TestAgent.server.raft: votes: needed=1 +38:07.174 [DEBUG] TestAgent.server.raft: vote granted: from=77be7c51-a91a-d2e0-5ae5-eaf3f142e90c term=2 tally=1 +38:07.174 [INFO] TestAgent.server.raft: election won: tally=1 +38:07.174 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:28405 [Leader]" +38:07.174 [INFO] TestAgent.server: cluster leadership acquired +38:07.183 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:28405 +38:07.184 [INFO] TestAgent.server: New leader elected: payload=Node-77be7c51-a91a-d2e0-5ae5-eaf3f142e90c +38:07.187 [INFO] TestAgent: Synced node info +38:07.187 [DEBUG] TestAgent: Node info in sync +38:07.187 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +38:07.238 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +38:07.238 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +38:07.238 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +38:07.238 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +38:07.238 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +38:07.238 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-77be7c51-a91a-d2e0-5ae5-eaf3f142e90c +38:07.238 [INFO] TestAgent.server: member joined, marking health alive: member=Node-77be7c51-a91a-d2e0-5ae5-eaf3f142e90c +38:07.239 [INFO] TestAgent.server: federation state anti-entropy synced +38:07.332 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:50680 latency=1.547023ms +38:07.335 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:50680 latency=784.562µs +38:07.336 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/8b0eabed-f19d-dc46-0d72-3e807b231964?acquire=8b0eabed-f19d-dc46-0d72-3e807b231964&flags=16210313421097356768 from=127.0.0.1:50680 latency=422.581µs +38:07.337 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?recurse=&wait=10000ms from=127.0.0.1:50680 latency=157.301µs +38:07.338 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?cas=0&flags=16210313421097356768 from=127.0.0.1:50680 latency=453.781µs +38:07.339 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?consistent=&recurse= from=127.0.0.1:50680 latency=207.081µs +38:07.344 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:50684 latency=125.821µs +38:07.345 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?cas=15&flags=16210313421097356768 from=127.0.0.1:50684 latency=559.821µs +38:07.345 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?consistent=&index=15&recurse= from=127.0.0.1:50680 latency=5.712534ms +38:07.346 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/8b0eabed-f19d-dc46-0d72-3e807b231964 from=127.0.0.1:50684 latency=442.941µs +38:07.347 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?recurse= from=127.0.0.1:50684 latency=154.42µs +38:07.347 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/8b0eabed-f19d-dc46-0d72-3e807b231964 from=127.0.0.1:50680 latency=663.802µs +38:07.348 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=16 from=127.0.0.1:50684 latency=529.042µs +38:07.348 [INFO] TestAgent: Requesting shutdown +38:07.348 [INFO] TestAgent.server: shutting down server +38:07.348 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +38:07.348 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +38:07.348 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +38:07.348 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +38:07.348 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +38:07.348 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +38:07.348 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +38:07.348 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +38:07.348 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +38:07.348 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +38:07.348 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:38:07.349Z [INFO] agent.router.manager: shutting down +2021-08-13T15:38:07.349Z [INFO] agent.router.manager: shutting down +38:07.349 [INFO] TestAgent: consul server down +38:07.349 [INFO] TestAgent: shutdown complete +38:07.349 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28400 network=tcp +38:07.349 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28400 network=udp +38:07.349 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:28401 network=tcp +38:07.850 [INFO] TestAgent: Waiting for endpoints to shut down +38:07.850 [INFO] TestAgent: Endpoints down +--- PASS: TestLockCommand_TrySemaphore (0.95s) +2021-08-13T15:38:07.096Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:38:07.205Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:07.208 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5ada238a-7d4e-136b-b8f3-72a8d06926b4 Address:127.0.0.1:28426}]" +38:07.209 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5ada238a-7d4e-136b-b8f3-72a8d06926b4.dc1 127.0.0.1 +38:07.209 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:28426 [Follower]" leader= +38:07.210 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5ada238a-7d4e-136b-b8f3-72a8d06926b4 127.0.0.1 +2021-08-13T15:38:07.210Z [INFO] agent.router: Initializing LAN area manager +38:07.210 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28421 network=udp +38:07.210 [INFO] TestAgent.server: Adding LAN server: server="Node-5ada238a-7d4e-136b-b8f3-72a8d06926b4 (Addr: tcp/127.0.0.1:28426) (DC: dc1)" +38:07.210 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5ada238a-7d4e-136b-b8f3-72a8d06926b4.dc1 area=wan +38:07.211 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28421 network=tcp +38:07.212 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:28422 network=tcp +38:07.212 [INFO] TestAgent: started state syncer +38:07.212 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:28427 network=tcp +38:07.258 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +38:07.258 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:28426 [Candidate]" term=2 +38:07.259 [DEBUG] TestAgent.server.raft: votes: needed=1 +38:07.259 [DEBUG] TestAgent.server.raft: vote granted: from=5ada238a-7d4e-136b-b8f3-72a8d06926b4 term=2 tally=1 +38:07.259 [INFO] TestAgent.server.raft: election won: tally=1 +38:07.259 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:28426 [Leader]" +38:07.259 [INFO] TestAgent.server: cluster leadership acquired +38:07.259 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:28426 +38:07.259 [INFO] TestAgent.server: New leader elected: payload=Node-5ada238a-7d4e-136b-b8f3-72a8d06926b4 +38:07.260 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +38:07.263 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +38:07.263 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +38:07.263 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +38:07.263 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +38:07.263 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +38:07.263 [INFO] TestAgent.server: federation state anti-entropy synced +38:07.275 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5ada238a-7d4e-136b-b8f3-72a8d06926b4 +38:07.275 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5ada238a-7d4e-136b-b8f3-72a8d06926b4 +38:07.338 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:52076 latency=1.513603ms +38:07.341 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:52076 latency=697.081µs +38:07.342 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/da45ef94-6707-63f6-2759-9aaf08aec798?acquire=da45ef94-6707-63f6-2759-9aaf08aec798&flags=16210313421097356768 from=127.0.0.1:52076 latency=417.901µs +38:07.343 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?recurse=&wait=15000ms from=127.0.0.1:52076 latency=148.381µs +38:07.344 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?cas=0&flags=16210313421097356768 from=127.0.0.1:52076 latency=422.461µs +38:07.344 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?consistent=&recurse= from=127.0.0.1:52076 latency=157.921µs +38:07.350 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:52080 latency=136.22µs +38:07.351 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?cas=14&flags=16210313421097356768 from=127.0.0.1:52080 latency=561.802µs +38:07.351 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?consistent=&index=14&recurse= from=127.0.0.1:52076 latency=5.942914ms +38:07.352 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/da45ef94-6707-63f6-2759-9aaf08aec798 from=127.0.0.1:52080 latency=501.101µs +38:07.352 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?recurse= from=127.0.0.1:52080 latency=148.74µs +38:07.353 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/da45ef94-6707-63f6-2759-9aaf08aec798 from=127.0.0.1:52076 latency=513.582µs +38:07.353 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=15 from=127.0.0.1:52080 latency=440.661µs +38:07.354 [INFO] TestAgent: Requesting shutdown +38:07.354 [INFO] TestAgent.server: shutting down server +38:07.354 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +38:07.354 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +38:07.354 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +38:07.354 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +38:07.354 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +38:07.354 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +38:07.354 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +38:07.354 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +38:07.354 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +38:07.354 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +38:07.354 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +38:07.354 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:38:07.354Z [INFO] agent.router.manager: shutting down +2021-08-13T15:38:07.354Z [INFO] agent.router.manager: shutting down +38:07.354 [INFO] TestAgent: consul server down +38:07.354 [INFO] TestAgent: shutdown complete +38:07.354 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28421 network=tcp +38:07.354 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28421 network=udp +38:07.354 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:28422 network=tcp +38:07.855 [INFO] TestAgent: Waiting for endpoints to shut down +38:07.855 [INFO] TestAgent: Endpoints down +--- PASS: TestLockCommand_MonitorRetry_Semaphore_Arg (0.96s) +2021-08-13T15:38:06.988Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:38:07.074Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:07.076 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8a5c6e6c-d441-f05d-91bb-1111f0f64d7b Address:127.0.0.1:28398}]" +38:07.077 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8a5c6e6c-d441-f05d-91bb-1111f0f64d7b.dc1 127.0.0.1 +38:07.079 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8a5c6e6c-d441-f05d-91bb-1111f0f64d7b 127.0.0.1 +2021-08-13T15:38:07.080Z [INFO] agent.router: Initializing LAN area manager +38:07.080 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28393 network=udp +38:07.081 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28393 network=tcp +38:07.082 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:28394 network=tcp +38:07.082 [INFO] TestAgent: started state syncer +38:07.083 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:28399 network=tcp +38:07.083 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:28398 [Follower]" leader= +38:07.083 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8a5c6e6c-d441-f05d-91bb-1111f0f64d7b.dc1 area=wan +38:07.084 [INFO] TestAgent.server: Adding LAN server: server="Node-8a5c6e6c-d441-f05d-91bb-1111f0f64d7b (Addr: tcp/127.0.0.1:28398) (DC: dc1)" +38:07.178 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +38:07.178 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:28398 [Candidate]" term=2 +38:07.178 [DEBUG] TestAgent.server.raft: votes: needed=1 +38:07.178 [DEBUG] TestAgent.server.raft: vote granted: from=8a5c6e6c-d441-f05d-91bb-1111f0f64d7b term=2 tally=1 +38:07.178 [INFO] TestAgent.server.raft: election won: tally=1 +38:07.178 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:28398 [Leader]" +38:07.179 [INFO] TestAgent.server: cluster leadership acquired +38:07.186 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:28398 +38:07.199 [INFO] TestAgent.server: New leader elected: payload=Node-8a5c6e6c-d441-f05d-91bb-1111f0f64d7b +38:07.200 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +38:07.242 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +38:07.242 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +38:07.242 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +38:07.242 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +38:07.242 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +38:07.242 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8a5c6e6c-d441-f05d-91bb-1111f0f64d7b +38:07.242 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8a5c6e6c-d441-f05d-91bb-1111f0f64d7b +38:07.244 [INFO] TestAgent.server: federation state anti-entropy synced +38:07.261 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:57760 latency=1.767504ms +38:07.278 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:57760 latency=3.889749ms +38:07.279 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?wait=15000ms from=127.0.0.1:57760 latency=82.2µs +38:07.280 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?acquire=77b1e5a5-0ef6-1364-f8c7-a3757998a993&flags=3304740253564472344 from=127.0.0.1:57760 latency=552.361µs +38:07.286 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent= from=127.0.0.1:57760 latency=3.488768ms +38:07.310 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?flags=3304740253564472344&release=77b1e5a5-0ef6-1364-f8c7-a3757998a993 from=127.0.0.1:57774 latency=549.622µs +38:07.311 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent=&index=13 from=127.0.0.1:57760 latency=23.840135ms +38:07.312 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:57774 latency=159.641µs +38:07.313 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/77b1e5a5-0ef6-1364-f8c7-a3757998a993 from=127.0.0.1:57778 latency=509.142µs +38:07.313 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=14 from=127.0.0.1:57774 latency=448.181µs +38:07.359 [INFO] TestAgent: Requesting shutdown +38:07.359 [INFO] TestAgent.server: shutting down server +38:07.359 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +38:07.359 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +38:07.359 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +38:07.359 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +38:07.359 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +38:07.359 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +38:07.359 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +38:07.359 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +38:07.359 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +38:07.359 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +38:07.359 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +38:07.359 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:38:07.359Z [INFO] agent.router.manager: shutting down +2021-08-13T15:38:07.359Z [INFO] agent.router.manager: shutting down +38:07.360 [INFO] TestAgent: consul server down +38:07.360 [INFO] TestAgent: shutdown complete +38:07.360 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28393 network=tcp +38:07.360 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28393 network=udp +38:07.360 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:28394 network=tcp +38:07.860 [INFO] TestAgent: Waiting for endpoints to shut down +38:07.860 [INFO] TestAgent: Endpoints down +--- PASS: TestLockCommand_NoShell (0.96s) +2021-08-13T15:38:07.002Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:38:07.166Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:07.196 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7b09ac50-45f8-7324-7151-d54df07e1a11 Address:127.0.0.1:28412}]" +38:07.198 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7b09ac50-45f8-7324-7151-d54df07e1a11.dc1 127.0.0.1 +38:07.199 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:28412 [Follower]" leader= +38:07.201 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7b09ac50-45f8-7324-7151-d54df07e1a11 127.0.0.1 +2021-08-13T15:38:07.201Z [INFO] agent.router: Initializing LAN area manager +38:07.202 [INFO] TestAgent.server: Adding LAN server: server="Node-7b09ac50-45f8-7324-7151-d54df07e1a11 (Addr: tcp/127.0.0.1:28412) (DC: dc1)" +38:07.210 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28407 network=tcp +38:07.211 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28407 network=udp +38:07.212 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:28408 network=tcp +38:07.212 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7b09ac50-45f8-7324-7151-d54df07e1a11.dc1 area=wan +38:07.212 [INFO] TestAgent: started state syncer +38:07.212 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:28413 network=tcp +38:07.264 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +38:07.264 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:28412 [Candidate]" term=2 +38:07.264 [DEBUG] TestAgent.server.raft: votes: needed=1 +38:07.264 [DEBUG] TestAgent.server.raft: vote granted: from=7b09ac50-45f8-7324-7151-d54df07e1a11 term=2 tally=1 +38:07.264 [INFO] TestAgent.server.raft: election won: tally=1 +38:07.264 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:28412 [Leader]" +38:07.264 [INFO] TestAgent.server: cluster leadership acquired +38:07.264 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:28412 +38:07.265 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +38:07.274 [INFO] TestAgent.server: New leader elected: payload=Node-7b09ac50-45f8-7324-7151-d54df07e1a11 +38:07.307 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +38:07.307 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +38:07.307 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +38:07.307 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +38:07.307 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +38:07.308 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7b09ac50-45f8-7324-7151-d54df07e1a11 +38:07.308 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7b09ac50-45f8-7324-7151-d54df07e1a11 +38:07.309 [INFO] TestAgent.server: federation state anti-entropy synced +38:07.449 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +38:07.450 [INFO] TestAgent: Synced node info +38:07.455 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +38:07.456 [DEBUG] TestAgent: Node info in sync +38:07.456 [DEBUG] TestAgent: Node info in sync +38:07.475 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:35718 latency=2.630287ms +38:07.483 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:35718 latency=1.934245ms +38:07.484 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?wait=10000ms from=127.0.0.1:35718 latency=85.62µs +38:07.485 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?acquire=6dcb366c-390b-699d-6a08-6828f6b4cb7d&flags=3304740253564472344 from=127.0.0.1:35718 latency=439.461µs +38:07.486 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent= from=127.0.0.1:35718 latency=159.86µs +38:07.491 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?flags=3304740253564472344&release=6dcb366c-390b-699d-6a08-6828f6b4cb7d from=127.0.0.1:35720 latency=558.301µs +38:07.491 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent=&index=14 from=127.0.0.1:35718 latency=4.45131ms +38:07.491 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:35720 latency=112.34µs +38:07.492 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/6dcb366c-390b-699d-6a08-6828f6b4cb7d from=127.0.0.1:35718 latency=539.462µs +38:07.492 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=15 from=127.0.0.1:35720 latency=437.481µs +38:07.493 [INFO] TestAgent: Requesting shutdown +38:07.493 [INFO] TestAgent.server: shutting down server +38:07.493 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +38:07.493 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +38:07.493 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +38:07.493 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +38:07.493 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +38:07.493 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +38:07.493 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +38:07.493 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +38:07.493 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +38:07.493 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +38:07.493 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:38:07.493Z [INFO] agent.router.manager: shutting down +2021-08-13T15:38:07.493Z [INFO] agent.router.manager: shutting down +38:07.493 [INFO] TestAgent: consul server down +38:07.493 [INFO] TestAgent: shutdown complete +38:07.493 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28407 network=tcp +38:07.493 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28407 network=udp +38:07.493 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:28408 network=tcp +38:07.994 [INFO] TestAgent: Waiting for endpoints to shut down +38:07.994 [INFO] TestAgent: Endpoints down +--- PASS: TestLockCommand_TryLock (1.10s) +2021-08-13T15:38:07.017Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:38:07.097Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:07.099 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6bb6d8e8-c542-37d1-ffbd-651b950d7fe3 Address:127.0.0.1:28419}]" +38:07.100 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6bb6d8e8-c542-37d1-ffbd-651b950d7fe3.dc1 127.0.0.1 +38:07.100 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:28419 [Follower]" leader= +38:07.168 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6bb6d8e8-c542-37d1-ffbd-651b950d7fe3 127.0.0.1 +38:07.182 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +38:07.185 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:28419 [Candidate]" term=2 +38:07.185 [WARN] TestAgent.server.raft: unable to get address for sever, using fallback address: id=6bb6d8e8-c542-37d1-ffbd-651b950d7fe3 fallback=127.0.0.1:28419 error="Could not find address for server id 6bb6d8e8-c542-37d1-ffbd-651b950d7fe3" +38:07.185 [DEBUG] TestAgent.server.raft: votes: needed=1 +38:07.185 [DEBUG] TestAgent.server.raft: vote granted: from=6bb6d8e8-c542-37d1-ffbd-651b950d7fe3 term=2 tally=1 +38:07.185 [INFO] TestAgent.server.raft: election won: tally=1 +38:07.185 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:28419 [Leader]" +2021-08-13T15:38:07.186Z [INFO] agent.router: Initializing LAN area manager +38:07.202 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28414 network=udp +38:07.202 [INFO] TestAgent.server: Adding LAN server: server="Node-6bb6d8e8-c542-37d1-ffbd-651b950d7fe3 (Addr: tcp/127.0.0.1:28419) (DC: dc1)" +38:07.207 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28414 network=tcp +38:07.208 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:28415 network=tcp +38:07.208 [INFO] TestAgent: started state syncer +38:07.208 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:28420 network=tcp +38:07.209 [INFO] TestAgent: Synced node info +38:07.209 [DEBUG] TestAgent: Node info in sync +38:07.210 [INFO] TestAgent.server: cluster leadership acquired +38:07.210 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:28419 +38:07.211 [INFO] TestAgent.server: New leader elected: payload=Node-6bb6d8e8-c542-37d1-ffbd-651b950d7fe3 +38:07.212 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +38:07.215 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6bb6d8e8-c542-37d1-ffbd-651b950d7fe3.dc1 area=wan +38:07.239 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +38:07.239 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +38:07.239 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +38:07.239 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +38:07.239 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +38:07.240 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6bb6d8e8-c542-37d1-ffbd-651b950d7fe3 +38:07.240 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6bb6d8e8-c542-37d1-ffbd-651b950d7fe3 +38:07.241 [INFO] TestAgent.server: federation state anti-entropy synced +38:07.241 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6bb6d8e8-c542-37d1-ffbd-651b950d7fe3 +38:07.273 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:40602 latency=7.759759ms +38:07.286 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:40602 latency=1.523123ms +38:07.287 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?wait=15000ms from=127.0.0.1:40602 latency=277.001µs +38:07.288 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?acquire=16bf61f5-df5d-1112-3c34-5be030c7c0fa&flags=3304740253564472344 from=127.0.0.1:40602 latency=633.462µs +38:07.289 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent= from=127.0.0.1:40602 latency=204.361µs +38:07.311 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?flags=3304740253564472344&release=16bf61f5-df5d-1112-3c34-5be030c7c0fa from=127.0.0.1:40602 latency=489.341µs +38:07.312 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent=&index=14 from=127.0.0.1:40616 latency=2.280505ms +38:07.312 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:40602 latency=100.32µs +38:07.314 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=15 from=127.0.0.1:40602 latency=443.801µs +38:07.315 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/16bf61f5-df5d-1112-3c34-5be030c7c0fa from=127.0.0.1:40616 latency=636.341µs +38:07.359 [INFO] TestAgent: Requesting shutdown +38:07.359 [INFO] TestAgent.server: shutting down server +38:07.359 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +38:07.359 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +38:07.359 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +38:07.359 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +38:07.359 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +38:07.359 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +38:07.359 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +38:07.359 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +38:07.359 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +38:07.359 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +38:07.359 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2021-08-13T15:38:07.359Z [INFO] agent.router.manager: shutting down +2021-08-13T15:38:07.360Z [INFO] agent.router.manager: shutting down +38:07.360 [INFO] TestAgent: consul server down +38:07.360 [INFO] TestAgent: shutdown complete +38:07.360 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28414 network=tcp +38:07.360 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28414 network=udp +38:07.360 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:28415 network=tcp +38:08.360 [WARN] TestAgent: Timeout stopping server: protocol=HTTP address=127.0.0.1:28415 network=tcp +38:08.360 [INFO] TestAgent: Waiting for endpoints to shut down +38:08.360 [INFO] TestAgent: Endpoints down +--- PASS: TestLockCommand (1.46s) +2021-08-13T15:38:06.997Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:38:07.025Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:07.074 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bd135621-73ce-c866-1262-82ae0fafb187 Address:127.0.0.1:28433}]" +38:07.074 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:28433 [Follower]" leader= +38:07.075 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bd135621-73ce-c866-1262-82ae0fafb187.dc1 127.0.0.1 +38:07.076 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bd135621-73ce-c866-1262-82ae0fafb187 127.0.0.1 +2021-08-13T15:38:07.076Z [INFO] agent.router: Initializing LAN area manager +38:07.076 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28428 network=udp +38:07.076 [INFO] TestAgent.server: Adding LAN server: server="Node-bd135621-73ce-c866-1262-82ae0fafb187 (Addr: tcp/127.0.0.1:28433) (DC: dc1)" +38:07.076 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bd135621-73ce-c866-1262-82ae0fafb187.dc1 area=wan +38:07.077 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28428 network=tcp +38:07.083 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:28429 network=tcp +38:07.083 [INFO] TestAgent: started state syncer +38:07.083 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:28434 network=tcp +38:07.176 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +38:07.176 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:28433 [Candidate]" term=2 +38:07.176 [DEBUG] TestAgent.server.raft: votes: needed=1 +38:07.176 [DEBUG] TestAgent.server.raft: vote granted: from=bd135621-73ce-c866-1262-82ae0fafb187 term=2 tally=1 +38:07.176 [INFO] TestAgent.server.raft: election won: tally=1 +38:07.176 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:28433 [Leader]" +38:07.177 [INFO] TestAgent.server: cluster leadership acquired +38:07.186 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:28433 +38:07.198 [INFO] TestAgent.server: New leader elected: payload=Node-bd135621-73ce-c866-1262-82ae0fafb187 +38:07.212 [INFO] TestAgent: Synced node info +38:07.212 [DEBUG] TestAgent: Node info in sync +38:07.213 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +38:07.239 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +38:07.239 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +38:07.239 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +38:07.239 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +38:07.239 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +38:07.239 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bd135621-73ce-c866-1262-82ae0fafb187 +38:07.239 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bd135621-73ce-c866-1262-82ae0fafb187 +38:07.241 [INFO] TestAgent.server: federation state anti-entropy synced +38:07.308 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:59568 latency=18.099262ms +38:07.315 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:59568 latency=4.07885ms +38:07.316 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?wait=15000ms from=127.0.0.1:59568 latency=72.46µs +38:07.317 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?acquire=eff69466-5647-844f-9d58-350d86db4676&flags=3304740253564472344 from=127.0.0.1:59568 latency=591.382µs +38:07.317 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent= from=127.0.0.1:59568 latency=327.861µs +38:07.366 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?flags=3304740253564472344&release=eff69466-5647-844f-9d58-350d86db4676 from=127.0.0.1:59598 latency=542.401µs +38:07.366 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent=&index=14 from=127.0.0.1:59568 latency=48.493973ms +38:07.367 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:59598 latency=110.1µs +38:07.368 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/eff69466-5647-844f-9d58-350d86db4676 from=127.0.0.1:59568 latency=565.101µs +38:07.368 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=15 from=127.0.0.1:59598 latency=564.302µs +38:07.368 [INFO] TestAgent: Requesting shutdown +38:07.368 [INFO] TestAgent.server: shutting down server +38:07.368 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +38:07.368 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +38:07.368 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +38:07.368 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +38:07.368 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +38:07.368 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +38:07.368 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +38:07.368 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +38:07.368 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +38:07.368 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +38:07.369 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:38:07.369Z [INFO] agent.router.manager: shutting down +2021-08-13T15:38:07.369Z [INFO] agent.router.manager: shutting down +38:07.369 [INFO] TestAgent: consul server down +38:07.369 [INFO] TestAgent: shutdown complete +38:07.369 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28428 network=tcp +38:07.369 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28428 network=udp +38:07.369 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:28429 network=tcp +38:08.369 [WARN] TestAgent: Timeout stopping server: protocol=HTTP address=127.0.0.1:28429 network=tcp +38:08.369 [INFO] TestAgent: Waiting for endpoints to shut down +38:08.369 [INFO] TestAgent: Endpoints down +--- PASS: TestLockCommand_MonitorRetry_Lock_Arg (1.47s) +2021-08-13T15:38:07.043Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:38:07.074Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:07.087 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1154eef9-0924-78b2-72a0-549195a16553 Address:127.0.0.1:28391}]" +38:07.087 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:28391 [Follower]" leader= +38:07.088 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1154eef9-0924-78b2-72a0-549195a16553.dc1 127.0.0.1 +38:07.168 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1154eef9-0924-78b2-72a0-549195a16553 127.0.0.1 +2021-08-13T15:38:07.168Z [INFO] agent.router: Initializing LAN area manager +38:07.169 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28386 network=udp +38:07.179 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +38:07.179 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:28391 [Candidate]" term=2 +38:07.179 [WARN] TestAgent.server.raft: unable to get address for sever, using fallback address: id=1154eef9-0924-78b2-72a0-549195a16553 fallback=127.0.0.1:28391 error="Could not find address for server id 1154eef9-0924-78b2-72a0-549195a16553" +38:07.180 [DEBUG] TestAgent.server.raft: votes: needed=1 +38:07.180 [DEBUG] TestAgent.server.raft: vote granted: from=1154eef9-0924-78b2-72a0-549195a16553 term=2 tally=1 +38:07.180 [INFO] TestAgent.server.raft: election won: tally=1 +38:07.180 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:28391 [Leader]" +38:07.181 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1154eef9-0924-78b2-72a0-549195a16553.dc1 area=wan +38:07.181 [INFO] TestAgent.server: cluster leadership acquired +38:07.184 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28386 network=tcp +38:07.185 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:28387 network=tcp +38:07.185 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=1 found=false leader=127.0.0.1:28391 +38:07.195 [INFO] TestAgent.server: Adding LAN server: server="Node-1154eef9-0924-78b2-72a0-549195a16553 (Addr: tcp/127.0.0.1:28391) (DC: dc1)" +38:07.195 [INFO] TestAgent.server: New leader elected: payload=Node-1154eef9-0924-78b2-72a0-549195a16553 +38:07.195 [INFO] TestAgent: started state syncer +38:07.196 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:28392 network=tcp +38:07.199 [INFO] TestAgent: Synced node info +38:07.202 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +38:07.237 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +38:07.237 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +38:07.237 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +38:07.237 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +38:07.237 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +38:07.237 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1154eef9-0924-78b2-72a0-549195a16553 +38:07.237 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1154eef9-0924-78b2-72a0-549195a16553 +38:07.239 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1154eef9-0924-78b2-72a0-549195a16553 +38:07.240 [INFO] TestAgent.server: federation state anti-entropy synced +38:07.253 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:53924 latency=2.244405ms +38:07.257 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:53924 latency=1.183282ms +38:07.258 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?wait=15000ms from=127.0.0.1:53924 latency=88.94µs +38:07.259 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?acquire=4e080daf-9000-2bc9-6af1-64b057b3171e&flags=3304740253564472344 from=127.0.0.1:53924 latency=570.981µs +38:07.275 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent= from=127.0.0.1:53924 latency=10.430324ms +38:07.277 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?flags=3304740253564472344&release=4e080daf-9000-2bc9-6af1-64b057b3171e from=127.0.0.1:53930 latency=998.422µs +38:07.277 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent=&index=14 from=127.0.0.1:53924 latency=266.22µs +38:07.278 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:53930 latency=145.46µs +38:07.282 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/4e080daf-9000-2bc9-6af1-64b057b3171e from=127.0.0.1:53924 latency=2.839467ms +38:07.284 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=15 from=127.0.0.1:53924 latency=626.821µs +38:07.310 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:53938 latency=1.561283ms +38:07.316 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:53938 latency=1.902844ms +38:07.317 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?wait=15000ms from=127.0.0.1:53938 latency=70.16µs +38:07.317 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?acquire=acfaaf3e-26d0-1fbb-9c5b-aaf7a39a38de&flags=3304740253564472344 from=127.0.0.1:53938 latency=392.381µs +38:07.318 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent= from=127.0.0.1:53938 latency=128.44µs +38:07.363 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?flags=3304740253564472344&release=acfaaf3e-26d0-1fbb-9c5b-aaf7a39a38de from=127.0.0.1:53958 latency=646.761µs +38:07.363 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent=&index=19 from=127.0.0.1:53938 latency=44.319023ms +38:07.364 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:53958 latency=141.3µs +38:07.364 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/acfaaf3e-26d0-1fbb-9c5b-aaf7a39a38de from=127.0.0.1:53938 latency=564.081µs +38:07.365 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=20 from=127.0.0.1:53958 latency=449.601µs +38:07.368 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:53964 latency=1.530563ms +38:07.372 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:53964 latency=752.602µs +38:07.372 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?wait=15000ms from=127.0.0.1:53964 latency=77.861µs +38:07.373 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?acquire=d0288034-9816-4252-c82a-3df42b96461b&flags=3304740253564472344 from=127.0.0.1:53964 latency=484.702µs +38:07.374 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent= from=127.0.0.1:53964 latency=158.24µs +38:07.380 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?flags=3304740253564472344&release=d0288034-9816-4252-c82a-3df42b96461b from=127.0.0.1:53968 latency=655.361µs +38:07.380 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent=&index=24 from=127.0.0.1:53964 latency=5.491532ms +38:07.381 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:53968 latency=131.681µs +38:07.381 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/d0288034-9816-4252-c82a-3df42b96461b from=127.0.0.1:53964 latency=532.021µs +38:07.382 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=25 from=127.0.0.1:53968 latency=454.421µs +38:07.382 [INFO] TestAgent: Requesting shutdown +38:07.382 [INFO] TestAgent.server: shutting down server +38:07.382 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +38:07.382 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +38:07.382 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +38:07.382 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +38:07.382 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +38:07.382 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +38:07.382 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +38:07.382 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +38:07.382 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +38:07.382 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +38:07.382 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:38:07.382Z [INFO] agent.router.manager: shutting down +2021-08-13T15:38:07.382Z [INFO] agent.router.manager: shutting down +38:07.383 [INFO] TestAgent: consul server down +38:07.383 [INFO] TestAgent: shutdown complete +38:07.383 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28386 network=tcp +38:07.383 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28386 network=udp +38:07.383 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:28387 network=tcp +38:08.383 [WARN] TestAgent: Timeout stopping server: protocol=HTTP address=127.0.0.1:28387 network=tcp +38:08.383 [INFO] TestAgent: Waiting for endpoints to shut down +38:08.383 [INFO] TestAgent: Endpoints down +--- PASS: TestLockCommand_ChildExitCode (1.49s) --- PASS: TestLockCommand_ChildExitCode/clean_exit (0.04s) - --- PASS: TestLockCommand_ChildExitCode/error_exit (0.04s) - --- PASS: TestLockCommand_ChildExitCode/not_propagated (0.03s) -2022-09-15T21:45:46.196Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:46.292Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:46.308 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6168e055-028e-829e-885e-8f44d8c8c3c6 Address:127.0.0.1:11041}]" -45:46.322 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6168e055-028e-829e-885e-8f44d8c8c3c6.dc1 127.0.0.1 -45:46.354 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:11041 [Follower]" leader= -45:46.363 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6168e055-028e-829e-885e-8f44d8c8c3c6 127.0.0.1 -2022-09-15T21:45:46.363Z [INFO] agent.router: Initializing LAN area manager -45:46.363 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11036 network=udp -45:46.364 [INFO] TestAgent.server: Adding LAN server: server="Node-6168e055-028e-829e-885e-8f44d8c8c3c6 (Addr: tcp/127.0.0.1:11041) (DC: dc1)" -45:46.364 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6168e055-028e-829e-885e-8f44d8c8c3c6.dc1 area=wan -45:46.364 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11036 network=tcp -45:46.365 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:11037 network=tcp -45:46.365 [INFO] TestAgent: started state syncer -45:46.365 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:11042 network=tcp -45:46.415 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:46.415 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:11041 [Candidate]" term=2 -45:46.416 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:46.416 [DEBUG] TestAgent.server.raft: vote granted: from=6168e055-028e-829e-885e-8f44d8c8c3c6 term=2 tally=1 -45:46.416 [INFO] TestAgent.server.raft: election won: tally=1 -45:46.416 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:11041 [Leader]" -45:46.418 [INFO] TestAgent.server: cluster leadership acquired -45:46.419 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:11041 -45:46.466 [INFO] TestAgent.server: New leader elected: payload=Node-6168e055-028e-829e-885e-8f44d8c8c3c6 -45:46.528 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:46.535 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:46.535 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:46.535 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:46.535 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:46.535 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:46.537 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6168e055-028e-829e-885e-8f44d8c8c3c6 -45:46.537 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6168e055-028e-829e-885e-8f44d8c8c3c6 -45:46.563 [INFO] TestAgent.server: federation state anti-entropy synced -45:46.624 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -45:46.625 [INFO] TestAgent: Synced node info -45:46.838 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:50652 latency=2.92812ms -45:46.842 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:50652 latency=998.72µs -45:46.844 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/cd748637-2d99-c553-6240-fd674a984123?acquire=cd748637-2d99-c553-6240-fd674a984123&flags=16210313421097356768 from=127.0.0.1:50652 latency=502.2µs -45:46.845 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?recurse=&wait=15000ms from=127.0.0.1:50652 latency=177.56µs -45:46.846 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?cas=0&flags=16210313421097356768 from=127.0.0.1:50652 latency=665.1µs -45:46.853 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?consistent=&recurse= from=127.0.0.1:50652 latency=592.98µs -45:46.865 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:50654 latency=233.92µs -45:46.867 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?cas=15&flags=16210313421097356768 from=127.0.0.1:50654 latency=679.44µs -45:46.867 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?consistent=&index=15&recurse= from=127.0.0.1:50652 latency=13.48936ms -45:46.869 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/cd748637-2d99-c553-6240-fd674a984123 from=127.0.0.1:50654 latency=489.4µs -45:46.870 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?recurse= from=127.0.0.1:50654 latency=169.38µs -45:46.871 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=16 from=127.0.0.1:50654 latency=688.16µs -45:46.871 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/cd748637-2d99-c553-6240-fd674a984123 from=127.0.0.1:50652 latency=2.0821ms -45:46.872 [INFO] TestAgent: Requesting shutdown -45:46.872 [INFO] TestAgent.server: shutting down server -45:46.872 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:46.872 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:46.872 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:46.872 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:46.872 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:46.872 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:46.872 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:46.872 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:46.872 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:46.872 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:46.872 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:46.872Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:46.872Z [INFO] agent.router.manager: shutting down -45:46.872 [INFO] TestAgent: consul server down -45:46.872 [INFO] TestAgent: shutdown complete -45:46.872 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11036 network=tcp -45:46.872 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11036 network=udp -45:46.872 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:11037 network=tcp -45:47.372 [INFO] TestAgent: Waiting for endpoints to shut down -45:47.372 [INFO] TestAgent: Endpoints down ---- PASS: TestLockCommand_MonitorRetry_Semaphore_Arg (1.46s) -2022-09-15T21:45:46.276Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:46.460Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:46.471 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a99fe679-574d-be66-b770-83418108c9ab Address:127.0.0.1:11055}]" -45:46.498 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:11055 [Follower]" leader= -45:46.499 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a99fe679-574d-be66-b770-83418108c9ab.dc1 127.0.0.1 -45:46.565 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a99fe679-574d-be66-b770-83418108c9ab 127.0.0.1 -2022-09-15T21:45:46.565Z [INFO] agent.router: Initializing LAN area manager -45:46.565 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11050 network=udp -45:46.566 [INFO] TestAgent.server: Adding LAN server: server="Node-a99fe679-574d-be66-b770-83418108c9ab (Addr: tcp/127.0.0.1:11055) (DC: dc1)" -45:46.566 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a99fe679-574d-be66-b770-83418108c9ab.dc1 area=wan -45:46.566 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11050 network=tcp -45:46.567 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:11051 network=tcp -45:46.567 [INFO] TestAgent: started state syncer -45:46.567 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:11056 network=tcp -45:46.572 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:46.572 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:11055 [Candidate]" term=2 -45:46.572 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:46.572 [DEBUG] TestAgent.server.raft: vote granted: from=a99fe679-574d-be66-b770-83418108c9ab term=2 tally=1 -45:46.572 [INFO] TestAgent.server.raft: election won: tally=1 -45:46.572 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:11055 [Leader]" -45:46.573 [INFO] TestAgent.server: cluster leadership acquired -45:46.573 [INFO] TestAgent.server: New leader elected: payload=Node-a99fe679-574d-be66-b770-83418108c9ab -45:46.573 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:11055 -45:46.574 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:46.577 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:46.577 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:46.577 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:46.577 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:46.577 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:46.577 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a99fe679-574d-be66-b770-83418108c9ab -45:46.577 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a99fe679-574d-be66-b770-83418108c9ab -45:46.578 [INFO] TestAgent.server: federation state anti-entropy synced -45:46.656 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -45:46.656 [INFO] TestAgent: Synced node info -45:46.888 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:43198 latency=1.48956ms -45:46.892 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:43198 latency=839.36µs -45:46.895 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/2997c733-2312-d9bd-348f-60fc5df4a2b4?acquire=2997c733-2312-d9bd-348f-60fc5df4a2b4&flags=16210313421097356768 from=127.0.0.1:43198 latency=492.22µs -45:46.895 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?recurse=&wait=15000ms from=127.0.0.1:43198 latency=172.1µs -45:46.897 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?cas=0&flags=16210313421097356768 from=127.0.0.1:43198 latency=575.88µs -45:46.908 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?consistent=&recurse= from=127.0.0.1:43198 latency=369.72µs -45:46.927 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:43200 latency=213.16µs -45:46.928 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?cas=15&flags=16210313421097356768 from=127.0.0.1:43200 latency=746.4µs -45:46.929 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?consistent=&index=15&recurse= from=127.0.0.1:43198 latency=19.15024ms -45:46.929 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/2997c733-2312-d9bd-348f-60fc5df4a2b4 from=127.0.0.1:43200 latency=477.14µs -45:46.930 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?recurse= from=127.0.0.1:43200 latency=163.66µs -45:46.930 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/2997c733-2312-d9bd-348f-60fc5df4a2b4 from=127.0.0.1:43198 latency=546.9µs -45:46.931 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=16 from=127.0.0.1:43200 latency=411.76µs -45:46.931 [INFO] TestAgent: Requesting shutdown -45:46.931 [INFO] TestAgent.server: shutting down server -45:46.931 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:46.931 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:46.931 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:46.931 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:46.931 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:46.932 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:46.932 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:46.932 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:46.932 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:46.932 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:46.932 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:46.932Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:46.932Z [INFO] agent.router.manager: shutting down -45:46.932 [INFO] TestAgent: consul server down -45:46.932 [INFO] TestAgent: shutdown complete -45:46.932 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11050 network=tcp -45:46.932 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11050 network=udp -45:46.932 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:11051 network=tcp -45:47.433 [INFO] TestAgent: Waiting for endpoints to shut down -45:47.433 [INFO] TestAgent: Endpoints down ---- PASS: TestLockCommand_MonitorRetry_Semaphore_Default (1.52s) -2022-09-15T21:45:46.368Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:46.434Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:46.571 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:52888c51-c5f9-243e-e9a5-628f42440393 Address:127.0.0.1:11069}]" -45:46.572 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:11069 [Follower]" leader= -45:46.572 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-52888c51-c5f9-243e-e9a5-628f42440393.dc1 127.0.0.1 -45:46.573 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-52888c51-c5f9-243e-e9a5-628f42440393 127.0.0.1 -2022-09-15T21:45:46.573Z [INFO] agent.router: Initializing LAN area manager -45:46.573 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11064 network=udp -45:46.574 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-52888c51-c5f9-243e-e9a5-628f42440393.dc1 area=wan -45:46.574 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11064 network=tcp -45:46.575 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:11065 network=tcp -45:46.575 [INFO] TestAgent: started state syncer -45:46.575 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:11070 network=tcp -45:46.575 [INFO] TestAgent.server: Adding LAN server: server="Node-52888c51-c5f9-243e-e9a5-628f42440393 (Addr: tcp/127.0.0.1:11069) (DC: dc1)" -45:46.622 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:46.622 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:11069 [Candidate]" term=2 -45:46.623 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:46.623 [DEBUG] TestAgent.server.raft: vote granted: from=52888c51-c5f9-243e-e9a5-628f42440393 term=2 tally=1 -45:46.623 [INFO] TestAgent.server.raft: election won: tally=1 -45:46.623 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:11069 [Leader]" -45:46.623 [INFO] TestAgent.server: cluster leadership acquired -45:46.624 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:11069 -45:46.624 [INFO] TestAgent.server: New leader elected: payload=Node-52888c51-c5f9-243e-e9a5-628f42440393 -45:46.625 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:46.633 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:46.633 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:46.633 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:46.633 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:46.633 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:46.633 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-52888c51-c5f9-243e-e9a5-628f42440393 -45:46.633 [INFO] TestAgent.server: member joined, marking health alive: member=Node-52888c51-c5f9-243e-e9a5-628f42440393 -45:46.654 [INFO] TestAgent.server: federation state anti-entropy synced -45:46.854 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -45:46.854 [INFO] TestAgent: Synced node info -45:46.854 [DEBUG] TestAgent: Node info in sync -45:47.000 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:45568 latency=1.8589ms -45:47.005 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:45568 latency=1.0077ms -45:47.009 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?wait=15000ms from=127.0.0.1:45568 latency=142.6µs -45:47.012 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?acquire=e693eede-ed3c-84b4-a0f2-776b77749472&flags=3304740253564472344 from=127.0.0.1:45568 latency=688.72µs -45:47.013 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent= from=127.0.0.1:45568 latency=213.38µs -45:47.039 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?flags=3304740253564472344&release=e693eede-ed3c-84b4-a0f2-776b77749472 from=127.0.0.1:45572 latency=737.44µs -45:47.039 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent=&index=14 from=127.0.0.1:45568 latency=24.86016ms -45:47.042 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:45572 latency=174.7µs -45:47.043 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/e693eede-ed3c-84b4-a0f2-776b77749472 from=127.0.0.1:45574 latency=742.4µs -45:47.044 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=15 from=127.0.0.1:45572 latency=482.72µs -45:47.045 [INFO] TestAgent: Requesting shutdown -45:47.045 [INFO] TestAgent.server: shutting down server -45:47.045 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:47.045 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:47.045 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:47.045 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:47.045 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:47.045 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:47.045 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:47.045 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:47.045 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:47.045 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:47.045 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:47.045Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:47.045Z [INFO] agent.router.manager: shutting down -45:47.050 [INFO] TestAgent: consul server down -45:47.050 [INFO] TestAgent: shutdown complete -45:47.050 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11064 network=tcp -45:47.050 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11064 network=udp -45:47.050 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:11065 network=tcp -45:47.554 [INFO] TestAgent: Waiting for endpoints to shut down -45:47.554 [INFO] TestAgent: Endpoints down ---- PASS: TestLockCommand_NoShell (1.64s) -2022-09-15T21:45:46.360Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:46.573Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:46.577 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:873d76ad-d110-f3cd-658e-8d5378a3f566 Address:127.0.0.1:11076}]" -45:46.578 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:11076 [Follower]" leader= -45:46.583 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-873d76ad-d110-f3cd-658e-8d5378a3f566.dc1 127.0.0.1 -45:46.584 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-873d76ad-d110-f3cd-658e-8d5378a3f566 127.0.0.1 -2022-09-15T21:45:46.584Z [INFO] agent.router: Initializing LAN area manager -45:46.584 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11071 network=udp -45:46.586 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11071 network=tcp -45:46.587 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:11072 network=tcp -45:46.587 [INFO] TestAgent: started state syncer -45:46.587 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:11077 network=tcp -45:46.590 [INFO] TestAgent.server: Adding LAN server: server="Node-873d76ad-d110-f3cd-658e-8d5378a3f566 (Addr: tcp/127.0.0.1:11076) (DC: dc1)" -45:46.594 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-873d76ad-d110-f3cd-658e-8d5378a3f566.dc1 area=wan -45:46.637 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:46.637 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:11076 [Candidate]" term=2 -45:46.637 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:46.637 [DEBUG] TestAgent.server.raft: vote granted: from=873d76ad-d110-f3cd-658e-8d5378a3f566 term=2 tally=1 -45:46.637 [INFO] TestAgent.server.raft: election won: tally=1 -45:46.637 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:11076 [Leader]" -45:46.637 [INFO] TestAgent.server: cluster leadership acquired -45:46.638 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:11076 -45:46.638 [INFO] TestAgent.server: New leader elected: payload=Node-873d76ad-d110-f3cd-658e-8d5378a3f566 -45:46.639 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:46.654 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:46.654 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:46.654 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:46.654 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:46.654 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:46.655 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-873d76ad-d110-f3cd-658e-8d5378a3f566 -45:46.655 [INFO] TestAgent.server: member joined, marking health alive: member=Node-873d76ad-d110-f3cd-658e-8d5378a3f566 -45:46.656 [INFO] TestAgent.server: federation state anti-entropy synced -45:46.704 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -45:46.705 [INFO] TestAgent: Synced node info -45:46.705 [DEBUG] TestAgent: Node info in sync -45:46.727 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -45:46.727 [DEBUG] TestAgent: Node info in sync -45:47.020 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:59188 latency=1.77696ms -45:47.043 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:59188 latency=777.66µs -45:47.045 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?wait=10000ms from=127.0.0.1:59188 latency=93.06µs -45:47.046 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?acquire=336846e4-5666-79fe-a44d-6aae82fc8bb8&flags=3304740253564472344 from=127.0.0.1:59188 latency=515.94µs -45:47.051 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent= from=127.0.0.1:59188 latency=233.44µs -45:47.072 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?flags=3304740253564472344&release=336846e4-5666-79fe-a44d-6aae82fc8bb8 from=127.0.0.1:59194 latency=722.06µs -45:47.072 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent=&index=14 from=127.0.0.1:59188 latency=14.11234ms -45:47.074 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/336846e4-5666-79fe-a44d-6aae82fc8bb8 from=127.0.0.1:59188 latency=842.06µs -45:47.074 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:59194 latency=483.26µs -45:47.075 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=15 from=127.0.0.1:59194 latency=460.74µs -45:47.075 [INFO] TestAgent: Requesting shutdown -45:47.075 [INFO] TestAgent.server: shutting down server -45:47.075 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:47.075 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:47.075 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:47.075 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:47.075 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:47.075 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:47.076 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:47.076 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:47.076 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:47.076 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:47.076 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:47.076Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:47.076Z [INFO] agent.router.manager: shutting down -45:47.076 [INFO] TestAgent: consul server down -45:47.076 [INFO] TestAgent: shutdown complete -45:47.076 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11071 network=tcp -45:47.076 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11071 network=udp -45:47.076 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:11072 network=tcp -45:47.577 [INFO] TestAgent: Waiting for endpoints to shut down -45:47.577 [INFO] TestAgent: Endpoints down ---- PASS: TestLockCommand_TryLock (1.66s) -2022-09-15T21:45:47.089Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:47.110Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:47.113 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1a9b37fe-caa9-b772-b530-9d27d1f606b5 Address:127.0.0.1:11083}]" -45:47.114 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:11083 [Follower]" leader= -45:47.115 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1a9b37fe-caa9-b772-b530-9d27d1f606b5.dc1 127.0.0.1 -45:47.116 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1a9b37fe-caa9-b772-b530-9d27d1f606b5 127.0.0.1 -2022-09-15T21:45:47.116Z [INFO] agent.router: Initializing LAN area manager -45:47.116 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11078 network=udp -45:47.116 [INFO] TestAgent.server: Adding LAN server: server="Node-1a9b37fe-caa9-b772-b530-9d27d1f606b5 (Addr: tcp/127.0.0.1:11083) (DC: dc1)" -45:47.116 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1a9b37fe-caa9-b772-b530-9d27d1f606b5.dc1 area=wan -45:47.117 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11078 network=tcp -45:47.118 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:11079 network=tcp -45:47.118 [INFO] TestAgent: started state syncer -45:47.118 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:11084 network=tcp -45:47.158 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:47.158 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:11083 [Candidate]" term=2 -45:47.159 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:47.159 [DEBUG] TestAgent.server.raft: vote granted: from=1a9b37fe-caa9-b772-b530-9d27d1f606b5 term=2 tally=1 -45:47.159 [INFO] TestAgent.server.raft: election won: tally=1 -45:47.159 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:11083 [Leader]" -45:47.159 [INFO] TestAgent.server: cluster leadership acquired -45:47.159 [INFO] TestAgent.server: New leader elected: payload=Node-1a9b37fe-caa9-b772-b530-9d27d1f606b5 -45:47.159 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:11083 -45:47.161 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:47.164 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:47.164 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:47.164 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:47.164 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:47.164 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:47.164 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1a9b37fe-caa9-b772-b530-9d27d1f606b5 -45:47.164 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1a9b37fe-caa9-b772-b530-9d27d1f606b5 -45:47.165 [INFO] TestAgent.server: federation state anti-entropy synced -45:47.178 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -45:47.179 [INFO] TestAgent: Synced node info -45:47.179 [DEBUG] TestAgent: Node info in sync -45:47.463 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:48860 latency=1.93934ms -45:47.468 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:48860 latency=814.42µs -45:47.468 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?wait=15000ms from=127.0.0.1:48860 latency=87.66µs -45:47.469 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?acquire=fcfc7a4e-f262-c922-923e-252afae55df4&flags=3304740253564472344 from=127.0.0.1:48860 latency=494.7µs -45:47.482 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent= from=127.0.0.1:48860 latency=294.06µs -45:47.496 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?flags=3304740253564472344&release=fcfc7a4e-f262-c922-923e-252afae55df4 from=127.0.0.1:48862 latency=795.96µs -45:47.496 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent=&index=14 from=127.0.0.1:48860 latency=9.9521ms -45:47.498 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:48862 latency=233.36µs -45:47.498 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/fcfc7a4e-f262-c922-923e-252afae55df4 from=127.0.0.1:48860 latency=949.34µs -45:47.503 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=15 from=127.0.0.1:48862 latency=649.2µs -45:47.503 [INFO] TestAgent: Requesting shutdown -45:47.504 [INFO] TestAgent.server: shutting down server -45:47.504 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:47.504 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:47.504 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:47.504 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:47.504 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:47.504 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:47.504 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:47.504 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:47.504 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:47.504 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:47.504 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:47.504Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:47.504Z [INFO] agent.router.manager: shutting down -45:47.504 [INFO] TestAgent: consul server down -45:47.504 [INFO] TestAgent: shutdown complete -45:47.504 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11078 network=tcp -45:47.505 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11078 network=udp -45:47.505 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:11079 network=tcp -45:48.005 [INFO] TestAgent: Waiting for endpoints to shut down -45:48.005 [INFO] TestAgent: Endpoints down ---- PASS: TestLockCommand (0.93s) + --- PASS: TestLockCommand_ChildExitCode/error_exit (0.08s) + --- PASS: TestLockCommand_ChildExitCode/not_propagated (0.02s) +2021-08-13T15:38:07.844Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:38:07.860Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:07.862 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b757b3e3-5cad-9b15-bc4e-b2a3ff548faa Address:127.0.0.1:28440}]" +38:07.862 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:28440 [Follower]" leader= +38:07.863 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b757b3e3-5cad-9b15-bc4e-b2a3ff548faa.dc1 127.0.0.1 +38:07.864 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b757b3e3-5cad-9b15-bc4e-b2a3ff548faa 127.0.0.1 +2021-08-13T15:38:07.864Z [INFO] agent.router: Initializing LAN area manager +38:07.864 [INFO] TestAgent.server: Adding LAN server: server="Node-b757b3e3-5cad-9b15-bc4e-b2a3ff548faa (Addr: tcp/127.0.0.1:28440) (DC: dc1)" +38:07.864 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b757b3e3-5cad-9b15-bc4e-b2a3ff548faa.dc1 area=wan +38:07.864 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28435 network=tcp +38:07.864 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28435 network=udp +38:07.865 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:28436 network=tcp +38:07.866 [INFO] TestAgent: started state syncer +38:07.866 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:28441 network=tcp +38:07.926 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +38:07.926 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:28440 [Candidate]" term=2 +38:07.926 [DEBUG] TestAgent.server.raft: votes: needed=1 +38:07.926 [DEBUG] TestAgent.server.raft: vote granted: from=b757b3e3-5cad-9b15-bc4e-b2a3ff548faa term=2 tally=1 +38:07.926 [INFO] TestAgent.server.raft: election won: tally=1 +38:07.926 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:28440 [Leader]" +38:07.926 [INFO] TestAgent.server: cluster leadership acquired +38:07.927 [INFO] TestAgent.server: New leader elected: payload=Node-b757b3e3-5cad-9b15-bc4e-b2a3ff548faa +38:07.927 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:28440 +38:07.928 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +38:07.931 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +38:07.931 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +38:07.931 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +38:07.931 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +38:07.931 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +38:07.931 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b757b3e3-5cad-9b15-bc4e-b2a3ff548faa +38:07.931 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b757b3e3-5cad-9b15-bc4e-b2a3ff548faa +38:07.931 [INFO] TestAgent.server: federation state anti-entropy synced +38:08.015 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +38:08.015 [INFO] TestAgent: Synced node info +38:08.015 [DEBUG] TestAgent: Node info in sync +38:08.074 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:55838 latency=4.793371ms +38:08.082 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:55838 latency=868.183µs +38:08.088 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/52e5ee97-a7a1-9cbd-164e-49d276eb4544?acquire=52e5ee97-a7a1-9cbd-164e-49d276eb4544&flags=16210313421097356768 from=127.0.0.1:55838 latency=638.262µs +38:08.089 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?recurse=&wait=15000ms from=127.0.0.1:55838 latency=178.22µs +38:08.090 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?cas=0&flags=16210313421097356768 from=127.0.0.1:55838 latency=483.721µs +38:08.091 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?consistent=&recurse= from=127.0.0.1:55838 latency=210.54µs +38:08.100 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:55840 latency=206.981µs +38:08.101 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?cas=15&flags=16210313421097356768 from=127.0.0.1:55840 latency=629.402µs +38:08.101 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?consistent=&index=15&recurse= from=127.0.0.1:55838 latency=9.324382ms +38:08.102 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/52e5ee97-a7a1-9cbd-164e-49d276eb4544 from=127.0.0.1:55840 latency=429.101µs +38:08.103 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?recurse= from=127.0.0.1:55840 latency=136.68µs +38:08.103 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/52e5ee97-a7a1-9cbd-164e-49d276eb4544 from=127.0.0.1:55838 latency=559.021µs +38:08.104 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=16 from=127.0.0.1:55840 latency=456.721µs +38:08.104 [INFO] TestAgent: Requesting shutdown +38:08.104 [INFO] TestAgent.server: shutting down server +38:08.104 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +38:08.104 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +38:08.104 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +38:08.104 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +38:08.104 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +38:08.104 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +38:08.104 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +38:08.104 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +38:08.104 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +38:08.104 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +38:08.104 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:38:08.105Z [INFO] agent.router.manager: shutting down +2021-08-13T15:38:08.105Z [INFO] agent.router.manager: shutting down +38:08.105 [INFO] TestAgent: consul server down +38:08.105 [INFO] TestAgent: shutdown complete +38:08.105 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28435 network=tcp +38:08.105 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28435 network=udp +38:08.105 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:28436 network=tcp +38:08.605 [INFO] TestAgent: Waiting for endpoints to shut down +38:08.605 [INFO] TestAgent: Endpoints down +--- PASS: TestLockCommand_MonitorRetry_Semaphore_Default (0.78s) PASS -ok github.com/hashicorp/consul/command/lock 2.241s +ok github.com/hashicorp/consul/command/lock 1.805s === RUN TestLoginCommand_noTabs === PAUSE TestLoginCommand_noTabs === RUN TestLoginCommand @@ -88867,17 +88763,13 @@ === RUN TestLoginCommand_jwt === PAUSE TestLoginCommand_jwt === CONT TestLoginCommand_noTabs ---- PASS: TestLoginCommand_noTabs (0.00s) === CONT TestLoginCommand_jwt -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestLoginCommand_k8s +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestLoginCommand +--- PASS: TestLoginCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -=== RUN TestLoginCommand_jwt/success_-_jwt_static_keys -=== RUN TestLoginCommand_jwt/success_-_jwt_jwks -=== RUN TestLoginCommand_jwt/success_-_jwt_oidc_discovery -=== RUN TestLoginCommand_k8s/try_login_with_method_configured_and_binding_rules === RUN TestLoginCommand/method_is_required === RUN TestLoginCommand/token-sink-file_is_required === RUN TestLoginCommand/bearer-token-file_is_required @@ -88885,246 +88777,88 @@ === RUN TestLoginCommand/try_login_with_no_method_configured === RUN TestLoginCommand/try_login_with_method_configured_but_no_binding_rules === RUN TestLoginCommand/try_login_with_method_configured_and_binding_rules -2022-09-15T21:45:51.343Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:51.371Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:51.375 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:04fb4823-d7bb-50ea-759b-3caafbe8d7c5 Address:127.0.0.1:12048}]" -45:51.376 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:12048 [Follower]" leader= -45:51.377 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-04fb4823-d7bb-50ea-759b-3caafbe8d7c5.dc1 127.0.0.1 -45:51.378 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-04fb4823-d7bb-50ea-759b-3caafbe8d7c5 127.0.0.1 -2022-09-15T21:45:51.378Z [INFO] agent.router: Initializing LAN area manager -45:51.378 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12043 network=udp -45:51.379 [INFO] TestAgent.server: Adding LAN server: server="Node-04fb4823-d7bb-50ea-759b-3caafbe8d7c5 (Addr: tcp/127.0.0.1:12048) (DC: dc1)" -45:51.379 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12043 network=tcp -45:51.380 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-04fb4823-d7bb-50ea-759b-3caafbe8d7c5.dc1 area=wan -45:51.380 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:12044 network=tcp -45:51.380 [INFO] TestAgent: started state syncer -45:51.380 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:12049 network=tcp -45:51.416 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:51.416 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:12048 [Candidate]" term=2 -45:51.416 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:51.416 [DEBUG] TestAgent.server.raft: vote granted: from=04fb4823-d7bb-50ea-759b-3caafbe8d7c5 term=2 tally=1 -45:51.416 [INFO] TestAgent.server.raft: election won: tally=1 -45:51.416 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:12048 [Leader]" -45:51.416 [INFO] TestAgent.server: cluster leadership acquired -45:51.417 [INFO] TestAgent.server: initializing acls -45:51.417 [INFO] TestAgent.server: Created ACL 'global-management' policy -45:51.417 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -45:51.417 [INFO] TestAgent.server: New leader elected: payload=Node-04fb4823-d7bb-50ea-759b-3caafbe8d7c5 -45:51.418 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -45:51.419 [INFO] TestAgent.server: Created ACL anonymous token from configuration -45:51.419 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -45:51.419 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -45:51.419 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-04fb4823-d7bb-50ea-759b-3caafbe8d7c5 -45:51.419 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-04fb4823-d7bb-50ea-759b-3caafbe8d7c5.dc1 -45:51.419 [INFO] TestAgent.server: Updating LAN server: server="Node-04fb4823-d7bb-50ea-759b-3caafbe8d7c5 (Addr: tcp/127.0.0.1:12048) (DC: dc1)" -45:51.419 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-04fb4823-d7bb-50ea-759b-3caafbe8d7c5.dc1 area=wan -45:51.430 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:51.438 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:51.438 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:51.438 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:51.438 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:51.438 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:51.438 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-04fb4823-d7bb-50ea-759b-3caafbe8d7c5 -45:51.438 [INFO] TestAgent.server: member joined, marking health alive: member=Node-04fb4823-d7bb-50ea-759b-3caafbe8d7c5 -45:51.441 [INFO] TestAgent.server: federation state anti-entropy synced -45:51.441 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-04fb4823-d7bb-50ea-759b-3caafbe8d7c5 -45:51.452 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:44318 latency=2.27916ms -45:51.455 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:44318 latency=2.18908ms -45:51.462 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/login from=127.0.0.1:44320 latency=3.08098ms -45:51.472 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:44318 latency=8.18378ms -45:51.488 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:44318 latency=14.578ms -45:51.500 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/login from=127.0.0.1:44326 latency=9.31286ms -45:51.518 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:44318 latency=16.09626ms -45:51.535 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:44318 latency=15.64192ms -45:51.562 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/login from=127.0.0.1:44338 latency=5.3878ms -45:51.563 [INFO] TestAgent: Requesting shutdown -45:51.563 [INFO] TestAgent.server: shutting down server -45:51.563 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -45:51.563 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -45:51.563 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:51.563 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:51.563 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:51.563 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:51.563 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:51.563 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:51.563 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:51.563 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -45:51.563 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -45:51.563 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -45:51.563 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:51.563 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:51.563 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:51.563 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:51.563Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:51.563Z [INFO] agent.router.manager: shutting down -45:51.564 [INFO] TestAgent: consul server down -45:51.564 [INFO] TestAgent: shutdown complete -45:51.564 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12043 network=tcp -45:51.564 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12043 network=udp -45:51.564 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:12044 network=tcp -45:52.064 [INFO] TestAgent: Waiting for endpoints to shut down -45:52.064 [INFO] TestAgent: Endpoints down ---- PASS: TestLoginCommand_jwt (0.81s) - --- PASS: TestLoginCommand_jwt/success_-_jwt_static_keys (0.02s) - --- PASS: TestLoginCommand_jwt/success_-_jwt_jwks (0.04s) - --- PASS: TestLoginCommand_jwt/success_-_jwt_oidc_discovery (0.06s) -2022-09-15T21:45:51.343Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:51.367Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:51.375 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7afe3a8d-ee6a-4351-2b7c-471176e78769 Address:127.0.0.1:12055}]" -45:51.378 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:12055 [Follower]" leader= -45:51.379 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7afe3a8d-ee6a-4351-2b7c-471176e78769.dc1 127.0.0.1 -45:51.379 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7afe3a8d-ee6a-4351-2b7c-471176e78769 127.0.0.1 -2022-09-15T21:45:51.379Z [INFO] agent.router: Initializing LAN area manager -45:51.380 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12050 network=udp -45:51.381 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7afe3a8d-ee6a-4351-2b7c-471176e78769.dc1 area=wan -45:51.381 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12050 network=tcp -45:51.382 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:12051 network=tcp -45:51.382 [INFO] TestAgent: started state syncer -45:51.382 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:12056 network=tcp -45:51.383 [INFO] TestAgent.server: Adding LAN server: server="Node-7afe3a8d-ee6a-4351-2b7c-471176e78769 (Addr: tcp/127.0.0.1:12055) (DC: dc1)" -45:51.431 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -45:51.442 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:51.442 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:12055 [Candidate]" term=2 -45:51.442 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:51.442 [DEBUG] TestAgent.server.raft: vote granted: from=7afe3a8d-ee6a-4351-2b7c-471176e78769 term=2 tally=1 -45:51.442 [INFO] TestAgent.server.raft: election won: tally=1 -45:51.442 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:12055 [Leader]" -45:51.442 [INFO] TestAgent.server: cluster leadership acquired -45:51.442 [INFO] TestAgent.server: New leader elected: payload=Node-7afe3a8d-ee6a-4351-2b7c-471176e78769 -45:51.443 [INFO] TestAgent.server: initializing acls -45:51.443 [INFO] TestAgent.server: Created ACL 'global-management' policy -45:51.443 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -45:51.444 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -45:51.445 [INFO] TestAgent.server: Created ACL anonymous token from configuration -45:51.445 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -45:51.445 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -45:51.445 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-7afe3a8d-ee6a-4351-2b7c-471176e78769 -45:51.445 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-7afe3a8d-ee6a-4351-2b7c-471176e78769.dc1 -45:51.448 [INFO] TestAgent.server: Updating LAN server: server="Node-7afe3a8d-ee6a-4351-2b7c-471176e78769 (Addr: tcp/127.0.0.1:12055) (DC: dc1)" -45:51.448 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-7afe3a8d-ee6a-4351-2b7c-471176e78769.dc1 area=wan -45:51.448 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:51.451 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:51.451 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:51.451 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:51.451 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:51.451 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:51.451 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7afe3a8d-ee6a-4351-2b7c-471176e78769 -45:51.451 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7afe3a8d-ee6a-4351-2b7c-471176e78769 -45:51.452 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7afe3a8d-ee6a-4351-2b7c-471176e78769 -45:51.452 [INFO] TestAgent.server: federation state anti-entropy synced -45:51.622 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:54830 latency=3.65934ms -45:51.630 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:54830 latency=6.23006ms -45:51.655 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/login from=127.0.0.1:54832 latency=15.67206ms -45:51.657 [INFO] TestAgent: Requesting shutdown -45:51.657 [INFO] TestAgent.server: shutting down server -45:51.657 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -45:51.657 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -45:51.657 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:51.657 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:51.657 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:51.657 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:51.657 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:51.657 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:51.657 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:51.657 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -45:51.657 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -45:51.657 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -45:51.657 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:51.657 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:51.657 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:51.657 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:51.657Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:51.657Z [INFO] agent.router.manager: shutting down -45:51.657 [INFO] TestAgent: consul server down -45:51.657 [INFO] TestAgent: shutdown complete -45:51.657 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12050 network=tcp -45:51.658 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12050 network=udp -45:51.658 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:12051 network=tcp -45:52.162 [INFO] TestAgent: Waiting for endpoints to shut down -45:52.162 [INFO] TestAgent: Endpoints down ---- PASS: TestLoginCommand_k8s (0.91s) - --- PASS: TestLoginCommand_k8s/try_login_with_method_configured_and_binding_rules (0.02s) -2022-09-15T21:45:51.345Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:51.378Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:51.382 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:485b4b6a-9982-a676-a07f-a0571b9d7fd0 Address:127.0.0.1:12062}]" -45:51.383 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:12062 [Follower]" leader= -45:51.384 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-485b4b6a-9982-a676-a07f-a0571b9d7fd0.dc1 127.0.0.1 -45:51.384 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-485b4b6a-9982-a676-a07f-a0571b9d7fd0 127.0.0.1 -2022-09-15T21:45:51.384Z [INFO] agent.router: Initializing LAN area manager -45:51.385 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12057 network=udp -45:51.385 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-485b4b6a-9982-a676-a07f-a0571b9d7fd0.dc1 area=wan -45:51.385 [INFO] TestAgent.server: Adding LAN server: server="Node-485b4b6a-9982-a676-a07f-a0571b9d7fd0 (Addr: tcp/127.0.0.1:12062) (DC: dc1)" -45:51.385 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12057 network=tcp -45:51.386 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:12058 network=tcp -45:51.386 [INFO] TestAgent: started state syncer -45:51.386 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:12063 network=tcp -45:51.435 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -45:51.442 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:51.442 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:12062 [Candidate]" term=2 -45:51.442 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:51.442 [DEBUG] TestAgent.server.raft: vote granted: from=485b4b6a-9982-a676-a07f-a0571b9d7fd0 term=2 tally=1 -45:51.442 [INFO] TestAgent.server.raft: election won: tally=1 -45:51.443 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:12062 [Leader]" -45:51.443 [INFO] TestAgent.server: cluster leadership acquired -45:51.443 [INFO] TestAgent.server: initializing acls -45:51.443 [INFO] TestAgent.server: Created ACL 'global-management' policy -45:51.443 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -45:51.445 [INFO] TestAgent.server: New leader elected: payload=Node-485b4b6a-9982-a676-a07f-a0571b9d7fd0 -45:51.445 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -45:51.448 [INFO] TestAgent.server: Created ACL anonymous token from configuration -45:51.448 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -45:51.448 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -45:51.449 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-485b4b6a-9982-a676-a07f-a0571b9d7fd0 -45:51.449 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-485b4b6a-9982-a676-a07f-a0571b9d7fd0.dc1 -45:51.450 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:51.451 [INFO] TestAgent.server: Updating LAN server: server="Node-485b4b6a-9982-a676-a07f-a0571b9d7fd0 (Addr: tcp/127.0.0.1:12062) (DC: dc1)" -45:51.451 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-485b4b6a-9982-a676-a07f-a0571b9d7fd0.dc1 area=wan -45:51.452 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:51.452 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:51.452 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:51.452 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:51.452 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:51.452 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-485b4b6a-9982-a676-a07f-a0571b9d7fd0 -45:51.452 [INFO] TestAgent.server: member joined, marking health alive: member=Node-485b4b6a-9982-a676-a07f-a0571b9d7fd0 -45:51.453 [INFO] TestAgent.server: federation state anti-entropy synced -45:51.453 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-485b4b6a-9982-a676-a07f-a0571b9d7fd0 -45:51.559 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -45:51.559 [INFO] TestAgent: Synced node info -45:51.559 [DEBUG] TestAgent: Node info in sync -45:51.714 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/login from=127.0.0.1:58310 error="ACL not found" -45:51.714 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/login from=127.0.0.1:58310 latency=337.58µs -45:51.717 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:58312 latency=1.53614ms -45:51.720 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/login from=127.0.0.1:58314 error="Permission denied" -45:51.720 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/login from=127.0.0.1:58314 latency=339.64µs -45:51.722 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:58312 latency=1.45604ms -45:51.732 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/login from=127.0.0.1:58316 latency=1.81506ms -45:51.733 [INFO] TestAgent: Requesting shutdown -45:51.733 [INFO] TestAgent.server: shutting down server -45:51.733 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:51.733 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:51.733 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:51.733 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -45:51.733 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -45:51.733 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:51.733 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:51.733 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:51.733 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:51.733 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:51.733 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:51.733 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:51.733 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -45:51.733 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -45:51.733 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:51.733Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:51.733Z [INFO] agent.router.manager: shutting down -45:51.734 [INFO] TestAgent: consul server down -45:51.734 [INFO] TestAgent: shutdown complete -45:51.734 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12057 network=tcp -45:51.734 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12057 network=udp -45:51.734 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:12058 network=tcp -45:52.235 [INFO] TestAgent: Waiting for endpoints to shut down -45:52.235 [INFO] TestAgent: Endpoints down ---- PASS: TestLoginCommand (0.97s) +=== RUN TestLoginCommand_jwt/success_-_jwt_static_keys +=== RUN TestLoginCommand_jwt/success_-_jwt_jwks +=== RUN TestLoginCommand_jwt/success_-_jwt_oidc_discovery +=== RUN TestLoginCommand_k8s/try_login_with_method_configured_and_binding_rules +2021-08-13T15:38:12.112Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:38:12.134Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:12.137 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:053e370c-5424-a760-8549-b222304993f1 Address:127.0.0.1:23293}]" +38:12.138 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23293 [Follower]" leader= +38:12.138 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-053e370c-5424-a760-8549-b222304993f1.dc1 127.0.0.1 +38:12.139 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-053e370c-5424-a760-8549-b222304993f1 127.0.0.1 +2021-08-13T15:38:12.139Z [INFO] agent.router: Initializing LAN area manager +38:12.140 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23288 network=udp +38:12.141 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23288 network=tcp +38:12.142 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23289 network=tcp +38:12.142 [INFO] TestAgent: started state syncer +38:12.142 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23294 network=tcp +38:12.148 [INFO] TestAgent.server: Adding LAN server: server="Node-053e370c-5424-a760-8549-b222304993f1 (Addr: tcp/127.0.0.1:23293) (DC: dc1)" +38:12.148 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-053e370c-5424-a760-8549-b222304993f1.dc1 area=wan +38:12.178 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +38:12.178 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23293 [Candidate]" term=2 +38:12.178 [DEBUG] TestAgent.server.raft: votes: needed=1 +38:12.178 [DEBUG] TestAgent.server.raft: vote granted: from=053e370c-5424-a760-8549-b222304993f1 term=2 tally=1 +38:12.178 [INFO] TestAgent.server.raft: election won: tally=1 +38:12.178 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23293 [Leader]" +38:12.178 [INFO] TestAgent.server: cluster leadership acquired +38:12.178 [INFO] TestAgent.server: initializing acls +38:12.179 [INFO] TestAgent.server: New leader elected: payload=Node-053e370c-5424-a760-8549-b222304993f1 +38:12.179 [INFO] TestAgent.server: Created ACL 'global-management' policy +38:12.179 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +38:12.180 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +38:12.180 [INFO] TestAgent.server: Created ACL anonymous token from configuration +38:12.180 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +38:12.180 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +38:12.181 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-053e370c-5424-a760-8549-b222304993f1 +38:12.181 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-053e370c-5424-a760-8549-b222304993f1.dc1 +38:12.181 [INFO] TestAgent.server: Updating LAN server: server="Node-053e370c-5424-a760-8549-b222304993f1 (Addr: tcp/127.0.0.1:23293) (DC: dc1)" +38:12.181 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-053e370c-5424-a760-8549-b222304993f1.dc1 area=wan +38:12.182 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +38:12.184 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +38:12.184 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +38:12.184 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +38:12.184 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +38:12.184 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +38:12.184 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-053e370c-5424-a760-8549-b222304993f1 +38:12.184 [INFO] TestAgent.server: member joined, marking health alive: member=Node-053e370c-5424-a760-8549-b222304993f1 +38:12.185 [INFO] TestAgent.server: federation state anti-entropy synced +38:12.186 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-053e370c-5424-a760-8549-b222304993f1 +38:12.262 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/login from=127.0.0.1:37310 error="ACL not found" +38:12.264 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/login from=127.0.0.1:37310 latency=2.435106ms +38:12.271 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:37312 latency=2.834567ms +38:12.274 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/login from=127.0.0.1:37314 error="Permission denied" +38:12.274 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/login from=127.0.0.1:37314 latency=420.181µs +38:12.276 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:37312 latency=1.193302ms +38:12.279 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/login from=127.0.0.1:37316 latency=1.531864ms +38:12.280 [INFO] TestAgent: Requesting shutdown +38:12.280 [INFO] TestAgent.server: shutting down server +38:12.280 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +38:12.280 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +38:12.280 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +38:12.280 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +38:12.280 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +38:12.280 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +38:12.280 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +38:12.280 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +38:12.280 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +38:12.280 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +38:12.280 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +38:12.280 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +38:12.280 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +38:12.280 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +38:12.280 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +38:12.280 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:38:12.281Z [INFO] agent.router.manager: shutting down +2021-08-13T15:38:12.281Z [INFO] agent.router.manager: shutting down +38:12.281 [INFO] TestAgent: consul server down +38:12.281 [INFO] TestAgent: shutdown complete +38:12.281 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23288 network=tcp +38:12.281 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23288 network=udp +38:12.281 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23289 network=tcp +38:12.781 [INFO] TestAgent: Waiting for endpoints to shut down +38:12.781 [INFO] TestAgent: Endpoints down +--- PASS: TestLoginCommand (0.73s) --- PASS: TestLoginCommand/method_is_required (0.00s) --- PASS: TestLoginCommand/token-sink-file_is_required (0.00s) --- PASS: TestLoginCommand/bearer-token-file_is_required (0.00s) @@ -89132,8 +88866,169 @@ --- PASS: TestLoginCommand/try_login_with_no_method_configured (0.01s) --- PASS: TestLoginCommand/try_login_with_method_configured_but_no_binding_rules (0.00s) --- PASS: TestLoginCommand/try_login_with_method_configured_and_binding_rules (0.00s) +2021-08-13T15:38:12.116Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:38:12.137Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:12.143 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c63ce9e2-3f35-247b-0183-4442028cef74 Address:127.0.0.1:23279}]" +38:12.145 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23279 [Follower]" leader= +38:12.145 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c63ce9e2-3f35-247b-0183-4442028cef74.dc1 127.0.0.1 +38:12.148 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c63ce9e2-3f35-247b-0183-4442028cef74 127.0.0.1 +2021-08-13T15:38:12.148Z [INFO] agent.router: Initializing LAN area manager +38:12.148 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23274 network=udp +38:12.148 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23274 network=tcp +38:12.148 [INFO] TestAgent.server: Adding LAN server: server="Node-c63ce9e2-3f35-247b-0183-4442028cef74 (Addr: tcp/127.0.0.1:23279) (DC: dc1)" +38:12.148 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c63ce9e2-3f35-247b-0183-4442028cef74.dc1 area=wan +38:12.149 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23275 network=tcp +38:12.149 [INFO] TestAgent: started state syncer +38:12.149 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23280 network=tcp +38:12.190 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +38:12.190 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23279 [Candidate]" term=2 +38:12.190 [DEBUG] TestAgent.server.raft: votes: needed=1 +38:12.190 [DEBUG] TestAgent.server.raft: vote granted: from=c63ce9e2-3f35-247b-0183-4442028cef74 term=2 tally=1 +38:12.190 [INFO] TestAgent.server.raft: election won: tally=1 +38:12.190 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23279 [Leader]" +38:12.190 [INFO] TestAgent.server: cluster leadership acquired +38:12.191 [INFO] TestAgent.server: initializing acls +38:12.191 [INFO] TestAgent.server: New leader elected: payload=Node-c63ce9e2-3f35-247b-0183-4442028cef74 +38:12.191 [INFO] TestAgent.server: Created ACL 'global-management' policy +38:12.191 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +38:12.191 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +38:12.192 [INFO] TestAgent.server: Created ACL anonymous token from configuration +38:12.192 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +38:12.192 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +38:12.192 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-c63ce9e2-3f35-247b-0183-4442028cef74 +38:12.192 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-c63ce9e2-3f35-247b-0183-4442028cef74.dc1 +38:12.193 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +38:12.195 [INFO] TestAgent.server: Updating LAN server: server="Node-c63ce9e2-3f35-247b-0183-4442028cef74 (Addr: tcp/127.0.0.1:23279) (DC: dc1)" +38:12.195 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-c63ce9e2-3f35-247b-0183-4442028cef74.dc1 area=wan +38:12.195 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +38:12.195 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +38:12.195 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +38:12.195 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +38:12.195 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +38:12.195 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c63ce9e2-3f35-247b-0183-4442028cef74 +38:12.195 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c63ce9e2-3f35-247b-0183-4442028cef74 +38:12.196 [INFO] TestAgent.server: federation state anti-entropy synced +38:12.196 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c63ce9e2-3f35-247b-0183-4442028cef74 +38:12.291 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +38:12.291 [INFO] TestAgent: Synced node info +38:12.430 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:46860 latency=1.726444ms +38:12.433 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:46860 latency=1.761984ms +38:12.438 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/login from=127.0.0.1:46862 latency=2.612247ms +38:12.446 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:46860 latency=7.011777ms +38:12.454 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:46860 latency=7.032456ms +38:12.464 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/login from=127.0.0.1:46868 latency=8.092819ms +38:12.479 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:46860 latency=13.05629ms +38:12.498 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:46860 latency=17.765202ms +38:12.505 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/login from=127.0.0.1:46880 latency=4.659031ms +38:12.506 [INFO] TestAgent: Requesting shutdown +38:12.506 [INFO] TestAgent.server: shutting down server +38:12.506 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +38:12.506 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +38:12.506 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +38:12.506 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +38:12.506 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +38:12.506 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +38:12.506 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +38:12.506 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +38:12.506 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +38:12.506 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +38:12.506 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +38:12.506 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +38:12.506 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +38:12.506 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +38:12.506 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:38:12.506Z [INFO] agent.router.manager: shutting down +2021-08-13T15:38:12.506Z [INFO] agent.router.manager: shutting down +38:12.506 [INFO] TestAgent: consul server down +38:12.506 [INFO] TestAgent: shutdown complete +38:12.506 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23274 network=tcp +38:12.506 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23274 network=udp +38:12.506 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23275 network=tcp +38:13.006 [INFO] TestAgent: Waiting for endpoints to shut down +38:13.006 [INFO] TestAgent: Endpoints down +--- PASS: TestLoginCommand_jwt (0.96s) + --- PASS: TestLoginCommand_jwt/success_-_jwt_static_keys (0.01s) + --- PASS: TestLoginCommand_jwt/success_-_jwt_jwks (0.03s) + --- PASS: TestLoginCommand_jwt/success_-_jwt_oidc_discovery (0.04s) +2021-08-13T15:38:12.110Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:38:12.134Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:12.138 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c9cce9e1-9ffe-23c3-e4f6-caf27cb27933 Address:127.0.0.1:23286}]" +38:12.139 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c9cce9e1-9ffe-23c3-e4f6-caf27cb27933.dc1 127.0.0.1 +38:12.139 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23286 [Follower]" leader= +38:12.143 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c9cce9e1-9ffe-23c3-e4f6-caf27cb27933 127.0.0.1 +2021-08-13T15:38:12.143Z [INFO] agent.router: Initializing LAN area manager +38:12.146 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c9cce9e1-9ffe-23c3-e4f6-caf27cb27933.dc1 area=wan +38:12.146 [INFO] TestAgent.server: Adding LAN server: server="Node-c9cce9e1-9ffe-23c3-e4f6-caf27cb27933 (Addr: tcp/127.0.0.1:23286) (DC: dc1)" +38:12.147 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23281 network=tcp +38:12.147 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23281 network=udp +38:12.148 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23282 network=tcp +38:12.148 [INFO] TestAgent: started state syncer +38:12.148 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23287 network=tcp +38:12.178 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +38:12.178 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23286 [Candidate]" term=2 +38:12.178 [DEBUG] TestAgent.server.raft: votes: needed=1 +38:12.178 [DEBUG] TestAgent.server.raft: vote granted: from=c9cce9e1-9ffe-23c3-e4f6-caf27cb27933 term=2 tally=1 +38:12.178 [INFO] TestAgent.server.raft: election won: tally=1 +38:12.178 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23286 [Leader]" +38:12.178 [INFO] TestAgent.server: cluster leadership acquired +38:12.179 [INFO] TestAgent.server: New leader elected: payload=Node-c9cce9e1-9ffe-23c3-e4f6-caf27cb27933 +38:12.179 [INFO] TestAgent.server: initializing acls +38:12.179 [INFO] TestAgent.server: Created ACL 'global-management' policy +38:12.179 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +38:12.180 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +38:12.180 [INFO] TestAgent.server: Created ACL anonymous token from configuration +38:12.180 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +38:12.180 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +38:12.181 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-c9cce9e1-9ffe-23c3-e4f6-caf27cb27933 +38:12.181 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-c9cce9e1-9ffe-23c3-e4f6-caf27cb27933.dc1 +38:12.181 [INFO] TestAgent.server: Updating LAN server: server="Node-c9cce9e1-9ffe-23c3-e4f6-caf27cb27933 (Addr: tcp/127.0.0.1:23286) (DC: dc1)" +38:12.181 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-c9cce9e1-9ffe-23c3-e4f6-caf27cb27933.dc1 area=wan +38:12.182 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +38:12.184 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +38:12.184 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +38:12.184 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +38:12.184 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +38:12.184 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +38:12.184 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c9cce9e1-9ffe-23c3-e4f6-caf27cb27933 +38:12.184 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c9cce9e1-9ffe-23c3-e4f6-caf27cb27933 +38:12.185 [INFO] TestAgent.server: federation state anti-entropy synced +38:12.186 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c9cce9e1-9ffe-23c3-e4f6-caf27cb27933 +38:12.196 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +38:12.197 [INFO] TestAgent: Synced node info +38:12.197 [DEBUG] TestAgent: Node info in sync +38:12.555 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:60740 latency=3.230367ms +38:12.559 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:60740 latency=2.382186ms +38:12.573 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/login from=127.0.0.1:60742 latency=11.684668ms +38:12.574 [INFO] TestAgent: Requesting shutdown +38:12.574 [INFO] TestAgent.server: shutting down server +38:12.574 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +38:12.574 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +38:12.574 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +38:12.574 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +38:12.574 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +38:12.574 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +38:12.574 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +38:12.574 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +38:12.574 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +38:12.574 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +38:12.574 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +38:12.574 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +38:12.574 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +38:12.574 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +38:12.574 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:38:12.574Z [INFO] agent.router.manager: shutting down +2021-08-13T15:38:12.574Z [INFO] agent.router.manager: shutting down +38:12.574 [INFO] TestAgent: consul server down +38:12.574 [INFO] TestAgent: shutdown complete +38:12.574 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23281 network=tcp +38:12.574 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23281 network=udp +38:12.574 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23282 network=tcp +38:13.075 [INFO] TestAgent: Waiting for endpoints to shut down +38:13.075 [INFO] TestAgent: Endpoints down +--- PASS: TestLoginCommand_k8s (1.03s) + --- PASS: TestLoginCommand_k8s/try_login_with_method_configured_and_binding_rules (0.01s) PASS -ok github.com/hashicorp/consul/command/login 1.085s +ok github.com/hashicorp/consul/command/login 1.115s === RUN TestLogout_noTabs === PAUSE TestLogout_noTabs === RUN TestLogoutCommand @@ -89141,197 +89036,198 @@ === RUN TestLogoutCommand_k8s === PAUSE TestLogoutCommand_k8s === CONT TestLogout_noTabs ---- PASS: TestLogout_noTabs (0.00s) === CONT TestLogoutCommand_k8s [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestLogout_noTabs (0.00s) === CONT TestLogoutCommand [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range === RUN TestLogoutCommand_k8s/no_token_specified === RUN TestLogoutCommand_k8s/logout_of_deleted_token === RUN TestLogoutCommand_k8s/logout_of_ordinary_token -=== RUN TestLogoutCommand_k8s/logout_of_login_token === RUN TestLogoutCommand/no_token_specified === RUN TestLogoutCommand/logout_of_deleted_token === RUN TestLogoutCommand/logout_of_ordinary_token +=== RUN TestLogoutCommand_k8s/logout_of_login_token === RUN TestLogoutCommand/logout_of_login_token -2022-09-15T21:45:52.148Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:52.178Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:52.183 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:dd04bc7f-0f78-319c-f793-c3fae27efa97 Address:127.0.0.1:29405}]" -45:52.194 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29405 [Follower]" leader= -45:52.207 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-dd04bc7f-0f78-319c-f793-c3fae27efa97.dc1 127.0.0.1 -45:52.219 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-dd04bc7f-0f78-319c-f793-c3fae27efa97 127.0.0.1 -2022-09-15T21:45:52.219Z [INFO] agent.router: Initializing LAN area manager -45:52.219 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29400 network=udp -45:52.220 [INFO] TestAgent.server: Adding LAN server: server="Node-dd04bc7f-0f78-319c-f793-c3fae27efa97 (Addr: tcp/127.0.0.1:29405) (DC: dc1)" -45:52.220 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-dd04bc7f-0f78-319c-f793-c3fae27efa97.dc1 area=wan -45:52.221 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29400 network=tcp -45:52.222 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29401 network=tcp -45:52.222 [INFO] TestAgent: started state syncer -45:52.222 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29406 network=tcp -45:52.244 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:52.244 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29405 [Candidate]" term=2 -45:52.244 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:52.244 [DEBUG] TestAgent.server.raft: vote granted: from=dd04bc7f-0f78-319c-f793-c3fae27efa97 term=2 tally=1 -45:52.244 [INFO] TestAgent.server.raft: election won: tally=1 -45:52.244 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29405 [Leader]" -45:52.245 [INFO] TestAgent.server: cluster leadership acquired -45:52.245 [INFO] TestAgent.server: initializing acls -45:52.245 [INFO] TestAgent.server: New leader elected: payload=Node-dd04bc7f-0f78-319c-f793-c3fae27efa97 -45:52.246 [INFO] TestAgent.server: Created ACL 'global-management' policy -45:52.246 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -45:52.247 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -45:52.247 [INFO] TestAgent.server: Created ACL anonymous token from configuration -45:52.247 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -45:52.247 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -45:52.248 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-dd04bc7f-0f78-319c-f793-c3fae27efa97 -45:52.248 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-dd04bc7f-0f78-319c-f793-c3fae27efa97.dc1 -45:52.248 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-dd04bc7f-0f78-319c-f793-c3fae27efa97.dc1 area=wan -45:52.248 [INFO] TestAgent.server: Updating LAN server: server="Node-dd04bc7f-0f78-319c-f793-c3fae27efa97 (Addr: tcp/127.0.0.1:29405) (DC: dc1)" -45:52.250 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:52.291 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:52.292 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:52.292 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:52.292 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:52.292 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:52.292 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-dd04bc7f-0f78-319c-f793-c3fae27efa97 -45:52.292 [INFO] TestAgent.server: member joined, marking health alive: member=Node-dd04bc7f-0f78-319c-f793-c3fae27efa97 -45:52.293 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-dd04bc7f-0f78-319c-f793-c3fae27efa97 -45:52.294 [INFO] TestAgent.server: federation state anti-entropy synced -45:52.454 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -45:52.455 [INFO] TestAgent: Synced node info -45:52.492 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/logout from=127.0.0.1:35742 error="ACL not found" -45:52.494 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/logout from=127.0.0.1:35742 latency=2.19436ms -45:52.504 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/logout from=127.0.0.1:35744 error="ACL not found" -45:52.506 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/logout from=127.0.0.1:35744 latency=1.87408ms -45:52.521 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:35746 latency=4.43558ms -45:52.535 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/logout from=127.0.0.1:35748 error="Permission denied" -45:52.535 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/logout from=127.0.0.1:35748 latency=418.96µs -45:52.540 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:35746 latency=3.4783ms -45:52.544 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:35746 latency=2.28714ms -45:52.564 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/login from=127.0.0.1:35746 latency=18.49132ms -45:52.569 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/logout from=127.0.0.1:35752 latency=669.86µs -45:52.570 [INFO] TestAgent: Requesting shutdown -45:52.570 [INFO] TestAgent.server: shutting down server -45:52.570 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:52.570 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:52.570 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:52.570 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:52.570 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -45:52.570 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -45:52.570 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:52.570 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:52.570 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:52.570 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:52.570 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:52.570 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:52.570 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -45:52.570 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -45:52.571 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:52.571Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:52.571Z [INFO] agent.router.manager: shutting down -45:52.571 [INFO] TestAgent: consul server down -45:52.571 [INFO] TestAgent: shutdown complete -45:52.571 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29400 network=tcp -45:52.571 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29400 network=udp -45:52.571 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29401 network=tcp -45:53.071 [INFO] TestAgent: Waiting for endpoints to shut down -45:53.071 [INFO] TestAgent: Endpoints down ---- PASS: TestLogoutCommand_k8s (1.02s) - --- PASS: TestLogoutCommand_k8s/no_token_specified (0.01s) +2021-08-13T15:38:28.771Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:38:28.796Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:28.803 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:96a435d9-bb61-56af-5b76-5b439688c27b Address:127.0.0.1:19195}]" +38:28.804 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-96a435d9-bb61-56af-5b76-5b439688c27b.dc1 127.0.0.1 +38:28.804 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19195 [Follower]" leader= +38:28.806 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-96a435d9-bb61-56af-5b76-5b439688c27b 127.0.0.1 +2021-08-13T15:38:28.806Z [INFO] agent.router: Initializing LAN area manager +38:28.806 [INFO] TestAgent.server: Adding LAN server: server="Node-96a435d9-bb61-56af-5b76-5b439688c27b (Addr: tcp/127.0.0.1:19195) (DC: dc1)" +38:28.806 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-96a435d9-bb61-56af-5b76-5b439688c27b.dc1 area=wan +38:28.807 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=udp +38:28.807 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=tcp +38:28.808 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19191 network=tcp +38:28.808 [INFO] TestAgent: started state syncer +38:28.808 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19196 network=tcp +38:28.857 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +38:28.860 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +38:28.860 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19195 [Candidate]" term=2 +38:28.860 [DEBUG] TestAgent.server.raft: votes: needed=1 +38:28.860 [DEBUG] TestAgent.server.raft: vote granted: from=96a435d9-bb61-56af-5b76-5b439688c27b term=2 tally=1 +38:28.860 [INFO] TestAgent.server.raft: election won: tally=1 +38:28.860 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19195 [Leader]" +38:28.860 [INFO] TestAgent.server: cluster leadership acquired +38:28.860 [INFO] TestAgent.server: New leader elected: payload=Node-96a435d9-bb61-56af-5b76-5b439688c27b +38:28.860 [INFO] TestAgent.server: initializing acls +38:28.861 [INFO] TestAgent.server: Created ACL 'global-management' policy +38:28.861 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +38:28.861 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +38:28.862 [INFO] TestAgent.server: Created ACL anonymous token from configuration +38:28.862 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +38:28.862 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +38:28.862 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-96a435d9-bb61-56af-5b76-5b439688c27b +38:28.862 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-96a435d9-bb61-56af-5b76-5b439688c27b.dc1 +38:28.863 [INFO] TestAgent.server: Updating LAN server: server="Node-96a435d9-bb61-56af-5b76-5b439688c27b (Addr: tcp/127.0.0.1:19195) (DC: dc1)" +38:28.863 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-96a435d9-bb61-56af-5b76-5b439688c27b.dc1 area=wan +38:28.864 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +38:28.887 [INFO] TestAgent: Synced node info +38:28.887 [DEBUG] TestAgent: Node info in sync +38:28.894 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +38:28.894 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +38:28.894 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +38:28.894 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +38:28.894 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +38:28.894 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-96a435d9-bb61-56af-5b76-5b439688c27b +38:28.894 [INFO] TestAgent.server: member joined, marking health alive: member=Node-96a435d9-bb61-56af-5b76-5b439688c27b +38:28.896 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-96a435d9-bb61-56af-5b76-5b439688c27b +38:28.896 [INFO] TestAgent.server: federation state anti-entropy synced +38:28.924 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/logout from=127.0.0.1:35974 error="ACL not found" +38:28.924 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/logout from=127.0.0.1:35974 latency=358.661µs +38:28.926 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/logout from=127.0.0.1:35976 error="ACL not found" +38:28.926 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/logout from=127.0.0.1:35976 latency=166.36µs +38:28.929 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:35978 latency=1.944322ms +38:28.931 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/logout from=127.0.0.1:35980 error="Permission denied" +38:28.931 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/logout from=127.0.0.1:35980 latency=195.84µs +38:28.936 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:35978 latency=3.616725ms +38:28.948 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:35978 latency=3.469025ms +38:28.962 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/login from=127.0.0.1:35978 latency=12.927898ms +38:28.964 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/logout from=127.0.0.1:35992 latency=614.241µs +38:28.965 [INFO] TestAgent: Requesting shutdown +38:28.965 [INFO] TestAgent.server: shutting down server +38:28.965 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +38:28.965 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +38:28.965 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +38:28.965 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +38:28.965 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +38:28.965 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +38:28.965 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +38:28.965 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +38:28.965 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +38:28.965 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +38:28.965 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +38:28.965 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +38:28.965 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +38:28.965 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +38:28.965 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:38:28.965Z [INFO] agent.router.manager: shutting down +2021-08-13T15:38:28.965Z [INFO] agent.router.manager: shutting down +38:28.965 [INFO] TestAgent: consul server down +38:28.965 [INFO] TestAgent: shutdown complete +38:28.965 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=tcp +38:28.965 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=udp +38:28.965 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19191 network=tcp +38:29.466 [INFO] TestAgent: Waiting for endpoints to shut down +38:29.466 [INFO] TestAgent: Endpoints down +--- PASS: TestLogoutCommand_k8s (0.75s) + --- PASS: TestLogoutCommand_k8s/no_token_specified (0.00s) --- PASS: TestLogoutCommand_k8s/logout_of_deleted_token (0.00s) - --- PASS: TestLogoutCommand_k8s/logout_of_ordinary_token (0.01s) + --- PASS: TestLogoutCommand_k8s/logout_of_ordinary_token (0.00s) --- PASS: TestLogoutCommand_k8s/logout_of_login_token (0.00s) -2022-09-15T21:45:52.167Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:45:52.193Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -45:52.201 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:dd1f7bb7-70bf-b620-5a10-03673b350590 Address:127.0.0.1:29412}]" -45:52.207 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29412 [Follower]" leader= -45:52.220 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-dd1f7bb7-70bf-b620-5a10-03673b350590.dc1 127.0.0.1 -45:52.221 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-dd1f7bb7-70bf-b620-5a10-03673b350590 127.0.0.1 -2022-09-15T21:45:52.221Z [INFO] agent.router: Initializing LAN area manager -45:52.221 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29407 network=udp -45:52.221 [INFO] TestAgent.server: Adding LAN server: server="Node-dd1f7bb7-70bf-b620-5a10-03673b350590 (Addr: tcp/127.0.0.1:29412) (DC: dc1)" -45:52.222 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29407 network=tcp -45:52.222 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-dd1f7bb7-70bf-b620-5a10-03673b350590.dc1 area=wan -45:52.222 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29408 network=tcp -45:52.223 [INFO] TestAgent: started state syncer -45:52.223 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29413 network=tcp -45:52.245 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -45:52.245 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29412 [Candidate]" term=2 -45:52.245 [DEBUG] TestAgent.server.raft: votes: needed=1 -45:52.245 [DEBUG] TestAgent.server.raft: vote granted: from=dd1f7bb7-70bf-b620-5a10-03673b350590 term=2 tally=1 -45:52.246 [INFO] TestAgent.server.raft: election won: tally=1 -45:52.246 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29412 [Leader]" -45:52.246 [INFO] TestAgent.server: cluster leadership acquired -45:52.247 [INFO] TestAgent.server: New leader elected: payload=Node-dd1f7bb7-70bf-b620-5a10-03673b350590 -45:52.248 [INFO] TestAgent.server: initializing acls -45:52.249 [INFO] TestAgent.server: Created ACL 'global-management' policy -45:52.249 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -45:52.254 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -45:52.255 [INFO] TestAgent.server: Created ACL anonymous token from configuration -45:52.255 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -45:52.255 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -45:52.255 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-dd1f7bb7-70bf-b620-5a10-03673b350590 -45:52.255 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-dd1f7bb7-70bf-b620-5a10-03673b350590.dc1 -45:52.256 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -45:52.257 [INFO] TestAgent.server: Updating LAN server: server="Node-dd1f7bb7-70bf-b620-5a10-03673b350590 (Addr: tcp/127.0.0.1:29412) (DC: dc1)" -45:52.257 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-dd1f7bb7-70bf-b620-5a10-03673b350590.dc1 area=wan -45:52.296 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -45:52.296 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -45:52.296 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -45:52.296 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -45:52.296 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -45:52.296 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-dd1f7bb7-70bf-b620-5a10-03673b350590 -45:52.296 [INFO] TestAgent.server: member joined, marking health alive: member=Node-dd1f7bb7-70bf-b620-5a10-03673b350590 -45:52.297 [INFO] TestAgent.server: federation state anti-entropy synced -45:52.297 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-dd1f7bb7-70bf-b620-5a10-03673b350590 -45:52.486 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -45:52.487 [INFO] TestAgent: Synced node info -45:52.487 [DEBUG] TestAgent: Node info in sync -45:52.613 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/logout from=127.0.0.1:38396 error="ACL not found" -45:52.614 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/logout from=127.0.0.1:38396 latency=232.02µs -45:52.616 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/logout from=127.0.0.1:38398 error="ACL not found" -45:52.616 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/logout from=127.0.0.1:38398 latency=277.68µs -45:52.628 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:38400 latency=9.45596ms -45:52.631 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/logout from=127.0.0.1:38402 error="Permission denied" -45:52.631 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/logout from=127.0.0.1:38402 latency=308.1µs -45:52.633 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:38400 latency=1.17008ms -45:52.640 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:38400 latency=6.51568ms -45:52.643 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/login from=127.0.0.1:38400 latency=1.40056ms -45:52.653 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/logout from=127.0.0.1:38404 latency=834.1µs -45:52.653 [INFO] TestAgent: Requesting shutdown -45:52.653 [INFO] TestAgent.server: shutting down server -45:52.653 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -45:52.653 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -45:52.653 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -45:52.653 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -45:52.653 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -45:52.653 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -45:52.653 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -45:52.654 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -45:52.654 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -45:52.654 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -45:52.654 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -45:52.654 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -45:52.654 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -45:52.654 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -45:52.654 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:45:52.654Z [INFO] agent.router.manager: shutting down -2022-09-15T21:45:52.654Z [INFO] agent.router.manager: shutting down -45:52.654 [INFO] TestAgent: consul server down -45:52.654 [INFO] TestAgent: shutdown complete -45:52.654 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29407 network=tcp -45:52.654 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29407 network=udp -45:52.654 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29408 network=tcp -45:53.170 [INFO] TestAgent: Waiting for endpoints to shut down -45:53.170 [INFO] TestAgent: Endpoints down ---- PASS: TestLogoutCommand (1.12s) - --- PASS: TestLogoutCommand/no_token_specified (0.01s) +2021-08-13T15:38:28.774Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:38:28.802Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:28.804 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e4099f90-2cea-2858-2053-145802e02fa1 Address:127.0.0.1:19202}]" +38:28.805 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19202 [Follower]" leader= +38:28.805 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e4099f90-2cea-2858-2053-145802e02fa1.dc1 127.0.0.1 +38:28.806 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e4099f90-2cea-2858-2053-145802e02fa1 127.0.0.1 +2021-08-13T15:38:28.806Z [INFO] agent.router: Initializing LAN area manager +38:28.806 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19197 network=udp +38:28.806 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e4099f90-2cea-2858-2053-145802e02fa1.dc1 area=wan +38:28.807 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19197 network=tcp +38:28.807 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19198 network=tcp +38:28.807 [INFO] TestAgent: started state syncer +38:28.808 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19203 network=tcp +38:28.808 [INFO] TestAgent.server: Adding LAN server: server="Node-e4099f90-2cea-2858-2053-145802e02fa1 (Addr: tcp/127.0.0.1:19202) (DC: dc1)" +38:28.856 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +38:28.871 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +38:28.871 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19202 [Candidate]" term=2 +38:28.871 [DEBUG] TestAgent.server.raft: votes: needed=1 +38:28.871 [DEBUG] TestAgent.server.raft: vote granted: from=e4099f90-2cea-2858-2053-145802e02fa1 term=2 tally=1 +38:28.871 [INFO] TestAgent.server.raft: election won: tally=1 +38:28.871 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19202 [Leader]" +38:28.871 [INFO] TestAgent.server: cluster leadership acquired +38:28.871 [INFO] TestAgent.server: initializing acls +38:28.872 [INFO] TestAgent.server: Created ACL 'global-management' policy +38:28.872 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +38:28.872 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +38:28.880 [INFO] TestAgent.server: New leader elected: payload=Node-e4099f90-2cea-2858-2053-145802e02fa1 +38:28.883 [INFO] TestAgent.server: Created ACL anonymous token from configuration +38:28.883 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +38:28.883 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +38:28.884 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-e4099f90-2cea-2858-2053-145802e02fa1 +38:28.884 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-e4099f90-2cea-2858-2053-145802e02fa1.dc1 +38:28.885 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +38:28.886 [INFO] TestAgent.server: Updating LAN server: server="Node-e4099f90-2cea-2858-2053-145802e02fa1 (Addr: tcp/127.0.0.1:19202) (DC: dc1)" +38:28.886 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-e4099f90-2cea-2858-2053-145802e02fa1.dc1 area=wan +38:28.894 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +38:28.894 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +38:28.894 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +38:28.894 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +38:28.894 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +38:28.894 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e4099f90-2cea-2858-2053-145802e02fa1 +38:28.894 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e4099f90-2cea-2858-2053-145802e02fa1 +38:28.895 [INFO] TestAgent.server: federation state anti-entropy synced +38:28.896 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e4099f90-2cea-2858-2053-145802e02fa1 +38:28.950 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/logout from=127.0.0.1:49300 error="ACL not found" +38:28.951 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/logout from=127.0.0.1:49300 latency=985.482µs +38:28.954 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/logout from=127.0.0.1:49304 error="ACL not found" +38:28.954 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/logout from=127.0.0.1:49304 latency=199.42µs +38:28.956 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:49306 latency=1.703262ms +38:28.958 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/logout from=127.0.0.1:49308 error="Permission denied" +38:28.958 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/logout from=127.0.0.1:49308 latency=118.94µs +38:28.964 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:49306 latency=5.242466ms +38:28.966 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:49306 latency=1.542802ms +38:28.968 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/login from=127.0.0.1:49306 latency=839.701µs +38:28.969 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +38:28.969 [INFO] TestAgent: Synced node info +38:28.970 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/logout from=127.0.0.1:49312 latency=512.32µs +38:28.970 [INFO] TestAgent: Requesting shutdown +38:28.970 [INFO] TestAgent.server: shutting down server +38:28.970 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +38:28.970 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +38:28.970 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +38:28.970 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +38:28.970 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +38:28.970 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +38:28.970 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +38:28.970 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +38:28.970 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +38:28.970 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +38:28.970 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +38:28.970 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +38:28.971 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +38:28.971 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +38:28.971 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:38:28.971Z [INFO] agent.router.manager: shutting down +2021-08-13T15:38:28.971Z [INFO] agent.router.manager: shutting down +38:28.971 [INFO] TestAgent: consul server down +38:28.971 [INFO] TestAgent: shutdown complete +38:28.971 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19197 network=tcp +38:28.971 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19197 network=udp +38:28.971 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19198 network=tcp +38:29.471 [INFO] TestAgent: Waiting for endpoints to shut down +38:29.471 [INFO] TestAgent: Endpoints down +--- PASS: TestLogoutCommand (0.76s) + --- PASS: TestLogoutCommand/no_token_specified (0.00s) --- PASS: TestLogoutCommand/logout_of_deleted_token (0.00s) --- PASS: TestLogoutCommand/logout_of_ordinary_token (0.00s) - --- PASS: TestLogoutCommand/logout_of_login_token (0.01s) + --- PASS: TestLogoutCommand/logout_of_login_token (0.00s) PASS -ok github.com/hashicorp/consul/command/logout 1.242s +ok github.com/hashicorp/consul/command/logout 0.825s === RUN TestMaintCommand_noTabs === PAUSE TestMaintCommand_noTabs === RUN TestMaintCommand_ConflictingArgs @@ -89349,379 +89245,383 @@ === RUN TestMaintCommand_ServiceMaintenance_NoService === PAUSE TestMaintCommand_ServiceMaintenance_NoService === CONT TestMaintCommand_noTabs -=== CONT TestMaintCommand_EnableServiceMaintenance -=== CONT TestMaintCommand_ServiceMaintenance_NoService -=== CONT TestMaintCommand_DisableServiceMaintenance +=== CONT TestMaintCommand_DisableNodeMaintenance [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestMaintCommand_noTabs (0.00s) === CONT TestMaintCommand_EnableNodeMaintenance -=== CONT TestMaintCommand_DisableNodeMaintenance +=== CONT TestMaintCommand_DisableServiceMaintenance === CONT TestMaintCommand_NoArgs ---- PASS: TestMaintCommand_noTabs (0.00s) === CONT TestMaintCommand_ConflictingArgs +=== CONT TestMaintCommand_ServiceMaintenance_NoService +=== CONT TestMaintCommand_EnableServiceMaintenance --- PASS: TestMaintCommand_ConflictingArgs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:46:01.310Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:46:01.355Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -46:01.372 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:189b5bb4-81e2-9990-9ece-ae32a69e04ca Address:127.0.0.1:15139}]" -46:01.392 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-189b5bb4-81e2-9990-9ece-ae32a69e04ca.dc1 127.0.0.1 -46:01.394 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-189b5bb4-81e2-9990-9ece-ae32a69e04ca 127.0.0.1 -2022-09-15T21:46:01.394Z [INFO] agent.router: Initializing LAN area manager -46:01.395 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15134 network=udp -46:01.395 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:15139 [Follower]" leader= -46:01.396 [INFO] TestAgent.server: Adding LAN server: server="Node-189b5bb4-81e2-9990-9ece-ae32a69e04ca (Addr: tcp/127.0.0.1:15139) (DC: dc1)" -46:01.396 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-189b5bb4-81e2-9990-9ece-ae32a69e04ca.dc1 area=wan -46:01.396 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15134 network=tcp -46:01.401 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:15135 network=tcp -46:01.401 [INFO] TestAgent: started state syncer -46:01.401 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:15140 network=tcp -46:01.454 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -46:01.454 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:15139 [Candidate]" term=2 -46:01.455 [DEBUG] TestAgent.server.raft: votes: needed=1 -46:01.455 [DEBUG] TestAgent.server.raft: vote granted: from=189b5bb4-81e2-9990-9ece-ae32a69e04ca term=2 tally=1 -46:01.455 [INFO] TestAgent.server.raft: election won: tally=1 -46:01.455 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:15139 [Leader]" -46:01.455 [INFO] TestAgent.server: cluster leadership acquired -46:01.455 [INFO] TestAgent.server: New leader elected: payload=Node-189b5bb4-81e2-9990-9ece-ae32a69e04ca -46:01.456 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:15139 -46:01.468 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -46:01.476 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -46:01.476 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -46:01.476 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -46:01.476 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -46:01.476 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -46:01.476 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-189b5bb4-81e2-9990-9ece-ae32a69e04ca -46:01.476 [INFO] TestAgent.server: member joined, marking health alive: member=Node-189b5bb4-81e2-9990-9ece-ae32a69e04ca -46:01.477 [INFO] TestAgent.server: federation state anti-entropy synced -46:01.526 [INFO] TestAgent: Service entered maintenance mode: service=test -46:01.526 [INFO] TestAgent: Node entered maintenance mode -46:01.534 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:43984 latency=6.26524ms -46:01.539 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/checks from=127.0.0.1:43984 latency=328.72µs -46:01.541 [INFO] TestAgent: Requesting shutdown -46:01.541 [INFO] TestAgent.server: shutting down server -46:01.541 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -46:01.541 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -46:01.541 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -46:01.541 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -46:01.541 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -46:01.541 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -46:01.541 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -46:01.541 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -46:01.541 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -46:01.541 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -46:01.541 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -46:01.541 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:46:01.541Z [INFO] agent.router.manager: shutting down -2022-09-15T21:46:01.541Z [INFO] agent.router.manager: shutting down -46:01.542 [INFO] TestAgent: consul server down -46:01.542 [INFO] TestAgent: shutdown complete -46:01.542 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15134 network=tcp -46:01.542 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15134 network=udp -46:01.542 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:15135 network=tcp -46:02.043 [INFO] TestAgent: Waiting for endpoints to shut down -46:02.043 [INFO] TestAgent: Endpoints down -2022-09-15T21:46:01.272Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:46:01.372Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -46:01.388 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e4fe746a-cfad-5d35-4b39-01320e70d184 Address:127.0.0.1:15125}]" -46:01.389 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e4fe746a-cfad-5d35-4b39-01320e70d184.dc1 127.0.0.1 -46:01.390 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e4fe746a-cfad-5d35-4b39-01320e70d184 127.0.0.1 -2022-09-15T21:46:01.390Z [INFO] agent.router: Initializing LAN area manager -46:01.391 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15120 network=udp -46:01.395 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:15125 [Follower]" leader= -46:01.396 [INFO] TestAgent.server: Adding LAN server: server="Node-e4fe746a-cfad-5d35-4b39-01320e70d184 (Addr: tcp/127.0.0.1:15125) (DC: dc1)" -46:01.399 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15120 network=tcp -46:01.399 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:15121 network=tcp -46:01.400 [INFO] TestAgent: started state syncer -46:01.400 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:15126 network=tcp -46:01.400 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e4fe746a-cfad-5d35-4b39-01320e70d184.dc1 area=wan -46:01.459 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -46:01.459 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:15125 [Candidate]" term=2 -46:01.460 [DEBUG] TestAgent.server.raft: votes: needed=1 -46:01.460 [DEBUG] TestAgent.server.raft: vote granted: from=e4fe746a-cfad-5d35-4b39-01320e70d184 term=2 tally=1 -46:01.460 [INFO] TestAgent.server.raft: election won: tally=1 -46:01.460 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:15125 [Leader]" -46:01.460 [INFO] TestAgent.server: cluster leadership acquired -46:01.460 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:15125 -46:01.466 [INFO] TestAgent.server: New leader elected: payload=Node-e4fe746a-cfad-5d35-4b39-01320e70d184 -46:01.467 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -46:01.471 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -46:01.471 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -46:01.471 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -46:01.471 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -46:01.471 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -46:01.471 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e4fe746a-cfad-5d35-4b39-01320e70d184 -46:01.471 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e4fe746a-cfad-5d35-4b39-01320e70d184 -46:01.478 [INFO] TestAgent.server: federation state anti-entropy synced -46:01.539 [INFO] TestAgent: Node entered maintenance mode -46:01.540 [INFO] TestAgent: Synced node info -46:01.541 [INFO] TestAgent: Synced check: check=_node_maintenance -46:01.541 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/maintenance?enable=true&reason=broken from=127.0.0.1:39208 latency=1.72302ms -46:01.541 [INFO] TestAgent: Requesting shutdown -46:01.541 [INFO] TestAgent.server: shutting down server -46:01.541 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -46:01.541 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -46:01.541 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -46:01.541 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -46:01.541 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -46:01.541 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -46:01.541 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -46:01.541 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -46:01.541 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -46:01.541 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -46:01.541 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -46:01.541 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:46:01.541Z [INFO] agent.router.manager: shutting down -2022-09-15T21:46:01.541Z [INFO] agent.router.manager: shutting down -46:01.541 [INFO] TestAgent: consul server down -46:01.541 [INFO] TestAgent: shutdown complete -46:01.541 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15120 network=tcp -46:01.542 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15120 network=udp -46:01.542 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:15121 network=tcp -46:02.043 [INFO] TestAgent: Waiting for endpoints to shut down -46:02.043 [INFO] TestAgent: Endpoints down ---- PASS: TestMaintCommand_NoArgs (0.89s) ---- PASS: TestMaintCommand_EnableNodeMaintenance (0.89s) -2022-09-15T21:46:01.345Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:46:01.402Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -46:01.455 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5c4774e2-20fb-5ff1-c4b7-dca8d77c37b3 Address:127.0.0.1:15111}]" -46:01.455 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:15111 [Follower]" leader= -46:01.458 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5c4774e2-20fb-5ff1-c4b7-dca8d77c37b3.dc1 127.0.0.1 -46:01.459 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5c4774e2-20fb-5ff1-c4b7-dca8d77c37b3 127.0.0.1 -2022-09-15T21:46:01.459Z [INFO] agent.router: Initializing LAN area manager -46:01.460 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15106 network=udp -46:01.460 [INFO] TestAgent.server: Adding LAN server: server="Node-5c4774e2-20fb-5ff1-c4b7-dca8d77c37b3 (Addr: tcp/127.0.0.1:15111) (DC: dc1)" -46:01.460 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5c4774e2-20fb-5ff1-c4b7-dca8d77c37b3.dc1 area=wan -46:01.461 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15106 network=tcp -46:01.469 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:15107 network=tcp -46:01.469 [INFO] TestAgent: started state syncer -46:01.469 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:15112 network=tcp -46:01.497 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -46:01.497 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:15111 [Candidate]" term=2 -46:01.497 [DEBUG] TestAgent.server.raft: votes: needed=1 -46:01.497 [DEBUG] TestAgent.server.raft: vote granted: from=5c4774e2-20fb-5ff1-c4b7-dca8d77c37b3 term=2 tally=1 -46:01.497 [INFO] TestAgent.server.raft: election won: tally=1 -46:01.497 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:15111 [Leader]" -46:01.497 [INFO] TestAgent.server: cluster leadership acquired -46:01.498 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:15111 -46:01.498 [INFO] TestAgent.server: New leader elected: payload=Node-5c4774e2-20fb-5ff1-c4b7-dca8d77c37b3 -46:01.499 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -46:01.501 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -46:01.501 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -46:01.501 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -46:01.501 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -46:01.502 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -46:01.502 [INFO] TestAgent.server: federation state anti-entropy synced -46:01.502 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5c4774e2-20fb-5ff1-c4b7-dca8d77c37b3 -46:01.502 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5c4774e2-20fb-5ff1-c4b7-dca8d77c37b3 -46:01.586 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -46:01.586 [INFO] TestAgent: Synced node info -46:01.808 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/maintenance/redis?enable=true&reason=broken from=127.0.0.1:38322 latency=13.21902ms -46:01.808 [INFO] TestAgent: Requesting shutdown -46:01.809 [INFO] TestAgent.server: shutting down server -46:01.809 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -46:01.809 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -46:01.809 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -46:01.809 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -46:01.809 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -46:01.809 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -46:01.809 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -46:01.809 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -46:01.809 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -46:01.809 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -46:01.809 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:46:01.809Z [INFO] agent.router.manager: shutting down -2022-09-15T21:46:01.809Z [INFO] agent.router.manager: shutting down -46:01.809 [INFO] TestAgent: consul server down -46:01.809 [INFO] TestAgent: shutdown complete -46:01.809 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15106 network=tcp -46:01.815 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15106 network=udp -46:01.815 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:15107 network=tcp -46:02.315 [INFO] TestAgent: Waiting for endpoints to shut down -46:02.315 [INFO] TestAgent: Endpoints down ---- PASS: TestMaintCommand_ServiceMaintenance_NoService (1.16s) -2022-09-15T21:46:01.332Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:46:01.377Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -46:01.380 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5bd7bb69-0261-bacb-76ad-6d71f76737b1 Address:127.0.0.1:15118}]" -46:01.381 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5bd7bb69-0261-bacb-76ad-6d71f76737b1.dc1 127.0.0.1 -46:01.382 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:15118 [Follower]" leader= -46:01.396 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5bd7bb69-0261-bacb-76ad-6d71f76737b1 127.0.0.1 -2022-09-15T21:46:01.396Z [INFO] agent.router: Initializing LAN area manager -46:01.397 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15113 network=udp -46:01.397 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5bd7bb69-0261-bacb-76ad-6d71f76737b1.dc1 area=wan -46:01.397 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15113 network=tcp -46:01.398 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:15114 network=tcp -46:01.398 [INFO] TestAgent: started state syncer -46:01.398 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:15119 network=tcp -46:01.400 [INFO] TestAgent.server: Adding LAN server: server="Node-5bd7bb69-0261-bacb-76ad-6d71f76737b1 (Addr: tcp/127.0.0.1:15118) (DC: dc1)" -46:01.433 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -46:01.433 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:15118 [Candidate]" term=2 -46:01.434 [DEBUG] TestAgent.server.raft: votes: needed=1 -46:01.434 [DEBUG] TestAgent.server.raft: vote granted: from=5bd7bb69-0261-bacb-76ad-6d71f76737b1 term=2 tally=1 -46:01.434 [INFO] TestAgent.server.raft: election won: tally=1 -46:01.434 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:15118 [Leader]" -46:01.434 [INFO] TestAgent.server: cluster leadership acquired -46:01.434 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:15118 -46:01.435 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -46:01.435 [INFO] TestAgent.server: New leader elected: payload=Node-5bd7bb69-0261-bacb-76ad-6d71f76737b1 -46:01.470 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -46:01.470 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -46:01.470 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -46:01.470 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -46:01.470 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -46:01.471 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5bd7bb69-0261-bacb-76ad-6d71f76737b1 -46:01.471 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5bd7bb69-0261-bacb-76ad-6d71f76737b1 -46:01.471 [INFO] TestAgent.server: federation state anti-entropy synced -46:01.614 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -46:01.615 [INFO] TestAgent: Synced node info -46:01.827 [DEBUG] TestAgent: Node info in sync -46:01.829 [INFO] TestAgent: Synced service: service=test -46:01.829 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/maintenance/test?enable=false from=127.0.0.1:41908 latency=1.30896ms -46:01.829 [INFO] TestAgent: Requesting shutdown -46:01.829 [INFO] TestAgent.server: shutting down server -46:01.829 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -46:01.829 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -46:01.829 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -46:01.829 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -46:01.829 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -46:01.830 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -46:01.830 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -46:01.830 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -46:01.830 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -46:01.830 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -46:01.830 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -2022-09-15T21:46:01.830Z [INFO] agent.router.manager: shutting down -2022-09-15T21:46:01.830Z [INFO] agent.router.manager: shutting down -46:01.830 [INFO] TestAgent: consul server down -46:01.830 [INFO] TestAgent: shutdown complete -46:01.830 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15113 network=tcp -46:01.830 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15113 network=udp -46:01.830 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:15114 network=tcp -46:02.331 [INFO] TestAgent: Waiting for endpoints to shut down -46:02.331 [INFO] TestAgent: Endpoints down ---- PASS: TestMaintCommand_DisableServiceMaintenance (1.17s) -2022-09-15T21:46:01.324Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:46:01.379Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -46:01.382 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:797b5da9-5f1a-7996-12e6-ed51be4576be Address:127.0.0.1:15146}]" -46:01.382 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:15146 [Follower]" leader= -46:01.394 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-797b5da9-5f1a-7996-12e6-ed51be4576be.dc1 127.0.0.1 -46:01.398 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-797b5da9-5f1a-7996-12e6-ed51be4576be 127.0.0.1 -2022-09-15T21:46:01.398Z [INFO] agent.router: Initializing LAN area manager -46:01.399 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15141 network=udp -46:01.399 [INFO] TestAgent.server: Adding LAN server: server="Node-797b5da9-5f1a-7996-12e6-ed51be4576be (Addr: tcp/127.0.0.1:15146) (DC: dc1)" -46:01.399 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-797b5da9-5f1a-7996-12e6-ed51be4576be.dc1 area=wan -46:01.399 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15141 network=tcp -46:01.400 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:15142 network=tcp -46:01.400 [INFO] TestAgent: started state syncer -46:01.400 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:15147 network=tcp -46:01.424 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -46:01.425 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:15146 [Candidate]" term=2 -46:01.425 [DEBUG] TestAgent.server.raft: votes: needed=1 -46:01.425 [DEBUG] TestAgent.server.raft: vote granted: from=797b5da9-5f1a-7996-12e6-ed51be4576be term=2 tally=1 -46:01.425 [INFO] TestAgent.server.raft: election won: tally=1 -46:01.425 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:15146 [Leader]" -46:01.425 [INFO] TestAgent.server: cluster leadership acquired -46:01.425 [INFO] TestAgent.server: New leader elected: payload=Node-797b5da9-5f1a-7996-12e6-ed51be4576be -46:01.425 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:15146 -46:01.427 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -46:01.461 [INFO] TestAgent: Synced node info -46:01.462 [DEBUG] TestAgent: Node info in sync -46:01.469 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -46:01.469 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -46:01.470 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -46:01.470 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -46:01.470 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -46:01.470 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-797b5da9-5f1a-7996-12e6-ed51be4576be -46:01.477 [INFO] TestAgent.server: member joined, marking health alive: member=Node-797b5da9-5f1a-7996-12e6-ed51be4576be -46:01.471 [INFO] TestAgent.server: federation state anti-entropy synced -46:01.839 [INFO] TestAgent: Service entered maintenance mode: service=test -46:01.839 [DEBUG] TestAgent: Node info in sync -46:01.840 [INFO] TestAgent: Synced service: service=test -46:01.840 [DEBUG] TestAgent: Check in sync: check=_service_maintenance:test -46:01.840 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/maintenance/test?enable=true&reason=broken from=127.0.0.1:34124 latency=1.6291ms -46:01.840 [INFO] TestAgent: Requesting shutdown -46:01.840 [INFO] TestAgent.server: shutting down server -46:01.840 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -46:01.840 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -46:01.840 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -46:01.840 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -46:01.840 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -46:01.841 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -46:01.841 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -46:01.841 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -46:01.841 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -46:01.841 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -46:01.841 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:46:01.841Z [INFO] agent.router.manager: shutting down -2022-09-15T21:46:01.841Z [INFO] agent.router.manager: shutting down -46:01.841 [INFO] TestAgent: consul server down -46:01.841 [INFO] TestAgent: shutdown complete -46:01.841 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15141 network=tcp -46:01.841 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15141 network=udp -46:01.841 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:15142 network=tcp -46:02.343 [INFO] TestAgent: Waiting for endpoints to shut down -46:02.343 [INFO] TestAgent: Endpoints down ---- PASS: TestMaintCommand_EnableServiceMaintenance (1.19s) -2022-09-15T21:46:01.353Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:46:01.398Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -46:01.400 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:facfe830-a348-a243-3a73-4db0b7d933c5 Address:127.0.0.1:15132}]" -46:01.401 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:15132 [Follower]" leader= -46:01.401 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-facfe830-a348-a243-3a73-4db0b7d933c5.dc1 127.0.0.1 -46:01.402 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-facfe830-a348-a243-3a73-4db0b7d933c5 127.0.0.1 -2022-09-15T21:46:01.402Z [INFO] agent.router: Initializing LAN area manager -46:01.402 [INFO] TestAgent.server: Adding LAN server: server="Node-facfe830-a348-a243-3a73-4db0b7d933c5 (Addr: tcp/127.0.0.1:15132) (DC: dc1)" -46:01.402 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-facfe830-a348-a243-3a73-4db0b7d933c5.dc1 area=wan -46:01.402 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15127 network=udp -46:01.450 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15127 network=tcp -46:01.451 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:15128 network=tcp -46:01.454 [INFO] TestAgent: started state syncer -46:01.454 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:15133 network=tcp -46:01.466 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -46:01.466 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:15132 [Candidate]" term=2 -46:01.466 [DEBUG] TestAgent.server.raft: votes: needed=1 -46:01.466 [DEBUG] TestAgent.server.raft: vote granted: from=facfe830-a348-a243-3a73-4db0b7d933c5 term=2 tally=1 -46:01.466 [INFO] TestAgent.server.raft: election won: tally=1 -46:01.466 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:15132 [Leader]" -46:01.466 [INFO] TestAgent.server: cluster leadership acquired -46:01.467 [INFO] TestAgent.server: New leader elected: payload=Node-facfe830-a348-a243-3a73-4db0b7d933c5 -46:01.467 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:15132 -46:01.471 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -46:01.474 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -46:01.474 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -46:01.474 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -46:01.474 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -46:01.474 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -46:01.474 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-facfe830-a348-a243-3a73-4db0b7d933c5 -46:01.474 [INFO] TestAgent.server: member joined, marking health alive: member=Node-facfe830-a348-a243-3a73-4db0b7d933c5 -46:01.476 [INFO] TestAgent.server: federation state anti-entropy synced -46:01.578 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -46:01.578 [INFO] TestAgent: Synced node info -46:01.855 [DEBUG] TestAgent: Node info in sync -46:01.855 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/maintenance?enable=false from=127.0.0.1:57582 latency=40.24µs -46:01.855 [INFO] TestAgent: Requesting shutdown -46:01.856 [INFO] TestAgent.server: shutting down server -46:01.856 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -46:01.856 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -46:01.856 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -46:01.856 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -46:01.856 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -46:01.856 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -46:01.856 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -46:01.856 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -46:01.856 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -46:01.856 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -46:01.856 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:46:01.856Z [INFO] agent.router.manager: shutting down -2022-09-15T21:46:01.856Z [INFO] agent.router.manager: shutting down -46:01.856 [INFO] TestAgent: consul server down -46:01.856 [INFO] TestAgent: shutdown complete -46:01.856 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15127 network=tcp -46:01.856 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15127 network=udp -46:01.856 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:15128 network=tcp -46:02.357 [INFO] TestAgent: Waiting for endpoints to shut down -46:02.357 [INFO] TestAgent: Endpoints down ---- PASS: TestMaintCommand_DisableNodeMaintenance (1.20s) +2021-08-13T15:38:35.132Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:38:35.214Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:35.217 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c38093c2-7534-ba70-7263-4fee33e4bb4e Address:127.0.0.1:24314}]" +38:35.217 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24314 [Follower]" leader= +38:35.218 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c38093c2-7534-ba70-7263-4fee33e4bb4e.dc1 127.0.0.1 +38:35.218 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c38093c2-7534-ba70-7263-4fee33e4bb4e 127.0.0.1 +2021-08-13T15:38:35.219Z [INFO] agent.router: Initializing LAN area manager +38:35.219 [INFO] TestAgent.server: Adding LAN server: server="Node-c38093c2-7534-ba70-7263-4fee33e4bb4e (Addr: tcp/127.0.0.1:24314) (DC: dc1)" +38:35.219 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c38093c2-7534-ba70-7263-4fee33e4bb4e.dc1 area=wan +38:35.219 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24309 network=tcp +38:35.457 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +38:35.457 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24314 [Candidate]" term=2 +38:35.458 [DEBUG] TestAgent.server.raft: votes: needed=1 +38:35.458 [DEBUG] TestAgent.server.raft: vote granted: from=c38093c2-7534-ba70-7263-4fee33e4bb4e term=2 tally=1 +38:35.458 [INFO] TestAgent.server.raft: election won: tally=1 +38:35.458 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24314 [Leader]" +38:35.458 [INFO] TestAgent.server: cluster leadership acquired +38:35.458 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24314 +38:35.460 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +38:35.460 [INFO] TestAgent.server: New leader elected: payload=Node-c38093c2-7534-ba70-7263-4fee33e4bb4e +38:35.462 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24309 network=udp +38:35.464 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24310 network=tcp +38:35.464 [INFO] TestAgent: started state syncer +38:35.466 [INFO] TestAgent: Service entered maintenance mode: service=test +38:35.466 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24315 network=tcp +38:35.467 [INFO] TestAgent: Node entered maintenance mode +38:35.481 [INFO] TestAgent: Synced node info +38:35.483 [INFO] TestAgent: Synced service: service=test +38:35.483 [DEBUG] TestAgent: Check in sync: check=_service_maintenance:test +38:35.495 [INFO] TestAgent: Synced check: check=_node_maintenance +38:35.495 [DEBUG] TestAgent: Node info in sync +38:35.495 [DEBUG] TestAgent: Service in sync: service=test +38:35.495 [DEBUG] TestAgent: Check in sync: check=_service_maintenance:test +38:35.495 [DEBUG] TestAgent: Check in sync: check=_node_maintenance +38:35.497 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:39220 latency=17.574984ms +38:35.499 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +38:35.499 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +38:35.499 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +38:35.499 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +38:35.499 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +38:35.499 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c38093c2-7534-ba70-7263-4fee33e4bb4e +38:35.499 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c38093c2-7534-ba70-7263-4fee33e4bb4e +38:35.502 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/checks from=127.0.0.1:39220 latency=372.981µs +38:35.503 [INFO] TestAgent: Requesting shutdown +38:35.503 [INFO] TestAgent.server: shutting down server +38:35.503 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +38:35.503 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +38:35.503 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +38:35.503 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +38:35.503 [INFO] TestAgent.server: federation state anti-entropy synced +38:35.503 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +38:35.503 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +38:35.503 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +38:35.504 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +38:35.504 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +38:35.504 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:38:35.504Z [INFO] agent.router.manager: shutting down +2021-08-13T15:38:35.504Z [INFO] agent.router.manager: shutting down +38:35.504 [INFO] TestAgent: consul server down +38:35.504 [INFO] TestAgent: shutdown complete +38:35.504 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24309 network=tcp +38:35.504 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24309 network=udp +38:35.504 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24310 network=tcp +38:36.004 [INFO] TestAgent: Waiting for endpoints to shut down +38:36.004 [INFO] TestAgent: Endpoints down +--- PASS: TestMaintCommand_NoArgs (0.94s) +2021-08-13T15:38:35.152Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:38:35.178Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:35.183 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:155a5fdb-fbf5-8e03-63c8-eec0318a715f Address:127.0.0.1:24328}]" +38:35.183 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24328 [Follower]" leader= +38:35.184 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-155a5fdb-fbf5-8e03-63c8-eec0318a715f.dc1 127.0.0.1 +38:35.185 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-155a5fdb-fbf5-8e03-63c8-eec0318a715f 127.0.0.1 +2021-08-13T15:38:35.185Z [INFO] agent.router: Initializing LAN area manager +38:35.185 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24323 network=udp +38:35.195 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-155a5fdb-fbf5-8e03-63c8-eec0318a715f.dc1 area=wan +38:35.199 [INFO] TestAgent.server: Adding LAN server: server="Node-155a5fdb-fbf5-8e03-63c8-eec0318a715f (Addr: tcp/127.0.0.1:24328) (DC: dc1)" +38:35.199 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24323 network=tcp +38:35.201 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24324 network=tcp +38:35.201 [INFO] TestAgent: started state syncer +38:35.201 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24329 network=tcp +38:35.231 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +38:35.231 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24328 [Candidate]" term=2 +38:35.232 [DEBUG] TestAgent.server.raft: votes: needed=1 +38:35.232 [DEBUG] TestAgent.server.raft: vote granted: from=155a5fdb-fbf5-8e03-63c8-eec0318a715f term=2 tally=1 +38:35.232 [INFO] TestAgent.server.raft: election won: tally=1 +38:35.232 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24328 [Leader]" +38:35.232 [INFO] TestAgent.server: cluster leadership acquired +38:35.232 [INFO] TestAgent.server: New leader elected: payload=Node-155a5fdb-fbf5-8e03-63c8-eec0318a715f +38:35.232 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24328 +38:35.233 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +38:35.502 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +38:35.502 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +38:35.502 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +38:35.502 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +38:35.502 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +38:35.502 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-155a5fdb-fbf5-8e03-63c8-eec0318a715f +38:35.502 [INFO] TestAgent.server: member joined, marking health alive: member=Node-155a5fdb-fbf5-8e03-63c8-eec0318a715f +38:35.505 [INFO] TestAgent.server: federation state anti-entropy synced +38:35.516 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/maintenance/redis?enable=true&reason=broken from=127.0.0.1:47298 latency=180.16µs +38:35.517 [INFO] TestAgent: Requesting shutdown +38:35.517 [INFO] TestAgent.server: shutting down server +38:35.517 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +38:35.517 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +38:35.517 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +38:35.517 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +38:35.517 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +38:35.517 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +38:35.517 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +38:35.517 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +38:35.517 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +38:35.517 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +38:35.517 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +38:35.517 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:38:35.517Z [INFO] agent.router.manager: shutting down +2021-08-13T15:38:35.517Z [INFO] agent.router.manager: shutting down +38:35.517 [INFO] TestAgent: consul server down +38:35.517 [INFO] TestAgent: shutdown complete +38:35.517 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24323 network=tcp +38:35.517 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24323 network=udp +38:35.517 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24324 network=tcp +38:36.017 [INFO] TestAgent: Waiting for endpoints to shut down +38:36.017 [INFO] TestAgent: Endpoints down +--- PASS: TestMaintCommand_ServiceMaintenance_NoService (0.95s) +2021-08-13T15:38:35.168Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:38:35.193Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:35.196 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5e549185-2934-051f-089a-f7536626ff18 Address:127.0.0.1:24307}]" +38:35.202 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24307 [Follower]" leader= +38:35.203 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5e549185-2934-051f-089a-f7536626ff18.dc1 127.0.0.1 +38:35.204 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5e549185-2934-051f-089a-f7536626ff18 127.0.0.1 +2021-08-13T15:38:35.204Z [INFO] agent.router: Initializing LAN area manager +38:35.205 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24302 network=udp +38:35.209 [INFO] TestAgent.server: Adding LAN server: server="Node-5e549185-2934-051f-089a-f7536626ff18 (Addr: tcp/127.0.0.1:24307) (DC: dc1)" +38:35.209 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24302 network=tcp +38:35.210 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24303 network=tcp +38:35.210 [INFO] TestAgent: started state syncer +38:35.210 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5e549185-2934-051f-089a-f7536626ff18.dc1 area=wan +38:35.210 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24308 network=tcp +38:35.465 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +38:35.465 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24307 [Candidate]" term=2 +38:35.465 [DEBUG] TestAgent.server.raft: votes: needed=1 +38:35.466 [DEBUG] TestAgent.server.raft: vote granted: from=5e549185-2934-051f-089a-f7536626ff18 term=2 tally=1 +38:35.466 [INFO] TestAgent.server.raft: election won: tally=1 +38:35.466 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24307 [Leader]" +38:35.466 [INFO] TestAgent.server: cluster leadership acquired +38:35.472 [INFO] TestAgent.server: New leader elected: payload=Node-5e549185-2934-051f-089a-f7536626ff18 +38:35.473 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24307 +38:35.491 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +38:35.500 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +38:35.500 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +38:35.500 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +38:35.500 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +38:35.500 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +38:35.500 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5e549185-2934-051f-089a-f7536626ff18 +38:35.500 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5e549185-2934-051f-089a-f7536626ff18 +38:35.504 [INFO] TestAgent.server: federation state anti-entropy synced +38:35.550 [INFO] TestAgent: Node entered maintenance mode +38:35.550 [INFO] TestAgent: Synced node info +38:35.551 [INFO] TestAgent: Synced check: check=_node_maintenance +38:35.551 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/maintenance?enable=true&reason=broken from=127.0.0.1:41746 latency=1.557243ms +38:35.551 [INFO] TestAgent: Requesting shutdown +38:35.551 [INFO] TestAgent.server: shutting down server +38:35.551 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +38:35.551 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +38:35.551 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +38:35.551 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +38:35.551 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +38:35.551 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +38:35.551 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +38:35.551 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +38:35.551 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +38:35.551 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +38:35.551 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +38:35.551 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:38:35.552Z [INFO] agent.router.manager: shutting down +2021-08-13T15:38:35.552Z [INFO] agent.router.manager: shutting down +38:35.552 [INFO] TestAgent: consul server down +38:35.552 [INFO] TestAgent: shutdown complete +38:35.552 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24302 network=tcp +38:35.552 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24302 network=udp +38:35.552 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24303 network=tcp +38:36.052 [INFO] TestAgent: Waiting for endpoints to shut down +38:36.052 [INFO] TestAgent: Endpoints down +--- PASS: TestMaintCommand_EnableNodeMaintenance (0.98s) +2021-08-13T15:38:35.128Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:38:35.155Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:35.168 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8b72c7d7-35f5-12a2-e0c7-9e4cb4be514e Address:127.0.0.1:24300}]" +38:35.169 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8b72c7d7-35f5-12a2-e0c7-9e4cb4be514e.dc1 127.0.0.1 +38:35.170 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8b72c7d7-35f5-12a2-e0c7-9e4cb4be514e 127.0.0.1 +2021-08-13T15:38:35.170Z [INFO] agent.router: Initializing LAN area manager +38:35.171 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24295 network=udp +38:35.171 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24300 [Follower]" leader= +38:35.171 [INFO] TestAgent.server: Adding LAN server: server="Node-8b72c7d7-35f5-12a2-e0c7-9e4cb4be514e (Addr: tcp/127.0.0.1:24300) (DC: dc1)" +38:35.171 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8b72c7d7-35f5-12a2-e0c7-9e4cb4be514e.dc1 area=wan +38:35.172 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24295 network=tcp +38:35.173 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24296 network=tcp +38:35.173 [INFO] TestAgent: started state syncer +38:35.173 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24301 network=tcp +38:35.459 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +38:35.459 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24300 [Candidate]" term=2 +38:35.459 [DEBUG] TestAgent.server.raft: votes: needed=1 +38:35.459 [DEBUG] TestAgent.server.raft: vote granted: from=8b72c7d7-35f5-12a2-e0c7-9e4cb4be514e term=2 tally=1 +38:35.459 [INFO] TestAgent.server.raft: election won: tally=1 +38:35.459 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24300 [Leader]" +38:35.460 [INFO] TestAgent: Synced node info +38:35.460 [DEBUG] TestAgent: Node info in sync +38:35.460 [INFO] TestAgent.server: cluster leadership acquired +38:35.461 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24300 +38:35.462 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +38:35.462 [INFO] TestAgent.server: New leader elected: payload=Node-8b72c7d7-35f5-12a2-e0c7-9e4cb4be514e +38:35.500 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +38:35.500 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +38:35.500 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +38:35.500 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +38:35.500 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +38:35.500 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8b72c7d7-35f5-12a2-e0c7-9e4cb4be514e +38:35.500 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8b72c7d7-35f5-12a2-e0c7-9e4cb4be514e +38:35.502 [INFO] TestAgent.server: federation state anti-entropy synced +38:35.597 [DEBUG] TestAgent: Node info in sync +38:35.597 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/maintenance?enable=false from=127.0.0.1:35364 latency=33.36µs +38:35.597 [INFO] TestAgent: Requesting shutdown +38:35.598 [INFO] TestAgent.server: shutting down server +38:35.598 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +38:35.598 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +38:35.598 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +38:35.598 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +38:35.598 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +38:35.598 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +38:35.598 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +38:35.598 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +38:35.598 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +38:35.598 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +38:35.598 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:38:35.598Z [INFO] agent.router.manager: shutting down +2021-08-13T15:38:35.598Z [INFO] agent.router.manager: shutting down +38:35.598 [INFO] TestAgent: consul server down +38:35.598 [INFO] TestAgent: shutdown complete +38:35.598 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24295 network=tcp +38:35.598 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24295 network=udp +38:35.598 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24296 network=tcp +38:36.098 [INFO] TestAgent: Waiting for endpoints to shut down +38:36.098 [INFO] TestAgent: Endpoints down +--- PASS: TestMaintCommand_DisableNodeMaintenance (1.03s) +2021-08-13T15:38:35.145Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:38:35.195Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:35.204 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:64af3f22-0ec6-301b-7248-4ba1dbd48cf7 Address:127.0.0.1:24335}]" +38:35.205 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-64af3f22-0ec6-301b-7248-4ba1dbd48cf7.dc1 127.0.0.1 +38:35.206 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24335 [Follower]" leader= +38:35.211 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-64af3f22-0ec6-301b-7248-4ba1dbd48cf7 127.0.0.1 +2021-08-13T15:38:35.211Z [INFO] agent.router: Initializing LAN area manager +38:35.211 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24330 network=udp +38:35.211 [INFO] TestAgent.server: Adding LAN server: server="Node-64af3f22-0ec6-301b-7248-4ba1dbd48cf7 (Addr: tcp/127.0.0.1:24335) (DC: dc1)" +38:35.211 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-64af3f22-0ec6-301b-7248-4ba1dbd48cf7.dc1 area=wan +38:35.212 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24330 network=tcp +38:35.212 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24331 network=tcp +38:35.213 [INFO] TestAgent: started state syncer +38:35.213 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24336 network=tcp +38:35.463 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +38:35.463 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24335 [Candidate]" term=2 +38:35.463 [DEBUG] TestAgent.server.raft: votes: needed=1 +38:35.463 [DEBUG] TestAgent.server.raft: vote granted: from=64af3f22-0ec6-301b-7248-4ba1dbd48cf7 term=2 tally=1 +38:35.463 [INFO] TestAgent.server.raft: election won: tally=1 +38:35.463 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24335 [Leader]" +38:35.464 [INFO] TestAgent.server: cluster leadership acquired +38:35.464 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24335 +38:35.465 [INFO] TestAgent.server: New leader elected: payload=Node-64af3f22-0ec6-301b-7248-4ba1dbd48cf7 +38:35.466 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +38:35.500 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +38:35.500 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +38:35.500 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +38:35.500 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +38:35.500 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +38:35.500 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-64af3f22-0ec6-301b-7248-4ba1dbd48cf7 +38:35.500 [INFO] TestAgent.server: member joined, marking health alive: member=Node-64af3f22-0ec6-301b-7248-4ba1dbd48cf7 +38:35.503 [INFO] TestAgent.server: federation state anti-entropy synced +38:35.620 [INFO] TestAgent: Service entered maintenance mode: service=test +38:35.621 [INFO] TestAgent: Synced node info +38:35.622 [INFO] TestAgent: Synced service: service=test +38:35.622 [DEBUG] TestAgent: Check in sync: check=_service_maintenance:test +38:35.622 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/maintenance/test?enable=true&reason=broken from=127.0.0.1:43116 latency=1.955522ms +38:35.622 [INFO] TestAgent: Requesting shutdown +38:35.622 [INFO] TestAgent.server: shutting down server +38:35.622 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +38:35.622 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +38:35.622 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +38:35.622 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +38:35.622 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +38:35.622 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +38:35.622 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +38:35.622 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +38:35.622 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +38:35.622 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +38:35.622 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +38:35.622 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:38:35.622Z [INFO] agent.router.manager: shutting down +2021-08-13T15:38:35.622Z [INFO] agent.router.manager: shutting down +38:35.623 [INFO] TestAgent: consul server down +38:35.623 [INFO] TestAgent: shutdown complete +38:35.623 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24330 network=tcp +38:35.623 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24330 network=udp +38:35.623 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24331 network=tcp +38:36.123 [INFO] TestAgent: Waiting for endpoints to shut down +38:36.123 [INFO] TestAgent: Endpoints down +--- PASS: TestMaintCommand_EnableServiceMaintenance (1.05s) +2021-08-13T15:38:35.213Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:38:35.462Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:35.466 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fc0322c1-118d-c051-f02b-90f9d6b7ab63 Address:127.0.0.1:24321}]" +38:35.467 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24321 [Follower]" leader= +38:35.494 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fc0322c1-118d-c051-f02b-90f9d6b7ab63.dc1 127.0.0.1 +38:35.503 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fc0322c1-118d-c051-f02b-90f9d6b7ab63 127.0.0.1 +2021-08-13T15:38:35.503Z [INFO] agent.router: Initializing LAN area manager +38:35.503 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24316 network=udp +38:35.504 [INFO] TestAgent.server: Adding LAN server: server="Node-fc0322c1-118d-c051-f02b-90f9d6b7ab63 (Addr: tcp/127.0.0.1:24321) (DC: dc1)" +38:35.504 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fc0322c1-118d-c051-f02b-90f9d6b7ab63.dc1 area=wan +38:35.504 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24316 network=tcp +38:35.505 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24317 network=tcp +38:35.505 [INFO] TestAgent: started state syncer +38:35.505 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24322 network=tcp +38:35.526 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +38:35.526 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24321 [Candidate]" term=2 +38:35.527 [DEBUG] TestAgent.server.raft: votes: needed=1 +38:35.527 [DEBUG] TestAgent.server.raft: vote granted: from=fc0322c1-118d-c051-f02b-90f9d6b7ab63 term=2 tally=1 +38:35.527 [INFO] TestAgent.server.raft: election won: tally=1 +38:35.527 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24321 [Leader]" +38:35.527 [INFO] TestAgent.server: cluster leadership acquired +38:35.527 [INFO] TestAgent.server: New leader elected: payload=Node-fc0322c1-118d-c051-f02b-90f9d6b7ab63 +38:35.527 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24321 +38:35.536 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +38:35.539 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +38:35.539 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +38:35.539 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +38:35.539 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +38:35.539 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +38:35.539 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fc0322c1-118d-c051-f02b-90f9d6b7ab63 +38:35.539 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fc0322c1-118d-c051-f02b-90f9d6b7ab63 +38:35.539 [INFO] TestAgent.server: federation state anti-entropy synced +38:35.546 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +38:35.546 [INFO] TestAgent: Synced node info +38:35.742 [DEBUG] TestAgent: Node info in sync +38:35.743 [INFO] TestAgent: Synced service: service=test +38:35.743 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/maintenance/test?enable=false from=127.0.0.1:53516 latency=925.741µs +38:35.745 [INFO] TestAgent: Requesting shutdown +38:35.745 [INFO] TestAgent.server: shutting down server +38:35.745 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +38:35.745 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +38:35.745 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +38:35.745 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +38:35.745 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +38:35.745 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +38:35.745 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +38:35.745 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +38:35.745 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +38:35.745 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +38:35.745 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:38:35.746Z [INFO] agent.router.manager: shutting down +2021-08-13T15:38:35.746Z [INFO] agent.router.manager: shutting down +38:35.746 [INFO] TestAgent: consul server down +38:35.746 [INFO] TestAgent: shutdown complete +38:35.746 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24316 network=tcp +38:35.746 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24316 network=udp +38:35.746 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24317 network=tcp +38:36.246 [INFO] TestAgent: Waiting for endpoints to shut down +38:36.246 [INFO] TestAgent: Endpoints down +--- PASS: TestMaintCommand_DisableServiceMaintenance (1.18s) PASS -ok github.com/hashicorp/consul/command/maint 1.298s +ok github.com/hashicorp/consul/command/maint 1.243s === RUN TestMembersCommand_noTabs === PAUSE TestMembersCommand_noTabs === RUN TestMembersCommand @@ -89735,391 +89635,390 @@ === RUN TestMembersCommand_verticalBar === PAUSE TestMembersCommand_verticalBar === CONT TestMembersCommand_noTabs ---- PASS: TestMembersCommand_noTabs (0.00s) -=== CONT TestMembersCommand_verticalBar -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... -=== CONT TestMembersCommand_statusFilter === CONT TestMembersCommand_statusFilter_failed +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestMembersCommand_WAN +=== CONT TestMembersCommand_statusFilter +--- PASS: TestMembersCommand_noTabs (0.00s) === CONT TestMembersCommand +=== CONT TestMembersCommand_verticalBar [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:46:01.484Z [WARN] agent: Node name "name|with|bars" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. -2022-09-15T21:46:01.484Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:46:01.516Z [WARN] agent.auto_config: Node name "name|with|bars" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. -2022-09-15T21:46:01.516Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -46:01.521 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ac69f223-0826-5650-1c5b-85bfd2ad382d Address:127.0.0.1:16132}]" -46:01.522 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16132 [Follower]" leader= -46:01.522 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: name|with|bars.dc1 127.0.0.1 -46:01.523 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: name|with|bars 127.0.0.1 -2022-09-15T21:46:01.523Z [INFO] agent.router: Initializing LAN area manager -46:01.523 [INFO] TestAgent.server: Adding LAN server: server="name|with|bars (Addr: tcp/127.0.0.1:16132) (DC: dc1)" -46:01.523 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16127 network=udp -46:01.523 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=name|with|bars.dc1 area=wan -46:01.523 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16127 network=tcp -46:01.524 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16128 network=tcp -46:01.524 [INFO] TestAgent: started state syncer -46:01.525 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16133 network=tcp -46:01.570 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -46:01.570 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16132 [Candidate]" term=2 -46:01.571 [DEBUG] TestAgent.server.raft: votes: needed=1 -46:01.571 [DEBUG] TestAgent.server.raft: vote granted: from=ac69f223-0826-5650-1c5b-85bfd2ad382d term=2 tally=1 -46:01.571 [INFO] TestAgent.server.raft: election won: tally=1 -46:01.571 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16132 [Leader]" -46:01.571 [INFO] TestAgent.server: cluster leadership acquired -46:01.571 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16132 -46:01.573 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -46:01.573 [INFO] TestAgent.server: New leader elected: payload=name|with|bars -46:01.588 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -46:01.588 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -46:01.588 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -46:01.588 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -46:01.588 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -46:01.588 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=name|with|bars -46:01.588 [INFO] TestAgent.server: member joined, marking health alive: member=name|with|bars -46:01.588 [INFO] TestAgent.server: federation state anti-entropy synced -46:01.650 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/members?segment=_all from=127.0.0.1:47696 latency=552.02µs -46:01.651 [INFO] TestAgent: Requesting shutdown -46:01.651 [INFO] TestAgent.server: shutting down server -46:01.651 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -46:01.651 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -46:01.651 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -46:01.651 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -46:01.652 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -46:01.652 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -46:01.652 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -46:01.652 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -46:01.652 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -46:01.652 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -46:01.652 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -46:01.652 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:46:01.652Z [INFO] agent.router.manager: shutting down -2022-09-15T21:46:01.652Z [INFO] agent.router.manager: shutting down -46:01.652 [INFO] TestAgent: consul server down -46:01.652 [INFO] TestAgent: shutdown complete -46:01.652 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16127 network=tcp -46:01.652 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16127 network=udp -46:01.652 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16128 network=tcp -46:02.153 [INFO] TestAgent: Waiting for endpoints to shut down -46:02.153 [INFO] TestAgent: Endpoints down ---- PASS: TestMembersCommand_verticalBar (0.76s) -2022-09-15T21:46:01.495Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:46:01.549Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -46:01.554 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e08d5e0b-aa93-8589-bf90-25992415de40 Address:127.0.0.1:16160}]" -46:01.554 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16160 [Follower]" leader= -46:01.555 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e08d5e0b-aa93-8589-bf90-25992415de40.dc1 127.0.0.1 -46:01.556 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e08d5e0b-aa93-8589-bf90-25992415de40 127.0.0.1 -2022-09-15T21:46:01.556Z [INFO] agent.router: Initializing LAN area manager -46:01.556 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16155 network=udp -46:01.556 [INFO] TestAgent.server: Adding LAN server: server="Node-e08d5e0b-aa93-8589-bf90-25992415de40 (Addr: tcp/127.0.0.1:16160) (DC: dc1)" -46:01.556 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e08d5e0b-aa93-8589-bf90-25992415de40.dc1 area=wan -46:01.590 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16155 network=tcp -46:01.591 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16156 network=tcp -46:01.591 [INFO] TestAgent: started state syncer -46:01.591 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16161 network=tcp -46:01.618 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -46:01.618 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16160 [Candidate]" term=2 -46:01.618 [DEBUG] TestAgent.server.raft: votes: needed=1 -46:01.618 [DEBUG] TestAgent.server.raft: vote granted: from=e08d5e0b-aa93-8589-bf90-25992415de40 term=2 tally=1 -46:01.618 [INFO] TestAgent.server.raft: election won: tally=1 -46:01.618 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16160 [Leader]" -46:01.619 [INFO] TestAgent.server: cluster leadership acquired -46:01.619 [INFO] TestAgent.server: New leader elected: payload=Node-e08d5e0b-aa93-8589-bf90-25992415de40 -46:01.619 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16160 -46:01.621 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -46:01.625 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -46:01.625 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -46:01.625 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -46:01.625 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -46:01.625 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -46:01.625 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e08d5e0b-aa93-8589-bf90-25992415de40 -46:01.625 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e08d5e0b-aa93-8589-bf90-25992415de40 -46:01.627 [INFO] TestAgent.server: federation state anti-entropy synced -46:01.759 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -46:01.760 [INFO] TestAgent: Synced node info -46:01.760 [DEBUG] TestAgent: Node info in sync -46:01.804 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/members?segment=_all from=127.0.0.1:42438 latency=240.56µs -46:01.814 [INFO] TestAgent: Requesting shutdown -46:01.814 [INFO] TestAgent.server: shutting down server -46:01.814 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -46:01.814 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -46:01.814 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -46:01.814 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -46:01.814 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -46:01.814 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -46:01.814 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -46:01.814 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -46:01.814 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -46:01.814 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -46:01.814 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:46:01.815Z [INFO] agent.router.manager: shutting down -2022-09-15T21:46:01.815Z [INFO] agent.router.manager: shutting down -46:01.815 [INFO] TestAgent: consul server down -46:01.815 [INFO] TestAgent: shutdown complete -46:01.815 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16155 network=tcp -46:01.815 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16155 network=udp -46:01.815 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16156 network=tcp -46:02.316 [INFO] TestAgent: Waiting for endpoints to shut down -46:02.316 [INFO] TestAgent: Endpoints down +2021-08-13T15:38:34.418Z [WARN] agent: Node name "name|with|bars" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. +2021-08-13T15:38:34.418Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:38:34.443Z [WARN] agent.auto_config: Node name "name|with|bars" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. +2021-08-13T15:38:34.443Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:34.446 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:15b7cf0a-4717-6cff-767c-5557044534e0 Address:127.0.0.1:31468}]" +38:34.446 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31468 [Follower]" leader= +38:34.447 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: name|with|bars.dc1 127.0.0.1 +38:34.448 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: name|with|bars 127.0.0.1 +2021-08-13T15:38:34.448Z [INFO] agent.router: Initializing LAN area manager +38:34.448 [INFO] TestAgent.server: Adding LAN server: server="name|with|bars (Addr: tcp/127.0.0.1:31468) (DC: dc1)" +38:34.448 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=name|with|bars.dc1 area=wan +38:34.448 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31463 network=udp +38:34.503 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +38:34.503 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31468 [Candidate]" term=2 +38:34.503 [DEBUG] TestAgent.server.raft: votes: needed=1 +38:34.503 [DEBUG] TestAgent.server.raft: vote granted: from=15b7cf0a-4717-6cff-767c-5557044534e0 term=2 tally=1 +38:34.503 [INFO] TestAgent.server.raft: election won: tally=1 +38:34.503 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31468 [Leader]" +38:34.503 [INFO] TestAgent.server: cluster leadership acquired +38:34.503 [INFO] TestAgent.server: New leader elected: payload=name|with|bars +38:34.503 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:31468 +38:34.505 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +38:34.507 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +38:34.507 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +38:34.507 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +38:34.507 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +38:34.507 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +38:34.507 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=name|with|bars +38:34.507 [INFO] TestAgent.server: member joined, marking health alive: member=name|with|bars +38:34.507 [INFO] TestAgent.server: federation state anti-entropy synced +38:34.527 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31463 network=tcp +38:34.536 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31464 network=tcp +38:34.536 [INFO] TestAgent: started state syncer +38:34.536 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31469 network=tcp +38:34.536 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +38:34.537 [INFO] TestAgent: Synced node info +38:34.537 [DEBUG] TestAgent: Node info in sync +38:34.538 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/members?segment=_all from=127.0.0.1:48474 latency=645.141µs +38:34.539 [INFO] TestAgent: Requesting shutdown +38:34.539 [INFO] TestAgent.server: shutting down server +38:34.539 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +38:34.539 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +38:34.539 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +38:34.539 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +38:34.539 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +38:34.539 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +38:34.539 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +38:34.539 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +38:34.539 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +38:34.539 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +38:34.540 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:38:34.540Z [INFO] agent.router.manager: shutting down +2021-08-13T15:38:34.540Z [INFO] agent.router.manager: shutting down +38:34.540 [INFO] TestAgent: consul server down +38:34.540 [INFO] TestAgent: shutdown complete +38:34.540 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31463 network=tcp +38:34.540 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31463 network=udp +38:34.540 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31464 network=tcp +38:35.040 [INFO] TestAgent: Waiting for endpoints to shut down +38:35.040 [INFO] TestAgent: Endpoints down +--- PASS: TestMembersCommand_verticalBar (0.75s) +2021-08-13T15:38:34.380Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:38:34.421Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:34.427 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2d5fbf4e-680f-391e-45ee-8564ca237010 Address:127.0.0.1:31475}]" +38:34.428 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2d5fbf4e-680f-391e-45ee-8564ca237010.dc1 127.0.0.1 +38:34.429 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2d5fbf4e-680f-391e-45ee-8564ca237010 127.0.0.1 +2021-08-13T15:38:34.429Z [INFO] agent.router: Initializing LAN area manager +38:34.430 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31470 network=udp +38:34.430 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31475 [Follower]" leader= +38:34.432 [INFO] TestAgent.server: Adding LAN server: server="Node-2d5fbf4e-680f-391e-45ee-8564ca237010 (Addr: tcp/127.0.0.1:31475) (DC: dc1)" +38:34.434 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31470 network=tcp +38:34.435 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31471 network=tcp +38:34.435 [INFO] TestAgent: started state syncer +38:34.435 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31476 network=tcp +38:34.437 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2d5fbf4e-680f-391e-45ee-8564ca237010.dc1 area=wan +38:34.486 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +38:34.486 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31475 [Candidate]" term=2 +38:34.487 [DEBUG] TestAgent.server.raft: votes: needed=1 +38:34.487 [DEBUG] TestAgent.server.raft: vote granted: from=2d5fbf4e-680f-391e-45ee-8564ca237010 term=2 tally=1 +38:34.487 [INFO] TestAgent.server.raft: election won: tally=1 +38:34.487 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31475 [Leader]" +38:34.487 [INFO] TestAgent.server: cluster leadership acquired +38:34.487 [INFO] TestAgent.server: New leader elected: payload=Node-2d5fbf4e-680f-391e-45ee-8564ca237010 +38:34.487 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:31475 +38:34.488 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +38:34.490 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +38:34.490 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +38:34.490 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +38:34.490 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +38:34.490 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +38:34.490 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2d5fbf4e-680f-391e-45ee-8564ca237010 +38:34.490 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2d5fbf4e-680f-391e-45ee-8564ca237010 +38:34.491 [INFO] TestAgent.server: federation state anti-entropy synced +38:34.705 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/members?segment=_all from=127.0.0.1:53174 latency=296.96µs +38:34.706 [INFO] TestAgent: Requesting shutdown +38:34.706 [INFO] TestAgent.server: shutting down server +38:34.706 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +38:34.706 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +38:34.706 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +38:34.706 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +38:34.706 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +38:34.706 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +38:34.706 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +38:34.706 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +38:34.706 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +38:34.706 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +38:34.706 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +38:34.706 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:38:34.706Z [INFO] agent.router.manager: shutting down +2021-08-13T15:38:34.706Z [INFO] agent.router.manager: shutting down +38:34.706 [INFO] TestAgent: consul server down +38:34.706 [INFO] TestAgent: shutdown complete +38:34.706 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31470 network=tcp +38:34.707 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31470 network=udp +38:34.707 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31471 network=tcp +38:35.207 [INFO] TestAgent: Waiting for endpoints to shut down +38:35.207 [INFO] TestAgent: Endpoints down +--- PASS: TestMembersCommand_statusFilter (0.91s) +2021-08-13T15:38:34.361Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:38:34.383Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:34.415 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c7d7fa34-4cdc-6d7e-30b5-1e9fd37800e3 Address:127.0.0.1:31461}]" +38:34.416 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c7d7fa34-4cdc-6d7e-30b5-1e9fd37800e3.dc1 127.0.0.1 +38:34.417 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c7d7fa34-4cdc-6d7e-30b5-1e9fd37800e3 127.0.0.1 +2021-08-13T15:38:34.417Z [INFO] agent.router: Initializing LAN area manager +38:34.417 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31456 network=udp +38:34.417 [INFO] TestAgent.server: Adding LAN server: server="Node-c7d7fa34-4cdc-6d7e-30b5-1e9fd37800e3 (Addr: tcp/127.0.0.1:31461) (DC: dc1)" +38:34.417 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c7d7fa34-4cdc-6d7e-30b5-1e9fd37800e3.dc1 area=wan +38:34.418 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31456 network=tcp +38:34.419 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31457 network=tcp +38:34.419 [INFO] TestAgent: started state syncer +38:34.419 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31462 network=tcp +38:34.419 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31461 [Follower]" leader= +38:34.458 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +38:34.458 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31461 [Candidate]" term=2 +38:34.459 [DEBUG] TestAgent.server.raft: votes: needed=1 +38:34.459 [DEBUG] TestAgent.server.raft: vote granted: from=c7d7fa34-4cdc-6d7e-30b5-1e9fd37800e3 term=2 tally=1 +38:34.459 [INFO] TestAgent.server.raft: election won: tally=1 +38:34.459 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31461 [Leader]" +38:34.459 [INFO] TestAgent.server: cluster leadership acquired +38:34.459 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:31461 +38:34.459 [INFO] TestAgent.server: New leader elected: payload=Node-c7d7fa34-4cdc-6d7e-30b5-1e9fd37800e3 +38:34.460 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +38:34.485 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +38:34.485 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +38:34.485 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +38:34.485 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +38:34.485 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +38:34.485 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c7d7fa34-4cdc-6d7e-30b5-1e9fd37800e3 +38:34.485 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c7d7fa34-4cdc-6d7e-30b5-1e9fd37800e3 +38:34.487 [INFO] TestAgent.server: federation state anti-entropy synced +38:34.695 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +38:34.696 [INFO] TestAgent: Synced node info +38:34.718 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/members?segment=_all from=127.0.0.1:42072 latency=239.86µs +38:34.718 [INFO] TestAgent: Requesting shutdown +38:34.718 [INFO] TestAgent.server: shutting down server +38:34.718 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +38:34.718 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +38:34.718 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +38:34.718 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +38:34.718 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +38:34.719 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +38:34.719 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +38:34.718 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +38:34.719 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +38:34.719 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +38:34.719 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:38:34.719Z [INFO] agent.router.manager: shutting down +2021-08-13T15:38:34.719Z [INFO] agent.router.manager: shutting down +38:34.719 [INFO] TestAgent: consul server down +38:34.719 [INFO] TestAgent: shutdown complete +38:34.719 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31456 network=tcp +38:34.719 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31456 network=udp +38:34.719 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31457 network=tcp +38:35.223 [INFO] TestAgent: Waiting for endpoints to shut down +38:35.223 [INFO] TestAgent: Endpoints down --- PASS: TestMembersCommand (0.93s) -2022-09-15T21:46:01.497Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:46:01.531Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -46:01.546 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:373b5c85-c40a-cd3e-55c4-46752072232f Address:127.0.0.1:16139}]" -46:01.546 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16139 [Follower]" leader= -46:01.548 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-373b5c85-c40a-cd3e-55c4-46752072232f.dc1 127.0.0.1 -46:01.549 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-373b5c85-c40a-cd3e-55c4-46752072232f 127.0.0.1 -2022-09-15T21:46:01.549Z [INFO] agent.router: Initializing LAN area manager -46:01.550 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16134 network=udp -46:01.550 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-373b5c85-c40a-cd3e-55c4-46752072232f.dc1 area=wan -46:01.550 [INFO] TestAgent.server: Adding LAN server: server="Node-373b5c85-c40a-cd3e-55c4-46752072232f (Addr: tcp/127.0.0.1:16139) (DC: dc1)" -46:01.550 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16134 network=tcp -46:01.551 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16135 network=tcp -46:01.551 [INFO] TestAgent: started state syncer -46:01.551 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16140 network=tcp -46:01.606 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -46:01.606 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16139 [Candidate]" term=2 -46:01.607 [DEBUG] TestAgent.server.raft: votes: needed=1 -46:01.607 [DEBUG] TestAgent.server.raft: vote granted: from=373b5c85-c40a-cd3e-55c4-46752072232f term=2 tally=1 -46:01.607 [INFO] TestAgent.server.raft: election won: tally=1 -46:01.607 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16139 [Leader]" -46:01.607 [INFO] TestAgent.server: cluster leadership acquired -46:01.607 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16139 -46:01.608 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -46:01.610 [INFO] TestAgent.server: New leader elected: payload=Node-373b5c85-c40a-cd3e-55c4-46752072232f -46:01.613 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -46:01.613 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -46:01.613 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -46:01.613 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -46:01.613 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -46:01.613 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-373b5c85-c40a-cd3e-55c4-46752072232f -46:01.613 [INFO] TestAgent.server: member joined, marking health alive: member=Node-373b5c85-c40a-cd3e-55c4-46752072232f -46:01.615 [INFO] TestAgent.server: federation state anti-entropy synced -46:01.665 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -46:01.666 [INFO] TestAgent: Synced node info -46:01.666 [DEBUG] TestAgent: Node info in sync -46:01.840 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/members?segment=_all from=127.0.0.1:56490 latency=236.04µs -46:01.841 [INFO] TestAgent: Requesting shutdown -46:01.841 [INFO] TestAgent.server: shutting down server -46:01.841 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -46:01.841 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -46:01.841 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -46:01.841 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -46:01.841 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -46:01.841 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -46:01.841 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -46:01.842 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -46:01.842 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -46:01.842 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -46:01.842 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:46:01.842Z [INFO] agent.router.manager: shutting down -2022-09-15T21:46:01.842Z [INFO] agent.router.manager: shutting down -46:01.842 [INFO] TestAgent: consul server down -46:01.842 [INFO] TestAgent: shutdown complete -46:01.842 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16134 network=tcp -46:01.842 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16134 network=udp -46:01.842 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16135 network=tcp -46:02.342 [INFO] TestAgent: Waiting for endpoints to shut down -46:02.342 [INFO] TestAgent: Endpoints down ---- PASS: TestMembersCommand_statusFilter (0.95s) -2022-09-15T21:46:01.484Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:46:01.509Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -46:01.516 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:279278ae-8f44-4187-e864-06c7404458b0 Address:127.0.0.1:16153}]" -46:01.517 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16153 [Follower]" leader= -46:01.523 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-279278ae-8f44-4187-e864-06c7404458b0.dc1 127.0.0.1 -46:01.524 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-279278ae-8f44-4187-e864-06c7404458b0 127.0.0.1 -2022-09-15T21:46:01.524Z [INFO] agent.router: Initializing LAN area manager -46:01.524 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16148 network=udp -46:01.525 [INFO] TestAgent.server: Adding LAN server: server="Node-279278ae-8f44-4187-e864-06c7404458b0 (Addr: tcp/127.0.0.1:16153) (DC: dc1)" -46:01.525 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-279278ae-8f44-4187-e864-06c7404458b0.dc1 area=wan -46:01.525 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16148 network=tcp -46:01.526 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16149 network=tcp -46:01.526 [INFO] TestAgent: started state syncer -46:01.526 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16154 network=tcp -46:01.559 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -46:01.559 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16153 [Candidate]" term=2 -46:01.559 [DEBUG] TestAgent.server.raft: votes: needed=1 -46:01.559 [DEBUG] TestAgent.server.raft: vote granted: from=279278ae-8f44-4187-e864-06c7404458b0 term=2 tally=1 -46:01.559 [INFO] TestAgent.server.raft: election won: tally=1 -46:01.559 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16153 [Leader]" -46:01.559 [INFO] TestAgent.server: cluster leadership acquired -46:01.559 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16153 -46:01.560 [INFO] TestAgent.server: New leader elected: payload=Node-279278ae-8f44-4187-e864-06c7404458b0 -46:01.561 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -46:01.585 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -46:01.585 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -46:01.585 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -46:01.585 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -46:01.585 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -46:01.585 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-279278ae-8f44-4187-e864-06c7404458b0 -46:01.585 [INFO] TestAgent.server: member joined, marking health alive: member=Node-279278ae-8f44-4187-e864-06c7404458b0 -46:01.587 [INFO] TestAgent.server: federation state anti-entropy synced -46:01.669 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -46:01.669 [INFO] TestAgent: Synced node info -46:01.832 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/members?segment=_all&wan=1 from=127.0.0.1:45988 latency=295.62µs -46:01.833 [INFO] TestAgent: Requesting shutdown -46:01.833 [INFO] TestAgent.server: shutting down server -46:01.833 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -46:01.833 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -46:01.833 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -46:01.833 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -46:01.833 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -46:01.833 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -46:01.833 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -46:01.833 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -46:01.833 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -46:01.833 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -46:01.833 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:46:01.833Z [INFO] agent.router.manager: shutting down -2022-09-15T21:46:01.833Z [INFO] agent.router.manager: shutting down -46:01.846 [INFO] TestAgent: consul server down -46:01.846 [INFO] TestAgent: shutdown complete -46:01.846 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16148 network=tcp -46:01.846 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16148 network=udp -46:01.846 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16149 network=tcp -46:02.346 [INFO] TestAgent: Waiting for endpoints to shut down -46:02.346 [INFO] TestAgent: Endpoints down ---- PASS: TestMembersCommand_WAN (0.96s) -2022-09-15T21:46:01.464Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:46:01.534Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -46:01.546 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d557aa82-ec40-92b3-e717-ea0f1cd45692 Address:127.0.0.1:16146}]" -46:01.547 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d557aa82-ec40-92b3-e717-ea0f1cd45692.dc1 127.0.0.1 -46:01.547 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16146 [Follower]" leader= -46:01.548 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d557aa82-ec40-92b3-e717-ea0f1cd45692 127.0.0.1 -2022-09-15T21:46:01.548Z [INFO] agent.router: Initializing LAN area manager -46:01.548 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16141 network=udp -46:01.548 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d557aa82-ec40-92b3-e717-ea0f1cd45692.dc1 area=wan -46:01.548 [INFO] TestAgent.server: Adding LAN server: server="Node-d557aa82-ec40-92b3-e717-ea0f1cd45692 (Addr: tcp/127.0.0.1:16146) (DC: dc1)" -46:01.549 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16141 network=tcp -46:01.549 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16142 network=tcp -46:01.550 [INFO] TestAgent: started state syncer -46:01.550 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16147 network=tcp -46:01.615 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -46:01.615 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16146 [Candidate]" term=2 -46:01.615 [DEBUG] TestAgent.server.raft: votes: needed=1 -46:01.615 [DEBUG] TestAgent.server.raft: vote granted: from=d557aa82-ec40-92b3-e717-ea0f1cd45692 term=2 tally=1 -46:01.615 [INFO] TestAgent.server.raft: election won: tally=1 -46:01.615 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16146 [Leader]" -46:01.615 [INFO] TestAgent.server: cluster leadership acquired -46:01.616 [INFO] TestAgent.server: New leader elected: payload=Node-d557aa82-ec40-92b3-e717-ea0f1cd45692 -46:01.616 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16146 -46:01.618 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -46:01.621 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -46:01.621 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -46:01.621 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -46:01.621 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -46:01.621 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -46:01.621 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d557aa82-ec40-92b3-e717-ea0f1cd45692 -46:01.621 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d557aa82-ec40-92b3-e717-ea0f1cd45692 -46:01.622 [INFO] TestAgent.server: federation state anti-entropy synced -46:01.656 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -46:01.657 [INFO] TestAgent: Synced node info -46:01.941 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/members?segment=_all from=127.0.0.1:36462 latency=236.28µs -46:01.942 [INFO] TestAgent: Requesting shutdown -46:01.942 [INFO] TestAgent.server: shutting down server -46:01.942 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -46:01.942 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -46:01.942 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -46:01.942 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -46:01.942 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -46:01.942 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -46:01.942 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -46:01.942 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -46:01.942 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -46:01.942 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -46:01.942 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:46:01.942Z [INFO] agent.router.manager: shutting down -2022-09-15T21:46:01.942Z [INFO] agent.router.manager: shutting down -46:01.942 [INFO] TestAgent: consul server down -46:01.942 [INFO] TestAgent: shutdown complete -46:01.942 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16141 network=tcp -46:01.943 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16141 network=udp -46:01.943 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16142 network=tcp -46:02.446 [INFO] TestAgent: Waiting for endpoints to shut down -46:02.446 [INFO] TestAgent: Endpoints down ---- PASS: TestMembersCommand_statusFilter_failed (1.06s) +2021-08-13T15:38:34.356Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:38:34.399Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:34.402 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:dddb04f1-b38a-7e76-b577-fa8cfad204b1 Address:127.0.0.1:31447}]" +38:34.403 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-dddb04f1-b38a-7e76-b577-fa8cfad204b1.dc1 127.0.0.1 +38:34.403 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31447 [Follower]" leader= +38:34.404 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-dddb04f1-b38a-7e76-b577-fa8cfad204b1 127.0.0.1 +2021-08-13T15:38:34.404Z [INFO] agent.router: Initializing LAN area manager +38:34.404 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-dddb04f1-b38a-7e76-b577-fa8cfad204b1.dc1 area=wan +38:34.410 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31442 network=tcp +38:34.411 [INFO] TestAgent.server: Adding LAN server: server="Node-dddb04f1-b38a-7e76-b577-fa8cfad204b1 (Addr: tcp/127.0.0.1:31447) (DC: dc1)" +38:34.411 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31442 network=udp +38:34.412 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31443 network=tcp +38:34.413 [INFO] TestAgent: started state syncer +38:34.413 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31448 network=tcp +38:34.458 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +38:34.458 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31447 [Candidate]" term=2 +38:34.459 [DEBUG] TestAgent.server.raft: votes: needed=1 +38:34.459 [DEBUG] TestAgent.server.raft: vote granted: from=dddb04f1-b38a-7e76-b577-fa8cfad204b1 term=2 tally=1 +38:34.459 [INFO] TestAgent.server.raft: election won: tally=1 +38:34.459 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31447 [Leader]" +38:34.459 [INFO] TestAgent.server: cluster leadership acquired +38:34.459 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:31447 +38:34.459 [INFO] TestAgent.server: New leader elected: payload=Node-dddb04f1-b38a-7e76-b577-fa8cfad204b1 +38:34.464 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +38:34.485 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +38:34.485 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +38:34.485 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +38:34.485 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +38:34.485 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +38:34.485 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-dddb04f1-b38a-7e76-b577-fa8cfad204b1 +38:34.485 [INFO] TestAgent.server: member joined, marking health alive: member=Node-dddb04f1-b38a-7e76-b577-fa8cfad204b1 +38:34.488 [INFO] TestAgent.server: federation state anti-entropy synced +38:34.740 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/members?segment=_all from=127.0.0.1:60918 latency=229.22µs +38:34.741 [INFO] TestAgent: Requesting shutdown +38:34.741 [INFO] TestAgent.server: shutting down server +38:34.741 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +38:34.741 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +38:34.741 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +38:34.741 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +38:34.741 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +38:34.741 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +38:34.741 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +38:34.741 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +38:34.741 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +38:34.741 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +38:34.741 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +38:34.741 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:38:34.741Z [INFO] agent.router.manager: shutting down +2021-08-13T15:38:34.741Z [INFO] agent.router.manager: shutting down +38:34.741 [INFO] TestAgent: consul server down +38:34.741 [INFO] TestAgent: shutdown complete +38:34.741 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31442 network=tcp +38:34.741 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31442 network=udp +38:34.741 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31443 network=tcp +38:35.241 [INFO] TestAgent: Waiting for endpoints to shut down +38:35.241 [INFO] TestAgent: Endpoints down +--- PASS: TestMembersCommand_statusFilter_failed (0.95s) +2021-08-13T15:38:34.395Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:38:34.418Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:34.421 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:66f4c1bd-e363-b868-fe22-6fe3d413bda9 Address:127.0.0.1:31454}]" +38:34.422 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-66f4c1bd-e363-b868-fe22-6fe3d413bda9.dc1 127.0.0.1 +38:34.422 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-66f4c1bd-e363-b868-fe22-6fe3d413bda9 127.0.0.1 +2021-08-13T15:38:34.422Z [INFO] agent.router: Initializing LAN area manager +38:34.423 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31449 network=udp +38:34.423 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31454 [Follower]" leader= +38:34.423 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-66f4c1bd-e363-b868-fe22-6fe3d413bda9.dc1 area=wan +38:34.423 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31449 network=tcp +38:34.423 [INFO] TestAgent.server: Adding LAN server: server="Node-66f4c1bd-e363-b868-fe22-6fe3d413bda9 (Addr: tcp/127.0.0.1:31454) (DC: dc1)" +38:34.433 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31450 network=tcp +38:34.433 [INFO] TestAgent: started state syncer +38:34.433 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31455 network=tcp +38:34.493 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +38:34.493 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31454 [Candidate]" term=2 +38:34.494 [DEBUG] TestAgent.server.raft: votes: needed=1 +38:34.494 [DEBUG] TestAgent.server.raft: vote granted: from=66f4c1bd-e363-b868-fe22-6fe3d413bda9 term=2 tally=1 +38:34.494 [INFO] TestAgent.server.raft: election won: tally=1 +38:34.494 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31454 [Leader]" +38:34.494 [INFO] TestAgent.server: cluster leadership acquired +38:34.494 [INFO] TestAgent.server: New leader elected: payload=Node-66f4c1bd-e363-b868-fe22-6fe3d413bda9 +38:34.494 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:31454 +38:34.495 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +38:34.497 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +38:34.497 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +38:34.497 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +38:34.497 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +38:34.497 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +38:34.497 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-66f4c1bd-e363-b868-fe22-6fe3d413bda9 +38:34.497 [INFO] TestAgent.server: member joined, marking health alive: member=Node-66f4c1bd-e363-b868-fe22-6fe3d413bda9 +38:34.498 [INFO] TestAgent.server: federation state anti-entropy synced +38:34.685 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +38:34.686 [INFO] TestAgent: Synced node info +38:34.686 [DEBUG] TestAgent: Node info in sync +38:34.778 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/members?segment=_all&wan=1 from=127.0.0.1:55376 latency=226.08µs +38:34.778 [INFO] TestAgent: Requesting shutdown +38:34.778 [INFO] TestAgent.server: shutting down server +38:34.778 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +38:34.778 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +38:34.778 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +38:34.778 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +38:34.778 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +38:34.778 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +38:34.779 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +38:34.779 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +38:34.779 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +38:34.779 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +38:34.779 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:38:34.779Z [INFO] agent.router.manager: shutting down +2021-08-13T15:38:34.779Z [INFO] agent.router.manager: shutting down +38:34.779 [INFO] TestAgent: consul server down +38:34.779 [INFO] TestAgent: shutdown complete +38:34.779 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31449 network=tcp +38:34.779 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31449 network=udp +38:34.779 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31450 network=tcp +38:35.279 [INFO] TestAgent: Waiting for endpoints to shut down +38:35.279 [INFO] TestAgent: Endpoints down +--- PASS: TestMembersCommand_WAN (0.98s) PASS -ok github.com/hashicorp/consul/command/members 1.212s +ok github.com/hashicorp/consul/command/members 1.050s === RUN TestOperatorCommand_noTabs === PAUSE TestOperatorCommand_noTabs === CONT TestOperatorCommand_noTabs --- PASS: TestOperatorCommand_noTabs (0.00s) PASS -ok github.com/hashicorp/consul/command/operator 0.044s +ok github.com/hashicorp/consul/command/operator 0.006s === RUN TestOperatorAutopilotCommand_noTabs === PAUSE TestOperatorAutopilotCommand_noTabs === CONT TestOperatorAutopilotCommand_noTabs --- PASS: TestOperatorAutopilotCommand_noTabs (0.00s) PASS -ok github.com/hashicorp/consul/command/operator/autopilot 0.018s +ok github.com/hashicorp/consul/command/operator/autopilot 0.006s === RUN TestOperatorAutopilotGetConfigCommand_noTabs === PAUSE TestOperatorAutopilotGetConfigCommand_noTabs === RUN TestOperatorAutopilotGetConfigCommand === PAUSE TestOperatorAutopilotGetConfigCommand === CONT TestOperatorAutopilotGetConfigCommand_noTabs ---- PASS: TestOperatorAutopilotGetConfigCommand_noTabs (0.00s) === CONT TestOperatorAutopilotGetConfigCommand [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestOperatorAutopilotGetConfigCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:46:06.817Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:46:06.851Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -46:06.855 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:65d91df7-2abe-a156-9639-e5830c6a9399 Address:127.0.0.1:21237}]" -46:06.856 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-65d91df7-2abe-a156-9639-e5830c6a9399.dc1 127.0.0.1 -46:06.857 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-65d91df7-2abe-a156-9639-e5830c6a9399 127.0.0.1 -2022-09-15T21:46:06.857Z [INFO] agent.router: Initializing LAN area manager -46:06.858 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:21237 [Follower]" leader= -46:06.858 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21232 network=udp -46:06.858 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-65d91df7-2abe-a156-9639-e5830c6a9399.dc1 area=wan -46:06.858 [INFO] TestAgent.server: Adding LAN server: server="Node-65d91df7-2abe-a156-9639-e5830c6a9399 (Addr: tcp/127.0.0.1:21237) (DC: dc1)" -46:06.859 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21232 network=tcp -46:06.860 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:21233 network=tcp -46:06.860 [INFO] TestAgent: started state syncer -46:06.860 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:21238 network=tcp -46:06.920 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -46:06.921 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:21237 [Candidate]" term=2 -46:06.921 [DEBUG] TestAgent.server.raft: votes: needed=1 -46:06.921 [DEBUG] TestAgent.server.raft: vote granted: from=65d91df7-2abe-a156-9639-e5830c6a9399 term=2 tally=1 -46:06.921 [INFO] TestAgent.server.raft: election won: tally=1 -46:06.921 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:21237 [Leader]" -46:06.921 [INFO] TestAgent.server: cluster leadership acquired -46:06.921 [INFO] TestAgent.server: New leader elected: payload=Node-65d91df7-2abe-a156-9639-e5830c6a9399 -46:06.921 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:21237 -46:06.923 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -46:06.929 [INFO] TestAgent: Synced node info -46:06.949 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -46:06.949 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -46:06.949 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -46:06.949 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -46:06.949 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -46:06.950 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-65d91df7-2abe-a156-9639-e5830c6a9399 -46:06.950 [INFO] TestAgent.server: member joined, marking health alive: member=Node-65d91df7-2abe-a156-9639-e5830c6a9399 -46:06.950 [INFO] TestAgent.server: federation state anti-entropy synced -46:07.293 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/autopilot/configuration from=127.0.0.1:57644 latency=374.7µs -46:07.294 [INFO] TestAgent: Requesting shutdown -46:07.294 [INFO] TestAgent.server: shutting down server -46:07.294 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -46:07.294 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -46:07.294 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -46:07.294 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -46:07.294 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -46:07.294 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -46:07.294 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -46:07.294 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -46:07.294 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -46:07.294 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -46:07.294 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:46:07.294Z [INFO] agent.router.manager: shutting down -2022-09-15T21:46:07.294Z [INFO] agent.router.manager: shutting down -46:07.294 [INFO] TestAgent: consul server down -46:07.294 [INFO] TestAgent: shutdown complete -46:07.294 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21232 network=tcp -46:07.294 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21232 network=udp -46:07.295 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:21233 network=tcp -46:07.798 [INFO] TestAgent: Waiting for endpoints to shut down -46:07.798 [INFO] TestAgent: Endpoints down ---- PASS: TestOperatorAutopilotGetConfigCommand (1.09s) +2021-08-13T15:38:46.572Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:38:46.592Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:46.595 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3b1acea0-df4b-26ab-88dc-b8ac42839f01 Address:127.0.0.1:19195}]" +38:46.596 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19195 [Follower]" leader= +38:46.596 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3b1acea0-df4b-26ab-88dc-b8ac42839f01.dc1 127.0.0.1 +38:46.597 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3b1acea0-df4b-26ab-88dc-b8ac42839f01 127.0.0.1 +2021-08-13T15:38:46.597Z [INFO] agent.router: Initializing LAN area manager +38:46.598 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3b1acea0-df4b-26ab-88dc-b8ac42839f01.dc1 area=wan +38:46.598 [INFO] TestAgent.server: Adding LAN server: server="Node-3b1acea0-df4b-26ab-88dc-b8ac42839f01 (Addr: tcp/127.0.0.1:19195) (DC: dc1)" +38:46.598 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=udp +38:46.598 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=tcp +38:46.599 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19191 network=tcp +38:46.599 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19196 network=tcp +38:46.599 [INFO] TestAgent: started state syncer +38:46.640 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +38:46.640 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19195 [Candidate]" term=2 +38:46.640 [DEBUG] TestAgent.server.raft: votes: needed=1 +38:46.640 [DEBUG] TestAgent.server.raft: vote granted: from=3b1acea0-df4b-26ab-88dc-b8ac42839f01 term=2 tally=1 +38:46.640 [INFO] TestAgent.server.raft: election won: tally=1 +38:46.640 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19195 [Leader]" +38:46.641 [INFO] TestAgent.server: cluster leadership acquired +38:46.641 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:19195 +38:46.642 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +38:46.647 [INFO] TestAgent.server: New leader elected: payload=Node-3b1acea0-df4b-26ab-88dc-b8ac42839f01 +38:46.666 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +38:46.666 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +38:46.666 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +38:46.666 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +38:46.666 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +38:46.666 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3b1acea0-df4b-26ab-88dc-b8ac42839f01 +38:46.666 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3b1acea0-df4b-26ab-88dc-b8ac42839f01 +38:46.667 [INFO] TestAgent.server: federation state anti-entropy synced +38:46.688 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/autopilot/configuration from=127.0.0.1:36020 latency=1.356162ms +38:46.691 [INFO] TestAgent: Requesting shutdown +38:46.691 [INFO] TestAgent.server: shutting down server +38:46.691 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +38:46.692 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +38:46.692 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +38:46.692 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +38:46.692 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +38:46.692 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +38:46.692 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +38:46.692 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +38:46.692 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +38:46.692 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +38:46.692 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +38:46.692 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:38:46.692Z [INFO] agent.router.manager: shutting down +2021-08-13T15:38:46.692Z [INFO] agent.router.manager: shutting down +38:46.692 [INFO] TestAgent: consul server down +38:46.692 [INFO] TestAgent: shutdown complete +38:46.692 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=tcp +38:46.692 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=udp +38:46.692 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19191 network=tcp +38:47.192 [INFO] TestAgent: Waiting for endpoints to shut down +38:47.192 [INFO] TestAgent: Endpoints down +--- PASS: TestOperatorAutopilotGetConfigCommand (0.67s) PASS -ok github.com/hashicorp/consul/command/operator/autopilot/get 1.276s +ok github.com/hashicorp/consul/command/operator/autopilot/get 0.738s === RUN TestOperatorAutopilotSetConfigCommand_noTabs === PAUSE TestOperatorAutopilotSetConfigCommand_noTabs === RUN TestOperatorAutopilotSetConfigCommand @@ -90127,77 +90026,75 @@ === CONT TestOperatorAutopilotSetConfigCommand_noTabs === CONT TestOperatorAutopilotSetConfigCommand [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... ---- PASS: TestOperatorAutopilotSetConfigCommand_noTabs (0.01s) +--- PASS: TestOperatorAutopilotSetConfigCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:46:07.849Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:46:07.876Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -46:07.880 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ed69c3cd-b34e-5cbf-e40f-82ede1c3ced7 Address:127.0.0.1:20216}]" -46:07.880 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:20216 [Follower]" leader= -46:07.881 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ed69c3cd-b34e-5cbf-e40f-82ede1c3ced7.dc1 127.0.0.1 -46:07.882 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ed69c3cd-b34e-5cbf-e40f-82ede1c3ced7 127.0.0.1 -2022-09-15T21:46:07.882Z [INFO] agent.router: Initializing LAN area manager -46:07.883 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20211 network=udp -46:07.883 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ed69c3cd-b34e-5cbf-e40f-82ede1c3ced7.dc1 area=wan -46:07.883 [INFO] TestAgent.server: Adding LAN server: server="Node-ed69c3cd-b34e-5cbf-e40f-82ede1c3ced7 (Addr: tcp/127.0.0.1:20216) (DC: dc1)" -46:07.883 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20211 network=tcp -46:07.884 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:20212 network=tcp -46:07.884 [INFO] TestAgent: started state syncer -46:07.884 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:20217 network=tcp -46:07.918 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -46:07.918 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:20216 [Candidate]" term=2 -46:07.919 [DEBUG] TestAgent.server.raft: votes: needed=1 -46:07.919 [DEBUG] TestAgent.server.raft: vote granted: from=ed69c3cd-b34e-5cbf-e40f-82ede1c3ced7 term=2 tally=1 -46:07.919 [INFO] TestAgent.server.raft: election won: tally=1 -46:07.919 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:20216 [Leader]" -46:07.919 [INFO] TestAgent.server: cluster leadership acquired -46:07.919 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:20216 -46:07.921 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -46:07.922 [INFO] TestAgent.server: New leader elected: payload=Node-ed69c3cd-b34e-5cbf-e40f-82ede1c3ced7 -46:07.945 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -46:07.945 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -46:07.945 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -46:07.945 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -46:07.945 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -46:07.945 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ed69c3cd-b34e-5cbf-e40f-82ede1c3ced7 -46:07.945 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ed69c3cd-b34e-5cbf-e40f-82ede1c3ced7 -46:07.947 [INFO] TestAgent.server: federation state anti-entropy synced -46:07.986 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -46:07.986 [INFO] TestAgent: Synced node info -46:07.986 [DEBUG] TestAgent: Node info in sync -46:08.125 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/autopilot/configuration from=127.0.0.1:56736 latency=528.1µs -46:08.127 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/operator/autopilot/configuration?cas=4 from=127.0.0.1:56736 latency=634.86µs -46:08.128 [INFO] TestAgent: Requesting shutdown -46:08.128 [INFO] TestAgent.server: shutting down server -46:08.128 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -46:08.128 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -46:08.128 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -46:08.128 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -46:08.128 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -46:08.128 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -46:08.128 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -46:08.128 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -46:08.128 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -46:08.128 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -46:08.128 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:46:08.128Z [INFO] agent.router.manager: shutting down -2022-09-15T21:46:08.128Z [INFO] agent.router.manager: shutting down -46:08.129 [INFO] TestAgent: consul server down -46:08.129 [INFO] TestAgent: shutdown complete -46:08.129 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20211 network=tcp -46:08.129 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20211 network=udp -46:08.129 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:20212 network=tcp -46:08.629 [INFO] TestAgent: Waiting for endpoints to shut down -46:08.629 [INFO] TestAgent: Endpoints down ---- PASS: TestOperatorAutopilotSetConfigCommand (0.87s) +2021-08-13T15:38:41.414Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:38:41.432Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:41.437 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c9ed7af1-8319-6803-d3c6-1ab19ee0be27 Address:127.0.0.1:19195}]" +38:41.437 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19195 [Follower]" leader= +38:41.438 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c9ed7af1-8319-6803-d3c6-1ab19ee0be27.dc1 127.0.0.1 +38:41.438 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c9ed7af1-8319-6803-d3c6-1ab19ee0be27 127.0.0.1 +2021-08-13T15:38:41.439Z [INFO] agent.router: Initializing LAN area manager +38:41.439 [INFO] TestAgent.server: Adding LAN server: server="Node-c9ed7af1-8319-6803-d3c6-1ab19ee0be27 (Addr: tcp/127.0.0.1:19195) (DC: dc1)" +38:41.439 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c9ed7af1-8319-6803-d3c6-1ab19ee0be27.dc1 area=wan +38:41.439 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=tcp +38:41.439 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=udp +38:41.440 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19191 network=tcp +38:41.440 [INFO] TestAgent: started state syncer +38:41.440 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19196 network=tcp +38:41.480 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +38:41.480 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19195 [Candidate]" term=2 +38:41.481 [DEBUG] TestAgent.server.raft: votes: needed=1 +38:41.481 [DEBUG] TestAgent.server.raft: vote granted: from=c9ed7af1-8319-6803-d3c6-1ab19ee0be27 term=2 tally=1 +38:41.481 [INFO] TestAgent.server.raft: election won: tally=1 +38:41.481 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19195 [Leader]" +38:41.481 [INFO] TestAgent.server: cluster leadership acquired +38:41.481 [INFO] TestAgent.server: New leader elected: payload=Node-c9ed7af1-8319-6803-d3c6-1ab19ee0be27 +38:41.481 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:19195 +38:41.483 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +38:41.506 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +38:41.506 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +38:41.506 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +38:41.506 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +38:41.506 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +38:41.506 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c9ed7af1-8319-6803-d3c6-1ab19ee0be27 +38:41.506 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c9ed7af1-8319-6803-d3c6-1ab19ee0be27 +38:41.507 [INFO] TestAgent.server: federation state anti-entropy synced +38:41.535 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/autopilot/configuration from=127.0.0.1:36018 latency=1.759942ms +38:41.540 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/operator/autopilot/configuration?cas=4 from=127.0.0.1:36018 latency=808.101µs +38:41.541 [INFO] TestAgent: Requesting shutdown +38:41.541 [INFO] TestAgent.server: shutting down server +38:41.541 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +38:41.541 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +38:41.541 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +38:41.541 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +38:41.541 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +38:41.541 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +38:41.541 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +38:41.541 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +38:41.541 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +38:41.541 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +38:41.541 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +38:41.541 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:38:41.541Z [INFO] agent.router.manager: shutting down +2021-08-13T15:38:41.541Z [INFO] agent.router.manager: shutting down +38:41.541 [INFO] TestAgent: consul server down +38:41.541 [INFO] TestAgent: shutdown complete +38:41.541 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=tcp +38:41.541 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=udp +38:41.542 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19191 network=tcp +38:42.042 [INFO] TestAgent: Waiting for endpoints to shut down +38:42.042 [INFO] TestAgent: Endpoints down +--- PASS: TestOperatorAutopilotSetConfigCommand (0.68s) PASS -ok github.com/hashicorp/consul/command/operator/autopilot/set 0.995s +ok github.com/hashicorp/consul/command/operator/autopilot/set 0.742s === RUN TestOperatorRaftCommand_noTabs === PAUSE TestOperatorRaftCommand_noTabs === CONT TestOperatorRaftCommand_noTabs --- PASS: TestOperatorRaftCommand_noTabs (0.00s) PASS -ok github.com/hashicorp/consul/command/operator/raft 0.024s +ok github.com/hashicorp/consul/command/operator/raft 0.006s === RUN TestOperatorRaftListPeersCommand_noTabs === PAUSE TestOperatorRaftListPeersCommand_noTabs === RUN TestOperatorRaftListPeersCommand @@ -90206,251 +90103,268 @@ === PAUSE TestOperatorRaftListPeersCommand_verticalBar === CONT TestOperatorRaftListPeersCommand_noTabs === CONT TestOperatorRaftListPeersCommand_verticalBar +--- PASS: TestOperatorRaftListPeersCommand_noTabs (0.00s) === CONT TestOperatorRaftListPeersCommand [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... ---- PASS: TestOperatorRaftListPeersCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:46:15.417Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:46:15.443Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -46:15.448 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:17796bcd-b8fe-94c3-6023-598381728bfc Address:127.0.0.1:22258}]" -46:15.450 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22258 [Follower]" leader= -46:15.451 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-17796bcd-b8fe-94c3-6023-598381728bfc.dc1 127.0.0.1 -46:15.459 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-17796bcd-b8fe-94c3-6023-598381728bfc 127.0.0.1 -2022-09-15T21:46:15.459Z [INFO] agent.router: Initializing LAN area manager -46:15.460 [INFO] TestAgent.server: Adding LAN server: server="Node-17796bcd-b8fe-94c3-6023-598381728bfc (Addr: tcp/127.0.0.1:22258) (DC: dc1)" -46:15.460 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-17796bcd-b8fe-94c3-6023-598381728bfc.dc1 area=wan -46:15.461 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22253 network=tcp -46:15.461 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22253 network=udp -46:15.462 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22254 network=tcp -46:15.462 [INFO] TestAgent: started state syncer -46:15.462 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22259 network=tcp -46:15.500 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -46:15.500 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22258 [Candidate]" term=2 -46:15.500 [DEBUG] TestAgent.server.raft: votes: needed=1 -46:15.500 [DEBUG] TestAgent.server.raft: vote granted: from=17796bcd-b8fe-94c3-6023-598381728bfc term=2 tally=1 -46:15.500 [INFO] TestAgent.server.raft: election won: tally=1 -46:15.500 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22258 [Leader]" -46:15.501 [INFO] TestAgent.server: cluster leadership acquired -46:15.501 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22258 -46:15.502 [INFO] TestAgent.server: New leader elected: payload=Node-17796bcd-b8fe-94c3-6023-598381728bfc -46:15.502 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -46:15.532 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -46:15.532 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -46:15.532 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -46:15.532 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -46:15.532 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -46:15.533 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-17796bcd-b8fe-94c3-6023-598381728bfc -46:15.533 [INFO] TestAgent.server: member joined, marking health alive: member=Node-17796bcd-b8fe-94c3-6023-598381728bfc -46:15.534 [INFO] TestAgent.server: federation state anti-entropy synced -46:15.579 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/raft/configuration from=127.0.0.1:49884 latency=2.27532ms -46:15.580 [INFO] TestAgent: Requesting shutdown -46:15.580 [INFO] TestAgent.server: shutting down server -46:15.580 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -46:15.580 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -46:15.580 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -46:15.580 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -46:15.580 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -46:15.580 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -46:15.580 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -46:15.580 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -46:15.580 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -46:15.580 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -46:15.580 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -46:15.580 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:46:15.581Z [INFO] agent.router.manager: shutting down -2022-09-15T21:46:15.581Z [INFO] agent.router.manager: shutting down -46:15.581 [INFO] TestAgent: consul server down -46:15.581 [INFO] TestAgent: shutdown complete -46:15.581 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22253 network=tcp -46:15.581 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22253 network=udp -46:15.581 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22254 network=tcp -46:16.082 [INFO] TestAgent: Waiting for endpoints to shut down -46:16.082 [INFO] TestAgent: Endpoints down ---- PASS: TestOperatorRaftListPeersCommand (0.73s) -2022-09-15T21:46:15.424Z [WARN] agent: Node name "name|with|bars" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. -2022-09-15T21:46:15.424Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:46:15.451Z [WARN] agent.auto_config: Node name "name|with|bars" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. -2022-09-15T21:46:15.451Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -46:15.461 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e0abfa21-8001-69a7-7e68-eeb1192661c3 Address:127.0.0.1:22265}]" -46:15.462 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22265 [Follower]" leader= -46:15.463 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: name|with|bars.dc1 127.0.0.1 -46:15.464 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: name|with|bars 127.0.0.1 -2022-09-15T21:46:15.464Z [INFO] agent.router: Initializing LAN area manager -46:15.464 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22260 network=udp -46:15.465 [INFO] TestAgent.server: Adding LAN server: server="name|with|bars (Addr: tcp/127.0.0.1:22265) (DC: dc1)" -46:15.465 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=name|with|bars.dc1 area=wan -46:15.465 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22260 network=tcp -46:15.467 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22261 network=tcp -46:15.467 [INFO] TestAgent: started state syncer -46:15.467 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22266 network=tcp -46:15.531 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -46:15.531 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22265 [Candidate]" term=2 -46:15.531 [DEBUG] TestAgent.server.raft: votes: needed=1 -46:15.531 [DEBUG] TestAgent.server.raft: vote granted: from=e0abfa21-8001-69a7-7e68-eeb1192661c3 term=2 tally=1 -46:15.531 [INFO] TestAgent.server.raft: election won: tally=1 -46:15.531 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22265 [Leader]" -46:15.531 [INFO] TestAgent.server: cluster leadership acquired -46:15.531 [INFO] TestAgent.server: New leader elected: payload=name|with|bars -46:15.532 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22265 -46:15.533 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -46:15.535 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -46:15.535 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -46:15.535 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -46:15.535 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -46:15.535 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -46:15.535 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=name|with|bars -46:15.535 [INFO] TestAgent.server: member joined, marking health alive: member=name|with|bars -46:15.537 [INFO] TestAgent.server: federation state anti-entropy synced -46:15.619 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/raft/configuration from=127.0.0.1:34678 latency=436.86µs -46:15.620 [INFO] TestAgent: Requesting shutdown -46:15.620 [INFO] TestAgent.server: shutting down server -46:15.620 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -46:15.620 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -46:15.620 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -46:15.620 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -46:15.620 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -46:15.620 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -46:15.620 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -46:15.620 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -46:15.620 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -46:15.620 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -46:15.620 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -46:15.620 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:46:15.620Z [INFO] agent.router.manager: shutting down -2022-09-15T21:46:15.620Z [INFO] agent.router.manager: shutting down -46:15.621 [INFO] TestAgent: consul server down -46:15.621 [INFO] TestAgent: shutdown complete -46:15.621 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22260 network=tcp -46:15.621 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22260 network=udp -46:15.621 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22261 network=tcp -46:16.122 [INFO] TestAgent: Waiting for endpoints to shut down -46:16.122 [INFO] TestAgent: Endpoints down +2021-08-13T15:38:56.724Z [WARN] agent: Node name "name|with|bars" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. +2021-08-13T15:38:56.724Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:38:56.768Z [WARN] agent.auto_config: Node name "name|with|bars" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. +2021-08-13T15:38:56.768Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:56.771 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:19fadcb4-aa4a-63fa-03c1-60db068cf7a5 Address:127.0.0.1:12055}]" +38:56.772 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: name|with|bars.dc1 127.0.0.1 +38:56.773 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: name|with|bars 127.0.0.1 +2021-08-13T15:38:56.773Z [INFO] agent.router: Initializing LAN area manager +38:56.773 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12050 network=udp +38:56.773 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:12055 [Follower]" leader= +38:56.774 [INFO] TestAgent.server: Adding LAN server: server="name|with|bars (Addr: tcp/127.0.0.1:12055) (DC: dc1)" +38:56.774 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=name|with|bars.dc1 area=wan +38:56.774 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12050 network=tcp +38:56.775 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:12051 network=tcp +38:56.775 [INFO] TestAgent: started state syncer +38:56.775 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:12056 network=tcp +38:56.831 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +38:56.831 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:12055 [Candidate]" term=2 +38:56.831 [DEBUG] TestAgent.server.raft: votes: needed=1 +38:56.831 [DEBUG] TestAgent.server.raft: vote granted: from=19fadcb4-aa4a-63fa-03c1-60db068cf7a5 term=2 tally=1 +38:56.831 [INFO] TestAgent.server.raft: election won: tally=1 +38:56.831 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:12055 [Leader]" +38:56.831 [INFO] TestAgent.server: cluster leadership acquired +38:56.832 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:12055 +38:56.832 [INFO] TestAgent.server: New leader elected: payload=name|with|bars +38:56.839 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +38:56.842 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +38:56.843 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +38:56.843 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +38:56.843 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +38:56.843 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +38:56.843 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=name|with|bars +38:56.843 [INFO] TestAgent.server: member joined, marking health alive: member=name|with|bars +38:56.844 [INFO] TestAgent.server: federation state anti-entropy synced +38:56.900 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/raft/configuration from=127.0.0.1:59446 latency=512.6µs +38:56.901 [INFO] TestAgent: Requesting shutdown +38:56.901 [INFO] TestAgent.server: shutting down server +38:56.901 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +38:56.901 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +38:56.901 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +38:56.901 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +38:56.901 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +38:56.901 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +38:56.901 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +38:56.901 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +38:56.901 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +38:56.901 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +38:56.901 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +38:56.901 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:38:56.901Z [INFO] agent.router.manager: shutting down +2021-08-13T15:38:56.901Z [INFO] agent.router.manager: shutting down +38:56.901 [INFO] TestAgent: consul server down +38:56.901 [INFO] TestAgent: shutdown complete +38:56.901 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12050 network=tcp +38:56.901 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12050 network=udp +38:56.901 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:12051 network=tcp +38:57.402 [INFO] TestAgent: Waiting for endpoints to shut down +38:57.402 [INFO] TestAgent: Endpoints down --- PASS: TestOperatorRaftListPeersCommand_verticalBar (0.77s) +2021-08-13T15:38:56.712Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:38:56.731Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:56.746 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:224d8a6a-4a5c-efe7-9929-15329504fa71 Address:127.0.0.1:12048}]" +38:56.747 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:12048 [Follower]" leader= +38:56.749 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-224d8a6a-4a5c-efe7-9929-15329504fa71.dc1 127.0.0.1 +38:56.750 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-224d8a6a-4a5c-efe7-9929-15329504fa71 127.0.0.1 +2021-08-13T15:38:56.750Z [INFO] agent.router: Initializing LAN area manager +38:56.751 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12043 network=tcp +38:56.751 [INFO] TestAgent.server: Adding LAN server: server="Node-224d8a6a-4a5c-efe7-9929-15329504fa71 (Addr: tcp/127.0.0.1:12048) (DC: dc1)" +38:56.751 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12043 network=udp +38:56.756 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-224d8a6a-4a5c-efe7-9929-15329504fa71.dc1 area=wan +38:56.757 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:12044 network=tcp +38:56.757 [INFO] TestAgent: started state syncer +38:56.757 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:12049 network=tcp +38:56.806 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +38:56.807 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:12048 [Candidate]" term=2 +38:56.807 [DEBUG] TestAgent.server.raft: votes: needed=1 +38:56.807 [DEBUG] TestAgent.server.raft: vote granted: from=224d8a6a-4a5c-efe7-9929-15329504fa71 term=2 tally=1 +38:56.807 [INFO] TestAgent.server.raft: election won: tally=1 +38:56.807 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:12048 [Leader]" +38:56.807 [INFO] TestAgent.server: cluster leadership acquired +38:56.807 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:12048 +38:56.809 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +38:56.811 [INFO] TestAgent.server: New leader elected: payload=Node-224d8a6a-4a5c-efe7-9929-15329504fa71 +38:56.832 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +38:56.832 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +38:56.832 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +38:56.832 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +38:56.832 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +38:56.832 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-224d8a6a-4a5c-efe7-9929-15329504fa71 +38:56.832 [INFO] TestAgent.server: member joined, marking health alive: member=Node-224d8a6a-4a5c-efe7-9929-15329504fa71 +38:56.843 [INFO] TestAgent.server: federation state anti-entropy synced +38:57.034 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/raft/configuration from=127.0.0.1:48506 latency=335.88µs +38:57.035 [INFO] TestAgent: Requesting shutdown +38:57.035 [INFO] TestAgent.server: shutting down server +38:57.035 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +38:57.035 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +38:57.035 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +38:57.035 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +38:57.035 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +38:57.035 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +38:57.035 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +38:57.035 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +38:57.035 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +38:57.035 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +38:57.035 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +38:57.035 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:38:57.036Z [INFO] agent.router.manager: shutting down +2021-08-13T15:38:57.036Z [INFO] agent.router.manager: shutting down +38:57.036 [INFO] TestAgent: consul server down +38:57.036 [INFO] TestAgent: shutdown complete +38:57.036 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12043 network=tcp +38:57.036 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12043 network=udp +38:57.036 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:12044 network=tcp +38:57.536 [INFO] TestAgent: Waiting for endpoints to shut down +38:57.536 [INFO] TestAgent: Endpoints down +--- PASS: TestOperatorRaftListPeersCommand (0.90s) PASS -ok github.com/hashicorp/consul/command/operator/raft/listpeers 0.857s +ok github.com/hashicorp/consul/command/operator/raft/listpeers 0.993s === RUN TestOperatorRaftRemovePeerCommand_noTabs === PAUSE TestOperatorRaftRemovePeerCommand_noTabs === RUN TestOperatorRaftRemovePeerCommand === PAUSE TestOperatorRaftRemovePeerCommand === CONT TestOperatorRaftRemovePeerCommand_noTabs +--- PASS: TestOperatorRaftRemovePeerCommand_noTabs (0.00s) === CONT TestOperatorRaftRemovePeerCommand [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... ---- PASS: TestOperatorRaftRemovePeerCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range === RUN TestOperatorRaftRemovePeerCommand/Test_the_remove-peer_subcommand_directly === RUN TestOperatorRaftRemovePeerCommand/Test_the_remove-peer_subcommand_with_-id -2022-09-15T21:46:16.816Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:46:16.847Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -46:16.851 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7d38a2af-b419-b755-eb72-d79bd0f13e04 Address:127.0.0.1:29405}]" -46:16.852 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7d38a2af-b419-b755-eb72-d79bd0f13e04.dc1 127.0.0.1 -46:16.852 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7d38a2af-b419-b755-eb72-d79bd0f13e04 127.0.0.1 -2022-09-15T21:46:16.852Z [INFO] agent.router: Initializing LAN area manager -46:16.853 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29400 network=udp -46:16.853 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29405 [Follower]" leader= -46:16.854 [INFO] TestAgent.server: Adding LAN server: server="Node-7d38a2af-b419-b755-eb72-d79bd0f13e04 (Addr: tcp/127.0.0.1:29405) (DC: dc1)" -46:16.854 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7d38a2af-b419-b755-eb72-d79bd0f13e04.dc1 area=wan -46:16.854 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29400 network=tcp -46:16.857 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29401 network=tcp -46:16.859 [INFO] TestAgent: started state syncer -46:16.859 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29406 network=tcp -46:16.917 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -46:16.917 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29405 [Candidate]" term=2 -46:16.917 [DEBUG] TestAgent.server.raft: votes: needed=1 -46:16.917 [DEBUG] TestAgent.server.raft: vote granted: from=7d38a2af-b419-b755-eb72-d79bd0f13e04 term=2 tally=1 -46:16.917 [INFO] TestAgent.server.raft: election won: tally=1 -46:16.917 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29405 [Leader]" -46:16.917 [INFO] TestAgent.server: cluster leadership acquired -46:16.918 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29405 -46:16.918 [INFO] TestAgent.server: New leader elected: payload=Node-7d38a2af-b419-b755-eb72-d79bd0f13e04 -46:16.920 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -46:16.925 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/operator/raft/peer?address=nope from=127.0.0.1:35794 error="address "nope" was not found in the Raft configuration" -46:16.930 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/operator/raft/peer?address=nope from=127.0.0.1:35794 latency=4.82132ms -46:16.933 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/operator/raft/peer?id=nope from=127.0.0.1:35796 error="id "nope" was not found in the Raft configuration" -46:16.940 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/operator/raft/peer?id=nope from=127.0.0.1:35796 latency=6.38316ms -46:16.941 [INFO] TestAgent: Requesting shutdown -46:16.954 [INFO] TestAgent.server: shutting down server -46:16.955 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -46:16.955 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -46:16.955 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:46:16.955Z [INFO] agent.router.manager: shutting down -2022-09-15T21:46:16.955Z [INFO] agent.router.manager: shutting down -46:16.955 [INFO] TestAgent: consul server down -46:16.955 [INFO] TestAgent: shutdown complete -46:16.955 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29400 network=tcp -46:16.955 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29400 network=udp -46:16.956 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29401 network=tcp -46:16.958 [ERROR] TestAgent.server: failed to establish leadership: error="error generating CA root certificate: error computing next serial number: raft is already shutdown" -46:16.958 [ERROR] TestAgent.server: failed to transfer leadership attempt, will retry: attempt=0 retry_limit=3 error="raft is already shutdown" -46:17.456 [INFO] TestAgent: Waiting for endpoints to shut down -46:17.456 [INFO] TestAgent: Endpoints down ---- PASS: TestOperatorRaftRemovePeerCommand (0.74s) - --- PASS: TestOperatorRaftRemovePeerCommand/Test_the_remove-peer_subcommand_directly (0.01s) - --- PASS: TestOperatorRaftRemovePeerCommand/Test_the_remove-peer_subcommand_with_-id (0.01s) +2021-08-13T15:38:49.675Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:38:49.697Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:49.702 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:76b47e4b-f968-723a-7fd0-abef6f5bc37b Address:127.0.0.1:19195}]" +38:49.702 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19195 [Follower]" leader= +38:49.703 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-76b47e4b-f968-723a-7fd0-abef6f5bc37b.dc1 127.0.0.1 +38:49.704 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-76b47e4b-f968-723a-7fd0-abef6f5bc37b 127.0.0.1 +2021-08-13T15:38:49.704Z [INFO] agent.router: Initializing LAN area manager +38:49.705 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=udp +38:49.705 [INFO] TestAgent.server: Adding LAN server: server="Node-76b47e4b-f968-723a-7fd0-abef6f5bc37b (Addr: tcp/127.0.0.1:19195) (DC: dc1)" +38:49.705 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-76b47e4b-f968-723a-7fd0-abef6f5bc37b.dc1 area=wan +38:49.705 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=tcp +38:49.706 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19191 network=tcp +38:49.706 [INFO] TestAgent: started state syncer +38:49.706 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19196 network=tcp +38:49.747 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +38:49.747 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19195 [Candidate]" term=2 +38:49.747 [DEBUG] TestAgent.server.raft: votes: needed=1 +38:49.747 [DEBUG] TestAgent.server.raft: vote granted: from=76b47e4b-f968-723a-7fd0-abef6f5bc37b term=2 tally=1 +38:49.747 [INFO] TestAgent.server.raft: election won: tally=1 +38:49.747 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19195 [Leader]" +38:49.748 [INFO] TestAgent.server: cluster leadership acquired +38:49.748 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:19195 +38:49.749 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +38:49.751 [INFO] TestAgent.server: New leader elected: payload=Node-76b47e4b-f968-723a-7fd0-abef6f5bc37b +38:49.780 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +38:49.781 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +38:49.781 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +38:49.781 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +38:49.781 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +38:49.781 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-76b47e4b-f968-723a-7fd0-abef6f5bc37b +38:49.781 [INFO] TestAgent.server: member joined, marking health alive: member=Node-76b47e4b-f968-723a-7fd0-abef6f5bc37b +38:49.782 [INFO] TestAgent.server: federation state anti-entropy synced +38:49.822 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +38:49.830 [INFO] TestAgent: Synced node info +38:49.830 [DEBUG] TestAgent: Node info in sync +38:49.896 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/operator/raft/peer?address=nope from=127.0.0.1:36022 error="address "nope" was not found in the Raft configuration" +38:49.897 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/operator/raft/peer?address=nope from=127.0.0.1:36022 latency=351.12µs +38:49.898 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/operator/raft/peer?id=nope from=127.0.0.1:36024 error="id "nope" was not found in the Raft configuration" +38:49.899 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/operator/raft/peer?id=nope from=127.0.0.1:36024 latency=288.78µs +38:49.899 [INFO] TestAgent: Requesting shutdown +38:49.899 [INFO] TestAgent.server: shutting down server +38:49.899 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +38:49.899 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +38:49.899 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +38:49.899 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +38:49.899 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +38:49.899 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +38:49.899 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +38:49.899 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +38:49.899 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +38:49.899 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +38:49.899 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:38:49.899Z [INFO] agent.router.manager: shutting down +2021-08-13T15:38:49.899Z [INFO] agent.router.manager: shutting down +38:49.900 [INFO] TestAgent: consul server down +38:49.900 [INFO] TestAgent: shutdown complete +38:49.900 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=tcp +38:49.900 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=udp +38:49.900 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19191 network=tcp +38:50.400 [INFO] TestAgent: Waiting for endpoints to shut down +38:50.400 [INFO] TestAgent: Endpoints down +--- PASS: TestOperatorRaftRemovePeerCommand (0.79s) + --- PASS: TestOperatorRaftRemovePeerCommand/Test_the_remove-peer_subcommand_directly (0.00s) + --- PASS: TestOperatorRaftRemovePeerCommand/Test_the_remove-peer_subcommand_with_-id (0.00s) PASS -ok github.com/hashicorp/consul/command/operator/raft/removepeer 0.931s +ok github.com/hashicorp/consul/command/operator/raft/removepeer 0.857s === RUN TestReloadCommand_noTabs === PAUSE TestReloadCommand_noTabs === RUN TestReloadCommand === PAUSE TestReloadCommand === CONT TestReloadCommand_noTabs +--- PASS: TestReloadCommand_noTabs (0.00s) === CONT TestReloadCommand [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... ---- PASS: TestReloadCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:46:20.180Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:46:20.204Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -46:20.208 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c6e46c7f-e9ea-346d-c9fe-605c4d08fd77 Address:127.0.0.1:13069}]" -46:20.209 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13069 [Follower]" leader= -46:20.210 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c6e46c7f-e9ea-346d-c9fe-605c4d08fd77.dc1 127.0.0.1 -46:20.210 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c6e46c7f-e9ea-346d-c9fe-605c4d08fd77 127.0.0.1 -2022-09-15T21:46:20.210Z [INFO] agent.router: Initializing LAN area manager -46:20.215 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c6e46c7f-e9ea-346d-c9fe-605c4d08fd77.dc1 area=wan -46:20.216 [INFO] TestAgent.server: Adding LAN server: server="Node-c6e46c7f-e9ea-346d-c9fe-605c4d08fd77 (Addr: tcp/127.0.0.1:13069) (DC: dc1)" -46:20.216 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13064 network=udp -46:20.216 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13064 network=tcp -46:20.218 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13065 network=tcp -46:20.219 [INFO] TestAgent: started state syncer -46:20.219 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13070 network=tcp -46:20.253 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -46:20.253 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13069 [Candidate]" term=2 -46:20.253 [DEBUG] TestAgent.server.raft: votes: needed=1 -46:20.253 [DEBUG] TestAgent.server.raft: vote granted: from=c6e46c7f-e9ea-346d-c9fe-605c4d08fd77 term=2 tally=1 -46:20.253 [INFO] TestAgent.server.raft: election won: tally=1 -46:20.253 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13069 [Leader]" -46:20.253 [INFO] TestAgent.server: cluster leadership acquired -46:20.254 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13069 -46:20.255 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -46:20.258 [INFO] TestAgent.server: New leader elected: payload=Node-c6e46c7f-e9ea-346d-c9fe-605c4d08fd77 -46:20.281 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -46:20.281 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -46:20.281 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -46:20.281 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -46:20.281 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -46:20.281 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c6e46c7f-e9ea-346d-c9fe-605c4d08fd77 -46:20.281 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c6e46c7f-e9ea-346d-c9fe-605c4d08fd77 -46:20.282 [INFO] TestAgent.server: federation state anti-entropy synced -2022-09-15T21:46:20.380Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -46:20.381 [INFO] TestAgent: Requesting shutdown -46:20.381 [INFO] TestAgent.server: shutting down server -46:20.381 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -46:20.381 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -46:20.381 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -46:20.381 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:46:20.382Z [INFO] agent.router.manager: shutting down -2022-09-15T21:46:20.382Z [INFO] agent.router.manager: shutting down -46:20.382 [INFO] TestAgent: consul server down -46:20.382 [INFO] TestAgent: shutdown complete -46:20.382 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13064 network=tcp -46:20.382 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13064 network=udp -46:20.382 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13065 network=tcp -46:20.883 [INFO] TestAgent: Waiting for endpoints to shut down -46:20.883 [INFO] TestAgent: Endpoints down ---- PASS: TestReloadCommand (0.77s) +2021-08-13T15:38:55.688Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:38:55.703Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:55.707 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ed27eadd-4930-bad1-7c9a-68faf71febd6 Address:127.0.0.1:26342}]" +38:55.707 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:26342 [Follower]" leader= +38:55.708 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ed27eadd-4930-bad1-7c9a-68faf71febd6.dc1 127.0.0.1 +38:55.710 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ed27eadd-4930-bad1-7c9a-68faf71febd6 127.0.0.1 +2021-08-13T15:38:55.710Z [INFO] agent.router: Initializing LAN area manager +38:55.710 [INFO] TestAgent: Started DNS server: address=127.0.0.1:26337 network=udp +38:55.711 [INFO] TestAgent.server: Adding LAN server: server="Node-ed27eadd-4930-bad1-7c9a-68faf71febd6 (Addr: tcp/127.0.0.1:26342) (DC: dc1)" +38:55.711 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ed27eadd-4930-bad1-7c9a-68faf71febd6.dc1 area=wan +38:55.711 [INFO] TestAgent: Started DNS server: address=127.0.0.1:26337 network=tcp +38:55.713 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:26338 network=tcp +38:55.713 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:26343 network=tcp +38:55.713 [INFO] TestAgent: started state syncer +38:55.777 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +38:55.777 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:26342 [Candidate]" term=2 +38:55.777 [DEBUG] TestAgent.server.raft: votes: needed=1 +38:55.777 [DEBUG] TestAgent.server.raft: vote granted: from=ed27eadd-4930-bad1-7c9a-68faf71febd6 term=2 tally=1 +38:55.777 [INFO] TestAgent.server.raft: election won: tally=1 +38:55.777 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:26342 [Leader]" +38:55.778 [INFO] TestAgent.server: cluster leadership acquired +38:55.778 [INFO] TestAgent.server: New leader elected: payload=Node-ed27eadd-4930-bad1-7c9a-68faf71febd6 +38:55.778 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:26342 +38:55.780 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +38:55.803 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +38:55.803 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +38:55.803 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +38:55.803 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +38:55.803 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +38:55.803 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ed27eadd-4930-bad1-7c9a-68faf71febd6 +38:55.803 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ed27eadd-4930-bad1-7c9a-68faf71febd6 +38:55.804 [INFO] TestAgent.server: federation state anti-entropy synced +2021-08-13T15:38:55.920Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +38:55.921 [INFO] TestAgent: Requesting shutdown +38:55.921 [INFO] TestAgent.server: shutting down server +38:55.921 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +38:55.921 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +38:55.921 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +38:55.921 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:38:55.921Z [INFO] agent.router.manager: shutting down +2021-08-13T15:38:55.921Z [INFO] agent.router.manager: shutting down +38:55.922 [INFO] TestAgent: consul server down +38:55.922 [INFO] TestAgent: shutdown complete +38:55.922 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:26337 network=tcp +38:55.922 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:26337 network=udp +38:55.922 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:26338 network=tcp +38:56.422 [INFO] TestAgent: Waiting for endpoints to shut down +38:56.422 [INFO] TestAgent: Endpoints down +--- PASS: TestReloadCommand (0.80s) PASS -ok github.com/hashicorp/consul/command/reload 0.882s +ok github.com/hashicorp/consul/command/reload 0.875s === RUN TestRTTCommand_noTabs === PAUSE TestRTTCommand_noTabs === RUN TestRTTCommand_BadArgs @@ -90461,11 +90375,11 @@ === PAUSE TestRTTCommand_WAN === CONT TestRTTCommand_noTabs === CONT TestRTTCommand_WAN +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestRTTCommand_noTabs (0.00s) === CONT TestRTTCommand_LAN === CONT TestRTTCommand_BadArgs === RUN TestRTTCommand_BadArgs/#00 -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... ---- PASS: TestRTTCommand_noTabs (0.00s) === RUN TestRTTCommand_BadArgs/node1_node2_node3 === RUN TestRTTCommand_BadArgs/-wan_node1_node2 === RUN TestRTTCommand_BadArgs/-wan_node1.dc1_node2 @@ -90478,129 +90392,133 @@ --- PASS: TestRTTCommand_BadArgs/-wan_node1_node2.dc1 (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:46:21.112Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:46:21.144Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -46:21.149 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:15009d9c-1d96-0b62-6ec1-c1686c40e2bd Address:127.0.0.1:19202}]" -46:21.151 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-15009d9c-1d96-0b62-6ec1-c1686c40e2bd.dc1 127.0.0.1 -46:21.152 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-15009d9c-1d96-0b62-6ec1-c1686c40e2bd 127.0.0.1 -2022-09-15T21:46:21.152Z [INFO] agent.router: Initializing LAN area manager -46:21.152 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19197 network=udp -46:21.152 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19202 [Follower]" leader= -46:21.153 [INFO] TestAgent.server: Adding LAN server: server="Node-15009d9c-1d96-0b62-6ec1-c1686c40e2bd (Addr: tcp/127.0.0.1:19202) (DC: dc1)" -46:21.153 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-15009d9c-1d96-0b62-6ec1-c1686c40e2bd.dc1 area=wan -46:21.153 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19197 network=tcp -46:21.157 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19198 network=tcp -46:21.157 [INFO] TestAgent: started state syncer -46:21.157 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19203 network=tcp -46:21.211 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -46:21.211 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19202 [Candidate]" term=2 -46:21.211 [DEBUG] TestAgent.server.raft: votes: needed=1 -46:21.212 [DEBUG] TestAgent.server.raft: vote granted: from=15009d9c-1d96-0b62-6ec1-c1686c40e2bd term=2 tally=1 -46:21.212 [INFO] TestAgent.server.raft: election won: tally=1 -46:21.212 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19202 [Leader]" -46:21.212 [INFO] TestAgent.server: cluster leadership acquired -46:21.212 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:19202 -46:21.213 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -46:21.213 [INFO] TestAgent.server: New leader elected: payload=Node-15009d9c-1d96-0b62-6ec1-c1686c40e2bd -46:21.230 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -46:21.230 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -46:21.230 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -46:21.230 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -46:21.230 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -46:21.230 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-15009d9c-1d96-0b62-6ec1-c1686c40e2bd -46:21.230 [INFO] TestAgent.server: member joined, marking health alive: member=Node-15009d9c-1d96-0b62-6ec1-c1686c40e2bd -46:21.231 [INFO] TestAgent.server: federation state anti-entropy synced -46:21.258 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/datacenters from=127.0.0.1:51368 latency=903.64µs -46:21.262 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:51370 latency=1.94726ms -46:21.269 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/datacenters from=127.0.0.1:51370 latency=3.87858ms -46:21.272 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/datacenters from=127.0.0.1:51372 latency=431.06µs -46:21.272 [INFO] TestAgent: Requesting shutdown -46:21.273 [INFO] TestAgent.server: shutting down server -46:21.273 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -46:21.273 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -46:21.273 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -46:21.273 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -46:21.273 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -46:21.273 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -46:21.273 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -46:21.273 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -46:21.273 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -46:21.273 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -46:21.273 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -46:21.273 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:46:21.273Z [INFO] agent.router.manager: shutting down -2022-09-15T21:46:21.273Z [INFO] agent.router.manager: shutting down -46:21.273 [INFO] TestAgent: consul server down -46:21.273 [INFO] TestAgent: shutdown complete -46:21.274 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19197 network=tcp -46:21.274 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19197 network=udp -46:21.274 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19198 network=tcp -46:21.774 [INFO] TestAgent: Waiting for endpoints to shut down -46:21.774 [INFO] TestAgent: Endpoints down ---- PASS: TestRTTCommand_WAN (0.74s) -2022-09-15T21:46:21.116Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:46:21.146Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -46:21.151 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6f779aed-f942-806a-7ac5-100ea8cb23be Address:127.0.0.1:19195}]" -46:21.152 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6f779aed-f942-806a-7ac5-100ea8cb23be.dc1 127.0.0.1 -46:21.152 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6f779aed-f942-806a-7ac5-100ea8cb23be 127.0.0.1 -2022-09-15T21:46:21.152Z [INFO] agent.router: Initializing LAN area manager -46:21.153 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=udp -46:21.153 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19195 [Follower]" leader= -46:21.153 [INFO] TestAgent.server: Adding LAN server: server="Node-6f779aed-f942-806a-7ac5-100ea8cb23be (Addr: tcp/127.0.0.1:19195) (DC: dc1)" -46:21.153 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6f779aed-f942-806a-7ac5-100ea8cb23be.dc1 area=wan -46:21.154 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=tcp -46:21.158 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19191 network=tcp -46:21.158 [INFO] TestAgent: started state syncer -46:21.158 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19196 network=tcp -46:21.203 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -46:21.203 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19195 [Candidate]" term=2 -46:21.203 [DEBUG] TestAgent.server.raft: votes: needed=1 -46:21.203 [DEBUG] TestAgent.server.raft: vote granted: from=6f779aed-f942-806a-7ac5-100ea8cb23be term=2 tally=1 -46:21.203 [INFO] TestAgent.server.raft: election won: tally=1 -46:21.203 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19195 [Leader]" -46:21.204 [INFO] TestAgent.server: cluster leadership acquired -46:21.204 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:19195 -46:21.205 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -46:21.210 [INFO] TestAgent.server: New leader elected: payload=Node-6f779aed-f942-806a-7ac5-100ea8cb23be -46:21.230 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -46:21.230 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -46:21.230 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -46:21.230 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -46:21.230 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -46:21.230 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6f779aed-f942-806a-7ac5-100ea8cb23be -46:21.230 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6f779aed-f942-806a-7ac5-100ea8cb23be -46:21.233 [INFO] TestAgent.server: federation state anti-entropy synced -46:21.344 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/nodes from=127.0.0.1:41268 latency=170.62µs -46:21.371 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/nodes from=127.0.0.1:41270 latency=214µs -46:21.374 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:41272 latency=1.58ms -46:21.377 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/nodes from=127.0.0.1:41272 latency=280µs -46:21.383 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/nodes from=127.0.0.1:41274 latency=194.62µs -46:21.384 [INFO] TestAgent: Requesting shutdown -46:21.384 [INFO] TestAgent.server: shutting down server -46:21.384 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -46:21.384 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -46:21.384 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -46:21.384 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -46:21.384 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -46:21.384 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -46:21.384 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -46:21.384 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -46:21.384 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -46:21.384 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -46:21.384 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -46:21.385 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:46:21.385Z [INFO] agent.router.manager: shutting down -2022-09-15T21:46:21.385Z [INFO] agent.router.manager: shutting down -46:21.385 [INFO] TestAgent: consul server down -46:21.385 [INFO] TestAgent: shutdown complete -46:21.385 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=tcp -46:21.385 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=udp -46:21.385 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19191 network=tcp -46:21.885 [INFO] TestAgent: Waiting for endpoints to shut down -46:21.885 [INFO] TestAgent: Endpoints down ---- PASS: TestRTTCommand_LAN (0.85s) +2021-08-13T15:39:03.492Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:39:03.511Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +39:03.517 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:86c6ac6c-be01-2144-8e99-36096e866381 Address:127.0.0.1:25321}]" +39:03.517 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25321 [Follower]" leader= +39:03.519 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-86c6ac6c-be01-2144-8e99-36096e866381.dc1 127.0.0.1 +39:03.520 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-86c6ac6c-be01-2144-8e99-36096e866381 127.0.0.1 +2021-08-13T15:39:03.520Z [INFO] agent.router: Initializing LAN area manager +39:03.520 [INFO] TestAgent.server: Adding LAN server: server="Node-86c6ac6c-be01-2144-8e99-36096e866381 (Addr: tcp/127.0.0.1:25321) (DC: dc1)" +39:03.522 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-86c6ac6c-be01-2144-8e99-36096e866381.dc1 area=wan +39:03.522 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25316 network=tcp +39:03.522 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25316 network=udp +39:03.523 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25317 network=tcp +39:03.523 [INFO] TestAgent: started state syncer +39:03.523 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25322 network=tcp +39:03.583 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +39:03.583 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25321 [Candidate]" term=2 +39:03.583 [DEBUG] TestAgent.server.raft: votes: needed=1 +39:03.583 [DEBUG] TestAgent.server.raft: vote granted: from=86c6ac6c-be01-2144-8e99-36096e866381 term=2 tally=1 +39:03.583 [INFO] TestAgent.server.raft: election won: tally=1 +39:03.583 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25321 [Leader]" +39:03.583 [INFO] TestAgent.server: cluster leadership acquired +39:03.583 [INFO] TestAgent.server: New leader elected: payload=Node-86c6ac6c-be01-2144-8e99-36096e866381 +39:03.584 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25321 +39:03.585 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +39:03.595 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +39:03.595 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +39:03.595 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +39:03.596 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +39:03.596 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +39:03.596 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-86c6ac6c-be01-2144-8e99-36096e866381 +39:03.596 [INFO] TestAgent.server: member joined, marking health alive: member=Node-86c6ac6c-be01-2144-8e99-36096e866381 +39:03.596 [INFO] TestAgent.server: federation state anti-entropy synced +39:03.647 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/datacenters from=127.0.0.1:41862 latency=5.913948ms +39:03.651 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:41864 latency=1.830162ms +39:03.655 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/datacenters from=127.0.0.1:41864 latency=361.9µs +39:03.657 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/datacenters from=127.0.0.1:41866 latency=261.04µs +39:03.657 [INFO] TestAgent: Requesting shutdown +39:03.657 [INFO] TestAgent.server: shutting down server +39:03.657 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +39:03.657 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +39:03.657 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +39:03.657 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +39:03.657 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +39:03.657 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +39:03.657 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +39:03.657 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +39:03.657 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +39:03.657 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +39:03.657 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +39:03.658 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:39:03.658Z [INFO] agent.router.manager: shutting down +2021-08-13T15:39:03.658Z [INFO] agent.router.manager: shutting down +39:03.658 [INFO] TestAgent: consul server down +39:03.658 [INFO] TestAgent: shutdown complete +39:03.658 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25316 network=tcp +39:03.658 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25316 network=udp +39:03.658 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25317 network=tcp +39:04.158 [INFO] TestAgent: Waiting for endpoints to shut down +39:04.158 [INFO] TestAgent: Endpoints down +--- PASS: TestRTTCommand_WAN (0.72s) +2021-08-13T15:39:03.499Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:39:03.525Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +39:03.527 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0204853b-e871-5fcc-2af1-55c494e0bc69 Address:127.0.0.1:25328}]" +39:03.527 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25328 [Follower]" leader= +39:03.528 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0204853b-e871-5fcc-2af1-55c494e0bc69.dc1 127.0.0.1 +39:03.529 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0204853b-e871-5fcc-2af1-55c494e0bc69 127.0.0.1 +2021-08-13T15:39:03.529Z [INFO] agent.router: Initializing LAN area manager +39:03.529 [INFO] TestAgent.server: Adding LAN server: server="Node-0204853b-e871-5fcc-2af1-55c494e0bc69 (Addr: tcp/127.0.0.1:25328) (DC: dc1)" +39:03.529 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0204853b-e871-5fcc-2af1-55c494e0bc69.dc1 area=wan +39:03.529 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25323 network=udp +39:03.530 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25323 network=tcp +39:03.530 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25324 network=tcp +39:03.530 [INFO] TestAgent: started state syncer +39:03.531 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25329 network=tcp +39:03.568 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +39:03.568 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25328 [Candidate]" term=2 +39:03.568 [DEBUG] TestAgent.server.raft: votes: needed=1 +39:03.568 [DEBUG] TestAgent.server.raft: vote granted: from=0204853b-e871-5fcc-2af1-55c494e0bc69 term=2 tally=1 +39:03.568 [INFO] TestAgent.server.raft: election won: tally=1 +39:03.568 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25328 [Leader]" +39:03.568 [INFO] TestAgent.server: cluster leadership acquired +39:03.569 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25328 +39:03.570 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +39:03.575 [INFO] TestAgent.server: New leader elected: payload=Node-0204853b-e871-5fcc-2af1-55c494e0bc69 +39:03.594 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +39:03.594 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +39:03.594 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +39:03.594 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +39:03.594 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +39:03.594 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0204853b-e871-5fcc-2af1-55c494e0bc69 +39:03.594 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0204853b-e871-5fcc-2af1-55c494e0bc69 +39:03.595 [INFO] TestAgent.server: federation state anti-entropy synced +39:03.846 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/nodes from=127.0.0.1:35686 latency=266.101µs +39:03.873 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/nodes from=127.0.0.1:35688 latency=183.901µs +39:03.880 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +39:03.881 [INFO] TestAgent: Synced node info +39:03.881 [DEBUG] TestAgent: Node info in sync +39:03.900 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/nodes from=127.0.0.1:35690 latency=314.86µs +39:03.927 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/nodes from=127.0.0.1:35692 latency=222.881µs +39:03.930 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:35694 latency=1.487482ms +39:03.934 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/nodes from=127.0.0.1:35694 latency=171.38µs +39:03.936 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/nodes from=127.0.0.1:35696 latency=144.16µs +39:03.936 [INFO] TestAgent: Requesting shutdown +39:03.936 [INFO] TestAgent.server: shutting down server +39:03.936 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +39:03.936 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +39:03.936 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +39:03.936 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +39:03.936 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +39:03.936 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +39:03.936 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +39:03.936 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +39:03.936 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +39:03.936 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +39:03.936 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:39:03.937Z [INFO] agent.router.manager: shutting down +2021-08-13T15:39:03.937Z [INFO] agent.router.manager: shutting down +39:03.937 [INFO] TestAgent: consul server down +39:03.937 [INFO] TestAgent: shutdown complete +39:03.937 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25323 network=tcp +39:03.937 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25323 network=udp +39:03.937 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25324 network=tcp +39:04.437 [INFO] TestAgent: Waiting for endpoints to shut down +39:04.437 [INFO] TestAgent: Endpoints down +--- PASS: TestRTTCommand_LAN (1.00s) PASS -ok github.com/hashicorp/consul/command/rtt 0.961s +ok github.com/hashicorp/consul/command/rtt 1.069s === RUN TestDevModeHasNoServices === PAUSE TestDevModeHasNoServices === RUN TestStructsToAgentService @@ -90608,7 +90526,9 @@ === RUN TestCommand_noTabs === PAUSE TestCommand_noTabs === CONT TestDevModeHasNoServices +=== CONT TestCommand_noTabs === CONT TestStructsToAgentService +--- PASS: TestCommand_noTabs (0.00s) === RUN TestStructsToAgentService/Basic_service_with_port === PAUSE TestStructsToAgentService/Basic_service_with_port === RUN TestStructsToAgentService/Service_with_a_check @@ -90622,23 +90542,21 @@ === RUN TestStructsToAgentService/Proxy_service === PAUSE TestStructsToAgentService/Proxy_service === CONT TestStructsToAgentService/Basic_service_with_port -=== CONT TestCommand_noTabs ---- PASS: TestCommand_noTabs (0.00s) -=== CONT TestStructsToAgentService/Proxy_service -=== CONT TestStructsToAgentService/Service_with_checks === CONT TestStructsToAgentService/Service_with_a_zero-value_check === CONT TestStructsToAgentService/Service_with_an_unnamed_check === CONT TestStructsToAgentService/Service_with_a_check +=== CONT TestStructsToAgentService/Proxy_service +=== CONT TestStructsToAgentService/Service_with_checks --- PASS: TestStructsToAgentService (0.00s) - --- PASS: TestStructsToAgentService/Service_with_an_unnamed_check (0.00s) --- PASS: TestStructsToAgentService/Basic_service_with_port (0.00s) - --- PASS: TestStructsToAgentService/Service_with_a_zero-value_check (0.01s) - --- PASS: TestStructsToAgentService/Proxy_service (0.01s) - --- PASS: TestStructsToAgentService/Service_with_checks (0.01s) - --- PASS: TestStructsToAgentService/Service_with_a_check (0.01s) ---- PASS: TestDevModeHasNoServices (0.03s) + --- PASS: TestStructsToAgentService/Service_with_an_unnamed_check (0.00s) + --- PASS: TestStructsToAgentService/Service_with_a_zero-value_check (0.00s) + --- PASS: TestStructsToAgentService/Service_with_a_check (0.00s) + --- PASS: TestStructsToAgentService/Proxy_service (0.00s) + --- PASS: TestStructsToAgentService/Service_with_checks (0.00s) +--- PASS: TestDevModeHasNoServices (0.02s) PASS -ok github.com/hashicorp/consul/command/services 0.092s +ok github.com/hashicorp/consul/command/services 0.102s === RUN TestCommand_noTabs === PAUSE TestCommand_noTabs === RUN TestCommand_Validation @@ -90650,236 +90568,231 @@ === RUN TestCommand_Flag === PAUSE TestCommand_Flag === CONT TestCommand_noTabs -=== CONT TestCommand_File_nameOnly -=== CONT TestCommand_File_id -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestCommand_noTabs (0.00s) === CONT TestCommand_Flag +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +=== CONT TestCommand_File_id +=== CONT TestCommand_File_nameOnly === CONT TestCommand_Validation ---- PASS: TestCommand_noTabs (0.00s) -=== RUN TestCommand_Validation/no_args_or_id === RUN TestCommand_Validation/args_and_-id ---- PASS: TestCommand_Validation (0.00s) - --- PASS: TestCommand_Validation/no_args_or_id (0.00s) - --- PASS: TestCommand_Validation/args_and_-id (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:46:29.291Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:46:29.320Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -46:29.335 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a538b45e-3bee-ffcd-3922-8faeb508e52d Address:127.0.0.1:20223}]" -46:29.336 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:20223 [Follower]" leader= -46:29.336 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a538b45e-3bee-ffcd-3922-8faeb508e52d.dc1 127.0.0.1 -46:29.342 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a538b45e-3bee-ffcd-3922-8faeb508e52d 127.0.0.1 -2022-09-15T21:46:29.342Z [INFO] agent.router: Initializing LAN area manager -46:29.342 [INFO] TestAgent.server: Adding LAN server: server="Node-a538b45e-3bee-ffcd-3922-8faeb508e52d (Addr: tcp/127.0.0.1:20223) (DC: dc1)" -46:29.342 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a538b45e-3bee-ffcd-3922-8faeb508e52d.dc1 area=wan -46:29.343 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20218 network=udp -46:29.343 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20218 network=tcp -46:29.343 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:20219 network=tcp -46:29.343 [INFO] TestAgent: started state syncer -46:29.346 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:20224 network=tcp -46:29.388 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -46:29.388 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:20223 [Candidate]" term=2 -46:29.389 [DEBUG] TestAgent.server.raft: votes: needed=1 -46:29.389 [DEBUG] TestAgent.server.raft: vote granted: from=a538b45e-3bee-ffcd-3922-8faeb508e52d term=2 tally=1 -46:29.389 [INFO] TestAgent.server.raft: election won: tally=1 -46:29.389 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:20223 [Leader]" -46:29.389 [INFO] TestAgent.server: cluster leadership acquired -46:29.389 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:20223 -46:29.391 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -46:29.394 [INFO] TestAgent.server: New leader elected: payload=Node-a538b45e-3bee-ffcd-3922-8faeb508e52d -46:29.398 [INFO] TestAgent: Synced node info -46:29.398 [INFO] TestAgent: Synced service: service=web -46:29.398 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:58626 latency=2.7718ms -46:29.399 [DEBUG] TestAgent: Node info in sync -46:29.399 [DEBUG] TestAgent: Service in sync: service=web -46:29.400 [INFO] TestAgent: Synced service: service=db -46:29.400 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:58626 latency=1.04878ms -46:29.404 [DEBUG] TestAgent: Node info in sync -46:29.404 [DEBUG] TestAgent: Service in sync: service=web -46:29.404 [DEBUG] TestAgent: Service in sync: service=db -46:29.404 [DEBUG] TestAgent: Node info in sync -46:29.404 [DEBUG] TestAgent: Service in sync: service=web -46:29.404 [DEBUG] TestAgent: Service in sync: service=db -46:29.418 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -46:29.418 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -46:29.418 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -46:29.418 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -46:29.418 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -46:29.418 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a538b45e-3bee-ffcd-3922-8faeb508e52d -46:29.418 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a538b45e-3bee-ffcd-3922-8faeb508e52d -46:29.419 [INFO] TestAgent.server: federation state anti-entropy synced -46:29.425 [DEBUG] TestAgent: removed service: service=web -46:29.425 [DEBUG] TestAgent: Node info in sync -46:29.426 [INFO] TestAgent: Deregistered service: service=web -46:29.426 [DEBUG] TestAgent: Service in sync: service=db -46:29.426 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/deregister/web from=127.0.0.1:58628 latency=989.78µs -46:29.433 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:58626 latency=5.74604ms -46:29.435 [INFO] TestAgent: Requesting shutdown -46:29.435 [INFO] TestAgent.server: shutting down server -46:29.435 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -46:29.435 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -46:29.435 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -46:29.435 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -46:29.435 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -46:29.435 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -46:29.435 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -46:29.435 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -46:29.435 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -46:29.435 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -46:29.435 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:46:29.436Z [INFO] agent.router.manager: shutting down -2022-09-15T21:46:29.436Z [INFO] agent.router.manager: shutting down -46:29.436 [INFO] TestAgent: consul server down -46:29.436 [INFO] TestAgent: shutdown complete -46:29.436 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20218 network=tcp -46:29.436 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20218 network=udp -46:29.436 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:20219 network=tcp -46:29.936 [INFO] TestAgent: Waiting for endpoints to shut down -46:29.936 [INFO] TestAgent: Endpoints down ---- PASS: TestCommand_File_id (0.74s) -2022-09-15T21:46:29.307Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:46:29.335Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -46:29.338 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cab05828-6d62-b69f-6330-561683aa90fd Address:127.0.0.1:20230}]" -46:29.338 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:20230 [Follower]" leader= -46:29.339 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-cab05828-6d62-b69f-6330-561683aa90fd.dc1 127.0.0.1 -46:29.340 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-cab05828-6d62-b69f-6330-561683aa90fd 127.0.0.1 -2022-09-15T21:46:29.340Z [INFO] agent.router: Initializing LAN area manager -46:29.340 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20225 network=udp -46:29.340 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-cab05828-6d62-b69f-6330-561683aa90fd.dc1 area=wan -46:29.340 [INFO] TestAgent.server: Adding LAN server: server="Node-cab05828-6d62-b69f-6330-561683aa90fd (Addr: tcp/127.0.0.1:20230) (DC: dc1)" -46:29.341 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20225 network=tcp -46:29.341 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:20226 network=tcp -46:29.343 [INFO] TestAgent: started state syncer -46:29.346 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:20231 network=tcp -46:29.394 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -46:29.394 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:20230 [Candidate]" term=2 -46:29.394 [DEBUG] TestAgent.server.raft: votes: needed=1 -46:29.394 [DEBUG] TestAgent.server.raft: vote granted: from=cab05828-6d62-b69f-6330-561683aa90fd term=2 tally=1 -46:29.394 [INFO] TestAgent.server.raft: election won: tally=1 -46:29.394 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:20230 [Leader]" -46:29.394 [INFO] TestAgent.server: cluster leadership acquired -46:29.394 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:20230 -46:29.395 [INFO] TestAgent.server: New leader elected: payload=Node-cab05828-6d62-b69f-6330-561683aa90fd -46:29.396 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -46:29.419 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -46:29.419 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -46:29.419 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -46:29.419 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -46:29.419 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -46:29.419 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cab05828-6d62-b69f-6330-561683aa90fd -46:29.419 [INFO] TestAgent.server: member joined, marking health alive: member=Node-cab05828-6d62-b69f-6330-561683aa90fd -46:29.420 [INFO] TestAgent.server: federation state anti-entropy synced -46:29.620 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -46:29.621 [INFO] TestAgent: Synced node info -46:29.621 [DEBUG] TestAgent: Node info in sync -46:29.659 [DEBUG] TestAgent: Node info in sync -46:29.660 [INFO] TestAgent: Synced service: service=web -46:29.660 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:44784 latency=1.35666ms -46:29.661 [DEBUG] TestAgent: Node info in sync -46:29.661 [DEBUG] TestAgent: Service in sync: service=web -46:29.662 [INFO] TestAgent: Synced service: service=db -46:29.662 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:44784 latency=1.34586ms -46:29.664 [DEBUG] TestAgent: removed service: service=web -46:29.664 [DEBUG] TestAgent: Node info in sync -46:29.665 [INFO] TestAgent: Deregistered service: service=web -46:29.665 [DEBUG] TestAgent: Service in sync: service=db -46:29.665 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/deregister/web from=127.0.0.1:44786 latency=826.56µs -46:29.669 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:44784 latency=549.44µs -46:29.670 [INFO] TestAgent: Requesting shutdown -46:29.670 [INFO] TestAgent.server: shutting down server -46:29.670 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -46:29.670 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -46:29.670 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -46:29.670 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -46:29.670 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -46:29.670 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -46:29.670 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -46:29.670 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -46:29.670 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -46:29.670 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -46:29.671 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:46:29.671Z [INFO] agent.router.manager: shutting down -2022-09-15T21:46:29.671Z [INFO] agent.router.manager: shutting down -46:29.671 [INFO] TestAgent: consul server down -46:29.671 [INFO] TestAgent: shutdown complete -46:29.671 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20225 network=tcp -46:29.671 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20225 network=udp -46:29.671 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:20226 network=tcp -46:30.174 [INFO] TestAgent: Waiting for endpoints to shut down -46:30.174 [INFO] TestAgent: Endpoints down ---- PASS: TestCommand_Flag (0.98s) -2022-09-15T21:46:29.309Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:46:29.342Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -46:29.348 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a2571b83-dad4-0b9d-c869-642a2a168018 Address:127.0.0.1:20216}]" -46:29.349 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a2571b83-dad4-0b9d-c869-642a2a168018.dc1 127.0.0.1 -46:29.350 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:20216 [Follower]" leader= -46:29.351 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a2571b83-dad4-0b9d-c869-642a2a168018 127.0.0.1 -2022-09-15T21:46:29.351Z [INFO] agent.router: Initializing LAN area manager -46:29.352 [INFO] TestAgent.server: Adding LAN server: server="Node-a2571b83-dad4-0b9d-c869-642a2a168018 (Addr: tcp/127.0.0.1:20216) (DC: dc1)" -46:29.352 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20211 network=udp -46:29.352 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a2571b83-dad4-0b9d-c869-642a2a168018.dc1 area=wan -46:29.352 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20211 network=tcp -46:29.354 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:20212 network=tcp -46:29.354 [INFO] TestAgent: started state syncer -46:29.354 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:20217 network=tcp -46:29.417 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -46:29.417 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:20216 [Candidate]" term=2 -46:29.417 [DEBUG] TestAgent.server.raft: votes: needed=1 -46:29.417 [DEBUG] TestAgent.server.raft: vote granted: from=a2571b83-dad4-0b9d-c869-642a2a168018 term=2 tally=1 -46:29.417 [INFO] TestAgent.server.raft: election won: tally=1 -46:29.417 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:20216 [Leader]" -46:29.418 [INFO] TestAgent.server: cluster leadership acquired -46:29.418 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:20216 -46:29.419 [INFO] TestAgent.server: New leader elected: payload=Node-a2571b83-dad4-0b9d-c869-642a2a168018 -46:29.419 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -46:29.422 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -46:29.422 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -46:29.422 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -46:29.422 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -46:29.422 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -46:29.422 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a2571b83-dad4-0b9d-c869-642a2a168018 -46:29.422 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a2571b83-dad4-0b9d-c869-642a2a168018 -46:29.424 [INFO] TestAgent.server: federation state anti-entropy synced -46:29.665 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -46:29.665 [INFO] TestAgent: Synced node info -46:29.687 [DEBUG] TestAgent: Node info in sync -46:29.688 [INFO] TestAgent: Synced service: service=web -46:29.688 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:56772 latency=1.39478ms -46:29.689 [DEBUG] TestAgent: Node info in sync -46:29.689 [DEBUG] TestAgent: Service in sync: service=web -46:29.689 [INFO] TestAgent: Synced service: service=db -46:29.689 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:56772 latency=954.46µs -46:29.708 [DEBUG] TestAgent: removed service: service=web -46:29.708 [DEBUG] TestAgent: Node info in sync -46:29.708 [DEBUG] TestAgent: Service in sync: service=db -46:29.709 [INFO] TestAgent: Deregistered service: service=web -46:29.709 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/deregister/web from=127.0.0.1:56774 latency=934.16µs -46:29.710 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:56772 latency=229.2µs -46:29.711 [INFO] TestAgent: Requesting shutdown -46:29.711 [INFO] TestAgent.server: shutting down server -46:29.711 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -46:29.711 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -46:29.711 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -46:29.711 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -46:29.711 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -46:29.711 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -46:29.711 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -46:29.711 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -46:29.711 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -46:29.711 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -46:29.711 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:46:29.712Z [INFO] agent.router.manager: shutting down -2022-09-15T21:46:29.712Z [INFO] agent.router.manager: shutting down -46:29.712 [INFO] TestAgent: consul server down -46:29.712 [INFO] TestAgent: shutdown complete -46:29.712 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20211 network=tcp -46:29.712 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20211 network=udp -46:29.712 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:20212 network=tcp -46:30.213 [INFO] TestAgent: Waiting for endpoints to shut down -46:30.213 [INFO] TestAgent: Endpoints down ---- PASS: TestCommand_File_nameOnly (1.02s) +=== RUN TestCommand_Validation/no_args_or_id +--- PASS: TestCommand_Validation (0.01s) + --- PASS: TestCommand_Validation/args_and_-id (0.01s) + --- PASS: TestCommand_Validation/no_args_or_id (0.00s) +2021-08-13T15:39:09.224Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:39:09.247Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +39:09.266 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2af8b593-7b5f-2df9-1618-b8fa044cf33e Address:127.0.0.1:18181}]" +39:09.267 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:18181 [Follower]" leader= +39:09.268 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2af8b593-7b5f-2df9-1618-b8fa044cf33e.dc1 127.0.0.1 +39:09.269 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2af8b593-7b5f-2df9-1618-b8fa044cf33e 127.0.0.1 +2021-08-13T15:39:09.269Z [INFO] agent.router: Initializing LAN area manager +39:09.270 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18176 network=udp +39:09.270 [INFO] TestAgent.server: Adding LAN server: server="Node-2af8b593-7b5f-2df9-1618-b8fa044cf33e (Addr: tcp/127.0.0.1:18181) (DC: dc1)" +39:09.270 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2af8b593-7b5f-2df9-1618-b8fa044cf33e.dc1 area=wan +39:09.270 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18176 network=tcp +39:09.271 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:18177 network=tcp +39:09.271 [INFO] TestAgent: started state syncer +39:09.271 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:18182 network=tcp +39:09.319 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +39:09.319 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:18181 [Candidate]" term=2 +39:09.319 [DEBUG] TestAgent.server.raft: votes: needed=1 +39:09.319 [DEBUG] TestAgent.server.raft: vote granted: from=2af8b593-7b5f-2df9-1618-b8fa044cf33e term=2 tally=1 +39:09.319 [INFO] TestAgent.server.raft: election won: tally=1 +39:09.319 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:18181 [Leader]" +39:09.320 [INFO] TestAgent.server: cluster leadership acquired +39:09.320 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:18181 +39:09.335 [INFO] TestAgent.server: New leader elected: payload=Node-2af8b593-7b5f-2df9-1618-b8fa044cf33e +39:09.336 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +39:09.361 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +39:09.361 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +39:09.361 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +39:09.361 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +39:09.361 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +39:09.361 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2af8b593-7b5f-2df9-1618-b8fa044cf33e +39:09.361 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2af8b593-7b5f-2df9-1618-b8fa044cf33e +39:09.362 [INFO] TestAgent.server: federation state anti-entropy synced +39:09.444 [INFO] TestAgent: Synced node info +39:09.445 [INFO] TestAgent: Synced service: service=web +39:09.445 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:37042 latency=2.582443ms +39:09.446 [DEBUG] TestAgent: Node info in sync +39:09.446 [DEBUG] TestAgent: Service in sync: service=web +39:09.446 [INFO] TestAgent: Synced service: service=db +39:09.446 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:37042 latency=938.201µs +39:09.464 [DEBUG] TestAgent: removed service: service=web +39:09.464 [DEBUG] TestAgent: Node info in sync +39:09.465 [INFO] TestAgent: Deregistered service: service=web +39:09.465 [DEBUG] TestAgent: Service in sync: service=db +39:09.465 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/deregister/web from=127.0.0.1:37044 latency=968.922µs +39:09.466 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:37042 latency=502.9µs +39:09.467 [INFO] TestAgent: Requesting shutdown +39:09.467 [INFO] TestAgent.server: shutting down server +39:09.467 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +39:09.467 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +39:09.467 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +39:09.467 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +39:09.467 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +39:09.467 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +39:09.467 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +39:09.467 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +39:09.467 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +39:09.467 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +39:09.467 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +39:09.467 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:39:09.467Z [INFO] agent.router.manager: shutting down +2021-08-13T15:39:09.467Z [INFO] agent.router.manager: shutting down +39:09.467 [INFO] TestAgent: consul server down +39:09.467 [INFO] TestAgent: shutdown complete +39:09.467 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18176 network=tcp +39:09.467 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18176 network=udp +39:09.468 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:18177 network=tcp +39:09.968 [INFO] TestAgent: Waiting for endpoints to shut down +39:09.968 [INFO] TestAgent: Endpoints down +--- PASS: TestCommand_File_id (0.84s) +2021-08-13T15:39:09.236Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:39:09.264Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +39:09.267 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d5370aca-67af-2199-047c-6fb121be688a Address:127.0.0.1:18174}]" +39:09.267 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:18174 [Follower]" leader= +39:09.268 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d5370aca-67af-2199-047c-6fb121be688a.dc1 127.0.0.1 +39:09.269 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d5370aca-67af-2199-047c-6fb121be688a 127.0.0.1 +2021-08-13T15:39:09.269Z [INFO] agent.router: Initializing LAN area manager +39:09.269 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18169 network=udp +39:09.269 [INFO] TestAgent.server: Adding LAN server: server="Node-d5370aca-67af-2199-047c-6fb121be688a (Addr: tcp/127.0.0.1:18174) (DC: dc1)" +39:09.269 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d5370aca-67af-2199-047c-6fb121be688a.dc1 area=wan +39:09.270 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18169 network=tcp +39:09.271 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:18170 network=tcp +39:09.271 [INFO] TestAgent: started state syncer +39:09.271 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:18175 network=tcp +39:09.331 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +39:09.331 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:18174 [Candidate]" term=2 +39:09.332 [DEBUG] TestAgent.server.raft: votes: needed=1 +39:09.332 [DEBUG] TestAgent.server.raft: vote granted: from=d5370aca-67af-2199-047c-6fb121be688a term=2 tally=1 +39:09.332 [INFO] TestAgent.server.raft: election won: tally=1 +39:09.332 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:18174 [Leader]" +39:09.332 [INFO] TestAgent.server: cluster leadership acquired +39:09.332 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:18174 +39:09.334 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +39:09.334 [INFO] TestAgent.server: New leader elected: payload=Node-d5370aca-67af-2199-047c-6fb121be688a +39:09.361 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +39:09.361 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +39:09.361 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +39:09.361 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +39:09.361 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +39:09.361 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d5370aca-67af-2199-047c-6fb121be688a +39:09.361 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d5370aca-67af-2199-047c-6fb121be688a +39:09.362 [INFO] TestAgent.server: federation state anti-entropy synced +39:09.619 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +39:09.620 [INFO] TestAgent: Synced node info +39:09.620 [DEBUG] TestAgent: Node info in sync +39:09.623 [DEBUG] TestAgent: Node info in sync +39:09.624 [INFO] TestAgent: Synced service: service=web +39:09.624 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:37646 latency=1.404222ms +39:09.625 [DEBUG] TestAgent: Node info in sync +39:09.625 [INFO] TestAgent: Synced service: service=db +39:09.625 [DEBUG] TestAgent: Service in sync: service=web +39:09.625 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:37646 latency=1.013781ms +39:09.636 [DEBUG] TestAgent: removed service: service=web +39:09.636 [DEBUG] TestAgent: Node info in sync +39:09.637 [INFO] TestAgent: Deregistered service: service=web +39:09.637 [DEBUG] TestAgent: Service in sync: service=db +39:09.637 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/deregister/web from=127.0.0.1:37648 latency=1.186141ms +39:09.638 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:37646 latency=244.021µs +39:09.639 [INFO] TestAgent: Requesting shutdown +39:09.639 [INFO] TestAgent.server: shutting down server +39:09.639 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +39:09.639 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +39:09.639 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +39:09.639 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +39:09.639 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +39:09.640 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +39:09.640 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +39:09.640 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +39:09.640 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +39:09.640 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +39:09.640 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:39:09.640Z [INFO] agent.router.manager: shutting down +2021-08-13T15:39:09.640Z [INFO] agent.router.manager: shutting down +39:09.644 [INFO] TestAgent: consul server down +39:09.644 [INFO] TestAgent: shutdown complete +39:09.644 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18169 network=tcp +39:09.644 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18169 network=udp +39:09.644 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:18170 network=tcp +39:10.144 [INFO] TestAgent: Waiting for endpoints to shut down +39:10.144 [INFO] TestAgent: Endpoints down +--- PASS: TestCommand_Flag (1.02s) +2021-08-13T15:39:09.226Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:39:09.250Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +39:09.263 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e53dad9a-c138-d7cb-98cb-7e9bd7dd7d20 Address:127.0.0.1:18188}]" +39:09.263 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:18188 [Follower]" leader= +39:09.265 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e53dad9a-c138-d7cb-98cb-7e9bd7dd7d20.dc1 127.0.0.1 +39:09.266 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e53dad9a-c138-d7cb-98cb-7e9bd7dd7d20 127.0.0.1 +2021-08-13T15:39:09.266Z [INFO] agent.router: Initializing LAN area manager +39:09.266 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18183 network=udp +39:09.267 [INFO] TestAgent.server: Adding LAN server: server="Node-e53dad9a-c138-d7cb-98cb-7e9bd7dd7d20 (Addr: tcp/127.0.0.1:18188) (DC: dc1)" +39:09.267 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e53dad9a-c138-d7cb-98cb-7e9bd7dd7d20.dc1 area=wan +39:09.267 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18183 network=tcp +39:09.268 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:18184 network=tcp +39:09.268 [INFO] TestAgent: started state syncer +39:09.268 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:18189 network=tcp +39:09.306 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +39:09.306 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:18188 [Candidate]" term=2 +39:09.306 [DEBUG] TestAgent.server.raft: votes: needed=1 +39:09.306 [DEBUG] TestAgent.server.raft: vote granted: from=e53dad9a-c138-d7cb-98cb-7e9bd7dd7d20 term=2 tally=1 +39:09.306 [INFO] TestAgent.server.raft: election won: tally=1 +39:09.306 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:18188 [Leader]" +39:09.306 [INFO] TestAgent.server: cluster leadership acquired +39:09.307 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:18188 +39:09.307 [INFO] TestAgent.server: New leader elected: payload=Node-e53dad9a-c138-d7cb-98cb-7e9bd7dd7d20 +39:09.308 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +39:09.359 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +39:09.359 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +39:09.359 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +39:09.359 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +39:09.359 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +39:09.359 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e53dad9a-c138-d7cb-98cb-7e9bd7dd7d20 +39:09.359 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e53dad9a-c138-d7cb-98cb-7e9bd7dd7d20 +39:09.361 [INFO] TestAgent.server: federation state anti-entropy synced +39:09.371 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +39:09.372 [INFO] TestAgent: Synced node info +39:09.641 [DEBUG] TestAgent: Node info in sync +39:09.642 [INFO] TestAgent: Synced service: service=web +39:09.642 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:55362 latency=1.757342ms +39:09.644 [DEBUG] TestAgent: Node info in sync +39:09.644 [DEBUG] TestAgent: Service in sync: service=web +39:09.645 [INFO] TestAgent: Synced service: service=db +39:09.645 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:55362 latency=1.406722ms +39:09.669 [DEBUG] TestAgent: removed service: service=web +39:09.669 [DEBUG] TestAgent: Node info in sync +39:09.669 [DEBUG] TestAgent: Service in sync: service=db +39:09.670 [INFO] TestAgent: Deregistered service: service=web +39:09.670 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/deregister/web from=127.0.0.1:55364 latency=914.601µs +39:09.671 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:55362 latency=249.801µs +39:09.672 [INFO] TestAgent: Requesting shutdown +39:09.672 [INFO] TestAgent.server: shutting down server +39:09.672 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +39:09.672 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +39:09.672 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +39:09.672 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +39:09.672 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +39:09.672 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +39:09.672 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +39:09.672 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +39:09.672 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +39:09.672 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +39:09.672 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:39:09.672Z [INFO] agent.router.manager: shutting down +2021-08-13T15:39:09.672Z [INFO] agent.router.manager: shutting down +39:09.673 [INFO] TestAgent: consul server down +39:09.673 [INFO] TestAgent: shutdown complete +39:09.673 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18183 network=tcp +39:09.673 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18183 network=udp +39:09.673 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:18184 network=tcp +39:10.173 [INFO] TestAgent: Waiting for endpoints to shut down +39:10.173 [INFO] TestAgent: Endpoints down +--- PASS: TestCommand_File_nameOnly (1.05s) PASS -ok github.com/hashicorp/consul/command/services/deregister 1.147s +ok github.com/hashicorp/consul/command/services/deregister 1.128s === RUN TestCommand_noTabs === PAUSE TestCommand_noTabs === RUN TestCommand_Validation @@ -90893,272 +90806,273 @@ === RUN TestCommand_FileWithUnnamedCheck === PAUSE TestCommand_FileWithUnnamedCheck === CONT TestCommand_noTabs ---- PASS: TestCommand_noTabs (0.01s) +--- PASS: TestCommand_noTabs (0.00s) === CONT TestCommand_Flags_TaggedAddresses -=== CONT TestCommand_Validation +=== CONT TestCommand_File === CONT TestCommand_Flags [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... -=== CONT TestCommand_File +=== CONT TestCommand_Validation === CONT TestCommand_FileWithUnnamedCheck -=== RUN TestCommand_Validation/args_and_-name === RUN TestCommand_Validation/no_args_or_id +=== RUN TestCommand_Validation/args_and_-name +--- PASS: TestCommand_Validation (0.00s) + --- PASS: TestCommand_Validation/no_args_or_id (0.00s) + --- PASS: TestCommand_Validation/args_and_-name (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range ---- PASS: TestCommand_Validation (0.01s) - --- PASS: TestCommand_Validation/args_and_-name (0.00s) - --- PASS: TestCommand_Validation/no_args_or_id (0.01s) -2022-09-15T21:46:33.772Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:46:33.817Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -46:33.820 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:908b613b-3044-3ebc-6a1a-a0a73f2335cf Address:127.0.0.1:29419}]" -46:33.821 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-908b613b-3044-3ebc-6a1a-a0a73f2335cf.dc1 127.0.0.1 -46:33.822 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29419 [Follower]" leader= -46:33.822 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-908b613b-3044-3ebc-6a1a-a0a73f2335cf 127.0.0.1 -2022-09-15T21:46:33.822Z [INFO] agent.router: Initializing LAN area manager -46:33.823 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29414 network=udp -46:33.823 [INFO] TestAgent.server: Adding LAN server: server="Node-908b613b-3044-3ebc-6a1a-a0a73f2335cf (Addr: tcp/127.0.0.1:29419) (DC: dc1)" -46:33.823 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-908b613b-3044-3ebc-6a1a-a0a73f2335cf.dc1 area=wan -46:33.823 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29414 network=tcp -46:33.838 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29415 network=tcp -46:33.838 [INFO] TestAgent: started state syncer -46:33.838 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29420 network=tcp -46:33.876 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -46:33.876 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29419 [Candidate]" term=2 -46:33.876 [DEBUG] TestAgent.server.raft: votes: needed=1 -46:33.876 [DEBUG] TestAgent.server.raft: vote granted: from=908b613b-3044-3ebc-6a1a-a0a73f2335cf term=2 tally=1 -46:33.876 [INFO] TestAgent.server.raft: election won: tally=1 -46:33.876 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29419 [Leader]" -46:33.876 [INFO] TestAgent.server: cluster leadership acquired -46:33.876 [INFO] TestAgent.server: New leader elected: payload=Node-908b613b-3044-3ebc-6a1a-a0a73f2335cf -46:33.877 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29419 -46:33.878 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -46:33.881 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -46:33.881 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -46:33.881 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -46:33.881 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -46:33.881 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -46:33.881 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-908b613b-3044-3ebc-6a1a-a0a73f2335cf -46:33.881 [INFO] TestAgent.server: member joined, marking health alive: member=Node-908b613b-3044-3ebc-6a1a-a0a73f2335cf -46:33.885 [INFO] TestAgent.server: federation state anti-entropy synced -46:33.948 [INFO] TestAgent: Synced node info -46:33.949 [INFO] TestAgent: Synced service: service=web -46:33.949 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:35298 latency=2.53398ms -46:33.955 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:35300 latency=5.259ms -46:33.962 [INFO] TestAgent: Requesting shutdown -46:33.962 [INFO] TestAgent.server: shutting down server -46:33.962 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -46:33.962 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -46:33.962 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -46:33.962 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -46:33.962 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -46:33.962 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -46:33.962 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -46:33.963 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -46:33.963 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -46:33.963 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -46:33.963 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -46:33.963 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:46:33.963Z [INFO] agent.router.manager: shutting down -2022-09-15T21:46:33.963Z [INFO] agent.router.manager: shutting down -46:33.963 [INFO] TestAgent: consul server down -46:33.963 [INFO] TestAgent: shutdown complete -46:33.963 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29414 network=tcp -46:33.963 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29414 network=udp -46:33.963 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29415 network=tcp -46:34.463 [INFO] TestAgent: Waiting for endpoints to shut down -46:34.463 [INFO] TestAgent: Endpoints down ---- PASS: TestCommand_File (0.84s) -2022-09-15T21:46:33.721Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:46:33.760Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -46:33.767 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e09bbe24-9e7d-0660-7a9d-bb469741605a Address:127.0.0.1:29412}]" -46:33.769 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e09bbe24-9e7d-0660-7a9d-bb469741605a.dc1 127.0.0.1 -46:33.773 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29412 [Follower]" leader= -46:33.774 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e09bbe24-9e7d-0660-7a9d-bb469741605a 127.0.0.1 -2022-09-15T21:46:33.774Z [INFO] agent.router: Initializing LAN area manager -46:33.775 [INFO] TestAgent.server: Adding LAN server: server="Node-e09bbe24-9e7d-0660-7a9d-bb469741605a (Addr: tcp/127.0.0.1:29412) (DC: dc1)" -46:33.775 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e09bbe24-9e7d-0660-7a9d-bb469741605a.dc1 area=wan -46:33.775 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29407 network=udp -46:33.775 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29407 network=tcp -46:33.776 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29408 network=tcp -46:33.776 [INFO] TestAgent: started state syncer -46:33.776 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29413 network=tcp -46:33.814 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -46:33.814 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29412 [Candidate]" term=2 -46:33.814 [DEBUG] TestAgent.server.raft: votes: needed=1 -46:33.814 [DEBUG] TestAgent.server.raft: vote granted: from=e09bbe24-9e7d-0660-7a9d-bb469741605a term=2 tally=1 -46:33.814 [INFO] TestAgent.server.raft: election won: tally=1 -46:33.814 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29412 [Leader]" -46:33.814 [INFO] TestAgent.server: cluster leadership acquired -46:33.815 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29412 -46:33.816 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -46:33.818 [INFO] TestAgent.server: New leader elected: payload=Node-e09bbe24-9e7d-0660-7a9d-bb469741605a -46:33.866 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -46:33.866 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -46:33.866 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -46:33.866 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -46:33.866 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -46:33.866 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e09bbe24-9e7d-0660-7a9d-bb469741605a -46:33.866 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e09bbe24-9e7d-0660-7a9d-bb469741605a -46:33.866 [INFO] TestAgent.server: federation state anti-entropy synced -46:33.922 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -46:33.922 [INFO] TestAgent: Synced node info -46:34.102 [DEBUG] TestAgent: Node info in sync -46:34.103 [INFO] TestAgent: Synced service: service=web -46:34.103 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:38476 latency=1.32938ms -46:34.104 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:38478 latency=435.58µs -46:34.105 [INFO] TestAgent: Requesting shutdown -46:34.105 [INFO] TestAgent.server: shutting down server -46:34.105 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -46:34.105 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -46:34.105 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -46:34.105 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -46:34.105 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -46:34.105 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -46:34.105 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -46:34.105 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -46:34.105 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -46:34.105 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -46:34.105 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:46:34.105Z [INFO] agent.router.manager: shutting down -2022-09-15T21:46:34.105Z [INFO] agent.router.manager: shutting down -46:34.105 [INFO] TestAgent: consul server down -46:34.105 [INFO] TestAgent: shutdown complete -46:34.105 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29407 network=tcp -46:34.106 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29407 network=udp -46:34.106 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29408 network=tcp -46:34.606 [INFO] TestAgent: Waiting for endpoints to shut down -46:34.606 [INFO] TestAgent: Endpoints down ---- PASS: TestCommand_Flags (0.98s) -2022-09-15T21:46:33.806Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:46:33.844Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -46:33.849 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f523a5c6-7e22-514c-9284-d367fbe45a96 Address:127.0.0.1:29426}]" -46:33.849 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29426 [Follower]" leader= -46:33.861 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f523a5c6-7e22-514c-9284-d367fbe45a96.dc1 127.0.0.1 -46:33.863 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f523a5c6-7e22-514c-9284-d367fbe45a96 127.0.0.1 -2022-09-15T21:46:33.863Z [INFO] agent.router: Initializing LAN area manager -46:33.864 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29421 network=udp -46:33.864 [INFO] TestAgent.server: Adding LAN server: server="Node-f523a5c6-7e22-514c-9284-d367fbe45a96 (Addr: tcp/127.0.0.1:29426) (DC: dc1)" -46:33.864 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f523a5c6-7e22-514c-9284-d367fbe45a96.dc1 area=wan -46:33.864 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29421 network=tcp -46:33.865 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29422 network=tcp -46:33.865 [INFO] TestAgent: started state syncer -46:33.865 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29427 network=tcp -46:33.902 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -46:33.902 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29426 [Candidate]" term=2 -46:33.902 [DEBUG] TestAgent.server.raft: votes: needed=1 -46:33.902 [DEBUG] TestAgent.server.raft: vote granted: from=f523a5c6-7e22-514c-9284-d367fbe45a96 term=2 tally=1 -46:33.902 [INFO] TestAgent.server.raft: election won: tally=1 -46:33.902 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29426 [Leader]" -46:33.902 [INFO] TestAgent.server: cluster leadership acquired -46:33.902 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29426 -46:33.904 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -46:33.906 [INFO] TestAgent.server: New leader elected: payload=Node-f523a5c6-7e22-514c-9284-d367fbe45a96 -46:33.907 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -46:33.907 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -46:33.907 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -46:33.907 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -46:33.907 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -46:33.907 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f523a5c6-7e22-514c-9284-d367fbe45a96 -46:33.907 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f523a5c6-7e22-514c-9284-d367fbe45a96 -46:33.909 [INFO] TestAgent.server: federation state anti-entropy synced -46:34.160 [INFO] TestAgent: Synced node info -46:34.160 [INFO] TestAgent: Synced service: service=web -46:34.160 [DEBUG] TestAgent: Check in sync: check=service:web -46:34.160 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:50142 latency=2.72076ms -46:34.162 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:50144 latency=395.7µs -46:34.163 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/checks from=127.0.0.1:50144 latency=262.1µs -46:34.164 [INFO] TestAgent: Requesting shutdown -46:34.164 [INFO] TestAgent.server: shutting down server -46:34.164 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -46:34.164 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -46:34.164 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -46:34.164 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -46:34.164 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -46:34.164 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -46:34.164 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -46:34.164 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -46:34.164 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -46:34.164 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -46:34.164 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -46:34.164 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:46:34.164Z [INFO] agent.router.manager: shutting down -2022-09-15T21:46:34.164Z [INFO] agent.router.manager: shutting down -46:34.164 [INFO] TestAgent: consul server down -46:34.164 [INFO] TestAgent: shutdown complete -46:34.164 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29421 network=tcp -46:34.164 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29421 network=udp -46:34.165 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29422 network=tcp -46:34.665 [INFO] TestAgent: Waiting for endpoints to shut down -46:34.665 [INFO] TestAgent: Endpoints down ---- PASS: TestCommand_FileWithUnnamedCheck (1.04s) -2022-09-15T21:46:33.728Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:46:33.788Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -46:33.804 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:dde7c100-927e-2c31-590c-904a8d74581a Address:127.0.0.1:29405}]" -46:33.805 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-dde7c100-927e-2c31-590c-904a8d74581a.dc1 127.0.0.1 -46:33.806 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-dde7c100-927e-2c31-590c-904a8d74581a 127.0.0.1 -2022-09-15T21:46:33.806Z [INFO] agent.router: Initializing LAN area manager -46:33.807 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29400 network=udp -46:33.810 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29405 [Follower]" leader= -46:33.810 [INFO] TestAgent.server: Adding LAN server: server="Node-dde7c100-927e-2c31-590c-904a8d74581a (Addr: tcp/127.0.0.1:29405) (DC: dc1)" -46:33.812 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-dde7c100-927e-2c31-590c-904a8d74581a.dc1 area=wan -46:33.812 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29400 network=tcp -46:33.813 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29401 network=tcp -46:33.813 [INFO] TestAgent: started state syncer -46:33.813 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29406 network=tcp -46:33.880 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -46:33.880 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29405 [Candidate]" term=2 -46:33.880 [DEBUG] TestAgent.server.raft: votes: needed=1 -46:33.880 [DEBUG] TestAgent.server.raft: vote granted: from=dde7c100-927e-2c31-590c-904a8d74581a term=2 tally=1 -46:33.880 [INFO] TestAgent.server.raft: election won: tally=1 -46:33.880 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29405 [Leader]" -46:33.880 [INFO] TestAgent.server: cluster leadership acquired -46:33.880 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29405 -46:33.882 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -46:33.884 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -46:33.884 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -46:33.884 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -46:33.884 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -46:33.884 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -46:33.884 [INFO] TestAgent.server: New leader elected: payload=Node-dde7c100-927e-2c31-590c-904a8d74581a -46:33.884 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-dde7c100-927e-2c31-590c-904a8d74581a -46:33.884 [INFO] TestAgent.server: member joined, marking health alive: member=Node-dde7c100-927e-2c31-590c-904a8d74581a -46:33.885 [INFO] TestAgent.server: federation state anti-entropy synced -46:33.940 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -46:33.940 [INFO] TestAgent: Synced node info -46:34.180 [DEBUG] TestAgent: Node info in sync -46:34.181 [INFO] TestAgent: Synced service: service=web -46:34.181 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:35842 latency=1.41614ms -46:34.183 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:35844 latency=118.56µs -46:34.183 [INFO] TestAgent: Requesting shutdown -46:34.183 [INFO] TestAgent.server: shutting down server -46:34.183 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -46:34.183 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -46:34.183 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -46:34.183 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -46:34.183 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -46:34.183 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -46:34.184 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -46:34.184 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -46:34.184 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -46:34.184 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -46:34.184 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:46:34.184Z [INFO] agent.router.manager: shutting down -2022-09-15T21:46:34.184Z [INFO] agent.router.manager: shutting down -46:34.184 [INFO] TestAgent: consul server down -46:34.184 [INFO] TestAgent: shutdown complete -46:34.184 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29400 network=tcp -46:34.184 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29400 network=udp -46:34.184 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29401 network=tcp -46:34.684 [INFO] TestAgent: Waiting for endpoints to shut down -46:34.684 [INFO] TestAgent: Endpoints down ---- PASS: TestCommand_Flags_TaggedAddresses (1.06s) +2021-08-13T15:39:11.466Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:39:11.501Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +39:11.511 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:93baa9f7-4c90-25c9-f959-eac42f17862a Address:127.0.0.1:24321}]" +39:11.512 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24321 [Follower]" leader= +39:11.513 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-93baa9f7-4c90-25c9-f959-eac42f17862a.dc1 127.0.0.1 +39:11.514 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-93baa9f7-4c90-25c9-f959-eac42f17862a 127.0.0.1 +2021-08-13T15:39:11.514Z [INFO] agent.router: Initializing LAN area manager +39:11.514 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24316 network=udp +39:11.515 [INFO] TestAgent.server: Adding LAN server: server="Node-93baa9f7-4c90-25c9-f959-eac42f17862a (Addr: tcp/127.0.0.1:24321) (DC: dc1)" +39:11.515 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-93baa9f7-4c90-25c9-f959-eac42f17862a.dc1 area=wan +39:11.515 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24316 network=tcp +39:11.517 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24317 network=tcp +39:11.517 [INFO] TestAgent: started state syncer +39:11.517 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24322 network=tcp +39:11.550 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +39:11.550 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24321 [Candidate]" term=2 +39:11.550 [DEBUG] TestAgent.server.raft: votes: needed=1 +39:11.550 [DEBUG] TestAgent.server.raft: vote granted: from=93baa9f7-4c90-25c9-f959-eac42f17862a term=2 tally=1 +39:11.550 [INFO] TestAgent.server.raft: election won: tally=1 +39:11.550 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24321 [Leader]" +39:11.550 [INFO] TestAgent.server: cluster leadership acquired +39:11.550 [INFO] TestAgent.server: New leader elected: payload=Node-93baa9f7-4c90-25c9-f959-eac42f17862a +39:11.550 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24321 +39:11.552 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +39:11.576 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +39:11.576 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +39:11.576 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +39:11.576 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +39:11.576 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +39:11.576 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-93baa9f7-4c90-25c9-f959-eac42f17862a +39:11.576 [INFO] TestAgent.server: member joined, marking health alive: member=Node-93baa9f7-4c90-25c9-f959-eac42f17862a +39:11.577 [INFO] TestAgent.server: federation state anti-entropy synced +39:11.677 [INFO] TestAgent: Synced node info +39:11.678 [INFO] TestAgent: Synced service: service=web +39:11.678 [DEBUG] TestAgent: Check in sync: check=service:web +39:11.678 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:53564 latency=3.752686ms +39:11.680 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:53566 latency=381.141µs +39:11.681 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/checks from=127.0.0.1:53566 latency=250.541µs +39:11.681 [INFO] TestAgent: Requesting shutdown +39:11.682 [INFO] TestAgent.server: shutting down server +39:11.682 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +39:11.682 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +39:11.682 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +39:11.682 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +39:11.682 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +39:11.682 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +39:11.682 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +39:11.682 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +39:11.682 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +39:11.682 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +39:11.682 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +39:11.682 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:39:11.682Z [INFO] agent.router.manager: shutting down +2021-08-13T15:39:11.682Z [INFO] agent.router.manager: shutting down +39:11.682 [INFO] TestAgent: consul server down +39:11.682 [INFO] TestAgent: shutdown complete +39:11.682 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24316 network=tcp +39:11.682 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24316 network=udp +39:11.682 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24317 network=tcp +39:12.183 [INFO] TestAgent: Waiting for endpoints to shut down +39:12.183 [INFO] TestAgent: Endpoints down +--- PASS: TestCommand_FileWithUnnamedCheck (0.82s) +2021-08-13T15:39:11.491Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:39:11.516Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +39:11.519 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e93433d9-6b33-9e78-6956-b5e8c6448bdf Address:127.0.0.1:24314}]" +39:11.520 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24314 [Follower]" leader= +39:11.520 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e93433d9-6b33-9e78-6956-b5e8c6448bdf.dc1 127.0.0.1 +39:11.521 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e93433d9-6b33-9e78-6956-b5e8c6448bdf 127.0.0.1 +2021-08-13T15:39:11.521Z [INFO] agent.router: Initializing LAN area manager +39:11.522 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24309 network=udp +39:11.522 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e93433d9-6b33-9e78-6956-b5e8c6448bdf.dc1 area=wan +39:11.523 [INFO] TestAgent.server: Adding LAN server: server="Node-e93433d9-6b33-9e78-6956-b5e8c6448bdf (Addr: tcp/127.0.0.1:24314) (DC: dc1)" +39:11.527 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24309 network=tcp +39:11.531 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24310 network=tcp +39:11.531 [INFO] TestAgent: started state syncer +39:11.531 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24315 network=tcp +39:11.578 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +39:11.578 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24314 [Candidate]" term=2 +39:11.578 [DEBUG] TestAgent.server.raft: votes: needed=1 +39:11.578 [DEBUG] TestAgent.server.raft: vote granted: from=e93433d9-6b33-9e78-6956-b5e8c6448bdf term=2 tally=1 +39:11.578 [INFO] TestAgent.server.raft: election won: tally=1 +39:11.578 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24314 [Leader]" +39:11.578 [INFO] TestAgent.server: cluster leadership acquired +39:11.580 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24314 +39:11.582 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +39:11.583 [INFO] TestAgent.server: New leader elected: payload=Node-e93433d9-6b33-9e78-6956-b5e8c6448bdf +39:11.589 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +39:11.590 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +39:11.590 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +39:11.590 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +39:11.590 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +39:11.590 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e93433d9-6b33-9e78-6956-b5e8c6448bdf +39:11.590 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e93433d9-6b33-9e78-6956-b5e8c6448bdf +39:11.592 [INFO] TestAgent.server: federation state anti-entropy synced +39:11.627 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +39:11.627 [INFO] TestAgent: Synced node info +39:11.752 [DEBUG] TestAgent: Node info in sync +39:11.753 [INFO] TestAgent: Synced service: service=web +39:11.753 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:39282 latency=1.240322ms +39:11.754 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:39284 latency=105.06µs +39:11.754 [INFO] TestAgent: Requesting shutdown +39:11.754 [INFO] TestAgent.server: shutting down server +39:11.754 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +39:11.754 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +39:11.754 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +39:11.754 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +39:11.754 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +39:11.755 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +39:11.755 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +39:11.755 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +39:11.755 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +39:11.755 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +39:11.755 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:39:11.755Z [INFO] agent.router.manager: shutting down +2021-08-13T15:39:11.755Z [INFO] agent.router.manager: shutting down +39:11.755 [INFO] TestAgent: consul server down +39:11.755 [INFO] TestAgent: shutdown complete +39:11.755 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24309 network=tcp +39:11.755 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24309 network=udp +39:11.755 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24310 network=tcp +39:12.255 [INFO] TestAgent: Waiting for endpoints to shut down +39:12.256 [INFO] TestAgent: Endpoints down +--- PASS: TestCommand_Flags (0.89s) +2021-08-13T15:39:11.480Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:39:11.529Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +39:11.534 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7493602d-4153-c202-0171-b67241a3cb62 Address:127.0.0.1:24307}]" +39:11.535 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24307 [Follower]" leader= +39:11.535 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7493602d-4153-c202-0171-b67241a3cb62.dc1 127.0.0.1 +39:11.536 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7493602d-4153-c202-0171-b67241a3cb62 127.0.0.1 +2021-08-13T15:39:11.536Z [INFO] agent.router: Initializing LAN area manager +39:11.537 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24302 network=udp +39:11.537 [INFO] TestAgent.server: Adding LAN server: server="Node-7493602d-4153-c202-0171-b67241a3cb62 (Addr: tcp/127.0.0.1:24307) (DC: dc1)" +39:11.537 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7493602d-4153-c202-0171-b67241a3cb62.dc1 area=wan +39:11.537 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24302 network=tcp +39:11.538 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24303 network=tcp +39:11.538 [INFO] TestAgent: started state syncer +39:11.539 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24308 network=tcp +39:11.597 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +39:11.597 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24307 [Candidate]" term=2 +39:11.598 [DEBUG] TestAgent.server.raft: votes: needed=1 +39:11.598 [DEBUG] TestAgent.server.raft: vote granted: from=7493602d-4153-c202-0171-b67241a3cb62 term=2 tally=1 +39:11.598 [INFO] TestAgent.server.raft: election won: tally=1 +39:11.598 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24307 [Leader]" +39:11.598 [INFO] TestAgent.server: cluster leadership acquired +39:11.598 [INFO] TestAgent.server: New leader elected: payload=Node-7493602d-4153-c202-0171-b67241a3cb62 +39:11.598 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24307 +39:11.599 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +39:11.602 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +39:11.602 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +39:11.602 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +39:11.602 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +39:11.602 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +39:11.602 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7493602d-4153-c202-0171-b67241a3cb62 +39:11.602 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7493602d-4153-c202-0171-b67241a3cb62 +39:11.603 [INFO] TestAgent.server: federation state anti-entropy synced +39:11.929 [INFO] TestAgent: Synced node info +39:11.930 [INFO] TestAgent: Synced service: service=web +39:11.930 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:41808 latency=1.735502ms +39:11.931 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:41810 latency=197.761µs +39:11.931 [INFO] TestAgent: Requesting shutdown +39:11.931 [INFO] TestAgent.server: shutting down server +39:11.931 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +39:11.931 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +39:11.931 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +39:11.931 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +39:11.931 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +39:11.932 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +39:11.932 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +39:11.932 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +39:11.932 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +39:11.932 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +39:11.932 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +39:11.932 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:39:11.932Z [INFO] agent.router.manager: shutting down +2021-08-13T15:39:11.932Z [INFO] agent.router.manager: shutting down +39:11.932 [INFO] TestAgent: consul server down +39:11.932 [INFO] TestAgent: shutdown complete +39:11.932 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24302 network=tcp +39:11.932 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24302 network=udp +39:11.932 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24303 network=tcp +39:12.433 [INFO] TestAgent: Waiting for endpoints to shut down +39:12.433 [INFO] TestAgent: Endpoints down +--- PASS: TestCommand_File (1.07s) +2021-08-13T15:39:11.508Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:39:11.540Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +39:11.543 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3a22531c-919d-aa00-e090-57f3fc4717a0 Address:127.0.0.1:24300}]" +39:11.543 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24300 [Follower]" leader= +39:11.544 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3a22531c-919d-aa00-e090-57f3fc4717a0.dc1 127.0.0.1 +39:11.545 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3a22531c-919d-aa00-e090-57f3fc4717a0 127.0.0.1 +2021-08-13T15:39:11.545Z [INFO] agent.router: Initializing LAN area manager +39:11.546 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24295 network=udp +39:11.546 [INFO] TestAgent.server: Adding LAN server: server="Node-3a22531c-919d-aa00-e090-57f3fc4717a0 (Addr: tcp/127.0.0.1:24300) (DC: dc1)" +39:11.546 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3a22531c-919d-aa00-e090-57f3fc4717a0.dc1 area=wan +39:11.546 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24295 network=tcp +39:11.547 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24296 network=tcp +39:11.547 [INFO] TestAgent: started state syncer +39:11.548 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24301 network=tcp +39:11.605 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +39:11.605 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24300 [Candidate]" term=2 +39:11.605 [DEBUG] TestAgent.server.raft: votes: needed=1 +39:11.605 [DEBUG] TestAgent.server.raft: vote granted: from=3a22531c-919d-aa00-e090-57f3fc4717a0 term=2 tally=1 +39:11.605 [INFO] TestAgent.server.raft: election won: tally=1 +39:11.605 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24300 [Leader]" +39:11.605 [INFO] TestAgent.server: cluster leadership acquired +39:11.605 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24300 +39:11.606 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +39:11.607 [INFO] TestAgent.server: New leader elected: payload=Node-3a22531c-919d-aa00-e090-57f3fc4717a0 +39:11.609 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +39:11.609 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +39:11.609 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +39:11.609 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +39:11.609 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +39:11.609 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3a22531c-919d-aa00-e090-57f3fc4717a0 +39:11.609 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3a22531c-919d-aa00-e090-57f3fc4717a0 +39:11.610 [INFO] TestAgent.server: federation state anti-entropy synced +39:11.614 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +39:11.615 [INFO] TestAgent: Synced node info +39:11.615 [DEBUG] TestAgent: Node info in sync +39:11.955 [DEBUG] TestAgent: Node info in sync +39:11.956 [INFO] TestAgent: Synced service: service=web +39:11.956 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:35428 latency=1.456822ms +39:11.957 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:35430 latency=260.48µs +39:11.958 [INFO] TestAgent: Requesting shutdown +39:11.958 [INFO] TestAgent.server: shutting down server +39:11.958 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +39:11.958 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +39:11.958 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +39:11.958 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +39:11.958 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +39:11.958 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +39:11.958 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +39:11.958 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +39:11.958 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +39:11.958 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +39:11.959 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:39:11.959Z [INFO] agent.router.manager: shutting down +2021-08-13T15:39:11.959Z [INFO] agent.router.manager: shutting down +39:11.959 [INFO] TestAgent: consul server down +39:11.959 [INFO] TestAgent: shutdown complete +39:11.959 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24295 network=tcp +39:11.959 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24295 network=udp +39:11.959 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24296 network=tcp +39:12.463 [INFO] TestAgent: Waiting for endpoints to shut down +39:12.463 [INFO] TestAgent: Endpoints down +--- PASS: TestCommand_Flags_TaggedAddresses (1.10s) PASS -ok github.com/hashicorp/consul/command/services/register 1.165s +ok github.com/hashicorp/consul/command/services/register 1.252s === RUN TestSnapshotCommand_noTabs === PAUSE TestSnapshotCommand_noTabs === CONT TestSnapshotCommand_noTabs --- PASS: TestSnapshotCommand_noTabs (0.00s) PASS -ok github.com/hashicorp/consul/command/snapshot 0.018s +ok github.com/hashicorp/consul/command/snapshot 0.006s === RUN TestSnapshotInspectCommand_noTabs === PAUSE TestSnapshotInspectCommand_noTabs === RUN TestSnapshotInspectCommand_Validation @@ -91166,77 +91080,78 @@ === RUN TestSnapshotInspectCommand === PAUSE TestSnapshotInspectCommand === CONT TestSnapshotInspectCommand_noTabs -=== CONT TestSnapshotInspectCommand --- PASS: TestSnapshotInspectCommand_noTabs (0.00s) +=== CONT TestSnapshotInspectCommand +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestSnapshotInspectCommand_Validation --- PASS: TestSnapshotInspectCommand_Validation (0.00s) -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:46:35.021Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:46:35.039Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -46:35.043 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a08608fb-055f-9da8-92a0-2d9d29f6ab23 Address:127.0.0.1:18174}]" -46:35.043 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:18174 [Follower]" leader= -46:35.044 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a08608fb-055f-9da8-92a0-2d9d29f6ab23.dc1 127.0.0.1 -46:35.045 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a08608fb-055f-9da8-92a0-2d9d29f6ab23 127.0.0.1 -2022-09-15T21:46:35.045Z [INFO] agent.router: Initializing LAN area manager -46:35.045 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18169 network=udp -46:35.045 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a08608fb-055f-9da8-92a0-2d9d29f6ab23.dc1 area=wan -46:35.045 [INFO] TestAgent.server: Adding LAN server: server="Node-a08608fb-055f-9da8-92a0-2d9d29f6ab23 (Addr: tcp/127.0.0.1:18174) (DC: dc1)" -46:35.046 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18169 network=tcp -46:35.058 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:18170 network=tcp -46:35.058 [INFO] TestAgent: started state syncer -46:35.058 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:18175 network=tcp -46:35.090 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -46:35.091 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:18174 [Candidate]" term=2 -46:35.091 [DEBUG] TestAgent.server.raft: votes: needed=1 -46:35.091 [DEBUG] TestAgent.server.raft: vote granted: from=a08608fb-055f-9da8-92a0-2d9d29f6ab23 term=2 tally=1 -46:35.091 [INFO] TestAgent.server.raft: election won: tally=1 -46:35.091 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:18174 [Leader]" -46:35.091 [INFO] TestAgent.server: cluster leadership acquired -46:35.091 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:18174 -46:35.093 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -46:35.094 [INFO] TestAgent.server: New leader elected: payload=Node-a08608fb-055f-9da8-92a0-2d9d29f6ab23 -46:35.126 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -46:35.126 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -46:35.126 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -46:35.126 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -46:35.126 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -46:35.126 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a08608fb-055f-9da8-92a0-2d9d29f6ab23 -46:35.126 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a08608fb-055f-9da8-92a0-2d9d29f6ab23 -46:35.128 [INFO] TestAgent.server: federation state anti-entropy synced -46:35.348 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -46:35.349 [INFO] TestAgent: Synced node info -46:35.382 [INFO] TestAgent.server.fsm: snapshot created: duration=125.02µs -46:35.382 [INFO] TestAgent.server.raft: starting snapshot up to: index=12 -46:35.382 [INFO] TestAgent.server.snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshotInspectCommand-TestAgent-agent461532435/raft/snapshots/2-12-1663278395382.tmp -46:35.383 [INFO] TestAgent.server.raft: snapshot complete up to: index=12 -46:35.386 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/snapshot from=127.0.0.1:54796 latency=4.60752ms -46:35.391 [INFO] TestAgent: Requesting shutdown -46:35.391 [INFO] TestAgent.server: shutting down server -46:35.391 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -46:35.391 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -46:35.391 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -46:35.391 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -46:35.391 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -46:35.391 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -46:35.391 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -46:35.391 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -46:35.391 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -46:35.391 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -46:35.391 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:46:35.392Z [INFO] agent.router.manager: shutting down -2022-09-15T21:46:35.392Z [INFO] agent.router.manager: shutting down -46:35.392 [INFO] TestAgent: consul server down -46:35.392 [INFO] TestAgent: shutdown complete -46:35.392 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18169 network=tcp -46:35.392 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18169 network=udp -46:35.392 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:18170 network=tcp -46:35.892 [INFO] TestAgent: Waiting for endpoints to shut down -46:35.892 [INFO] TestAgent: Endpoints down ---- PASS: TestSnapshotInspectCommand (0.96s) +2021-08-13T15:39:17.769Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:39:17.791Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +39:17.795 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a0e575ae-68d3-7054-19ba-e3235aa4bf36 Address:127.0.0.1:29405}]" +39:17.795 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29405 [Follower]" leader= +39:17.796 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a0e575ae-68d3-7054-19ba-e3235aa4bf36.dc1 127.0.0.1 +39:17.797 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a0e575ae-68d3-7054-19ba-e3235aa4bf36 127.0.0.1 +2021-08-13T15:39:17.797Z [INFO] agent.router: Initializing LAN area manager +39:17.798 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29400 network=udp +39:17.798 [INFO] TestAgent.server: Adding LAN server: server="Node-a0e575ae-68d3-7054-19ba-e3235aa4bf36 (Addr: tcp/127.0.0.1:29405) (DC: dc1)" +39:17.798 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a0e575ae-68d3-7054-19ba-e3235aa4bf36.dc1 area=wan +39:17.799 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29400 network=tcp +39:17.800 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29401 network=tcp +39:17.800 [INFO] TestAgent: started state syncer +39:17.800 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29406 network=tcp +39:17.834 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +39:17.834 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29405 [Candidate]" term=2 +39:17.834 [DEBUG] TestAgent.server.raft: votes: needed=1 +39:17.834 [DEBUG] TestAgent.server.raft: vote granted: from=a0e575ae-68d3-7054-19ba-e3235aa4bf36 term=2 tally=1 +39:17.834 [INFO] TestAgent.server.raft: election won: tally=1 +39:17.834 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29405 [Leader]" +39:17.834 [INFO] TestAgent.server: cluster leadership acquired +39:17.834 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29405 +39:17.836 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +39:17.839 [INFO] TestAgent.server: New leader elected: payload=Node-a0e575ae-68d3-7054-19ba-e3235aa4bf36 +39:17.860 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +39:17.860 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +39:17.860 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +39:17.860 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +39:17.860 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +39:17.860 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a0e575ae-68d3-7054-19ba-e3235aa4bf36 +39:17.860 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a0e575ae-68d3-7054-19ba-e3235aa4bf36 +39:17.862 [INFO] TestAgent.server: federation state anti-entropy synced +39:17.930 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +39:17.931 [INFO] TestAgent: Synced node info +39:17.931 [DEBUG] TestAgent: Node info in sync +39:18.185 [INFO] TestAgent.server.fsm: snapshot created: duration=102.44µs +39:18.185 [INFO] TestAgent.server.raft: starting snapshot up to: index=12 +39:18.185 [INFO] TestAgent.server.snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshotInspectCommand-TestAgent-agent645121957/raft/snapshots/2-12-1628869158185.tmp +39:18.186 [INFO] TestAgent.server.raft: snapshot complete up to: index=12 +39:18.194 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/snapshot from=127.0.0.1:60920 latency=9.782294ms +39:18.198 [INFO] TestAgent: Requesting shutdown +39:18.198 [INFO] TestAgent.server: shutting down server +39:18.198 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +39:18.198 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +39:18.198 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +39:18.198 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +39:18.198 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +39:18.198 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +39:18.198 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +39:18.198 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +39:18.198 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +39:18.198 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +39:18.198 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:39:18.199Z [INFO] agent.router.manager: shutting down +2021-08-13T15:39:18.199Z [INFO] agent.router.manager: shutting down +39:18.199 [INFO] TestAgent: consul server down +39:18.199 [INFO] TestAgent: shutdown complete +39:18.199 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29400 network=tcp +39:18.199 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29400 network=udp +39:18.199 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29401 network=tcp +39:18.699 [INFO] TestAgent: Waiting for endpoints to shut down +39:18.699 [INFO] TestAgent: Endpoints down +--- PASS: TestSnapshotInspectCommand (1.00s) PASS -ok github.com/hashicorp/consul/command/snapshot/inspect 1.074s +ok github.com/hashicorp/consul/command/snapshot/inspect 1.065s === RUN TestSnapshotRestoreCommand_noTabs === PAUSE TestSnapshotRestoreCommand_noTabs === RUN TestSnapshotRestoreCommand_Validation @@ -91250,8 +91165,8 @@ === CONT TestSnapshotRestoreCommand_TruncatedSnapshot [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestSnapshotRestoreCommand_Validation ---- PASS: TestSnapshotRestoreCommand_Validation (0.00s) === CONT TestSnapshotRestoreCommand +--- PASS: TestSnapshotRestoreCommand_Validation (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range === RUN TestSnapshotRestoreCommand_TruncatedSnapshot/truncate_200_bytes_from_end @@ -91260,171 +91175,169 @@ === RUN TestSnapshotRestoreCommand_TruncatedSnapshot/truncate_4_bytes_from_end === RUN TestSnapshotRestoreCommand_TruncatedSnapshot/truncate_2_bytes_from_end === RUN TestSnapshotRestoreCommand_TruncatedSnapshot/truncate_1_bytes_from_end -2022-09-15T21:46:35.418Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:46:35.435Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -46:35.446 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7aa00cbb-6081-d97d-c9f3-df5c759f12d0 Address:127.0.0.1:25328}]" -46:35.446 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25328 [Follower]" leader= -46:35.454 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7aa00cbb-6081-d97d-c9f3-df5c759f12d0.dc1 127.0.0.1 -46:35.455 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7aa00cbb-6081-d97d-c9f3-df5c759f12d0 127.0.0.1 -2022-09-15T21:46:35.455Z [INFO] agent.router: Initializing LAN area manager -46:35.455 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7aa00cbb-6081-d97d-c9f3-df5c759f12d0.dc1 area=wan -46:35.455 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25323 network=udp -46:35.455 [INFO] TestAgent.server: Adding LAN server: server="Node-7aa00cbb-6081-d97d-c9f3-df5c759f12d0 (Addr: tcp/127.0.0.1:25328) (DC: dc1)" -46:35.456 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25323 network=tcp -46:35.456 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25324 network=tcp -46:35.457 [INFO] TestAgent: started state syncer -46:35.457 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25329 network=tcp -46:35.501 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -46:35.501 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25328 [Candidate]" term=2 -46:35.501 [DEBUG] TestAgent.server.raft: votes: needed=1 -46:35.501 [DEBUG] TestAgent.server.raft: vote granted: from=7aa00cbb-6081-d97d-c9f3-df5c759f12d0 term=2 tally=1 -46:35.501 [INFO] TestAgent.server.raft: election won: tally=1 -46:35.501 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25328 [Leader]" -46:35.501 [INFO] TestAgent.server: cluster leadership acquired -46:35.502 [INFO] TestAgent.server: New leader elected: payload=Node-7aa00cbb-6081-d97d-c9f3-df5c759f12d0 -46:35.502 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25328 -46:35.504 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -46:35.529 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -46:35.529 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -46:35.529 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -46:35.529 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -46:35.529 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -46:35.529 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7aa00cbb-6081-d97d-c9f3-df5c759f12d0 -46:35.529 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7aa00cbb-6081-d97d-c9f3-df5c759f12d0 -46:35.548 [INFO] TestAgent.server: federation state anti-entropy synced -46:35.689 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -46:35.690 [INFO] TestAgent: Synced node info -46:35.744 [INFO] TestAgent.server.fsm: snapshot created: duration=99.1µs -46:35.744 [INFO] TestAgent.server.raft: starting snapshot up to: index=12 -46:35.744 [INFO] TestAgent.server.snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshotRestoreCommand-TestAgent-agent375885833/raft/snapshots/2-12-1663278395744.tmp -46:35.746 [INFO] TestAgent.server.raft: snapshot complete up to: index=12 -46:35.749 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/snapshot from=127.0.0.1:55158 latency=4.73118ms -46:35.752 [INFO] TestAgent.server.snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshotRestoreCommand-TestAgent-agent375885833/raft/snapshots/2-13-1663278395752.tmp -46:35.753 [INFO] TestAgent.server.raft: copied to local snapshot: bytes=5146 -46:35.756 [INFO] TestAgent.server.raft: restored user snapshot: index=1 -46:35.756 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -46:35.756 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -46:35.756 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -46:35.756 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -46:35.756 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -46:35.756 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -46:35.756 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -46:35.756 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25328 -46:35.757 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -46:35.757 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -46:35.757 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -46:35.757 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -46:35.757 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/snapshot from=127.0.0.1:55160 latency=5.95222ms -46:35.758 [INFO] TestAgent: Requesting shutdown -46:35.758 [INFO] TestAgent.server: shutting down server -46:35.758 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -46:35.758 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -46:35.758 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -46:35.758 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -46:35.758 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -46:35.758 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -46:35.758 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -46:35.758 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -46:35.758 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -46:35.758 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -46:35.758 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -46:35.758 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -46:35.758 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:46:35.758Z [INFO] agent.router.manager: shutting down -2022-09-15T21:46:35.758Z [INFO] agent.router.manager: shutting down -46:35.758 [INFO] TestAgent: consul server down -46:35.758 [INFO] TestAgent: shutdown complete -46:35.758 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25323 network=tcp -46:35.758 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25323 network=udp -46:35.758 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25324 network=tcp -46:36.260 [INFO] TestAgent: Waiting for endpoints to shut down -46:36.260 [INFO] TestAgent: Endpoints down ---- PASS: TestSnapshotRestoreCommand (0.95s) -2022-09-15T21:46:35.422Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:46:35.445Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -46:35.449 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3dc0434d-d2ca-acf9-51d8-399798c0137e Address:127.0.0.1:25321}]" -46:35.449 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25321 [Follower]" leader= -46:35.450 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3dc0434d-d2ca-acf9-51d8-399798c0137e.dc1 127.0.0.1 -46:35.451 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3dc0434d-d2ca-acf9-51d8-399798c0137e 127.0.0.1 -2022-09-15T21:46:35.451Z [INFO] agent.router: Initializing LAN area manager -46:35.452 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25316 network=udp -46:35.452 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3dc0434d-d2ca-acf9-51d8-399798c0137e.dc1 area=wan -46:35.452 [INFO] TestAgent.server: Adding LAN server: server="Node-3dc0434d-d2ca-acf9-51d8-399798c0137e (Addr: tcp/127.0.0.1:25321) (DC: dc1)" -46:35.452 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25316 network=tcp -46:35.453 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25317 network=tcp -46:35.453 [INFO] TestAgent: started state syncer -46:35.454 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25322 network=tcp -46:35.518 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -46:35.518 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25321 [Candidate]" term=2 -46:35.518 [DEBUG] TestAgent.server.raft: votes: needed=1 -46:35.518 [DEBUG] TestAgent.server.raft: vote granted: from=3dc0434d-d2ca-acf9-51d8-399798c0137e term=2 tally=1 -46:35.518 [INFO] TestAgent.server.raft: election won: tally=1 -46:35.518 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25321 [Leader]" -46:35.518 [INFO] TestAgent.server: cluster leadership acquired -46:35.519 [INFO] TestAgent.server: New leader elected: payload=Node-3dc0434d-d2ca-acf9-51d8-399798c0137e -46:35.519 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25321 -46:35.520 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -46:35.528 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -46:35.528 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -46:35.528 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -46:35.528 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -46:35.528 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -46:35.528 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3dc0434d-d2ca-acf9-51d8-399798c0137e -46:35.528 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3dc0434d-d2ca-acf9-51d8-399798c0137e -46:35.529 [INFO] TestAgent.server: federation state anti-entropy synced -46:35.794 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -46:35.794 [INFO] TestAgent: Synced node info -46:35.816 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/blob from=127.0.0.1:37012 latency=1.4555ms -46:35.816 [INFO] TestAgent.server.fsm: snapshot created: duration=88.78µs -46:35.816 [INFO] TestAgent.server.raft: starting snapshot up to: index=13 -46:35.816 [INFO] TestAgent.server.snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshotRestoreCommand_TruncatedSnapshot-TestAgent-agent329781970/raft/snapshots/2-13-1663278395816.tmp -46:35.818 [INFO] TestAgent.server.raft: snapshot complete up to: index=13 -46:35.851 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/snapshot from=127.0.0.1:37012 latency=34.47164ms -46:35.860 [ERROR] TestAgent.server: Failed to close snapshot decompressor: error="unexpected EOF" -46:35.860 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/snapshot from=127.0.0.1:37014 error="failed to read snapshot file: failed reading snapshot: unexpected EOF" -46:35.860 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/snapshot from=127.0.0.1:37014 latency=3.7269ms -46:35.866 [ERROR] TestAgent.server: Failed to close snapshot decompressor: error="unexpected EOF" -46:35.866 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/snapshot from=127.0.0.1:37016 error="failed to read snapshot file: failed reading snapshot: unexpected EOF" -46:35.866 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/snapshot from=127.0.0.1:37016 latency=3.43966ms -46:35.871 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/snapshot from=127.0.0.1:37018 error="unexpected EOF" -46:35.871 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/snapshot from=127.0.0.1:37018 latency=3.43304ms -46:35.877 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/snapshot from=127.0.0.1:37020 error="unexpected EOF" -46:35.877 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/snapshot from=127.0.0.1:37020 latency=3.40212ms -46:35.882 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/snapshot from=127.0.0.1:37022 error="unexpected EOF" -46:35.882 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/snapshot from=127.0.0.1:37022 latency=3.40782ms -46:35.888 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/snapshot from=127.0.0.1:37024 error="unexpected EOF" -46:35.889 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/snapshot from=127.0.0.1:37024 latency=3.56458ms -46:35.894 [INFO] TestAgent: Requesting shutdown -46:35.894 [INFO] TestAgent.server: shutting down server -46:35.894 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -46:35.894 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -46:35.894 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -46:35.894 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -46:35.894 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -46:35.894 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -46:35.894 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -46:35.894 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -46:35.894 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -46:35.894 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -46:35.895 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:46:35.895Z [INFO] agent.router.manager: shutting down -2022-09-15T21:46:35.895Z [INFO] agent.router.manager: shutting down -46:35.895 [INFO] TestAgent: consul server down -46:35.895 [INFO] TestAgent: shutdown complete -46:35.895 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25316 network=tcp -46:35.895 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25316 network=udp -46:35.895 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25317 network=tcp -46:36.395 [INFO] TestAgent: Waiting for endpoints to shut down -46:36.395 [INFO] TestAgent: Endpoints down ---- PASS: TestSnapshotRestoreCommand_TruncatedSnapshot (1.09s) +2021-08-13T15:39:20.625Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:39:20.648Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +39:20.653 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:71181d05-8897-5ce7-e753-870c1c98e060 Address:127.0.0.1:13076}]" +39:20.654 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-71181d05-8897-5ce7-e753-870c1c98e060.dc1 127.0.0.1 +39:20.656 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13076 [Follower]" leader= +39:20.657 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-71181d05-8897-5ce7-e753-870c1c98e060 127.0.0.1 +2021-08-13T15:39:20.657Z [INFO] agent.router: Initializing LAN area manager +39:20.658 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13071 network=udp +39:20.658 [INFO] TestAgent.server: Adding LAN server: server="Node-71181d05-8897-5ce7-e753-870c1c98e060 (Addr: tcp/127.0.0.1:13076) (DC: dc1)" +39:20.658 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-71181d05-8897-5ce7-e753-870c1c98e060.dc1 area=wan +39:20.658 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13071 network=tcp +39:20.659 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13072 network=tcp +39:20.659 [INFO] TestAgent: started state syncer +39:20.660 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13077 network=tcp +39:20.701 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +39:20.701 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13076 [Candidate]" term=2 +39:20.701 [DEBUG] TestAgent.server.raft: votes: needed=1 +39:20.701 [DEBUG] TestAgent.server.raft: vote granted: from=71181d05-8897-5ce7-e753-870c1c98e060 term=2 tally=1 +39:20.701 [INFO] TestAgent.server.raft: election won: tally=1 +39:20.701 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13076 [Leader]" +39:20.702 [INFO] TestAgent.server: cluster leadership acquired +39:20.702 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13076 +39:20.702 [INFO] TestAgent.server: New leader elected: payload=Node-71181d05-8897-5ce7-e753-870c1c98e060 +39:20.703 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +39:20.734 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +39:20.734 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +39:20.734 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +39:20.734 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +39:20.734 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +39:20.734 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-71181d05-8897-5ce7-e753-870c1c98e060 +39:20.734 [INFO] TestAgent.server: member joined, marking health alive: member=Node-71181d05-8897-5ce7-e753-870c1c98e060 +39:20.740 [INFO] TestAgent.server: federation state anti-entropy synced +39:20.781 [INFO] TestAgent.server.fsm: snapshot created: duration=94µs +39:20.781 [INFO] TestAgent.server.raft: starting snapshot up to: index=11 +39:20.781 [INFO] TestAgent.server.snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshotRestoreCommand-TestAgent-agent649544659/raft/snapshots/2-11-1628869160781.tmp +39:20.782 [INFO] TestAgent.server.raft: snapshot complete up to: index=11 +39:20.785 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/snapshot from=127.0.0.1:51988 latency=4.395986ms +39:20.788 [INFO] TestAgent.server.snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshotRestoreCommand-TestAgent-agent649544659/raft/snapshots/2-12-1628869160788.tmp +39:20.789 [INFO] TestAgent.server.raft: copied to local snapshot: bytes=4876 +39:20.795 [INFO] TestAgent.server.raft: restored user snapshot: index=1 +39:20.796 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +39:20.796 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +39:20.796 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +39:20.796 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +39:20.796 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +39:20.796 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +39:20.796 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +39:20.796 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +39:20.796 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +39:20.796 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13076 +39:20.797 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +39:20.797 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +39:20.797 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +39:20.797 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +39:20.797 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/snapshot from=127.0.0.1:51990 latency=9.968134ms +39:20.798 [INFO] TestAgent: Requesting shutdown +39:20.798 [INFO] TestAgent.server: shutting down server +39:20.798 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +39:20.798 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +39:20.798 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +39:20.798 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +39:20.798 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +39:20.798 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +39:20.798 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +39:20.798 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +39:20.798 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +39:20.798 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +39:20.798 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +39:20.799 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:39:20.799Z [INFO] agent.router.manager: shutting down +2021-08-13T15:39:20.799Z [INFO] agent.router.manager: shutting down +39:20.799 [INFO] TestAgent: consul server down +39:20.799 [INFO] TestAgent: shutdown complete +39:20.799 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13071 network=tcp +39:20.799 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13071 network=udp +39:20.799 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13072 network=tcp +39:21.299 [INFO] TestAgent: Waiting for endpoints to shut down +39:21.299 [INFO] TestAgent: Endpoints down +--- PASS: TestSnapshotRestoreCommand (0.74s) +2021-08-13T15:39:20.631Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:39:20.656Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +39:20.658 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:43487680-5b28-ce30-24ff-6f299b2eeddd Address:127.0.0.1:13069}]" +39:20.659 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-43487680-5b28-ce30-24ff-6f299b2eeddd.dc1 127.0.0.1 +39:20.660 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-43487680-5b28-ce30-24ff-6f299b2eeddd 127.0.0.1 +2021-08-13T15:39:20.660Z [INFO] agent.router: Initializing LAN area manager +39:20.660 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13064 network=udp +39:20.660 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13069 [Follower]" leader= +39:20.661 [INFO] TestAgent.server: Adding LAN server: server="Node-43487680-5b28-ce30-24ff-6f299b2eeddd (Addr: tcp/127.0.0.1:13069) (DC: dc1)" +39:20.661 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-43487680-5b28-ce30-24ff-6f299b2eeddd.dc1 area=wan +39:20.661 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13064 network=tcp +39:20.662 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13065 network=tcp +39:20.662 [INFO] TestAgent: started state syncer +39:20.662 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13070 network=tcp +39:20.701 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +39:20.701 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13069 [Candidate]" term=2 +39:20.701 [DEBUG] TestAgent.server.raft: votes: needed=1 +39:20.701 [DEBUG] TestAgent.server.raft: vote granted: from=43487680-5b28-ce30-24ff-6f299b2eeddd term=2 tally=1 +39:20.701 [INFO] TestAgent.server.raft: election won: tally=1 +39:20.701 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13069 [Leader]" +39:20.702 [INFO] TestAgent.server: cluster leadership acquired +39:20.702 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13069 +39:20.702 [INFO] TestAgent.server: New leader elected: payload=Node-43487680-5b28-ce30-24ff-6f299b2eeddd +39:20.708 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +39:20.740 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +39:20.740 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +39:20.740 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +39:20.740 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +39:20.740 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +39:20.740 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-43487680-5b28-ce30-24ff-6f299b2eeddd +39:20.740 [INFO] TestAgent.server: member joined, marking health alive: member=Node-43487680-5b28-ce30-24ff-6f299b2eeddd +39:20.742 [INFO] TestAgent.server: federation state anti-entropy synced +39:20.910 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/blob from=127.0.0.1:36632 latency=1.479822ms +39:20.911 [INFO] TestAgent.server.fsm: snapshot created: duration=90.78µs +39:20.911 [INFO] TestAgent.server.raft: starting snapshot up to: index=12 +39:20.911 [INFO] TestAgent.server.snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshotRestoreCommand_TruncatedSnapshot-TestAgent-agent239985476/raft/snapshots/2-12-1628869160911.tmp +39:20.913 [INFO] TestAgent.server.raft: snapshot complete up to: index=12 +39:20.942 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/snapshot from=127.0.0.1:36632 latency=30.680423ms +39:20.948 [ERROR] TestAgent.server: Failed to close snapshot decompressor: error="unexpected EOF" +39:20.948 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/snapshot from=127.0.0.1:36634 error="failed to read snapshot file: failed reading snapshot: unexpected EOF" +39:20.948 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/snapshot from=127.0.0.1:36634 latency=3.312984ms +39:20.953 [ERROR] TestAgent.server: Failed to close snapshot decompressor: error="unexpected EOF" +39:20.953 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/snapshot from=127.0.0.1:36636 error="failed to read snapshot file: failed reading snapshot: unexpected EOF" +39:20.953 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/snapshot from=127.0.0.1:36636 latency=3.347584ms +39:20.958 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/snapshot from=127.0.0.1:36638 error="unexpected EOF" +39:20.958 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/snapshot from=127.0.0.1:36638 latency=3.256864ms +39:20.963 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/snapshot from=127.0.0.1:36640 error="unexpected EOF" +39:20.963 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/snapshot from=127.0.0.1:36640 latency=3.269784ms +39:20.968 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/snapshot from=127.0.0.1:36642 error="unexpected EOF" +39:20.968 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/snapshot from=127.0.0.1:36642 latency=3.455244ms +39:20.973 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/snapshot from=127.0.0.1:36644 error="unexpected EOF" +39:20.973 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/snapshot from=127.0.0.1:36644 latency=3.338805ms +39:20.973 [INFO] TestAgent: Requesting shutdown +39:20.974 [INFO] TestAgent.server: shutting down server +39:20.974 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +39:20.974 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +39:20.974 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +39:20.974 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +39:20.974 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +39:20.974 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +39:20.974 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +39:20.974 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +39:20.974 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +39:20.974 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +39:20.974 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +39:20.974 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2021-08-13T15:39:20.974Z [INFO] agent.router.manager: shutting down +2021-08-13T15:39:20.974Z [INFO] agent.router.manager: shutting down +39:20.974 [INFO] TestAgent: consul server down +39:20.974 [INFO] TestAgent: shutdown complete +39:20.974 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13064 network=tcp +39:20.974 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13064 network=udp +39:20.974 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13065 network=tcp +39:21.474 [INFO] TestAgent: Waiting for endpoints to shut down +39:21.474 [INFO] TestAgent: Endpoints down +--- PASS: TestSnapshotRestoreCommand_TruncatedSnapshot (0.91s) --- PASS: TestSnapshotRestoreCommand_TruncatedSnapshot/truncate_200_bytes_from_end (0.01s) - --- PASS: TestSnapshotRestoreCommand_TruncatedSnapshot/truncate_16_bytes_from_end (0.01s) - --- PASS: TestSnapshotRestoreCommand_TruncatedSnapshot/truncate_8_bytes_from_end (0.01s) - --- PASS: TestSnapshotRestoreCommand_TruncatedSnapshot/truncate_4_bytes_from_end (0.01s) + --- PASS: TestSnapshotRestoreCommand_TruncatedSnapshot/truncate_16_bytes_from_end (0.00s) + --- PASS: TestSnapshotRestoreCommand_TruncatedSnapshot/truncate_8_bytes_from_end (0.00s) + --- PASS: TestSnapshotRestoreCommand_TruncatedSnapshot/truncate_4_bytes_from_end (0.00s) --- PASS: TestSnapshotRestoreCommand_TruncatedSnapshot/truncate_2_bytes_from_end (0.01s) --- PASS: TestSnapshotRestoreCommand_TruncatedSnapshot/truncate_1_bytes_from_end (0.01s) PASS -ok github.com/hashicorp/consul/command/snapshot/restore 1.188s +ok github.com/hashicorp/consul/command/snapshot/restore 1.063s === RUN TestSnapshotSaveCommand_noTabs === PAUSE TestSnapshotSaveCommand_noTabs === RUN TestSnapshotSaveCommand_Validation @@ -91434,10 +91347,10 @@ === RUN TestSnapshotSaveCommand_TruncatedStream === PAUSE TestSnapshotSaveCommand_TruncatedStream === CONT TestSnapshotSaveCommand_noTabs -=== CONT TestSnapshotSaveCommand_TruncatedStream --- PASS: TestSnapshotSaveCommand_noTabs (0.00s) -=== CONT TestSnapshotSaveCommand +=== CONT TestSnapshotSaveCommand_TruncatedStream [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +=== CONT TestSnapshotSaveCommand === CONT TestSnapshotSaveCommand_Validation --- PASS: TestSnapshotSaveCommand_Validation (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] @@ -91448,156 +91361,155 @@ === RUN TestSnapshotSaveCommand_TruncatedStream/truncate_4_bytes_from_end === RUN TestSnapshotSaveCommand_TruncatedStream/truncate_2_bytes_from_end === RUN TestSnapshotSaveCommand_TruncatedStream/truncate_1_bytes_from_end -2022-09-15T21:46:42.297Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:46:42.315Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -46:42.330 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:64224304-8480-120b-2e28-8d1642120401 Address:127.0.0.1:13076}]" -46:42.332 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13076 [Follower]" leader= -46:42.333 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-64224304-8480-120b-2e28-8d1642120401.dc1 127.0.0.1 -46:42.334 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-64224304-8480-120b-2e28-8d1642120401 127.0.0.1 -2022-09-15T21:46:42.334Z [INFO] agent.router: Initializing LAN area manager -46:42.334 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13071 network=udp -46:42.335 [INFO] TestAgent.server: Adding LAN server: server="Node-64224304-8480-120b-2e28-8d1642120401 (Addr: tcp/127.0.0.1:13076) (DC: dc1)" -46:42.335 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-64224304-8480-120b-2e28-8d1642120401.dc1 area=wan -46:42.335 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13071 network=tcp -46:42.336 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13072 network=tcp -46:42.336 [INFO] TestAgent: started state syncer -46:42.336 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13077 network=tcp -46:42.391 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -46:42.391 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13076 [Candidate]" term=2 -46:42.391 [DEBUG] TestAgent.server.raft: votes: needed=1 -46:42.391 [DEBUG] TestAgent.server.raft: vote granted: from=64224304-8480-120b-2e28-8d1642120401 term=2 tally=1 -46:42.391 [INFO] TestAgent.server.raft: election won: tally=1 -46:42.391 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13076 [Leader]" -46:42.391 [INFO] TestAgent.server: cluster leadership acquired -46:42.391 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13076 -46:42.393 [INFO] TestAgent.server: New leader elected: payload=Node-64224304-8480-120b-2e28-8d1642120401 -46:42.393 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -46:42.425 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -46:42.425 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -46:42.425 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -46:42.425 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -46:42.425 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -46:42.425 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-64224304-8480-120b-2e28-8d1642120401 -46:42.425 [INFO] TestAgent.server: member joined, marking health alive: member=Node-64224304-8480-120b-2e28-8d1642120401 -46:42.426 [INFO] TestAgent.server: federation state anti-entropy synced -46:42.574 [INFO] TestAgent.server.fsm: snapshot created: duration=155.72µs -46:42.574 [INFO] TestAgent.server.raft: starting snapshot up to: index=11 -46:42.574 [INFO] TestAgent.server.snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshotSaveCommand-TestAgent-agent208801863/raft/snapshots/2-11-1663278402574.tmp -46:42.576 [INFO] TestAgent.server.raft: snapshot complete up to: index=11 -46:42.579 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/snapshot from=127.0.0.1:37116 latency=174.24244ms -46:42.586 [INFO] TestAgent.server.snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshotSaveCommand-TestAgent-agent208801863/raft/snapshots/2-12-1663278402586.tmp -46:42.587 [INFO] TestAgent.server.raft: copied to local snapshot: bytes=4876 -46:42.589 [INFO] TestAgent.server.raft: restored user snapshot: index=1 -46:42.597 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -46:42.597 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -46:42.597 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -46:42.597 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -46:42.597 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -46:42.597 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -46:42.597 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -46:42.597 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13076 -46:42.599 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -46:42.599 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -46:42.599 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -46:42.599 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -46:42.599 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/snapshot from=127.0.0.1:37118 latency=14.68812ms -46:42.599 [INFO] TestAgent: Requesting shutdown -46:42.600 [INFO] TestAgent.server: shutting down server -46:42.600 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -46:42.600 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -46:42.600 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -46:42.600 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -46:42.600 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -46:42.600 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -46:42.600 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -46:42.600 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -46:42.600 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -46:42.600 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -46:42.600 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -46:42.600 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -46:42.600 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -46:42.600 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:46:42.600Z [INFO] agent.router.manager: shutting down -2022-09-15T21:46:42.600Z [INFO] agent.router.manager: shutting down -46:42.600 [INFO] TestAgent: consul server down -46:42.600 [INFO] TestAgent: shutdown complete -46:42.600 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13071 network=tcp -46:42.601 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13071 network=udp -46:42.601 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13072 network=tcp -46:43.101 [INFO] TestAgent: Waiting for endpoints to shut down -46:43.101 [INFO] TestAgent: Endpoints down ---- PASS: TestSnapshotSaveCommand (0.88s) -2022-09-15T21:46:42.296Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:46:42.316Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -46:42.330 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e9ec7c7f-0676-0321-d19e-7785205cb4af Address:127.0.0.1:13069}]" -46:42.332 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13069 [Follower]" leader= -46:42.332 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e9ec7c7f-0676-0321-d19e-7785205cb4af.dc1 127.0.0.1 -46:42.334 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e9ec7c7f-0676-0321-d19e-7785205cb4af 127.0.0.1 -2022-09-15T21:46:42.334Z [INFO] agent.router: Initializing LAN area manager -46:42.334 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13064 network=udp -46:42.334 [INFO] TestAgent.server: Adding LAN server: server="Node-e9ec7c7f-0676-0321-d19e-7785205cb4af (Addr: tcp/127.0.0.1:13069) (DC: dc1)" -46:42.334 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e9ec7c7f-0676-0321-d19e-7785205cb4af.dc1 area=wan -46:42.335 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13064 network=tcp -46:42.335 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13065 network=tcp -46:42.336 [INFO] TestAgent: started state syncer -46:42.336 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13070 network=tcp -46:42.386 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -46:42.386 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13069 [Candidate]" term=2 -46:42.386 [DEBUG] TestAgent.server.raft: votes: needed=1 -46:42.386 [DEBUG] TestAgent.server.raft: vote granted: from=e9ec7c7f-0676-0321-d19e-7785205cb4af term=2 tally=1 -46:42.386 [INFO] TestAgent.server.raft: election won: tally=1 -46:42.386 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13069 [Leader]" -46:42.386 [INFO] TestAgent.server: cluster leadership acquired -46:42.387 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13069 -46:42.388 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -46:42.390 [INFO] TestAgent.server: New leader elected: payload=Node-e9ec7c7f-0676-0321-d19e-7785205cb4af -46:42.398 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/blob from=127.0.0.1:46788 latency=5.93076ms -46:42.428 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -46:42.428 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -46:42.428 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -46:42.428 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -46:42.428 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -46:42.428 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e9ec7c7f-0676-0321-d19e-7785205cb4af -46:42.428 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e9ec7c7f-0676-0321-d19e-7785205cb4af -46:42.430 [INFO] TestAgent.server: federation state anti-entropy synced -46:42.577 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -46:42.578 [INFO] TestAgent: Synced node info -46:42.770 [INFO] TestAgent.server.fsm: snapshot created: duration=126.44µs -46:42.770 [INFO] TestAgent.server.raft: starting snapshot up to: index=13 -46:42.770 [INFO] TestAgent.server.snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshotSaveCommand_TruncatedStream-TestAgent-agent181737608/raft/snapshots/2-13-1663278402770.tmp -46:42.771 [INFO] TestAgent.server.raft: snapshot complete up to: index=13 -46:42.793 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/snapshot from=127.0.0.1:46788 latency=388.1943ms -46:42.832 [INFO] TestAgent: Requesting shutdown -46:42.832 [INFO] TestAgent.server: shutting down server -46:42.832 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -46:42.832 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -46:42.832 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -46:42.833 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -46:42.833 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -46:42.833 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -46:42.833 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -46:42.833 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -46:42.833 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -46:42.833 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -46:42.833 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:46:42.833Z [INFO] agent.router.manager: shutting down -2022-09-15T21:46:42.833Z [INFO] agent.router.manager: shutting down -46:42.833 [INFO] TestAgent: consul server down -46:42.833 [INFO] TestAgent: shutdown complete -46:42.833 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13064 network=tcp -46:42.833 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13064 network=udp -46:42.833 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13065 network=tcp -46:43.334 [INFO] TestAgent: Waiting for endpoints to shut down -46:43.334 [INFO] TestAgent: Endpoints down ---- PASS: TestSnapshotSaveCommand_TruncatedStream (1.12s) +2021-08-13T15:39:24.477Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:39:24.509Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +39:24.512 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fe104835-5fd8-30dc-6a7d-c614551f4141 Address:127.0.0.1:30433}]" +39:24.512 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30433 [Follower]" leader= +39:24.513 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fe104835-5fd8-30dc-6a7d-c614551f4141.dc1 127.0.0.1 +39:24.513 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fe104835-5fd8-30dc-6a7d-c614551f4141 127.0.0.1 +2021-08-13T15:39:24.514Z [INFO] agent.router: Initializing LAN area manager +39:24.514 [INFO] TestAgent.server: Adding LAN server: server="Node-fe104835-5fd8-30dc-6a7d-c614551f4141 (Addr: tcp/127.0.0.1:30433) (DC: dc1)" +39:24.514 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fe104835-5fd8-30dc-6a7d-c614551f4141.dc1 area=wan +39:24.514 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30428 network=udp +39:24.514 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30428 network=tcp +39:24.515 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30429 network=tcp +39:24.515 [INFO] TestAgent: started state syncer +39:24.515 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30434 network=tcp +39:24.566 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +39:24.566 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30433 [Candidate]" term=2 +39:24.567 [DEBUG] TestAgent.server.raft: votes: needed=1 +39:24.567 [DEBUG] TestAgent.server.raft: vote granted: from=fe104835-5fd8-30dc-6a7d-c614551f4141 term=2 tally=1 +39:24.567 [INFO] TestAgent.server.raft: election won: tally=1 +39:24.567 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30433 [Leader]" +39:24.567 [INFO] TestAgent.server: cluster leadership acquired +39:24.567 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:30433 +39:24.568 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +39:24.568 [INFO] TestAgent.server: New leader elected: payload=Node-fe104835-5fd8-30dc-6a7d-c614551f4141 +39:24.588 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +39:24.588 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +39:24.588 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +39:24.588 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +39:24.588 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +39:24.588 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fe104835-5fd8-30dc-6a7d-c614551f4141 +39:24.588 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fe104835-5fd8-30dc-6a7d-c614551f4141 +39:24.589 [INFO] TestAgent.server: federation state anti-entropy synced +39:24.691 [INFO] TestAgent.server.fsm: snapshot created: duration=112.56µs +39:24.691 [INFO] TestAgent.server.raft: starting snapshot up to: index=11 +39:24.691 [INFO] TestAgent.server.snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshotSaveCommand-TestAgent-agent760470292/raft/snapshots/2-11-1628869164691.tmp +39:24.693 [INFO] TestAgent.server.raft: snapshot complete up to: index=11 +39:24.701 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/snapshot from=127.0.0.1:47544 latency=10.806875ms +39:24.727 [INFO] TestAgent.server.snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshotSaveCommand-TestAgent-agent760470292/raft/snapshots/2-12-1628869164727.tmp +39:24.728 [INFO] TestAgent.server.raft: copied to local snapshot: bytes=4876 +39:24.730 [INFO] TestAgent.server.raft: restored user snapshot: index=1 +39:24.731 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +39:24.731 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +39:24.731 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +39:24.731 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +39:24.731 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +39:24.731 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +39:24.731 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +39:24.731 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:30433 +39:24.733 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +39:24.733 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +39:24.733 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +39:24.733 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +39:24.733 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/snapshot from=127.0.0.1:47546 latency=23.143793ms +39:24.734 [INFO] TestAgent: Requesting shutdown +39:24.734 [INFO] TestAgent.server: shutting down server +39:24.734 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +39:24.734 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +39:24.734 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +39:24.734 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +39:24.734 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +39:24.734 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +39:24.734 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +39:24.734 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="shutdown waiting for leader" +39:24.734 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +39:24.734 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +39:24.734 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +39:24.734 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +39:24.734 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +39:24.734 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:39:24.734Z [INFO] agent.router.manager: shutting down +2021-08-13T15:39:24.735Z [INFO] agent.router.manager: shutting down +39:24.735 [INFO] TestAgent: consul server down +39:24.735 [INFO] TestAgent: shutdown complete +39:24.735 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30428 network=tcp +39:24.735 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30428 network=udp +39:24.735 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30429 network=tcp +39:25.235 [INFO] TestAgent: Waiting for endpoints to shut down +39:25.235 [INFO] TestAgent: Endpoints down +--- PASS: TestSnapshotSaveCommand (0.84s) +2021-08-13T15:39:24.474Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:39:24.490Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +39:24.495 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c7b12f5a-c686-cabd-4a32-11543ab8d633 Address:127.0.0.1:30426}]" +39:24.497 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c7b12f5a-c686-cabd-4a32-11543ab8d633.dc1 127.0.0.1 +39:24.498 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c7b12f5a-c686-cabd-4a32-11543ab8d633 127.0.0.1 +2021-08-13T15:39:24.498Z [INFO] agent.router: Initializing LAN area manager +39:24.502 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=udp +39:24.502 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30426 [Follower]" leader= +39:24.503 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=tcp +39:24.504 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c7b12f5a-c686-cabd-4a32-11543ab8d633.dc1 area=wan +39:24.504 [INFO] TestAgent.server: Adding LAN server: server="Node-c7b12f5a-c686-cabd-4a32-11543ab8d633 (Addr: tcp/127.0.0.1:30426) (DC: dc1)" +39:24.506 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30422 network=tcp +39:24.506 [INFO] TestAgent: started state syncer +39:24.506 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30427 network=tcp +39:24.562 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +39:24.562 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30426 [Candidate]" term=2 +39:24.562 [DEBUG] TestAgent.server.raft: votes: needed=1 +39:24.562 [DEBUG] TestAgent.server.raft: vote granted: from=c7b12f5a-c686-cabd-4a32-11543ab8d633 term=2 tally=1 +39:24.562 [INFO] TestAgent.server.raft: election won: tally=1 +39:24.562 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30426 [Leader]" +39:24.562 [INFO] TestAgent.server: cluster leadership acquired +39:24.563 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:30426 +39:24.564 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +39:24.566 [INFO] TestAgent.server: New leader elected: payload=Node-c7b12f5a-c686-cabd-4a32-11543ab8d633 +39:24.588 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +39:24.588 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +39:24.588 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +39:24.588 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +39:24.588 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +39:24.588 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c7b12f5a-c686-cabd-4a32-11543ab8d633 +39:24.588 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c7b12f5a-c686-cabd-4a32-11543ab8d633 +39:24.590 [INFO] TestAgent.server: federation state anti-entropy synced +39:24.780 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/blob from=127.0.0.1:49484 latency=1.753902ms +39:24.781 [INFO] TestAgent.server.fsm: snapshot created: duration=87.72µs +39:24.781 [INFO] TestAgent.server.raft: starting snapshot up to: index=12 +39:24.781 [INFO] TestAgent.server.snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshotSaveCommand_TruncatedStream-TestAgent-agent738491977/raft/snapshots/2-12-1628869164781.tmp +39:24.782 [INFO] TestAgent.server.raft: snapshot complete up to: index=12 +39:24.804 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/snapshot from=127.0.0.1:49484 latency=23.506873ms +39:24.876 [INFO] TestAgent: Requesting shutdown +39:24.877 [INFO] TestAgent.server: shutting down server +39:24.877 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +39:24.877 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +39:24.877 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +39:24.877 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +39:24.877 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +39:24.877 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +39:24.877 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +39:24.877 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +39:24.877 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +39:24.877 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +39:24.877 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +39:24.877 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:39:24.877Z [INFO] agent.router.manager: shutting down +2021-08-13T15:39:24.877Z [INFO] agent.router.manager: shutting down +39:24.877 [INFO] TestAgent: consul server down +39:24.877 [INFO] TestAgent: shutdown complete +39:24.877 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=tcp +39:24.877 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=udp +39:24.877 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30422 network=tcp +39:25.377 [INFO] TestAgent: Waiting for endpoints to shut down +39:25.377 [INFO] TestAgent: Endpoints down +--- PASS: TestSnapshotSaveCommand_TruncatedStream (0.99s) --- PASS: TestSnapshotSaveCommand_TruncatedStream/truncate_200_bytes_from_end (0.01s) - --- PASS: TestSnapshotSaveCommand_TruncatedStream/truncate_16_bytes_from_end (0.01s) + --- PASS: TestSnapshotSaveCommand_TruncatedStream/truncate_16_bytes_from_end (0.00s) --- PASS: TestSnapshotSaveCommand_TruncatedStream/truncate_8_bytes_from_end (0.00s) - --- PASS: TestSnapshotSaveCommand_TruncatedStream/truncate_4_bytes_from_end (0.00s) - --- PASS: TestSnapshotSaveCommand_TruncatedStream/truncate_2_bytes_from_end (0.01s) + --- PASS: TestSnapshotSaveCommand_TruncatedStream/truncate_4_bytes_from_end (0.01s) + --- PASS: TestSnapshotSaveCommand_TruncatedStream/truncate_2_bytes_from_end (0.00s) --- PASS: TestSnapshotSaveCommand_TruncatedStream/truncate_1_bytes_from_end (0.01s) PASS -ok github.com/hashicorp/consul/command/snapshot/save 1.224s +ok github.com/hashicorp/consul/command/snapshot/save 1.075s === RUN TestValidateCommand_noTabs === PAUSE TestValidateCommand_noTabs === RUN TestValidateCommand_FailOnEmptyFile @@ -91617,25 +91529,25 @@ === RUN TestValidateCommand_Quiet === PAUSE TestValidateCommand_Quiet === CONT TestValidateCommand_noTabs +=== CONT TestValidateCommand_SucceedWithJSONAsHCL --- PASS: TestValidateCommand_noTabs (0.00s) -=== CONT TestValidateCommand_Quiet === CONT TestValidateCommand_SucceedWithMinimalHCLConfigFormat +=== CONT TestValidateCommand_Quiet === CONT TestValidateCommand_FailForInvalidJSONConfigFormat === CONT TestValidateCommand_SucceedOnMinimalConfigDir -=== CONT TestValidateCommand_SucceedWithJSONAsHCL === CONT TestValidateCommand_SucceedOnMinimalConfigFile === CONT TestValidateCommand_SucceedWithMinimalJSONConfigFormat === CONT TestValidateCommand_FailOnEmptyFile ---- PASS: TestValidateCommand_SucceedOnMinimalConfigDir (0.05s) ---- PASS: TestValidateCommand_SucceedWithJSONAsHCL (0.06s) +--- PASS: TestValidateCommand_SucceedOnMinimalConfigDir (0.04s) +--- PASS: TestValidateCommand_SucceedOnMinimalConfigFile (0.04s) +--- PASS: TestValidateCommand_SucceedWithMinimalHCLConfigFormat (0.07s) +--- PASS: TestValidateCommand_FailForInvalidJSONConfigFormat (0.07s) --- PASS: TestValidateCommand_FailOnEmptyFile (0.07s) ---- PASS: TestValidateCommand_FailForInvalidJSONConfigFormat (0.09s) ---- PASS: TestValidateCommand_SucceedOnMinimalConfigFile (0.09s) ---- PASS: TestValidateCommand_SucceedWithMinimalJSONConfigFormat (0.09s) ---- PASS: TestValidateCommand_SucceedWithMinimalHCLConfigFormat (0.10s) ---- PASS: TestValidateCommand_Quiet (0.10s) +--- PASS: TestValidateCommand_Quiet (0.07s) +--- PASS: TestValidateCommand_SucceedWithJSONAsHCL (0.08s) +--- PASS: TestValidateCommand_SucceedWithMinimalJSONConfigFormat (0.08s) PASS -ok github.com/hashicorp/consul/command/validate 0.201s +ok github.com/hashicorp/consul/command/validate 0.171s === RUN TestFormat === RUN TestFormat/pretty === RUN TestFormat/json @@ -91647,7 +91559,7 @@ === CONT TestVersionCommand_noTabs --- PASS: TestVersionCommand_noTabs (0.00s) PASS -ok github.com/hashicorp/consul/command/version 0.088s +ok github.com/hashicorp/consul/command/version 0.060s === RUN TestWatchCommand_noTabs === PAUSE TestWatchCommand_noTabs === RUN TestWatchCommand @@ -91660,62 +91572,64 @@ === RUN TestWatchCommand_loadToken/token_env === RUN TestWatchCommand_loadToken/token_file_arg === RUN TestWatchCommand_loadToken/token_file_env -2022-09-15T21:46:48.868Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:46:48.891Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -46:48.895 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:76119b89-393c-23a8-2489-0788a321f0d7 Address:127.0.0.1:12048}]" -46:48.896 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:12048 [Follower]" leader= -46:48.897 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-76119b89-393c-23a8-2489-0788a321f0d7.dc1 127.0.0.1 -46:48.898 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-76119b89-393c-23a8-2489-0788a321f0d7 127.0.0.1 -2022-09-15T21:46:48.898Z [INFO] agent.router: Initializing LAN area manager -46:48.899 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12043 network=udp -46:48.899 [INFO] TestAgent.server: Adding LAN server: server="Node-76119b89-393c-23a8-2489-0788a321f0d7 (Addr: tcp/127.0.0.1:12048) (DC: dc1)" -46:48.899 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-76119b89-393c-23a8-2489-0788a321f0d7.dc1 area=wan -46:48.899 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12043 network=tcp -46:48.900 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:12044 network=tcp -46:48.901 [INFO] TestAgent: started state syncer -46:48.901 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:12049 network=tcp -46:48.945 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -46:48.945 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:12048 [Candidate]" term=2 -46:48.945 [DEBUG] TestAgent.server.raft: votes: needed=1 -46:48.945 [DEBUG] TestAgent.server.raft: vote granted: from=76119b89-393c-23a8-2489-0788a321f0d7 term=2 tally=1 -46:48.946 [INFO] TestAgent.server.raft: election won: tally=1 -46:48.946 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:12048 [Leader]" -46:48.946 [INFO] TestAgent.server: cluster leadership acquired -46:48.946 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:12048 -46:48.954 [INFO] TestAgent.server: New leader elected: payload=Node-76119b89-393c-23a8-2489-0788a321f0d7 -46:48.955 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -46:48.981 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -46:48.981 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -46:48.981 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -46:48.981 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -46:48.981 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -46:48.981 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-76119b89-393c-23a8-2489-0788a321f0d7 -46:48.981 [INFO] TestAgent.server: member joined, marking health alive: member=Node-76119b89-393c-23a8-2489-0788a321f0d7 -46:48.983 [INFO] TestAgent.server: federation state anti-entropy synced -46:49.006 [INFO] TestAgent: Requesting shutdown -46:49.006 [INFO] TestAgent.server: shutting down server -46:49.006 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -46:49.006 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -46:49.006 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -46:49.006 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -46:49.006 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -46:49.006 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -46:49.007 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -46:49.007 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -46:49.007 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -46:49.007 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -46:49.007 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -46:49.007 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:46:49.007Z [INFO] agent.router.manager: shutting down -2022-09-15T21:46:49.007Z [INFO] agent.router.manager: shutting down -46:49.007 [INFO] TestAgent: consul server down -46:49.007 [INFO] TestAgent: shutdown complete -46:49.007 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12043 network=tcp -46:49.007 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12043 network=udp -46:49.007 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:12044 network=tcp -46:49.507 [INFO] TestAgent: Waiting for endpoints to shut down -46:49.507 [INFO] TestAgent: Endpoints down ---- PASS: TestWatchCommand_loadToken (0.71s) +2021-08-13T15:39:34.609Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:39:34.641Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +39:34.647 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f550fe17-f81b-1413-0186-0f2bf5bf5c6d Address:127.0.0.1:17153}]" +39:34.647 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17153 [Follower]" leader= +39:34.648 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f550fe17-f81b-1413-0186-0f2bf5bf5c6d.dc1 127.0.0.1 +39:34.650 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f550fe17-f81b-1413-0186-0f2bf5bf5c6d 127.0.0.1 +2021-08-13T15:39:34.650Z [INFO] agent.router: Initializing LAN area manager +39:34.650 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17148 network=udp +39:34.650 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f550fe17-f81b-1413-0186-0f2bf5bf5c6d.dc1 area=wan +39:34.650 [INFO] TestAgent.server: Adding LAN server: server="Node-f550fe17-f81b-1413-0186-0f2bf5bf5c6d (Addr: tcp/127.0.0.1:17153) (DC: dc1)" +39:34.651 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17148 network=tcp +39:34.654 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17149 network=tcp +39:34.654 [INFO] TestAgent: started state syncer +39:34.654 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17154 network=tcp +39:34.717 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +39:34.717 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17153 [Candidate]" term=2 +39:34.723 [DEBUG] TestAgent.server.raft: votes: needed=1 +39:34.723 [DEBUG] TestAgent.server.raft: vote granted: from=f550fe17-f81b-1413-0186-0f2bf5bf5c6d term=2 tally=1 +39:34.723 [INFO] TestAgent.server.raft: election won: tally=1 +39:34.723 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17153 [Leader]" +39:34.724 [INFO] TestAgent.server: cluster leadership acquired +39:34.724 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17153 +39:34.724 [INFO] TestAgent.server: New leader elected: payload=Node-f550fe17-f81b-1413-0186-0f2bf5bf5c6d +39:34.726 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +39:34.750 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +39:34.750 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +39:34.750 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +39:34.750 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +39:34.750 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +39:34.751 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f550fe17-f81b-1413-0186-0f2bf5bf5c6d +39:34.751 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f550fe17-f81b-1413-0186-0f2bf5bf5c6d +39:34.751 [INFO] TestAgent.server: federation state anti-entropy synced +39:34.949 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +39:34.949 [INFO] TestAgent: Synced node info +39:34.949 [DEBUG] TestAgent: Node info in sync +39:35.083 [INFO] TestAgent: Requesting shutdown +39:35.083 [INFO] TestAgent.server: shutting down server +39:35.083 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +39:35.083 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +39:35.083 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +39:35.083 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +39:35.083 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +39:35.083 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +39:35.083 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +39:35.083 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +39:35.083 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +39:35.083 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +39:35.083 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:39:35.084Z [INFO] agent.router.manager: shutting down +2021-08-13T15:39:35.084Z [INFO] agent.router.manager: shutting down +39:35.084 [INFO] TestAgent: consul server down +39:35.084 [INFO] TestAgent: shutdown complete +39:35.084 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17148 network=tcp +39:35.084 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17148 network=udp +39:35.084 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17149 network=tcp +39:35.584 [INFO] TestAgent: Waiting for endpoints to shut down +39:35.584 [INFO] TestAgent: Endpoints down +--- PASS: TestWatchCommand_loadToken (1.05s) --- PASS: TestWatchCommand_loadToken/token_arg (0.00s) --- PASS: TestWatchCommand_loadToken/token_env (0.00s) --- PASS: TestWatchCommand_loadToken/token_file_arg (0.00s) @@ -91725,183 +91639,185 @@ === RUN TestWatchCommandNoAgentService === PAUSE TestWatchCommandNoAgentService === CONT TestWatchCommand_noTabs ---- PASS: TestWatchCommand_noTabs (0.00s) -=== CONT TestWatchCommandNoAgentService === CONT TestWatchCommandNoConnect +--- PASS: TestWatchCommand_noTabs (0.00s) === CONT TestWatchCommand -2022-09-15T21:46:49.536Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:46:49.566Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -46:49.585 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:92663fca-2f4f-b484-45bb-c6e710ec547f Address:127.0.0.1:12055}]" -46:49.586 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:12055 [Follower]" leader= -46:49.587 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-92663fca-2f4f-b484-45bb-c6e710ec547f.dc1 127.0.0.1 -46:49.588 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-92663fca-2f4f-b484-45bb-c6e710ec547f 127.0.0.1 -2022-09-15T21:46:49.588Z [INFO] agent.router: Initializing LAN area manager -46:49.589 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12050 network=udp -46:49.589 [INFO] TestAgent.server: Adding LAN server: server="Node-92663fca-2f4f-b484-45bb-c6e710ec547f (Addr: tcp/127.0.0.1:12055) (DC: dc1)" -46:49.589 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-92663fca-2f4f-b484-45bb-c6e710ec547f.dc1 area=wan -46:49.590 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12050 network=tcp -46:49.601 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:12051 network=tcp -46:49.601 [INFO] TestAgent: started state syncer -46:49.601 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:12056 network=tcp -46:49.634 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -46:49.634 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:12055 [Candidate]" term=2 -46:49.635 [DEBUG] TestAgent.server.raft: votes: needed=1 -46:49.635 [DEBUG] TestAgent.server.raft: vote granted: from=92663fca-2f4f-b484-45bb-c6e710ec547f term=2 tally=1 -46:49.635 [INFO] TestAgent.server.raft: election won: tally=1 -46:49.635 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:12055 [Leader]" -46:49.635 [INFO] TestAgent.server: cluster leadership acquired -46:49.635 [INFO] TestAgent.server: New leader elected: payload=Node-92663fca-2f4f-b484-45bb-c6e710ec547f -46:49.635 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:12055 -46:49.637 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -46:49.640 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -46:49.640 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -46:49.640 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -46:49.640 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -46:49.640 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -46:49.640 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-92663fca-2f4f-b484-45bb-c6e710ec547f -46:49.640 [INFO] TestAgent.server: member joined, marking health alive: member=Node-92663fca-2f4f-b484-45bb-c6e710ec547f -46:49.641 [INFO] TestAgent.server: federation state anti-entropy synced -46:49.765 [INFO] TestAgent: Requesting shutdown -46:49.766 [INFO] TestAgent.server: shutting down server -46:49.766 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -46:49.766 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -46:49.766 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -46:49.766 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -46:49.766 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -46:49.766 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -46:49.766 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -46:49.766 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -46:49.766 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -46:49.766 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -46:49.766 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -46:49.766 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:46:49.766Z [INFO] agent.router.manager: shutting down -2022-09-15T21:46:49.766Z [INFO] agent.router.manager: shutting down -46:49.766 [INFO] TestAgent: consul server down -46:49.766 [INFO] TestAgent: shutdown complete -46:49.766 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12050 network=tcp -46:49.766 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12050 network=udp -46:49.766 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:12051 network=tcp -46:50.268 [INFO] TestAgent: Waiting for endpoints to shut down -46:50.268 [INFO] TestAgent: Endpoints down ---- PASS: TestWatchCommandNoAgentService (0.76s) -2022-09-15T21:46:49.589Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:46:49.616Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -46:49.621 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f6668462-5a4e-874f-78a0-bbe5584125ab Address:127.0.0.1:12069}]" -46:49.622 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:12069 [Follower]" leader= -46:49.622 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f6668462-5a4e-874f-78a0-bbe5584125ab.dc1 127.0.0.1 -46:49.623 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f6668462-5a4e-874f-78a0-bbe5584125ab 127.0.0.1 -2022-09-15T21:46:49.623Z [INFO] agent.router: Initializing LAN area manager -46:49.624 [INFO] TestAgent.server: Adding LAN server: server="Node-f6668462-5a4e-874f-78a0-bbe5584125ab (Addr: tcp/127.0.0.1:12069) (DC: dc1)" -46:49.624 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12064 network=udp -46:49.624 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f6668462-5a4e-874f-78a0-bbe5584125ab.dc1 area=wan -46:49.624 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12064 network=tcp -46:49.625 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:12065 network=tcp -46:49.625 [INFO] TestAgent: started state syncer -46:49.625 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:12070 network=tcp -46:49.662 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -46:49.662 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:12069 [Candidate]" term=2 -46:49.662 [DEBUG] TestAgent.server.raft: votes: needed=1 -46:49.662 [DEBUG] TestAgent.server.raft: vote granted: from=f6668462-5a4e-874f-78a0-bbe5584125ab term=2 tally=1 -46:49.662 [INFO] TestAgent.server.raft: election won: tally=1 -46:49.662 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:12069 [Leader]" -46:49.663 [INFO] TestAgent.server: cluster leadership acquired -46:49.663 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:12069 -46:49.665 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -46:49.666 [INFO] TestAgent.server: New leader elected: payload=Node-f6668462-5a4e-874f-78a0-bbe5584125ab -46:49.667 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -46:49.668 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -46:49.668 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -46:49.668 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -46:49.668 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -46:49.668 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f6668462-5a4e-874f-78a0-bbe5584125ab -46:49.668 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f6668462-5a4e-874f-78a0-bbe5584125ab -46:49.669 [INFO] TestAgent.server: federation state anti-entropy synced -46:49.669 [INFO] TestAgent: Synced node info -46:49.838 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:39040 latency=3.58788ms -46:49.845 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/nodes from=127.0.0.1:39042 latency=428.8µs -46:49.846 [INFO] TestAgent: Requesting shutdown -46:49.846 [INFO] TestAgent.server: shutting down server -46:49.846 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -46:49.846 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -46:49.846 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -46:49.846 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -46:49.846 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -46:49.846 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -46:49.846 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -46:49.846 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -46:49.846 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -46:49.846 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -46:49.846 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:46:49.846Z [INFO] agent.router.manager: shutting down -2022-09-15T21:46:49.846Z [INFO] agent.router.manager: shutting down -46:49.847 [INFO] TestAgent: consul server down -46:49.847 [INFO] TestAgent: shutdown complete -46:49.847 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12064 network=tcp -46:49.847 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12064 network=udp -46:49.847 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:12065 network=tcp -46:50.347 [INFO] TestAgent: Waiting for endpoints to shut down -46:50.347 [INFO] TestAgent: Endpoints down ---- PASS: TestWatchCommand (0.84s) -2022-09-15T21:46:49.553Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:46:49.587Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -46:49.593 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cfb95d60-f128-5879-85f0-e53fb6b6552b Address:127.0.0.1:12062}]" -46:49.596 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-cfb95d60-f128-5879-85f0-e53fb6b6552b.dc1 127.0.0.1 -46:49.597 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-cfb95d60-f128-5879-85f0-e53fb6b6552b 127.0.0.1 -2022-09-15T21:46:49.597Z [INFO] agent.router: Initializing LAN area manager -46:49.602 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:12062 [Follower]" leader= -46:49.602 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12057 network=tcp -46:49.602 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-cfb95d60-f128-5879-85f0-e53fb6b6552b.dc1 area=wan -46:49.603 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12057 network=udp -46:49.603 [INFO] TestAgent.server: Adding LAN server: server="Node-cfb95d60-f128-5879-85f0-e53fb6b6552b (Addr: tcp/127.0.0.1:12062) (DC: dc1)" -46:49.604 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:12058 network=tcp -46:49.604 [INFO] TestAgent: started state syncer -46:49.605 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:12063 network=tcp -46:49.649 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -46:49.649 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:12062 [Candidate]" term=2 -46:49.650 [DEBUG] TestAgent.server.raft: votes: needed=1 -46:49.650 [DEBUG] TestAgent.server.raft: vote granted: from=cfb95d60-f128-5879-85f0-e53fb6b6552b term=2 tally=1 -46:49.650 [INFO] TestAgent.server.raft: election won: tally=1 -46:49.650 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:12062 [Leader]" -46:49.650 [INFO] TestAgent.server: cluster leadership acquired -46:49.650 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:12062 -46:49.651 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -46:49.654 [INFO] TestAgent.server: New leader elected: payload=Node-cfb95d60-f128-5879-85f0-e53fb6b6552b -46:49.654 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -46:49.654 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -46:49.654 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -46:49.654 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -46:49.654 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -46:49.654 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cfb95d60-f128-5879-85f0-e53fb6b6552b -46:49.654 [INFO] TestAgent.server: member joined, marking health alive: member=Node-cfb95d60-f128-5879-85f0-e53fb6b6552b -46:49.656 [INFO] TestAgent.server: federation state anti-entropy synced -46:49.814 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -46:49.815 [INFO] TestAgent: Synced node info -46:49.890 [INFO] TestAgent: Requesting shutdown -46:49.890 [INFO] TestAgent.server: shutting down server -46:49.890 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -46:49.890 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -46:49.890 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -46:49.890 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -46:49.890 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -46:49.890 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -46:49.891 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -46:49.891 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -46:49.891 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -46:49.891 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -46:49.891 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:46:49.891Z [INFO] agent.router.manager: shutting down -2022-09-15T21:46:49.891Z [INFO] agent.router.manager: shutting down -46:49.891 [INFO] TestAgent: consul server down -46:49.891 [INFO] TestAgent: shutdown complete -46:49.891 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12057 network=tcp -46:49.891 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12057 network=udp -46:49.891 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:12058 network=tcp -46:50.391 [INFO] TestAgent: Waiting for endpoints to shut down -46:50.391 [INFO] TestAgent: Endpoints down ---- PASS: TestWatchCommandNoConnect (0.88s) +=== CONT TestWatchCommandNoAgentService +2021-08-13T15:39:35.619Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:39:35.646Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +39:35.649 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ceda9818-f865-95cc-2ac5-a7d79f3a8ec2 Address:127.0.0.1:17174}]" +39:35.650 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ceda9818-f865-95cc-2ac5-a7d79f3a8ec2.dc1 127.0.0.1 +39:35.651 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17174 [Follower]" leader= +39:35.651 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ceda9818-f865-95cc-2ac5-a7d79f3a8ec2 127.0.0.1 +2021-08-13T15:39:35.651Z [INFO] agent.router: Initializing LAN area manager +39:35.657 [INFO] TestAgent.server: Adding LAN server: server="Node-ceda9818-f865-95cc-2ac5-a7d79f3a8ec2 (Addr: tcp/127.0.0.1:17174) (DC: dc1)" +39:35.657 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ceda9818-f865-95cc-2ac5-a7d79f3a8ec2.dc1 area=wan +39:35.658 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17169 network=udp +39:35.658 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17169 network=tcp +39:35.661 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17170 network=tcp +39:35.661 [INFO] TestAgent: started state syncer +39:35.662 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17175 network=tcp +39:35.697 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +39:35.697 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17174 [Candidate]" term=2 +39:35.698 [DEBUG] TestAgent.server.raft: votes: needed=1 +39:35.698 [DEBUG] TestAgent.server.raft: vote granted: from=ceda9818-f865-95cc-2ac5-a7d79f3a8ec2 term=2 tally=1 +39:35.698 [INFO] TestAgent.server.raft: election won: tally=1 +39:35.698 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17174 [Leader]" +39:35.698 [INFO] TestAgent.server: cluster leadership acquired +39:35.698 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17174 +39:35.699 [INFO] TestAgent.server: New leader elected: payload=Node-ceda9818-f865-95cc-2ac5-a7d79f3a8ec2 +39:35.700 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +39:35.702 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +39:35.702 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +39:35.702 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +39:35.702 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +39:35.702 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +39:35.702 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ceda9818-f865-95cc-2ac5-a7d79f3a8ec2 +39:35.702 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ceda9818-f865-95cc-2ac5-a7d79f3a8ec2 +39:35.703 [INFO] TestAgent.server: federation state anti-entropy synced +39:35.942 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +39:35.942 [INFO] TestAgent: Synced node info +39:35.942 [DEBUG] TestAgent: Node info in sync +39:35.966 [INFO] TestAgent: Requesting shutdown +39:35.966 [INFO] TestAgent.server: shutting down server +39:35.966 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +39:35.966 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +39:35.966 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +39:35.966 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +39:35.966 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +39:35.967 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +39:35.967 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +39:35.967 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +39:35.967 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +39:35.967 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +39:35.967 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:39:35.967Z [INFO] agent.router.manager: shutting down +2021-08-13T15:39:35.967Z [INFO] agent.router.manager: shutting down +39:35.967 [INFO] TestAgent: consul server down +39:35.967 [INFO] TestAgent: shutdown complete +39:35.967 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17169 network=tcp +39:35.967 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17169 network=udp +39:35.967 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17170 network=tcp +39:36.468 [INFO] TestAgent: Waiting for endpoints to shut down +39:36.468 [INFO] TestAgent: Endpoints down +--- PASS: TestWatchCommandNoAgentService (0.88s) +2021-08-13T15:39:35.618Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:39:35.665Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +39:35.669 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b813b3ca-3f2a-cec9-6ad0-326fed41883f Address:127.0.0.1:17167}]" +39:35.669 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17167 [Follower]" leader= +39:35.670 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b813b3ca-3f2a-cec9-6ad0-326fed41883f.dc1 127.0.0.1 +39:35.671 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b813b3ca-3f2a-cec9-6ad0-326fed41883f 127.0.0.1 +2021-08-13T15:39:35.671Z [INFO] agent.router: Initializing LAN area manager +39:35.671 [INFO] TestAgent.server: Adding LAN server: server="Node-b813b3ca-3f2a-cec9-6ad0-326fed41883f (Addr: tcp/127.0.0.1:17167) (DC: dc1)" +39:35.671 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b813b3ca-3f2a-cec9-6ad0-326fed41883f.dc1 area=wan +39:35.671 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17162 network=udp +39:35.671 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17162 network=tcp +39:35.672 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17163 network=tcp +39:35.672 [INFO] TestAgent: started state syncer +39:35.672 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17168 network=tcp +39:35.732 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +39:35.732 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17167 [Candidate]" term=2 +39:35.733 [DEBUG] TestAgent.server.raft: votes: needed=1 +39:35.733 [DEBUG] TestAgent.server.raft: vote granted: from=b813b3ca-3f2a-cec9-6ad0-326fed41883f term=2 tally=1 +39:35.733 [INFO] TestAgent.server.raft: election won: tally=1 +39:35.733 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17167 [Leader]" +39:35.733 [INFO] TestAgent.server: cluster leadership acquired +39:35.733 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17167 +39:35.734 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +39:35.736 [INFO] TestAgent.server: New leader elected: payload=Node-b813b3ca-3f2a-cec9-6ad0-326fed41883f +39:35.737 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +39:35.737 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +39:35.737 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +39:35.737 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +39:35.737 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +39:35.737 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b813b3ca-3f2a-cec9-6ad0-326fed41883f +39:35.737 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b813b3ca-3f2a-cec9-6ad0-326fed41883f +39:35.738 [INFO] TestAgent.server: federation state anti-entropy synced +39:36.004 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:35702 latency=2.421304ms +39:36.016 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/nodes from=127.0.0.1:35704 latency=866.221µs +39:36.017 [INFO] TestAgent: Requesting shutdown +39:36.018 [INFO] TestAgent.server: shutting down server +39:36.018 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +39:36.018 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +39:36.018 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +39:36.018 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +39:36.018 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +39:36.018 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +39:36.018 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +39:36.018 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +39:36.018 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +39:36.018 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +39:36.018 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +39:36.018 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:39:36.019Z [INFO] agent.router.manager: shutting down +2021-08-13T15:39:36.019Z [INFO] agent.router.manager: shutting down +39:36.020 [INFO] TestAgent: consul server down +39:36.020 [INFO] TestAgent: shutdown complete +39:36.020 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17162 network=tcp +39:36.020 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17162 network=udp +39:36.020 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17163 network=tcp +39:36.520 [INFO] TestAgent: Waiting for endpoints to shut down +39:36.520 [INFO] TestAgent: Endpoints down +--- PASS: TestWatchCommand (0.94s) +2021-08-13T15:39:35.616Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:39:35.647Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +39:35.649 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b8f6a5c4-c46f-ee00-f5da-53c990385317 Address:127.0.0.1:17160}]" +39:35.650 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b8f6a5c4-c46f-ee00-f5da-53c990385317.dc1 127.0.0.1 +39:35.650 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17160 [Follower]" leader= +39:35.651 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b8f6a5c4-c46f-ee00-f5da-53c990385317 127.0.0.1 +2021-08-13T15:39:35.651Z [INFO] agent.router: Initializing LAN area manager +39:35.651 [INFO] TestAgent.server: Adding LAN server: server="Node-b8f6a5c4-c46f-ee00-f5da-53c990385317 (Addr: tcp/127.0.0.1:17160) (DC: dc1)" +39:35.652 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17155 network=udp +39:35.655 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17155 network=tcp +39:35.657 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17156 network=tcp +39:35.657 [INFO] TestAgent: started state syncer +39:35.657 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17161 network=tcp +39:35.657 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b8f6a5c4-c46f-ee00-f5da-53c990385317.dc1 area=wan +39:35.698 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +39:35.698 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17160 [Candidate]" term=2 +39:35.698 [DEBUG] TestAgent.server.raft: votes: needed=1 +39:35.698 [DEBUG] TestAgent.server.raft: vote granted: from=b8f6a5c4-c46f-ee00-f5da-53c990385317 term=2 tally=1 +39:35.698 [INFO] TestAgent.server.raft: election won: tally=1 +39:35.698 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17160 [Leader]" +39:35.699 [INFO] TestAgent.server: cluster leadership acquired +39:35.699 [INFO] TestAgent.server: New leader elected: payload=Node-b8f6a5c4-c46f-ee00-f5da-53c990385317 +39:35.699 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17160 +39:35.700 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +39:35.702 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +39:35.702 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +39:35.702 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +39:35.702 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +39:35.702 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +39:35.702 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b8f6a5c4-c46f-ee00-f5da-53c990385317 +39:35.702 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b8f6a5c4-c46f-ee00-f5da-53c990385317 +39:35.703 [INFO] TestAgent.server: federation state anti-entropy synced +39:35.735 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +39:35.736 [INFO] TestAgent: Synced node info +39:36.028 [INFO] TestAgent: Requesting shutdown +39:36.028 [INFO] TestAgent.server: shutting down server +39:36.028 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +39:36.028 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +39:36.028 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +39:36.028 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +39:36.028 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +39:36.028 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +39:36.028 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +39:36.028 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +39:36.028 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +39:36.028 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +39:36.028 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:39:36.029Z [INFO] agent.router.manager: shutting down +2021-08-13T15:39:36.029Z [INFO] agent.router.manager: shutting down +39:36.029 [INFO] TestAgent: consul server down +39:36.029 [INFO] TestAgent: shutdown complete +39:36.029 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17155 network=tcp +39:36.029 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17155 network=udp +39:36.029 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17156 network=tcp +39:36.529 [INFO] TestAgent: Waiting for endpoints to shut down +39:36.529 [INFO] TestAgent: Endpoints down +--- PASS: TestWatchCommandNoConnect (0.94s) PASS -ok github.com/hashicorp/consul/command/watch 1.711s +ok github.com/hashicorp/consul/command/watch 2.071s === RUN TestStaticResolver_Resolve === RUN TestStaticResolver_Resolve/simples --- PASS: TestStaticResolver_Resolve (0.00s) @@ -91917,94 +91833,94 @@ === RUN TestConsulResolver_Resolve/timeout_errors === RUN TestConsulResolver_Resolve/prepared_query_by_id === RUN TestConsulResolver_Resolve/prepared_query_by_name -2022-09-15T21:46:54.907Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:46:54.937Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -46:54.943 [INFO] test-consul.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1d9ba38c-d597-8515-2c1d-f86f4a30c703 Address:127.0.0.1:28384}]" -46:54.944 [INFO] test-consul.server.raft: entering follower state: follower="Node at 127.0.0.1:28384 [Follower]" leader= -46:54.945 [INFO] test-consul.server.serf.wan: serf: EventMemberJoin: Node-1d9ba38c-d597-8515-2c1d-f86f4a30c703.dc1 127.0.0.1 -46:54.945 [INFO] test-consul.server.serf.lan: serf: EventMemberJoin: Node-1d9ba38c-d597-8515-2c1d-f86f4a30c703 127.0.0.1 -2022-09-15T21:46:54.946Z [INFO] agent.router: Initializing LAN area manager -46:54.946 [INFO] test-consul.server: Handled event for server in area: event=member-join server=Node-1d9ba38c-d597-8515-2c1d-f86f4a30c703.dc1 area=wan -46:54.946 [INFO] test-consul.server: Adding LAN server: server="Node-1d9ba38c-d597-8515-2c1d-f86f4a30c703 (Addr: tcp/127.0.0.1:28384) (DC: dc1)" -46:54.946 [INFO] test-consul: Started DNS server: address=127.0.0.1:28379 network=tcp -46:54.946 [INFO] test-consul: Started DNS server: address=127.0.0.1:28379 network=udp -46:54.947 [INFO] test-consul: Started HTTP server: address=127.0.0.1:28380 network=tcp -46:54.947 [INFO] test-consul: started state syncer -46:54.947 [INFO] test-consul: Started gRPC server: address=127.0.0.1:28385 network=tcp -46:54.985 [WARN] test-consul.server.raft: heartbeat timeout reached, starting election: last-leader= -46:54.985 [INFO] test-consul.server.raft: entering candidate state: node="Node at 127.0.0.1:28384 [Candidate]" term=2 -46:54.985 [DEBUG] test-consul.server.raft: votes: needed=1 -46:54.985 [DEBUG] test-consul.server.raft: vote granted: from=1d9ba38c-d597-8515-2c1d-f86f4a30c703 term=2 tally=1 -46:54.985 [INFO] test-consul.server.raft: election won: tally=1 -46:54.985 [INFO] test-consul.server.raft: entering leader state: leader="Node at 127.0.0.1:28384 [Leader]" -46:54.985 [INFO] test-consul.server: cluster leadership acquired -46:54.986 [DEBUG] test-consul.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:28384 -46:54.986 [INFO] test-consul.server: New leader elected: payload=Node-1d9ba38c-d597-8515-2c1d-f86f4a30c703 -46:54.988 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -46:55.043 [INFO] test-consul.server.connect: initialized primary datacenter CA with provider: provider=consul -46:55.043 [INFO] test-consul.leader: started routine: routine="federation state anti-entropy" -46:55.043 [INFO] test-consul.leader: started routine: routine="federation state pruning" -46:55.043 [INFO] test-consul.leader: started routine: routine="intermediate cert renew watch" -46:55.043 [INFO] test-consul.leader: started routine: routine="CA root pruning" -46:55.043 [DEBUG] test-consul.server: Skipping self join check for node since the cluster is too small: node=Node-1d9ba38c-d597-8515-2c1d-f86f4a30c703 -46:55.043 [INFO] test-consul.server: member joined, marking health alive: member=Node-1d9ba38c-d597-8515-2c1d-f86f4a30c703 -46:55.044 [INFO] test-consul.server: federation state anti-entropy synced -46:55.248 [INFO] test-consul: Synced node info -46:55.249 [INFO] test-consul: Synced service: service=web -46:55.249 [DEBUG] test-consul.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:58024 latency=5.69662ms -46:55.250 [DEBUG] test-consul: Node info in sync -46:55.250 [DEBUG] test-consul: Service in sync: service=web -46:55.251 [INFO] test-consul: Synced service: service=web-proxy -46:55.251 [DEBUG] test-consul.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:58024 latency=1.73742ms -46:55.256 [DEBUG] test-consul: Node info in sync -46:55.256 [DEBUG] test-consul: Service in sync: service=web -46:55.256 [DEBUG] test-consul: Service in sync: service=web-proxy -46:55.257 [INFO] test-consul: Synced service: service=web-proxy-2 -46:55.257 [DEBUG] test-consul.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:58024 latency=1.33464ms -46:55.258 [ERROR] test-consul.proxycfg: watch error: service_id=web-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" -46:55.259 [DEBUG] test-consul: Node info in sync -46:55.259 [DEBUG] test-consul: Service in sync: service=web -46:55.259 [DEBUG] test-consul: Service in sync: service=web-proxy -46:55.259 [DEBUG] test-consul: Service in sync: service=web-proxy-2 -46:55.259 [ERROR] test-consul.proxycfg: watch error: service_id=web-proxy-2 id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" -46:55.259 [ERROR] test-consul.proxycfg: watch error: service_id=web-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" -46:55.259 [INFO] test-consul: Synced service: service=db -46:55.259 [DEBUG] test-consul.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:58024 latency=2.10194ms -46:55.260 [ERROR] test-consul.proxycfg: watch error: service_id=web-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" -46:55.260 [ERROR] test-consul.proxycfg: watch error: service_id=web-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" -46:55.260 [ERROR] test-consul.proxycfg: watch error: service_id=web-proxy-2 id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" -46:55.261 [ERROR] test-consul.proxycfg: watch error: service_id=web-proxy-2 id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" -46:55.261 [ERROR] test-consul.proxycfg: watch error: service_id=web-proxy-2 id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" -46:55.267 [DEBUG] test-consul.http: Request finished: method=POST url=/v1/query from=127.0.0.1:58024 latency=6.99292ms -46:55.269 [DEBUG] test-consul.http: Request finished: method=GET url=/v1/health/connect/web?connect=true&passing=1&stale= from=127.0.0.1:58024 latency=802.76µs -46:55.272 [DEBUG] test-consul.http: Request finished: method=GET url=/v1/health/connect/db?connect=true&passing=1&stale= from=127.0.0.1:58024 latency=338.22µs -46:55.273 [DEBUG] test-consul.http: Request finished: method=GET url=/v1/health/connect/foo?connect=true&passing=1&stale= from=127.0.0.1:58024 latency=156.14µs -46:55.275 [DEBUG] test-consul.http: Request finished: method=GET url=/v1/query/2f436236-3e73-8b2d-0504-f2366a431df5/execute?connect=true&stale= from=127.0.0.1:58024 latency=546.6µs -46:55.277 [DEBUG] test-consul.http: Request finished: method=GET url=/v1/query/test-query/execute?connect=true&stale= from=127.0.0.1:58024 latency=388.2µs -46:55.277 [INFO] test-consul: Requesting shutdown -46:55.278 [INFO] test-consul.server: shutting down server -46:55.278 [DEBUG] test-consul.leader: stopping routine: routine="federation state anti-entropy" -46:55.278 [DEBUG] test-consul.leader: stopping routine: routine="federation state pruning" -46:55.278 [DEBUG] test-consul.leader: stopping routine: routine="intermediate cert renew watch" -46:55.278 [DEBUG] test-consul.leader: stopping routine: routine="CA root pruning" -46:55.278 [WARN] test-consul.server.serf.lan: serf: Shutdown without a Leave -46:55.278 [ERROR] test-consul.server: error performing anti-entropy sync of federation state: error="context canceled" -46:55.278 [DEBUG] test-consul.leader: stopped routine: routine="federation state anti-entropy" -46:55.278 [ERROR] test-consul.anti_entropy: failed to sync remote state: error="No cluster leader" -46:55.278 [DEBUG] test-consul.leader: stopped routine: routine="federation state pruning" -46:55.278 [DEBUG] test-consul.leader: stopped routine: routine="intermediate cert renew watch" -46:55.278 [DEBUG] test-consul.leader: stopped routine: routine="CA root pruning" -46:55.278 [WARN] test-consul.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:46:55.278Z [INFO] agent.router.manager: shutting down -2022-09-15T21:46:55.278Z [INFO] agent.router.manager: shutting down -46:55.279 [INFO] test-consul: consul server down -46:55.279 [INFO] test-consul: shutdown complete -46:55.279 [INFO] test-consul: Stopping server: protocol=DNS address=127.0.0.1:28379 network=tcp -46:55.279 [INFO] test-consul: Stopping server: protocol=DNS address=127.0.0.1:28379 network=udp -46:55.279 [INFO] test-consul: Stopping server: protocol=HTTP address=127.0.0.1:28380 network=tcp -46:55.780 [INFO] test-consul: Waiting for endpoints to shut down -46:55.780 [INFO] test-consul: Endpoints down ---- PASS: TestConsulResolver_Resolve (0.94s) +2021-08-13T15:39:36.544Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:39:36.566Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +39:36.570 [INFO] test-consul.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5a35c7a5-c854-a4b5-1b0b-49ecb7c49b08 Address:127.0.0.1:27363}]" +39:36.570 [INFO] test-consul.server.raft: entering follower state: follower="Node at 127.0.0.1:27363 [Follower]" leader= +39:36.572 [INFO] test-consul.server.serf.wan: serf: EventMemberJoin: Node-5a35c7a5-c854-a4b5-1b0b-49ecb7c49b08.dc1 127.0.0.1 +39:36.573 [INFO] test-consul.server.serf.lan: serf: EventMemberJoin: Node-5a35c7a5-c854-a4b5-1b0b-49ecb7c49b08 127.0.0.1 +2021-08-13T15:39:36.573Z [INFO] agent.router: Initializing LAN area manager +39:36.573 [INFO] test-consul: Started DNS server: address=127.0.0.1:27358 network=udp +39:36.573 [INFO] test-consul.server: Adding LAN server: server="Node-5a35c7a5-c854-a4b5-1b0b-49ecb7c49b08 (Addr: tcp/127.0.0.1:27363) (DC: dc1)" +39:36.573 [INFO] test-consul.server: Handled event for server in area: event=member-join server=Node-5a35c7a5-c854-a4b5-1b0b-49ecb7c49b08.dc1 area=wan +39:36.573 [INFO] test-consul: Started DNS server: address=127.0.0.1:27358 network=tcp +39:36.574 [INFO] test-consul: Started HTTP server: address=127.0.0.1:27359 network=tcp +39:36.574 [INFO] test-consul: started state syncer +39:36.575 [INFO] test-consul: Started gRPC server: address=127.0.0.1:27364 network=tcp +39:36.623 [WARN] test-consul.server.raft: heartbeat timeout reached, starting election: last-leader= +39:36.623 [INFO] test-consul.server.raft: entering candidate state: node="Node at 127.0.0.1:27363 [Candidate]" term=2 +39:36.623 [DEBUG] test-consul.server.raft: votes: needed=1 +39:36.623 [DEBUG] test-consul.server.raft: vote granted: from=5a35c7a5-c854-a4b5-1b0b-49ecb7c49b08 term=2 tally=1 +39:36.623 [INFO] test-consul.server.raft: election won: tally=1 +39:36.623 [INFO] test-consul.server.raft: entering leader state: leader="Node at 127.0.0.1:27363 [Leader]" +39:36.624 [INFO] test-consul.server: cluster leadership acquired +39:36.624 [DEBUG] test-consul.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:27363 +39:36.625 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +39:36.626 [INFO] test-consul.server: New leader elected: payload=Node-5a35c7a5-c854-a4b5-1b0b-49ecb7c49b08 +39:36.653 [INFO] test-consul.server.connect: initialized primary datacenter CA with provider: provider=consul +39:36.653 [INFO] test-consul.leader: started routine: routine="federation state anti-entropy" +39:36.653 [INFO] test-consul.leader: started routine: routine="federation state pruning" +39:36.653 [INFO] test-consul.leader: started routine: routine="intermediate cert renew watch" +39:36.653 [INFO] test-consul.leader: started routine: routine="CA root pruning" +39:36.653 [DEBUG] test-consul.server: Skipping self join check for node since the cluster is too small: node=Node-5a35c7a5-c854-a4b5-1b0b-49ecb7c49b08 +39:36.653 [INFO] test-consul.server: member joined, marking health alive: member=Node-5a35c7a5-c854-a4b5-1b0b-49ecb7c49b08 +39:36.659 [INFO] test-consul.server: federation state anti-entropy synced +39:36.767 [INFO] test-consul: Synced node info +39:36.768 [INFO] test-consul: Synced service: service=web +39:36.768 [DEBUG] test-consul.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:35264 latency=2.661424ms +39:36.769 [DEBUG] test-consul: Node info in sync +39:36.769 [DEBUG] test-consul: Service in sync: service=web +39:36.769 [INFO] test-consul: Synced service: service=web-proxy +39:36.769 [DEBUG] test-consul.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:35264 latency=1.174322ms +39:36.770 [ERROR] test-consul.proxycfg: watch error: service_id=web-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" +39:36.770 [ERROR] test-consul.proxycfg: watch error: service_id=web-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" +39:36.770 [ERROR] test-consul.proxycfg: watch error: service_id=web-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" +39:36.770 [ERROR] test-consul.proxycfg: watch error: service_id=web-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" +39:36.770 [DEBUG] test-consul: Node info in sync +39:36.770 [DEBUG] test-consul: Service in sync: service=web +39:36.770 [DEBUG] test-consul: Service in sync: service=web-proxy +39:36.771 [INFO] test-consul: Synced service: service=web-proxy-2 +39:36.771 [DEBUG] test-consul.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:35264 latency=1.086981ms +39:36.771 [ERROR] test-consul.proxycfg: watch error: service_id=web-proxy-2 id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" +39:36.772 [ERROR] test-consul.proxycfg: watch error: service_id=web-proxy-2 id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" +39:36.772 [ERROR] test-consul.proxycfg: watch error: service_id=web-proxy-2 id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" +39:36.772 [ERROR] test-consul.proxycfg: watch error: service_id=web-proxy-2 id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" +39:36.772 [DEBUG] test-consul: Node info in sync +39:36.773 [INFO] test-consul: Synced service: service=db +39:36.773 [DEBUG] test-consul: Service in sync: service=web +39:36.773 [DEBUG] test-consul: Service in sync: service=web-proxy +39:36.773 [DEBUG] test-consul: Service in sync: service=web-proxy-2 +39:36.773 [DEBUG] test-consul.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:35264 latency=1.130941ms +39:36.775 [DEBUG] test-consul.http: Request finished: method=POST url=/v1/query from=127.0.0.1:35264 latency=1.219602ms +39:36.777 [DEBUG] test-consul.http: Request finished: method=GET url=/v1/health/connect/web?connect=true&passing=1&stale= from=127.0.0.1:35264 latency=1.040721ms +39:36.780 [DEBUG] test-consul.http: Request finished: method=GET url=/v1/health/connect/db?connect=true&passing=1&stale= from=127.0.0.1:35264 latency=449.04µs +39:36.781 [DEBUG] test-consul.http: Request finished: method=GET url=/v1/health/connect/foo?connect=true&passing=1&stale= from=127.0.0.1:35264 latency=199.86µs +39:36.782 [DEBUG] test-consul.http: Request finished: method=GET url=/v1/query/c2e015f5-8211-4f21-3c9e-ce3dcb274d66/execute?connect=true&stale= from=127.0.0.1:35264 latency=538.321µs +39:36.787 [DEBUG] test-consul.http: Request finished: method=GET url=/v1/query/test-query/execute?connect=true&stale= from=127.0.0.1:35264 latency=804.661µs +39:36.788 [INFO] test-consul: Requesting shutdown +39:36.788 [INFO] test-consul.server: shutting down server +39:36.788 [DEBUG] test-consul.leader: stopping routine: routine="federation state anti-entropy" +39:36.788 [DEBUG] test-consul.leader: stopping routine: routine="federation state pruning" +39:36.788 [DEBUG] test-consul.leader: stopping routine: routine="intermediate cert renew watch" +39:36.788 [DEBUG] test-consul.leader: stopping routine: routine="CA root pruning" +39:36.788 [WARN] test-consul.server.serf.lan: serf: Shutdown without a Leave +39:36.789 [ERROR] test-consul.server: error performing anti-entropy sync of federation state: error="context canceled" +39:36.789 [DEBUG] test-consul.leader: stopped routine: routine="federation state anti-entropy" +39:36.789 [ERROR] test-consul.anti_entropy: failed to sync remote state: error="No cluster leader" +39:36.789 [DEBUG] test-consul.leader: stopped routine: routine="federation state pruning" +39:36.789 [DEBUG] test-consul.leader: stopped routine: routine="intermediate cert renew watch" +39:36.789 [DEBUG] test-consul.leader: stopped routine: routine="CA root pruning" +39:36.789 [WARN] test-consul.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:39:36.789Z [INFO] agent.router.manager: shutting down +2021-08-13T15:39:36.789Z [INFO] agent.router.manager: shutting down +39:36.789 [INFO] test-consul: consul server down +39:36.789 [INFO] test-consul: shutdown complete +39:36.789 [INFO] test-consul: Stopping server: protocol=DNS address=127.0.0.1:27358 network=tcp +39:36.789 [INFO] test-consul: Stopping server: protocol=DNS address=127.0.0.1:27358 network=udp +39:36.790 [INFO] test-consul: Stopping server: protocol=HTTP address=127.0.0.1:27359 network=tcp +39:37.290 [INFO] test-consul: Waiting for endpoints to shut down +39:37.290 [INFO] test-consul: Endpoints down +--- PASS: TestConsulResolver_Resolve (0.82s) --- PASS: TestConsulResolver_Resolve/basic_service_discovery (0.00s) --- PASS: TestConsulResolver_Resolve/basic_service_with_native_service (0.00s) --- PASS: TestConsulResolver_Resolve/Bad_Type_errors (0.00s) @@ -92045,20 +91961,20 @@ service_test.go:129: DM-skipped --- SKIP: TestService_ServerTLSConfig (0.00s) === RUN TestService_HTTPClient -2022/09/15 21:46:55 starting test connect HTTPS server on 127.0.0.1:28386 -2022/09/15 21:46:55 test connect service listening on 127.0.0.1:28386 +2021/08/13 15:39:37 starting test connect HTTPS server on 127.0.0.1:27365 +2021/08/13 15:39:37 test connect service listening on 127.0.0.1:27365 --- PASS: TestService_HTTPClient (0.02s) === RUN TestService_HasDefaultHTTPResolverFromAddr +2021-08-13T15:39:37.316Z [ERROR] connect.watch: Watch errored: service=foo type=connect_roots error="Get "http://127.0.0.1:8500/v1/agent/connect/ca/roots": dial tcp 127.0.0.1:8500: connect: connection refused" retry=5s --- PASS: TestService_HasDefaultHTTPResolverFromAddr (0.00s) === RUN Test_verifyServerCertMatchesURI -2022-09-15T21:46:55.803Z [ERROR] connect.watch: Watch errored: service=foo type=connect_leaf error="Get "http://127.0.0.1:8500/v1/agent/connect/ca/leaf/foo": dial tcp 127.0.0.1:8500: connect: connection refused" retry=5s -2022-09-15T21:46:55.803Z [ERROR] connect.watch: Watch errored: service=foo type=connect_roots error="Get "http://127.0.0.1:8500/v1/agent/connect/ca/roots": dial tcp 127.0.0.1:8500: connect: connection refused" retry=5s +2021-08-13T15:39:37.319Z [ERROR] connect.watch: Watch errored: service=foo type=connect_leaf error="Get "http://127.0.0.1:8500/v1/agent/connect/ca/leaf/foo": dial tcp 127.0.0.1:8500: connect: connection refused" retry=5s === RUN Test_verifyServerCertMatchesURI/simple_match === RUN Test_verifyServerCertMatchesURI/different_trust-domain_allowed === RUN Test_verifyServerCertMatchesURI/mismatch === RUN Test_verifyServerCertMatchesURI/no_certs === RUN Test_verifyServerCertMatchesURI/nil_certs ---- PASS: Test_verifyServerCertMatchesURI (0.00s) +--- PASS: Test_verifyServerCertMatchesURI (0.01s) --- PASS: Test_verifyServerCertMatchesURI/simple_match (0.00s) --- PASS: Test_verifyServerCertMatchesURI/different_trust-domain_allowed (0.00s) --- PASS: Test_verifyServerCertMatchesURI/mismatch (0.00s) @@ -92069,7 +91985,7 @@ === RUN TestClientSideVerifier/untrusted_CA === RUN TestClientSideVerifier/cross_signed_intermediate === RUN TestClientSideVerifier/cross_signed_without_intermediate ---- PASS: TestClientSideVerifier (0.02s) +--- PASS: TestClientSideVerifier (0.01s) --- PASS: TestClientSideVerifier/ok_service_ca1 (0.00s) --- PASS: TestClientSideVerifier/untrusted_CA (0.00s) --- PASS: TestClientSideVerifier/cross_signed_intermediate (0.00s) @@ -92077,77 +91993,76 @@ === RUN TestServerSideVerifier === RUN TestServerSideVerifier/ok_service_ca1,_allow === RUN TestServerSideVerifier/untrusted_CA -2022-09-15T21:46:56.320Z [ERROR] TestServerSideVerifier/untrusted_CA: failed TLS verification: error="x509: certificate signed by unknown authority" +2021-08-13T15:39:37.628Z [ERROR] TestServerSideVerifier/untrusted_CA: failed TLS verification: error="x509: certificate signed by unknown authority" === RUN TestServerSideVerifier/cross_signed_intermediate,_allow === RUN TestServerSideVerifier/cross_signed_without_intermediate -2022-09-15T21:46:56.323Z [ERROR] TestServerSideVerifier/cross_signed_without_intermediate: failed TLS verification: error="x509: certificate signed by unknown authority" +2021-08-13T15:39:37.630Z [ERROR] TestServerSideVerifier/cross_signed_without_intermediate: failed TLS verification: error="x509: certificate signed by unknown authority" === RUN TestServerSideVerifier/ok_service_ca1,_deny -2022-09-15T21:46:56.324Z [ERROR] TestServerSideVerifier/ok_service_ca1,_deny: authz call denied: reason="Matched intention: DENY default/* => default/db (ID: 4aaea26d-5125-78ea-2842-63d0146e8587, Precedence: 8)" +2021-08-13T15:39:37.632Z [ERROR] TestServerSideVerifier/ok_service_ca1,_deny: authz call denied: reason="Matched intention: DENY default/* => default/db (ID: d760dcdc-e112-c10e-b5da-e4821030455e, Precedence: 8)" === RUN TestServerSideVerifier/cross_signed_intermediate,_deny -2022-09-15T21:46:56.327Z [ERROR] TestServerSideVerifier/cross_signed_intermediate,_deny: authz call denied: reason="Matched intention: DENY default/* => default/db (ID: 4aaea26d-5125-78ea-2842-63d0146e8587, Precedence: 8)" -2022-09-15T21:46:55.847Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:46:55.865Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -46:55.868 [INFO] test-consul.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a48b2a69-42e2-f06f-2039-de16dfaf093f Address:127.0.0.1:28392}]" -46:55.869 [INFO] test-consul.server.serf.wan: serf: EventMemberJoin: Node-a48b2a69-42e2-f06f-2039-de16dfaf093f.dc1 127.0.0.1 -46:55.870 [INFO] test-consul.server.raft: entering follower state: follower="Node at 127.0.0.1:28392 [Follower]" leader= -46:55.875 [INFO] test-consul.server.serf.lan: serf: EventMemberJoin: Node-a48b2a69-42e2-f06f-2039-de16dfaf093f 127.0.0.1 -2022-09-15T21:46:55.875Z [INFO] agent.router: Initializing LAN area manager -46:55.875 [INFO] test-consul: Started DNS server: address=127.0.0.1:28387 network=udp -46:55.876 [INFO] test-consul.server: Adding LAN server: server="Node-a48b2a69-42e2-f06f-2039-de16dfaf093f (Addr: tcp/127.0.0.1:28392) (DC: dc1)" -46:55.876 [INFO] test-consul.server: Handled event for server in area: event=member-join server=Node-a48b2a69-42e2-f06f-2039-de16dfaf093f.dc1 area=wan -46:55.876 [INFO] test-consul: Started DNS server: address=127.0.0.1:28387 network=tcp -46:55.877 [INFO] test-consul: Started HTTP server: address=127.0.0.1:28388 network=tcp -46:55.877 [INFO] test-consul: started state syncer -46:55.877 [INFO] test-consul: Started gRPC server: address=127.0.0.1:28393 network=tcp -46:55.929 [WARN] test-consul.server.raft: heartbeat timeout reached, starting election: last-leader= -46:55.929 [INFO] test-consul.server.raft: entering candidate state: node="Node at 127.0.0.1:28392 [Candidate]" term=2 -46:55.930 [DEBUG] test-consul.server.raft: votes: needed=1 -46:55.930 [DEBUG] test-consul.server.raft: vote granted: from=a48b2a69-42e2-f06f-2039-de16dfaf093f term=2 tally=1 -46:55.930 [INFO] test-consul.server.raft: election won: tally=1 -46:55.930 [INFO] test-consul.server.raft: entering leader state: leader="Node at 127.0.0.1:28392 [Leader]" -46:55.930 [INFO] test-consul.server: cluster leadership acquired -46:55.930 [DEBUG] test-consul.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:28392 -46:55.931 [INFO] test-consul.server: New leader elected: payload=Node-a48b2a69-42e2-f06f-2039-de16dfaf093f -46:55.932 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -46:55.937 [INFO] test-consul.server.connect: initialized primary datacenter CA with provider: provider=consul -46:55.937 [INFO] test-consul.leader: started routine: routine="federation state anti-entropy" -46:55.937 [INFO] test-consul.leader: started routine: routine="federation state pruning" -46:55.937 [INFO] test-consul.leader: started routine: routine="intermediate cert renew watch" -46:55.937 [INFO] test-consul.leader: started routine: routine="CA root pruning" -46:55.939 [INFO] test-consul.server: federation state anti-entropy synced -46:55.939 [DEBUG] test-consul.server: Skipping self join check for node since the cluster is too small: node=Node-a48b2a69-42e2-f06f-2039-de16dfaf093f -46:55.939 [INFO] test-consul.server: member joined, marking health alive: member=Node-a48b2a69-42e2-f06f-2039-de16dfaf093f -46:55.954 [DEBUG] test-consul: Skipping remote check since it is managed automatically: check=serfHealth -46:55.955 [INFO] test-consul: Synced node info -46:56.301 [DEBUG] test-consul.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:37446 latency=4.02652ms -46:56.306 [DEBUG] test-consul.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:37446 latency=1.5469ms -46:56.319 [DEBUG] test-consul.http: Request finished: method=POST url=/v1/agent/connect/authorize from=127.0.0.1:37446 latency=1.01862ms -46:56.322 [DEBUG] test-consul.http: Request finished: method=POST url=/v1/agent/connect/authorize from=127.0.0.1:37446 latency=173.22µs -46:56.324 [DEBUG] test-consul.http: Request finished: method=POST url=/v1/agent/connect/authorize from=127.0.0.1:37446 latency=173.62µs -46:56.326 [DEBUG] test-consul.http: Request finished: method=POST url=/v1/agent/connect/authorize from=127.0.0.1:37446 latency=353.56µs -46:56.327 [INFO] test-consul: Requesting shutdown -46:56.327 [INFO] test-consul.server: shutting down server -46:56.327 [DEBUG] test-consul.leader: stopping routine: routine="federation state anti-entropy" -46:56.327 [DEBUG] test-consul.leader: stopping routine: routine="federation state pruning" -46:56.327 [DEBUG] test-consul.leader: stopping routine: routine="intermediate cert renew watch" -46:56.327 [DEBUG] test-consul.leader: stopping routine: routine="CA root pruning" -46:56.327 [WARN] test-consul.server.serf.lan: serf: Shutdown without a Leave -46:56.327 [ERROR] test-consul.server: error performing anti-entropy sync of federation state: error="context canceled" -46:56.327 [DEBUG] test-consul.leader: stopped routine: routine="federation state anti-entropy" -46:56.327 [DEBUG] test-consul.leader: stopped routine: routine="intermediate cert renew watch" -46:56.327 [DEBUG] test-consul.leader: stopped routine: routine="CA root pruning" -46:56.327 [DEBUG] test-consul.leader: stopped routine: routine="federation state pruning" -46:56.327 [WARN] test-consul.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:46:56.327Z [INFO] agent.router.manager: shutting down -2022-09-15T21:46:56.327Z [INFO] agent.router.manager: shutting down -46:56.327 [INFO] test-consul: consul server down -46:56.327 [INFO] test-consul: shutdown complete -46:56.327 [INFO] test-consul: Stopping server: protocol=DNS address=127.0.0.1:28387 network=tcp -46:56.327 [INFO] test-consul: Stopping server: protocol=DNS address=127.0.0.1:28387 network=udp -46:56.327 [INFO] test-consul: Stopping server: protocol=HTTP address=127.0.0.1:28388 network=tcp -46:56.828 [INFO] test-consul: Waiting for endpoints to shut down -46:56.828 [INFO] test-consul: Endpoints down ---- PASS: TestServerSideVerifier (1.00s) +2021-08-13T15:39:37.634Z [ERROR] TestServerSideVerifier/cross_signed_intermediate,_deny: authz call denied: reason="Matched intention: DENY default/* => default/db (ID: d760dcdc-e112-c10e-b5da-e4821030455e, Precedence: 8)" +2021-08-13T15:39:37.359Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:39:37.376Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +39:37.381 [INFO] test-consul.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a470e075-9727-ae7e-84da-29faa24cee22 Address:127.0.0.1:27371}]" +39:37.381 [INFO] test-consul.server.raft: entering follower state: follower="Node at 127.0.0.1:27371 [Follower]" leader= +39:37.382 [INFO] test-consul.server.serf.wan: serf: EventMemberJoin: Node-a470e075-9727-ae7e-84da-29faa24cee22.dc1 127.0.0.1 +39:37.384 [INFO] test-consul.server.serf.lan: serf: EventMemberJoin: Node-a470e075-9727-ae7e-84da-29faa24cee22 127.0.0.1 +2021-08-13T15:39:37.384Z [INFO] agent.router: Initializing LAN area manager +39:37.384 [INFO] test-consul: Started DNS server: address=127.0.0.1:27366 network=udp +39:37.384 [INFO] test-consul.server: Handled event for server in area: event=member-join server=Node-a470e075-9727-ae7e-84da-29faa24cee22.dc1 area=wan +39:37.384 [INFO] test-consul: Started DNS server: address=127.0.0.1:27366 network=tcp +39:37.384 [INFO] test-consul.server: Adding LAN server: server="Node-a470e075-9727-ae7e-84da-29faa24cee22 (Addr: tcp/127.0.0.1:27371) (DC: dc1)" +39:37.386 [INFO] test-consul: Started HTTP server: address=127.0.0.1:27367 network=tcp +39:37.386 [INFO] test-consul: started state syncer +39:37.386 [INFO] test-consul: Started gRPC server: address=127.0.0.1:27372 network=tcp +39:37.450 [WARN] test-consul.server.raft: heartbeat timeout reached, starting election: last-leader= +39:37.450 [INFO] test-consul.server.raft: entering candidate state: node="Node at 127.0.0.1:27371 [Candidate]" term=2 +39:37.450 [DEBUG] test-consul.server.raft: votes: needed=1 +39:37.450 [DEBUG] test-consul.server.raft: vote granted: from=a470e075-9727-ae7e-84da-29faa24cee22 term=2 tally=1 +39:37.450 [INFO] test-consul.server.raft: election won: tally=1 +39:37.450 [INFO] test-consul.server.raft: entering leader state: leader="Node at 127.0.0.1:27371 [Leader]" +39:37.450 [INFO] test-consul.server: cluster leadership acquired +39:37.450 [INFO] test-consul.server: New leader elected: payload=Node-a470e075-9727-ae7e-84da-29faa24cee22 +39:37.450 [DEBUG] test-consul.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:27371 +39:37.452 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +39:37.454 [INFO] test-consul.server.connect: initialized primary datacenter CA with provider: provider=consul +39:37.454 [INFO] test-consul.leader: started routine: routine="federation state anti-entropy" +39:37.454 [INFO] test-consul.leader: started routine: routine="federation state pruning" +39:37.454 [INFO] test-consul.leader: started routine: routine="intermediate cert renew watch" +39:37.454 [INFO] test-consul.leader: started routine: routine="CA root pruning" +39:37.454 [DEBUG] test-consul.server: Skipping self join check for node since the cluster is too small: node=Node-a470e075-9727-ae7e-84da-29faa24cee22 +39:37.454 [INFO] test-consul.server: member joined, marking health alive: member=Node-a470e075-9727-ae7e-84da-29faa24cee22 +39:37.455 [INFO] test-consul.server: federation state anti-entropy synced +39:37.613 [DEBUG] test-consul.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:49806 latency=1.360302ms +39:37.615 [DEBUG] test-consul.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:49806 latency=661.701µs +39:37.626 [DEBUG] test-consul.http: Request finished: method=POST url=/v1/agent/connect/authorize from=127.0.0.1:49806 latency=615.181µs +39:37.629 [DEBUG] test-consul.http: Request finished: method=POST url=/v1/agent/connect/authorize from=127.0.0.1:49806 latency=179.36µs +39:37.632 [DEBUG] test-consul.http: Request finished: method=POST url=/v1/agent/connect/authorize from=127.0.0.1:49806 latency=303.82µs +39:37.634 [DEBUG] test-consul.http: Request finished: method=POST url=/v1/agent/connect/authorize from=127.0.0.1:49806 latency=165.36µs +39:37.634 [INFO] test-consul: Requesting shutdown +39:37.634 [INFO] test-consul.server: shutting down server +39:37.634 [DEBUG] test-consul.leader: stopping routine: routine="federation state anti-entropy" +39:37.634 [DEBUG] test-consul.leader: stopping routine: routine="federation state pruning" +39:37.634 [DEBUG] test-consul.leader: stopping routine: routine="intermediate cert renew watch" +39:37.634 [DEBUG] test-consul.leader: stopping routine: routine="CA root pruning" +39:37.634 [WARN] test-consul.server.serf.lan: serf: Shutdown without a Leave +39:37.635 [ERROR] test-consul.server: error performing anti-entropy sync of federation state: error="context canceled" +39:37.635 [DEBUG] test-consul.leader: stopped routine: routine="federation state anti-entropy" +39:37.635 [ERROR] test-consul.anti_entropy: failed to sync remote state: error="No cluster leader" +39:37.635 [DEBUG] test-consul.leader: stopped routine: routine="federation state pruning" +39:37.635 [DEBUG] test-consul.leader: stopped routine: routine="intermediate cert renew watch" +39:37.635 [DEBUG] test-consul.leader: stopped routine: routine="CA root pruning" +39:37.635 [WARN] test-consul.server.serf.wan: serf: Shutdown without a Leave +2021-08-13T15:39:37.635Z [INFO] agent.router.manager: shutting down +2021-08-13T15:39:37.635Z [INFO] agent.router.manager: shutting down +39:37.635 [INFO] test-consul: consul server down +39:37.635 [INFO] test-consul: shutdown complete +39:37.635 [INFO] test-consul: Stopping server: protocol=DNS address=127.0.0.1:27366 network=tcp +39:37.635 [INFO] test-consul: Stopping server: protocol=DNS address=127.0.0.1:27366 network=udp +39:37.635 [INFO] test-consul: Stopping server: protocol=HTTP address=127.0.0.1:27367 network=tcp +39:38.136 [INFO] test-consul: Waiting for endpoints to shut down +39:38.136 [INFO] test-consul: Endpoints down +--- PASS: TestServerSideVerifier (0.80s) --- PASS: TestServerSideVerifier/ok_service_ca1,_allow (0.00s) --- PASS: TestServerSideVerifier/untrusted_CA (0.00s) --- PASS: TestServerSideVerifier/cross_signed_intermediate,_allow (0.00s) @@ -92159,7 +92074,7 @@ === RUN TestDynamicTLSConfig_Ready --- PASS: TestDynamicTLSConfig_Ready (0.01s) PASS -ok github.com/hashicorp/consul/connect 2.125s +ok github.com/hashicorp/consul/connect 1.797s ? github.com/hashicorp/consul/connect/certgen [no test files] === RUN TestUpstreamResolverFuncFromClient === PAUSE TestUpstreamResolverFuncFromClient @@ -92193,225 +92108,226 @@ === CONT TestConnSrcClosing === CONT TestAgentConfigWatcherSidecarProxy [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... ---- PASS: TestConnSrcClosing (0.01s) ---- PASS: TestConnDstClosing (0.01s) +--- PASS: TestConnSrcClosing (0.00s) +--- PASS: TestConnDstClosing (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2022-09-15T21:46:56.950Z [ERROR] TestAgentConfigWatcherSidecarProxy.watch: Watch errored: service_id=web-sidecar-proxy type=agent_service error="Get "http://127.0.0.1:31443/v1/agent/service/web-sidecar-proxy?hash=790f39b13effbdf2": dial tcp 127.0.0.1:31443: connect: connection refused" retry=5s -2022-09-15T21:46:56.745Z [WARN] agent: bootstrap = true: do not enable unless necessary -2022-09-15T21:46:56.769Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -46:56.773 [INFO] agent_smith.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:70092363-5470-148b-97fe-c69cf8244af9 Address:127.0.0.1:31447}]" -46:56.774 [INFO] agent_smith.server.raft: entering follower state: follower="Node at 127.0.0.1:31447 [Follower]" leader= -46:56.774 [INFO] agent_smith.server.serf.wan: serf: EventMemberJoin: Node-70092363-5470-148b-97fe-c69cf8244af9.dc1 127.0.0.1 -46:56.776 [INFO] agent_smith.server.serf.lan: serf: EventMemberJoin: Node-70092363-5470-148b-97fe-c69cf8244af9 127.0.0.1 -2022-09-15T21:46:56.776Z [INFO] agent.router: Initializing LAN area manager -46:56.779 [INFO] agent_smith.server: Adding LAN server: server="Node-70092363-5470-148b-97fe-c69cf8244af9 (Addr: tcp/127.0.0.1:31447) (DC: dc1)" -46:56.779 [INFO] agent_smith.server: Handled event for server in area: event=member-join server=Node-70092363-5470-148b-97fe-c69cf8244af9.dc1 area=wan -46:56.779 [INFO] agent_smith: Started DNS server: address=127.0.0.1:31442 network=udp -46:56.779 [INFO] agent_smith: Started DNS server: address=127.0.0.1:31442 network=tcp -46:56.781 [INFO] agent_smith: Started HTTP server: address=127.0.0.1:31443 network=tcp -46:56.781 [INFO] agent_smith: started state syncer -46:56.782 [INFO] agent_smith: Started gRPC server: address=127.0.0.1:31448 network=tcp -46:56.841 [WARN] agent_smith.server.raft: heartbeat timeout reached, starting election: last-leader= -46:56.841 [INFO] agent_smith.server.raft: entering candidate state: node="Node at 127.0.0.1:31447 [Candidate]" term=2 -46:56.842 [DEBUG] agent_smith.server.raft: votes: needed=1 -46:56.842 [DEBUG] agent_smith.server.raft: vote granted: from=70092363-5470-148b-97fe-c69cf8244af9 term=2 tally=1 -46:56.842 [INFO] agent_smith.server.raft: election won: tally=1 -46:56.842 [INFO] agent_smith.server.raft: entering leader state: leader="Node at 127.0.0.1:31447 [Leader]" -46:56.842 [INFO] agent_smith.server: cluster leadership acquired -46:56.842 [DEBUG] agent_smith.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:31447 -46:56.844 [INFO] agent_smith.server: New leader elected: payload=Node-70092363-5470-148b-97fe-c69cf8244af9 -46:56.844 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -46:56.878 [INFO] agent_smith.server.connect: initialized primary datacenter CA with provider: provider=consul -46:56.878 [INFO] agent_smith.leader: started routine: routine="federation state anti-entropy" -46:56.878 [INFO] agent_smith.leader: started routine: routine="federation state pruning" -46:56.878 [INFO] agent_smith.leader: started routine: routine="intermediate cert renew watch" -46:56.878 [INFO] agent_smith.leader: started routine: routine="CA root pruning" -46:56.878 [DEBUG] agent_smith.server: Skipping self join check for node since the cluster is too small: node=Node-70092363-5470-148b-97fe-c69cf8244af9 -46:56.878 [INFO] agent_smith.server: member joined, marking health alive: member=Node-70092363-5470-148b-97fe-c69cf8244af9 -46:56.880 [INFO] agent_smith.server: federation state anti-entropy synced -46:56.898 [INFO] agent_smith: Synced node info -46:56.898 [INFO] agent_smith: Synced service: service=web -46:56.900 [INFO] agent_smith: Synced service: service=web-sidecar-proxy -46:56.900 [DEBUG] agent_smith: Check in sync: check=service:web-sidecar-proxy:1 -46:56.900 [DEBUG] agent_smith: Check in sync: check=service:web-sidecar-proxy:2 -46:56.900 [DEBUG] agent_smith.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:60546 latency=5.38894ms -46:56.903 [DEBUG] agent_smith.http: Request finished: method=GET url=/v1/agent/service/web-sidecar-proxy from=127.0.0.1:60546 latency=686.28µs -46:56.927 [DEBUG] agent_smith: Node info in sync -46:56.928 [INFO] agent_smith: Synced service: service=web -46:56.930 [INFO] agent_smith: Synced service: service=web-sidecar-proxy -46:56.930 [DEBUG] agent_smith: Check in sync: check=service:web-sidecar-proxy:1 -46:56.930 [DEBUG] agent_smith: Check in sync: check=service:web-sidecar-proxy:2 -46:56.930 [DEBUG] agent_smith.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:60548 latency=4.7377ms -46:56.944 [DEBUG] agent_smith.http: Request finished: method=GET url=/v1/agent/service/web-sidecar-proxy?hash=e79fad2be70b72e8 from=127.0.0.1:60546 latency=40.1ms -46:56.946 [INFO] agent_smith: Requesting shutdown -46:56.946 [INFO] agent_smith.server: shutting down server -46:56.946 [DEBUG] agent_smith.leader: stopping routine: routine="federation state anti-entropy" -46:56.946 [DEBUG] agent_smith.leader: stopping routine: routine="federation state pruning" -46:56.946 [DEBUG] agent_smith.leader: stopping routine: routine="intermediate cert renew watch" -46:56.946 [DEBUG] agent_smith.leader: stopping routine: routine="CA root pruning" -46:56.946 [WARN] agent_smith.server.serf.lan: serf: Shutdown without a Leave -46:56.947 [ERROR] agent_smith.server: error performing anti-entropy sync of federation state: error="context canceled" -46:56.947 [DEBUG] agent_smith.leader: stopped routine: routine="federation state anti-entropy" -46:56.947 [ERROR] agent_smith.anti_entropy: failed to sync remote state: error="No cluster leader" -46:56.947 [DEBUG] agent_smith.leader: stopped routine: routine="federation state pruning" -46:56.947 [DEBUG] agent_smith.leader: stopped routine: routine="intermediate cert renew watch" -46:56.947 [DEBUG] agent_smith.leader: stopped routine: routine="CA root pruning" -46:56.947 [WARN] agent_smith.server.serf.wan: serf: Shutdown without a Leave -2022-09-15T21:46:56.947Z [INFO] agent.router.manager: shutting down -2022-09-15T21:46:56.948Z [INFO] agent.router.manager: shutting down -46:56.948 [INFO] agent_smith: consul server down -46:56.948 [INFO] agent_smith: shutdown complete -46:56.948 [INFO] agent_smith: Stopping server: protocol=DNS address=127.0.0.1:31442 network=tcp -46:56.948 [INFO] agent_smith: Stopping server: protocol=DNS address=127.0.0.1:31442 network=udp -46:56.948 [INFO] agent_smith: Stopping server: protocol=HTTP address=127.0.0.1:31443 network=tcp -46:56.948 [DEBUG] agent_smith.http: Request finished: method=GET url=/v1/agent/service/web-sidecar-proxy?hash=790f39b13effbdf2 from=127.0.0.1:60546 latency=1.2885ms -46:57.448 [INFO] agent_smith: Waiting for endpoints to shut down -46:57.448 [INFO] agent_smith: Endpoints down ---- PASS: TestAgentConfigWatcherSidecarProxy (0.77s) +2021-08-13T15:39:40.966Z [ERROR] TestAgentConfigWatcherSidecarProxy.watch: Watch errored: service_id=web-sidecar-proxy type=agent_service error="Get "http://127.0.0.1:18170/v1/agent/service/web-sidecar-proxy?hash=790f39b13effbdf2": dial tcp 127.0.0.1:18170: connect: connection refused" retry=5s +2021-08-13T15:39:40.704Z [WARN] agent: bootstrap = true: do not enable unless necessary +2021-08-13T15:39:40.720Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +39:40.728 [INFO] agent_smith.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:34b8ef3a-9dcb-2831-900a-15920dbc14cd Address:127.0.0.1:18174}]" +39:40.728 [INFO] agent_smith.server.raft: entering follower state: follower="Node at 127.0.0.1:18174 [Follower]" leader= +39:40.729 [INFO] agent_smith.server.serf.wan: serf: EventMemberJoin: Node-34b8ef3a-9dcb-2831-900a-15920dbc14cd.dc1 127.0.0.1 +39:40.730 [INFO] agent_smith.server.serf.lan: serf: EventMemberJoin: Node-34b8ef3a-9dcb-2831-900a-15920dbc14cd 127.0.0.1 +2021-08-13T15:39:40.730Z [INFO] agent.router: Initializing LAN area manager +39:40.730 [INFO] agent_smith.server: Adding LAN server: server="Node-34b8ef3a-9dcb-2831-900a-15920dbc14cd (Addr: tcp/127.0.0.1:18174) (DC: dc1)" +39:40.730 [INFO] agent_smith.server: Handled event for server in area: event=member-join server=Node-34b8ef3a-9dcb-2831-900a-15920dbc14cd.dc1 area=wan +39:40.731 [INFO] agent_smith: Started DNS server: address=127.0.0.1:18169 network=udp +39:40.731 [INFO] agent_smith: Started DNS server: address=127.0.0.1:18169 network=tcp +39:40.731 [INFO] agent_smith: Started HTTP server: address=127.0.0.1:18170 network=tcp +39:40.732 [INFO] agent_smith: started state syncer +39:40.732 [INFO] agent_smith: Started gRPC server: address=127.0.0.1:18175 network=tcp +39:40.782 [WARN] agent_smith.server.raft: heartbeat timeout reached, starting election: last-leader= +39:40.782 [INFO] agent_smith.server.raft: entering candidate state: node="Node at 127.0.0.1:18174 [Candidate]" term=2 +39:40.782 [DEBUG] agent_smith.server.raft: votes: needed=1 +39:40.782 [DEBUG] agent_smith.server.raft: vote granted: from=34b8ef3a-9dcb-2831-900a-15920dbc14cd term=2 tally=1 +39:40.782 [INFO] agent_smith.server.raft: election won: tally=1 +39:40.782 [INFO] agent_smith.server.raft: entering leader state: leader="Node at 127.0.0.1:18174 [Leader]" +39:40.782 [INFO] agent_smith.server: cluster leadership acquired +39:40.783 [INFO] agent_smith.server: New leader elected: payload=Node-34b8ef3a-9dcb-2831-900a-15920dbc14cd +39:40.783 [DEBUG] agent_smith.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:18174 +39:40.784 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +39:40.790 [INFO] agent_smith: Synced node info +39:40.790 [DEBUG] agent_smith: Node info in sync +39:40.808 [INFO] agent_smith.server.connect: initialized primary datacenter CA with provider: provider=consul +39:40.808 [INFO] agent_smith.leader: started routine: routine="federation state anti-entropy" +39:40.808 [INFO] agent_smith.leader: started routine: routine="federation state pruning" +39:40.808 [INFO] agent_smith.leader: started routine: routine="intermediate cert renew watch" +39:40.808 [INFO] agent_smith.leader: started routine: routine="CA root pruning" +39:40.808 [DEBUG] agent_smith.server: Skipping self join check for node since the cluster is too small: node=Node-34b8ef3a-9dcb-2831-900a-15920dbc14cd +39:40.808 [INFO] agent_smith.server: member joined, marking health alive: member=Node-34b8ef3a-9dcb-2831-900a-15920dbc14cd +39:40.809 [INFO] agent_smith.server: federation state anti-entropy synced +39:40.927 [DEBUG] agent_smith: Node info in sync +39:40.928 [INFO] agent_smith: Synced service: service=web +39:40.929 [INFO] agent_smith: Synced service: service=web-sidecar-proxy +39:40.929 [DEBUG] agent_smith: Check in sync: check=service:web-sidecar-proxy:1 +39:40.929 [DEBUG] agent_smith: Check in sync: check=service:web-sidecar-proxy:2 +39:40.929 [DEBUG] agent_smith.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:38238 latency=4.822847ms +39:40.933 [DEBUG] agent_smith.http: Request finished: method=GET url=/v1/agent/service/web-sidecar-proxy from=127.0.0.1:38238 latency=656.461µs +39:40.959 [DEBUG] agent_smith: Node info in sync +39:40.960 [INFO] agent_smith: Synced service: service=web +39:40.961 [INFO] agent_smith: Synced service: service=web-sidecar-proxy +39:40.961 [DEBUG] agent_smith: Check in sync: check=service:web-sidecar-proxy:1 +39:40.961 [DEBUG] agent_smith: Check in sync: check=service:web-sidecar-proxy:2 +39:40.961 [DEBUG] agent_smith.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:38240 latency=3.572405ms +39:40.962 [DEBUG] agent_smith.http: Request finished: method=GET url=/v1/agent/service/web-sidecar-proxy?hash=e79fad2be70b72e8 from=127.0.0.1:38238 latency=24.684056ms +39:40.963 [INFO] agent_smith: Requesting shutdown +39:40.963 [INFO] agent_smith.server: shutting down server +39:40.963 [DEBUG] agent_smith.leader: stopping routine: routine="CA root pruning" +39:40.963 [DEBUG] agent_smith.leader: stopping routine: routine="federation state anti-entropy" +39:40.963 [DEBUG] agent_smith.leader: stopping routine: routine="federation state pruning" +39:40.963 [DEBUG] agent_smith.leader: stopping routine: routine="intermediate cert renew watch" +39:40.963 [WARN] agent_smith.server.serf.lan: serf: Shutdown without a Leave +39:40.963 [ERROR] agent_smith.server: error performing anti-entropy sync of federation state: error="context canceled" +39:40.963 [DEBUG] agent_smith.leader: stopped routine: routine="federation state anti-entropy" +39:40.964 [DEBUG] agent_smith.leader: stopped routine: routine="CA root pruning" +39:40.964 [DEBUG] agent_smith.leader: stopped routine: routine="federation state pruning" +39:40.964 [WARN] agent_smith.server.serf.wan: serf: Shutdown without a Leave +39:40.964 [DEBUG] agent_smith.leader: stopped routine: routine="intermediate cert renew watch" +2021-08-13T15:39:40.964Z [INFO] agent.router.manager: shutting down +2021-08-13T15:39:40.964Z [INFO] agent.router.manager: shutting down +39:40.964 [INFO] agent_smith: consul server down +39:40.964 [INFO] agent_smith: shutdown complete +39:40.964 [INFO] agent_smith: Stopping server: protocol=DNS address=127.0.0.1:18169 network=tcp +39:40.964 [INFO] agent_smith: Stopping server: protocol=DNS address=127.0.0.1:18169 network=udp +39:40.964 [INFO] agent_smith: Stopping server: protocol=HTTP address=127.0.0.1:18170 network=tcp +39:40.964 [DEBUG] agent_smith.http: Request finished: method=GET url=/v1/agent/service/web-sidecar-proxy?hash=790f39b13effbdf2 from=127.0.0.1:38238 latency=1.236422ms +39:41.464 [INFO] agent_smith: Waiting for endpoints to shut down +39:41.464 [INFO] agent_smith: Endpoints down +--- PASS: TestAgentConfigWatcherSidecarProxy (0.81s) PASS -ok github.com/hashicorp/consul/connect/proxy 0.879s +ok github.com/hashicorp/consul/connect/proxy 0.895s === RUN TestConfigValidate -=== RUN TestConfigValidate/type=inferred_oidc/good_discovery_cert -=== RUN TestConfigValidate/type=inferred_oidc/valid_redirect_uris -=== RUN TestConfigValidate/type=jwt/jwks_bad_certs -=== RUN TestConfigValidate/type=jwt/too_many_methods_(discovery_+_pubkeys) -=== RUN TestConfigValidate/type=oidc/missing_required_OIDCClientID -=== RUN TestConfigValidate/type=oidc/incompatible_with_JWTValidationPubKeys -=== RUN TestConfigValidate/type=inferred_oidc/incompatible_with_JWTValidationPubKeys -=== RUN TestConfigValidate/type=inferred_oidc/incompatible_with_NotBeforeLeeway -=== RUN TestConfigValidate/type=oidc/valid_redirect_uris +=== RUN TestConfigValidate/type=oidc/incompatible_with_JWKSCACert +=== RUN TestConfigValidate/type=oidc/invalid_redirect_uris === RUN TestConfigValidate/type=oidc/all_required -=== RUN TestConfigValidate/type=jwt/invalid_repeated_value_claim_mappings -=== RUN TestConfigValidate/type=jwt/invalid_repeated_list_claim_mappings -=== RUN TestConfigValidate/type=inferred_oidc/incompatible_with_ExpirationLeeway -=== RUN TestConfigValidate/type=oidc/missing_required_OIDCDiscoveryURL -=== RUN TestConfigValidate/type=oidc/missing_required_OIDCClientSecret -=== RUN TestConfigValidate/type=inferred_oidc/invalid_redirect_uris -=== RUN TestConfigValidate/type=jwt/bad_discovery_cert -=== RUN TestConfigValidate/type=inferred_oidc/incompatible_with_JWKSCACert === RUN TestConfigValidate/type=oidc/garbage_discovery_cert === RUN TestConfigValidate/type=inferred_oidc/missing_required_OIDCDiscoveryURL -=== RUN TestConfigValidate/type=inferred_oidc/incompatible_with_ClockSkewLeeway -=== RUN TestConfigValidate/type=oidc/invalid_redirect_uris === RUN TestConfigValidate/type=oidc/incompatible_with_JWKSURL -=== RUN TestConfigValidate/type=jwt/all_required_for_oidc_discovery -=== RUN TestConfigValidate/type=jwt/all_required_for_jwks -=== RUN TestConfigValidate/type=oidc/invalid_algorithm +=== RUN TestConfigValidate/type=inferred_oidc/all_required === RUN TestConfigValidate/type=inferred_oidc/missing_required_AllowedRedirectURIs -=== RUN TestConfigValidate/type=inferred_oidc/valid_claim_mappings +=== RUN TestConfigValidate/type=inferred_oidc/incompatible_with_JWKSCACert === RUN TestConfigValidate/type=jwt/incompatible_with_OIDCScopes -=== RUN TestConfigValidate/type=jwt/good_discovery_cert -=== RUN TestConfigValidate/type=jwt/incompatible_with_VerboseOIDCLogging -=== RUN TestConfigValidate/type=inferred_oidc/invalid_repeated_value_claim_mappings -=== RUN TestConfigValidate/type=oidc/incompatible_with_NotBeforeLeeway -=== RUN TestConfigValidate/type=inferred_oidc/bad_discovery_cert -=== RUN TestConfigValidate/type=oidc/invalid_repeated_value_claim_mappings +=== RUN TestConfigValidate/type=jwt/incompatible_with_AllowedRedirectURIs +=== RUN TestConfigValidate/type=oidc/invalid_algorithm === RUN TestConfigValidate/type=jwt/too_many_methods_(jwks_+_pubkeys) -=== RUN TestConfigValidate/type=inferred_oidc/incompatible_with_BoundIssuer -=== RUN TestConfigValidate/type=inferred_oidc/missing_required_OIDCClientID -=== RUN TestConfigValidate/type=inferred_oidc/missing_required_OIDCClientSecret -=== RUN TestConfigValidate/type=inferred_oidc/all_required -=== RUN TestConfigValidate/type=jwt/incompatible_with_OIDCClientSecret -=== RUN TestConfigValidate/type=inferred_oidc/garbage_discovery_cert === RUN TestConfigValidate/type=oidc/valid_algorithm -=== RUN TestConfigValidate/type=oidc/valid_claim_mappings -=== RUN TestConfigValidate/type=inferred_oidc/incompatible_with_JWKSURL -=== RUN TestConfigValidate/type=jwt/jwks_mismatched_certs -=== RUN TestConfigValidate/type=jwt/incompatible_with_JWKSCACert -=== RUN TestConfigValidate/type=jwt/invalid_pubkey -=== RUN TestConfigValidate/type=inferred_oidc/valid_algorithm === RUN TestConfigValidate/type=inferred_oidc/invalid_repeated_list_claim_mappings === RUN TestConfigValidate/type=oidc/bad_discovery_cert -=== RUN TestConfigValidate/type=oidc/incompatible_with_JWKSCACert -=== RUN TestConfigValidate/type=oidc/good_discovery_cert -=== RUN TestConfigValidate/type=oidc/incompatible_with_ClockSkewLeeway -=== RUN TestConfigValidate/type=oidc/incompatible_with_BoundIssuer -=== RUN TestConfigValidate/type=jwt/invalid_algorithm -=== RUN TestConfigValidate/type=jwt/valid_claim_mappings -=== RUN TestConfigValidate/type=jwt/incompatible_with_OIDCClientID +=== RUN TestConfigValidate/type=inferred_oidc/incompatible_with_ExpirationLeeway +=== RUN TestConfigValidate/type=oidc/missing_required_OIDCClientID +=== RUN TestConfigValidate/type=oidc/incompatible_with_JWTValidationPubKeys +=== RUN TestConfigValidate/type=inferred_oidc/good_discovery_cert +=== RUN TestConfigValidate/type=inferred_oidc/incompatible_with_NotBeforeLeeway === RUN TestConfigValidate/type=jwt/too_many_methods_(discovery_+_jwks) +=== RUN TestConfigValidate/type=jwt/invalid_pubkey +=== RUN TestConfigValidate/type=jwt/jwks_bad_certs +=== RUN TestConfigValidate/type=inferred_oidc/incompatible_with_ClockSkewLeeway +=== RUN TestConfigValidate/type=jwt/incompatible_with_OIDCClientID +=== RUN TestConfigValidate/type=inferred_oidc/valid_algorithm +=== RUN TestConfigValidate/type=inferred_oidc/bad_discovery_cert +=== RUN TestConfigValidate/type=inferred_oidc/valid_claim_mappings +=== RUN TestConfigValidate/type=inferred_oidc/garbage_discovery_cert +=== RUN TestConfigValidate/type=jwt/invalid_repeated_value_claim_mappings +=== RUN TestConfigValidate/type=jwt/bad_discovery_cert +=== RUN TestConfigValidate/type=inferred_oidc/incompatible_with_JWTValidationPubKeys +=== RUN TestConfigValidate/type=jwt/incompatible_with_JWKSCACert +=== RUN TestConfigValidate/type=jwt/incompatible_with_VerboseOIDCLogging +=== RUN TestConfigValidate/type=oidc/incompatible_with_BoundIssuer +=== RUN TestConfigValidate/type=oidc/valid_claim_mappings +=== RUN TestConfigValidate/type=jwt/incompatible_with_OIDCDiscoveryCACert +=== RUN TestConfigValidate/type=jwt/all_required_for_jwks +=== RUN TestConfigValidate/type=jwt/too_many_methods_(discovery_+_jwks_+_pubkeys) +=== RUN TestConfigValidate/type=inferred_oidc/incompatible_with_JWKSURL === RUN TestConfigValidate/type=inferred_oidc/invalid_algorithm -=== RUN TestConfigValidate/type=oidc/missing_required_AllowedRedirectURIs === RUN TestConfigValidate/type=jwt/valid_algorithm -=== RUN TestConfigValidate/type=jwt/incompatible_with_OIDCDiscoveryCACert +=== RUN TestConfigValidate/type=oidc/valid_redirect_uris +=== RUN TestConfigValidate/type=jwt/good_discovery_cert +=== RUN TestConfigValidate/type=jwt/invalid_repeated_list_claim_mappings +=== RUN TestConfigValidate/type=jwt/too_many_methods_(discovery_+_pubkeys) === RUN TestConfigValidate/type=jwt/all_required_for_public_keys -=== RUN TestConfigValidate/type=jwt/incompatible_with_AllowedRedirectURIs -=== RUN TestConfigValidate/bad_type +=== RUN TestConfigValidate/type=jwt/jwks_mismatched_certs +=== RUN TestConfigValidate/type=oidc/good_discovery_cert === RUN TestConfigValidate/type=oidc/incompatible_with_ExpirationLeeway -=== RUN TestConfigValidate/type=jwt/too_many_methods_(discovery_+_jwks_+_pubkeys) +=== RUN TestConfigValidate/type=oidc/missing_required_OIDCClientSecret +=== RUN TestConfigValidate/type=inferred_oidc/invalid_repeated_value_claim_mappings +=== RUN TestConfigValidate/type=jwt/all_required_for_oidc_discovery +=== RUN TestConfigValidate/bad_type +=== RUN TestConfigValidate/type=oidc/missing_required_OIDCDiscoveryURL +=== RUN TestConfigValidate/type=inferred_oidc/missing_required_OIDCClientID +=== RUN TestConfigValidate/type=inferred_oidc/incompatible_with_BoundIssuer +=== RUN TestConfigValidate/type=oidc/incompatible_with_NotBeforeLeeway +=== RUN TestConfigValidate/type=inferred_oidc/missing_required_OIDCClientSecret +=== RUN TestConfigValidate/type=oidc/invalid_repeated_value_claim_mappings +=== RUN TestConfigValidate/type=jwt/invalid_algorithm +=== RUN TestConfigValidate/type=jwt/incompatible_with_OIDCClientSecret +=== RUN TestConfigValidate/type=inferred_oidc/valid_redirect_uris +=== RUN TestConfigValidate/type=inferred_oidc/invalid_redirect_uris === RUN TestConfigValidate/type=oidc/invalid_repeated_list_claim_mappings +=== RUN TestConfigValidate/type=jwt/valid_claim_mappings +=== RUN TestConfigValidate/type=oidc/incompatible_with_ClockSkewLeeway +=== RUN TestConfigValidate/type=oidc/missing_required_AllowedRedirectURIs === RUN TestConfigValidate/type=jwt/jwks_invalid_404 ---- PASS: TestConfigValidate (0.19s) - --- PASS: TestConfigValidate/type=inferred_oidc/good_discovery_cert (0.01s) - --- PASS: TestConfigValidate/type=inferred_oidc/valid_redirect_uris (0.01s) - --- PASS: TestConfigValidate/type=jwt/jwks_bad_certs (0.00s) - --- PASS: TestConfigValidate/type=jwt/too_many_methods_(discovery_+_pubkeys) (0.00s) - --- PASS: TestConfigValidate/type=oidc/missing_required_OIDCClientID (0.00s) - --- PASS: TestConfigValidate/type=oidc/incompatible_with_JWTValidationPubKeys (0.00s) - --- PASS: TestConfigValidate/type=inferred_oidc/incompatible_with_JWTValidationPubKeys (0.00s) - --- PASS: TestConfigValidate/type=inferred_oidc/incompatible_with_NotBeforeLeeway (0.00s) - --- PASS: TestConfigValidate/type=oidc/valid_redirect_uris (0.01s) +--- PASS: TestConfigValidate (0.18s) + --- PASS: TestConfigValidate/type=oidc/incompatible_with_JWKSCACert (0.00s) + --- PASS: TestConfigValidate/type=oidc/invalid_redirect_uris (0.00s) --- PASS: TestConfigValidate/type=oidc/all_required (0.01s) - --- PASS: TestConfigValidate/type=jwt/invalid_repeated_value_claim_mappings (0.00s) - --- PASS: TestConfigValidate/type=jwt/invalid_repeated_list_claim_mappings (0.00s) - --- PASS: TestConfigValidate/type=inferred_oidc/incompatible_with_ExpirationLeeway (0.00s) - --- PASS: TestConfigValidate/type=oidc/missing_required_OIDCDiscoveryURL (0.00s) - --- PASS: TestConfigValidate/type=oidc/missing_required_OIDCClientSecret (0.00s) - --- PASS: TestConfigValidate/type=inferred_oidc/invalid_redirect_uris (0.00s) - --- PASS: TestConfigValidate/type=jwt/bad_discovery_cert (0.01s) - --- PASS: TestConfigValidate/type=inferred_oidc/incompatible_with_JWKSCACert (0.00s) --- PASS: TestConfigValidate/type=oidc/garbage_discovery_cert (0.00s) --- PASS: TestConfigValidate/type=inferred_oidc/missing_required_OIDCDiscoveryURL (0.00s) - --- PASS: TestConfigValidate/type=inferred_oidc/incompatible_with_ClockSkewLeeway (0.00s) - --- PASS: TestConfigValidate/type=oidc/invalid_redirect_uris (0.00s) --- PASS: TestConfigValidate/type=oidc/incompatible_with_JWKSURL (0.00s) - --- PASS: TestConfigValidate/type=jwt/all_required_for_oidc_discovery (0.01s) - --- PASS: TestConfigValidate/type=jwt/all_required_for_jwks (0.01s) - --- PASS: TestConfigValidate/type=oidc/invalid_algorithm (0.01s) + --- PASS: TestConfigValidate/type=inferred_oidc/all_required (0.01s) --- PASS: TestConfigValidate/type=inferred_oidc/missing_required_AllowedRedirectURIs (0.00s) - --- PASS: TestConfigValidate/type=inferred_oidc/valid_claim_mappings (0.01s) + --- PASS: TestConfigValidate/type=inferred_oidc/incompatible_with_JWKSCACert (0.00s) --- PASS: TestConfigValidate/type=jwt/incompatible_with_OIDCScopes (0.00s) - --- PASS: TestConfigValidate/type=jwt/good_discovery_cert (0.01s) - --- PASS: TestConfigValidate/type=jwt/incompatible_with_VerboseOIDCLogging (0.00s) - --- PASS: TestConfigValidate/type=inferred_oidc/invalid_repeated_value_claim_mappings (0.01s) - --- PASS: TestConfigValidate/type=oidc/incompatible_with_NotBeforeLeeway (0.00s) - --- PASS: TestConfigValidate/type=inferred_oidc/bad_discovery_cert (0.01s) - --- PASS: TestConfigValidate/type=oidc/invalid_repeated_value_claim_mappings (0.01s) + --- PASS: TestConfigValidate/type=jwt/incompatible_with_AllowedRedirectURIs (0.00s) + --- PASS: TestConfigValidate/type=oidc/invalid_algorithm (0.01s) --- PASS: TestConfigValidate/type=jwt/too_many_methods_(jwks_+_pubkeys) (0.00s) - --- PASS: TestConfigValidate/type=inferred_oidc/incompatible_with_BoundIssuer (0.00s) - --- PASS: TestConfigValidate/type=inferred_oidc/missing_required_OIDCClientID (0.00s) - --- PASS: TestConfigValidate/type=inferred_oidc/missing_required_OIDCClientSecret (0.00s) - --- PASS: TestConfigValidate/type=inferred_oidc/all_required (0.01s) - --- PASS: TestConfigValidate/type=jwt/incompatible_with_OIDCClientSecret (0.00s) - --- PASS: TestConfigValidate/type=inferred_oidc/garbage_discovery_cert (0.00s) --- PASS: TestConfigValidate/type=oidc/valid_algorithm (0.01s) - --- PASS: TestConfigValidate/type=oidc/valid_claim_mappings (0.01s) - --- PASS: TestConfigValidate/type=inferred_oidc/incompatible_with_JWKSURL (0.00s) - --- PASS: TestConfigValidate/type=jwt/jwks_mismatched_certs (0.01s) - --- PASS: TestConfigValidate/type=jwt/incompatible_with_JWKSCACert (0.00s) - --- PASS: TestConfigValidate/type=jwt/invalid_pubkey (0.00s) - --- PASS: TestConfigValidate/type=inferred_oidc/valid_algorithm (0.01s) --- PASS: TestConfigValidate/type=inferred_oidc/invalid_repeated_list_claim_mappings (0.01s) --- PASS: TestConfigValidate/type=oidc/bad_discovery_cert (0.01s) - --- PASS: TestConfigValidate/type=oidc/incompatible_with_JWKSCACert (0.00s) - --- PASS: TestConfigValidate/type=oidc/good_discovery_cert (0.01s) - --- PASS: TestConfigValidate/type=oidc/incompatible_with_ClockSkewLeeway (0.00s) - --- PASS: TestConfigValidate/type=oidc/incompatible_with_BoundIssuer (0.00s) - --- PASS: TestConfigValidate/type=jwt/invalid_algorithm (0.00s) - --- PASS: TestConfigValidate/type=jwt/valid_claim_mappings (0.00s) - --- PASS: TestConfigValidate/type=jwt/incompatible_with_OIDCClientID (0.00s) + --- PASS: TestConfigValidate/type=inferred_oidc/incompatible_with_ExpirationLeeway (0.00s) + --- PASS: TestConfigValidate/type=oidc/missing_required_OIDCClientID (0.00s) + --- PASS: TestConfigValidate/type=oidc/incompatible_with_JWTValidationPubKeys (0.00s) + --- PASS: TestConfigValidate/type=inferred_oidc/good_discovery_cert (0.01s) + --- PASS: TestConfigValidate/type=inferred_oidc/incompatible_with_NotBeforeLeeway (0.00s) --- PASS: TestConfigValidate/type=jwt/too_many_methods_(discovery_+_jwks) (0.00s) + --- PASS: TestConfigValidate/type=jwt/invalid_pubkey (0.00s) + --- PASS: TestConfigValidate/type=jwt/jwks_bad_certs (0.00s) + --- PASS: TestConfigValidate/type=inferred_oidc/incompatible_with_ClockSkewLeeway (0.00s) + --- PASS: TestConfigValidate/type=jwt/incompatible_with_OIDCClientID (0.00s) + --- PASS: TestConfigValidate/type=inferred_oidc/valid_algorithm (0.01s) + --- PASS: TestConfigValidate/type=inferred_oidc/bad_discovery_cert (0.01s) + --- PASS: TestConfigValidate/type=inferred_oidc/valid_claim_mappings (0.01s) + --- PASS: TestConfigValidate/type=inferred_oidc/garbage_discovery_cert (0.00s) + --- PASS: TestConfigValidate/type=jwt/invalid_repeated_value_claim_mappings (0.00s) + --- PASS: TestConfigValidate/type=jwt/bad_discovery_cert (0.01s) + --- PASS: TestConfigValidate/type=inferred_oidc/incompatible_with_JWTValidationPubKeys (0.00s) + --- PASS: TestConfigValidate/type=jwt/incompatible_with_JWKSCACert (0.00s) + --- PASS: TestConfigValidate/type=jwt/incompatible_with_VerboseOIDCLogging (0.00s) + --- PASS: TestConfigValidate/type=oidc/incompatible_with_BoundIssuer (0.00s) + --- PASS: TestConfigValidate/type=oidc/valid_claim_mappings (0.01s) + --- PASS: TestConfigValidate/type=jwt/incompatible_with_OIDCDiscoveryCACert (0.00s) + --- PASS: TestConfigValidate/type=jwt/all_required_for_jwks (0.01s) + --- PASS: TestConfigValidate/type=jwt/too_many_methods_(discovery_+_jwks_+_pubkeys) (0.00s) + --- PASS: TestConfigValidate/type=inferred_oidc/incompatible_with_JWKSURL (0.00s) --- PASS: TestConfigValidate/type=inferred_oidc/invalid_algorithm (0.01s) - --- PASS: TestConfigValidate/type=oidc/missing_required_AllowedRedirectURIs (0.00s) --- PASS: TestConfigValidate/type=jwt/valid_algorithm (0.00s) - --- PASS: TestConfigValidate/type=jwt/incompatible_with_OIDCDiscoveryCACert (0.00s) + --- PASS: TestConfigValidate/type=oidc/valid_redirect_uris (0.01s) + --- PASS: TestConfigValidate/type=jwt/good_discovery_cert (0.01s) + --- PASS: TestConfigValidate/type=jwt/invalid_repeated_list_claim_mappings (0.00s) + --- PASS: TestConfigValidate/type=jwt/too_many_methods_(discovery_+_pubkeys) (0.00s) --- PASS: TestConfigValidate/type=jwt/all_required_for_public_keys (0.00s) - --- PASS: TestConfigValidate/type=jwt/incompatible_with_AllowedRedirectURIs (0.00s) - --- PASS: TestConfigValidate/bad_type (0.00s) + --- PASS: TestConfigValidate/type=jwt/jwks_mismatched_certs (0.01s) + --- PASS: TestConfigValidate/type=oidc/good_discovery_cert (0.01s) --- PASS: TestConfigValidate/type=oidc/incompatible_with_ExpirationLeeway (0.00s) - --- PASS: TestConfigValidate/type=jwt/too_many_methods_(discovery_+_jwks_+_pubkeys) (0.00s) + --- PASS: TestConfigValidate/type=oidc/missing_required_OIDCClientSecret (0.00s) + --- PASS: TestConfigValidate/type=inferred_oidc/invalid_repeated_value_claim_mappings (0.01s) + --- PASS: TestConfigValidate/type=jwt/all_required_for_oidc_discovery (0.01s) + --- PASS: TestConfigValidate/bad_type (0.00s) + --- PASS: TestConfigValidate/type=oidc/missing_required_OIDCDiscoveryURL (0.00s) + --- PASS: TestConfigValidate/type=inferred_oidc/missing_required_OIDCClientID (0.00s) + --- PASS: TestConfigValidate/type=inferred_oidc/incompatible_with_BoundIssuer (0.00s) + --- PASS: TestConfigValidate/type=oidc/incompatible_with_NotBeforeLeeway (0.00s) + --- PASS: TestConfigValidate/type=inferred_oidc/missing_required_OIDCClientSecret (0.00s) + --- PASS: TestConfigValidate/type=oidc/invalid_repeated_value_claim_mappings (0.01s) + --- PASS: TestConfigValidate/type=jwt/invalid_algorithm (0.00s) + --- PASS: TestConfigValidate/type=jwt/incompatible_with_OIDCClientSecret (0.00s) + --- PASS: TestConfigValidate/type=inferred_oidc/valid_redirect_uris (0.01s) + --- PASS: TestConfigValidate/type=inferred_oidc/invalid_redirect_uris (0.00s) --- PASS: TestConfigValidate/type=oidc/invalid_repeated_list_claim_mappings (0.01s) + --- PASS: TestConfigValidate/type=jwt/valid_claim_mappings (0.00s) + --- PASS: TestConfigValidate/type=oidc/incompatible_with_ClockSkewLeeway (0.00s) + --- PASS: TestConfigValidate/type=oidc/missing_required_AllowedRedirectURIs (0.00s) --- PASS: TestConfigValidate/type=jwt/jwks_invalid_404 (0.01s) === RUN TestJWT_OIDC_Functions_Fail === RUN TestJWT_OIDC_Functions_Fail/static @@ -92423,13 +92339,13 @@ === RUN TestJWT_OIDC_Functions_Fail/oidc_discovery === RUN TestJWT_OIDC_Functions_Fail/oidc_discovery/GetAuthCodeURL === RUN TestJWT_OIDC_Functions_Fail/oidc_discovery/ClaimsFromAuthCode ---- PASS: TestJWT_OIDC_Functions_Fail (0.10s) +--- PASS: TestJWT_OIDC_Functions_Fail (0.07s) --- PASS: TestJWT_OIDC_Functions_Fail/static (0.00s) --- PASS: TestJWT_OIDC_Functions_Fail/static/GetAuthCodeURL (0.00s) --- PASS: TestJWT_OIDC_Functions_Fail/static/ClaimsFromAuthCode (0.00s) - --- PASS: TestJWT_OIDC_Functions_Fail/JWKS (0.05s) - --- PASS: TestJWT_OIDC_Functions_Fail/JWKS/GetAuthCodeURL (0.04s) - --- PASS: TestJWT_OIDC_Functions_Fail/JWKS/ClaimsFromAuthCode (0.02s) + --- PASS: TestJWT_OIDC_Functions_Fail/JWKS (0.03s) + --- PASS: TestJWT_OIDC_Functions_Fail/JWKS/GetAuthCodeURL (0.02s) + --- PASS: TestJWT_OIDC_Functions_Fail/JWKS/ClaimsFromAuthCode (0.01s) --- PASS: TestJWT_OIDC_Functions_Fail/oidc_discovery (0.04s) --- PASS: TestJWT_OIDC_Functions_Fail/oidc_discovery/GetAuthCodeURL (0.02s) --- PASS: TestJWT_OIDC_Functions_Fail/oidc_discovery/ClaimsFromAuthCode (0.02s) @@ -92456,7 +92372,7 @@ === RUN TestJWT_ClaimsFromJWT/oidc_discovery/bad_signature === RUN TestJWT_ClaimsFromJWT/oidc_discovery/bad_issuer === RUN TestJWT_ClaimsFromJWT/oidc_discovery/bad_audience ---- PASS: TestJWT_ClaimsFromJWT (0.33s) +--- PASS: TestJWT_ClaimsFromJWT (0.25s) --- PASS: TestJWT_ClaimsFromJWT/static (0.01s) --- PASS: TestJWT_ClaimsFromJWT/static/missing_audience (0.00s) --- PASS: TestJWT_ClaimsFromJWT/static/valid_inputs (0.00s) @@ -92464,238 +92380,238 @@ --- PASS: TestJWT_ClaimsFromJWT/static/bad_signature (0.00s) --- PASS: TestJWT_ClaimsFromJWT/static/bad_issuer (0.00s) --- PASS: TestJWT_ClaimsFromJWT/static/bad_audience (0.00s) - --- PASS: TestJWT_ClaimsFromJWT/JWKS (0.19s) - --- PASS: TestJWT_ClaimsFromJWT/JWKS/missing_audience (0.03s) - --- PASS: TestJWT_ClaimsFromJWT/JWKS/valid_inputs (0.03s) + --- PASS: TestJWT_ClaimsFromJWT/JWKS (0.13s) + --- PASS: TestJWT_ClaimsFromJWT/JWKS/missing_audience (0.02s) + --- PASS: TestJWT_ClaimsFromJWT/JWKS/valid_inputs (0.02s) --- PASS: TestJWT_ClaimsFromJWT/JWKS/unusable_claims (0.02s) - --- PASS: TestJWT_ClaimsFromJWT/JWKS/bad_signature (0.04s) - --- PASS: TestJWT_ClaimsFromJWT/JWKS/bad_issuer (0.03s) + --- PASS: TestJWT_ClaimsFromJWT/JWKS/bad_signature (0.02s) + --- PASS: TestJWT_ClaimsFromJWT/JWKS/bad_issuer (0.02s) --- PASS: TestJWT_ClaimsFromJWT/JWKS/bad_audience (0.02s) - --- PASS: TestJWT_ClaimsFromJWT/oidc_discovery (0.13s) + --- PASS: TestJWT_ClaimsFromJWT/oidc_discovery (0.11s) --- SKIP: TestJWT_ClaimsFromJWT/oidc_discovery/missing_audience (0.00s) - --- PASS: TestJWT_ClaimsFromJWT/oidc_discovery/valid_inputs (0.02s) + --- PASS: TestJWT_ClaimsFromJWT/oidc_discovery/valid_inputs (0.03s) --- PASS: TestJWT_ClaimsFromJWT/oidc_discovery/unusable_claims (0.02s) --- PASS: TestJWT_ClaimsFromJWT/oidc_discovery/bad_signature (0.02s) - --- PASS: TestJWT_ClaimsFromJWT/oidc_discovery/bad_issuer (0.04s) - --- PASS: TestJWT_ClaimsFromJWT/oidc_discovery/bad_audience (0.03s) + --- PASS: TestJWT_ClaimsFromJWT/oidc_discovery/bad_issuer (0.02s) + --- PASS: TestJWT_ClaimsFromJWT/oidc_discovery/bad_audience (0.02s) === RUN TestJWT_ClaimsFromJWT_ExpiryClaims === RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static === PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static === RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS === PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS === CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_iat_with_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_iat_with_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_auto_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_auto_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_no_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_no_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_auto_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_iat_with_no_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_auto_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_iat_with_no_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_custom_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_custom_clock_skew_leeway === RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_iat_with_auto_clock_skew_leeway === PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_iat_with_auto_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_no_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_no_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_iat_with_no_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_iat_with_no_clock_skew_leeway === RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway === PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_nbf_with_no_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_nbf_with_no_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway === RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/no_expire_leeway_using_iat_with_clock_skew_leeway === PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/no_expire_leeway_using_iat_with_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_no_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_iat_with_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_iat_with_clock_skew_leeway === RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_nbf_with_clock_skew_leeway === PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_nbf_with_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_auto_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_auto_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_no_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_no_clock_skew_leeway === RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway === PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_iat_with_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_iat_with_clock_skew_leeway === RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway === PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway === RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway === PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_iat_with_custom_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_iat_with_custom_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_nbf_with_no_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_nbf_with_no_clock_skew_leeway === RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_nbf_with_no_clock_skew_leeway === PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_nbf_with_no_clock_skew_leeway === RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway === PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway === RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway === PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway === RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway === PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_auto_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_auto_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS === RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_custom_clock_skew_leeway === PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_custom_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_iat_with_custom_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_iat_with_custom_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_auto_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_auto_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_auto_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_auto_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_no_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_no_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_no_clock_skew_leeway === PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_no_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway === RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway === PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_iat_with_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_iat_with_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_nbf_with_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_nbf_with_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_custom_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_custom_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_custom_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_custom_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_no_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_no_clock_skew_leeway === RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_iat_with_clock_skew_leeway === PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_iat_with_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway === RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway === PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_custom_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_custom_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf === RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_no_clock_skew_leeway === PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_no_clock_skew_leeway === RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway === PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_no_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_no_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway === RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_iat_with_clock_skew_leeway === PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_iat_with_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_nbf_with_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_nbf_with_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_iat_with_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_iat_with_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_nbf_with_no_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_custom_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_auto_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_nbf_with_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_iat_with_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_iat_with_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_iat_with_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_iat_with_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_iat_with_auto_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway === CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_auto_clock_skew_leeway === CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_iat_with_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_iat_with_custom_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_iat_with_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_no_clock_skew_leeway === CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_auto_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_no_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_custom_clock_skew_leeway === CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_custom_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_no_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf === CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_nbf_with_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway === CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/no_expire_leeway_using_iat_with_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_no_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_nbf_with_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_iat_with_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_no_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway === CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_auto_clock_skew_leeway === CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_iat_with_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_no_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_iat_with_auto_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_nbf_with_no_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_custom_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway === CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_no_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_iat_with_no_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_no_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf === CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_iat_with_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_iat_with_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_iat_with_custom_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_custom_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_auto_clock_skew_leeway === CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway === CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_auto_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_nbf_with_no_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_nbf_with_no_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_nbf_with_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/no_expire_leeway_using_iat_with_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_iat_with_no_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_no_clock_skew_leeway --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims (0.00s) --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_iat_with_auto_clock_skew_leeway (0.01s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_iat_with_clock_skew_leeway (0.01s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_iat_with_clock_skew_leeway (0.00s) --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_iat_with_custom_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_custom_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_auto_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway (0.00s) --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_iat_with_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway (0.00s) --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway (0.00s) --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_nbf_with_no_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_custom_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_iat_with_custom_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway (0.01s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_nbf_with_no_clock_skew_leeway (0.00s) --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway (0.01s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/no_expire_leeway_using_iat_with_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_no_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_nbf_with_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_iat_with_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway (0.00s) --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_auto_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_iat_with_auto_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_nbf_with_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/no_expire_leeway_using_iat_with_clock_skew_leeway (0.00s) --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_nbf_with_no_clock_skew_leeway (0.01s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_iat_with_no_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway (0.02s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway (0.02s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_no_clock_skew_leeway (0.01s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_iat_with_no_clock_skew_leeway (0.01s) --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway (0.06s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_iat_with_clock_skew_leeway (0.06s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_auto_clock_skew_leeway (0.06s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway (0.06s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway (0.07s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_iat_with_clock_skew_leeway (0.10s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_custom_clock_skew_leeway (0.09s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_no_clock_skew_leeway (0.10s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway (0.09s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway (0.11s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf (0.11s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway (0.11s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_nbf_with_clock_skew_leeway (0.11s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf (0.08s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway (0.10s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway (0.08s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway (0.14s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway (0.04s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway (0.05s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_auto_clock_skew_leeway (0.06s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_iat_with_clock_skew_leeway (0.07s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_iat_with_clock_skew_leeway (0.07s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway (0.06s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway (0.09s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_nbf_with_clock_skew_leeway (0.12s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_auto_clock_skew_leeway (0.07s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway (0.09s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_no_clock_skew_leeway (0.12s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf (0.09s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_custom_clock_skew_leeway (0.12s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway (0.10s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_no_clock_skew_leeway (0.15s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway (0.09s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_custom_clock_skew_leeway (0.15s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_no_clock_skew_leeway (0.09s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_iat_with_clock_skew_leeway (0.04s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway (0.13s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway (0.11s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway (0.07s) --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_no_clock_skew_leeway (0.07s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway (0.10s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_no_clock_skew_leeway (0.09s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway (0.07s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_custom_clock_skew_leeway (0.10s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_iat_with_clock_skew_leeway (0.13s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_no_clock_skew_leeway (0.08s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway (0.06s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_auto_clock_skew_leeway (0.04s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway (0.07s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf (0.06s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway (0.11s) === RUN TestJWT_ClaimsFromJWT_NotBeforeClaims === RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static === PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static @@ -92703,175 +92619,175 @@ === PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS === CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static === CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/no_nbf_leeway_using_iat_with_auto_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/no_nbf_leeway_using_iat_with_auto_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_no_nbf_leeway_using_iat_with_custom_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_no_nbf_leeway_using_iat_with_custom_clock_skew_leeway === RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway === PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway === RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/no_nbf_leeway_using_iat_with_auto_clock_skew_leeway === PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/no_nbf_leeway_using_iat_with_custom_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/no_nbf_leeway_using_iat_with_auto_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/no_nbf_leeway_using_iat_with_auto_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_no_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_no_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/no_nbf_leeway_using_iat_with_auto_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_no_nbf_leeway_using_iat_with_custom_clock_skew_leeway === RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_no_nbf_leeway_using_iat_with_custom_clock_skew_leeway === PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_no_nbf_leeway_using_iat_with_auto_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_no_nbf_leeway_using_iat_with_auto_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/nbf_leeway_using_iat_with_no_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/nbf_leeway_using_iat_with_no_clock_skew_leeway === RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/nbf_leeway_using_iat_with_no_clock_skew_leeway === PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway === RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway === PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_no_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_no_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/no_nbf_leeway_using_iat_with_custom_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/no_nbf_leeway_using_iat_with_custom_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_nbf_leeway_using_iat_with_no_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_nbf_leeway_using_iat_with_no_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/no_nbf_leeway_using_iat_with_custom_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/nbf_leeway_using_iat_with_no_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_nbf_leeway_using_iat_with_no_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_nbf_leeway_using_iat_with_no_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/auto_nbf_leeway_using_exp_with_no_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/auto_nbf_leeway_using_exp_with_no_clock_skew_leeway === RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_auto_nbf_leeway_using_exp_with_no_clock_skew_leeway === PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_auto_nbf_leeway_using_exp_with_no_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_no_nbf_leeway_using_iat_with_custom_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_no_nbf_leeway_using_iat_with_custom_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway === RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_no_nbf_leeway_using_iat_with_auto_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway === PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_no_nbf_leeway_using_iat_with_auto_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/nbf_leeway_using_iat_with_no_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/nbf_leeway_using_iat_with_no_clock_skew_leeway === RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway === PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway === RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway === PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway === RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway === PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_nbf_leeway_using_iat_with_no_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_nbf_leeway_using_iat_with_no_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/auto_nbf_leeway_using_exp_with_no_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/auto_nbf_leeway_using_exp_with_no_clock_skew_leeway === RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway === PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_no_nbf_leeway_using_iat_with_custom_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_no_nbf_leeway_using_iat_with_custom_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/no_nbf_leeway_using_iat_with_custom_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/no_nbf_leeway_using_iat_with_custom_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway === RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway === PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway === RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway === PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/no_nbf_leeway_using_iat_with_auto_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_no_nbf_leeway_using_iat_with_custom_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_no_nbf_leeway_using_iat_with_auto_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/no_nbf_leeway_using_iat_with_custom_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway === CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/nbf_leeway_using_iat_with_no_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/nbf_leeway_using_iat_with_no_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_nbf_leeway_using_iat_with_no_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_nbf_leeway_using_iat_with_no_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_no_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_no_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/no_nbf_leeway_using_iat_with_auto_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/no_nbf_leeway_using_iat_with_auto_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_no_nbf_leeway_using_iat_with_auto_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_no_nbf_leeway_using_iat_with_auto_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/no_nbf_leeway_using_iat_with_custom_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/no_nbf_leeway_using_iat_with_custom_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_no_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_no_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_no_nbf_leeway_using_iat_with_custom_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway === CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_no_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_nbf_leeway_using_iat_with_no_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/nbf_leeway_using_iat_with_no_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway === CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway === CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_no_nbf_leeway_using_iat_with_custom_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/auto_nbf_leeway_using_exp_with_no_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_no_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway === CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/no_nbf_leeway_using_iat_with_custom_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_nbf_leeway_using_iat_with_no_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_no_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/nbf_leeway_using_iat_with_no_clock_skew_leeway === CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_no_nbf_leeway_using_iat_with_auto_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway === CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_nbf_leeway_using_iat_with_no_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_no_nbf_leeway_using_iat_with_auto_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/nbf_leeway_using_iat_with_no_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/no_nbf_leeway_using_iat_with_auto_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway === CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_auto_nbf_leeway_using_exp_with_no_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/no_nbf_leeway_using_iat_with_custom_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/auto_nbf_leeway_using_exp_with_no_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_no_clock_skew_leeway === CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/no_nbf_leeway_using_iat_with_auto_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_no_nbf_leeway_using_iat_with_custom_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/nbf_leeway_using_iat_with_no_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/no_nbf_leeway_using_iat_with_auto_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims (0.00s) --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/no_nbf_leeway_using_iat_with_auto_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_no_nbf_leeway_using_iat_with_auto_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/no_nbf_leeway_using_iat_with_custom_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway (0.00s) --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway (0.00s) --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway (0.00s) --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/auto_nbf_leeway_using_exp_with_no_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_no_nbf_leeway_using_iat_with_custom_clock_skew_leeway (0.01s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway (0.01s) --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_nbf_leeway_using_iat_with_no_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_no_nbf_leeway_using_iat_with_auto_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/nbf_leeway_using_iat_with_no_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/no_nbf_leeway_using_iat_with_custom_clock_skew_leeway (0.00s) --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_auto_nbf_leeway_using_exp_with_no_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/auto_nbf_leeway_using_exp_with_no_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_no_nbf_leeway_using_iat_with_custom_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/nbf_leeway_using_iat_with_no_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/no_nbf_leeway_using_iat_with_auto_clock_skew_leeway (0.00s) --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_no_clock_skew_leeway (0.07s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway (0.08s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway (0.08s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway (0.08s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_no_nbf_leeway_using_iat_with_custom_clock_skew_leeway (0.11s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway (0.12s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway (0.13s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway (0.17s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_nbf_leeway_using_iat_with_no_clock_skew_leeway (0.10s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway (0.11s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/no_nbf_leeway_using_iat_with_custom_clock_skew_leeway (0.13s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_no_nbf_leeway_using_iat_with_auto_clock_skew_leeway (0.10s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_no_clock_skew_leeway (0.14s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway (0.10s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/nbf_leeway_using_iat_with_no_clock_skew_leeway (0.14s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_nbf_leeway_using_iat_with_no_clock_skew_leeway (0.04s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/nbf_leeway_using_iat_with_no_clock_skew_leeway (0.06s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_no_clock_skew_leeway (0.05s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway (0.06s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway (0.07s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway (0.07s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_no_nbf_leeway_using_iat_with_custom_clock_skew_leeway (0.08s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway (0.09s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway (0.05s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway (0.08s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway (0.05s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/no_nbf_leeway_using_iat_with_custom_clock_skew_leeway (0.07s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/no_nbf_leeway_using_iat_with_auto_clock_skew_leeway (0.07s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_no_clock_skew_leeway (0.07s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_no_nbf_leeway_using_iat_with_auto_clock_skew_leeway (0.09s) --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway (0.08s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway (0.06s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway (0.04s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/no_nbf_leeway_using_iat_with_auto_clock_skew_leeway (0.05s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway (0.07s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway (0.04s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway (0.05s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway (0.07s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway (0.04s) === RUN TestParsePublicKeyPEM === RUN TestParsePublicKeyPEM/rsa === RUN TestParsePublicKeyPEM/ecdsa === RUN TestParsePublicKeyPEM/ed25519 ---- PASS: TestParsePublicKeyPEM (0.36s) - --- PASS: TestParsePublicKeyPEM/rsa (0.36s) +--- PASS: TestParsePublicKeyPEM (0.32s) + --- PASS: TestParsePublicKeyPEM/rsa (0.32s) --- PASS: TestParsePublicKeyPEM/ecdsa (0.00s) --- PASS: TestParsePublicKeyPEM/ed25519 (0.00s) === RUN TestOIDC_AuthURL @@ -92887,7 +92803,7 @@ --- PASS: TestOIDC_AuthURL (0.00s) --- PASS: TestOIDC_AuthURL/invalid_RedirectURI (0.03s) --- PASS: TestOIDC_AuthURL/missing_RedirectURI (0.03s) - --- PASS: TestOIDC_AuthURL/normal_case (0.03s) + --- PASS: TestOIDC_AuthURL/normal_case (0.04s) === RUN TestOIDC_JWT_Functions_Fail --- PASS: TestOIDC_JWT_Functions_Fail (0.02s) === RUN TestOIDC_ClaimsFromAuthCode @@ -92904,15 +92820,15 @@ === RUN TestOIDC_ClaimsFromAuthCode/no_response_from_provider === RUN TestOIDC_ClaimsFromAuthCode/invalid_bound_audience === CONT TestOIDC_ClaimsFromAuthCode/failed_login_-_bad_nonce ---- PASS: TestOIDC_ClaimsFromAuthCode (0.25s) +--- PASS: TestOIDC_ClaimsFromAuthCode (0.24s) --- PASS: TestOIDC_ClaimsFromAuthCode/successful_login (0.03s) - --- PASS: TestOIDC_ClaimsFromAuthCode/failed_login_unusable_claims (0.02s) - --- PASS: TestOIDC_ClaimsFromAuthCode/successful_login_-_no_userinfo (0.04s) + --- PASS: TestOIDC_ClaimsFromAuthCode/failed_login_unusable_claims (0.04s) + --- PASS: TestOIDC_ClaimsFromAuthCode/successful_login_-_no_userinfo (0.02s) --- PASS: TestOIDC_ClaimsFromAuthCode/missing_state (0.02s) --- PASS: TestOIDC_ClaimsFromAuthCode/unknown_state (0.02s) --- PASS: TestOIDC_ClaimsFromAuthCode/valid_state,_missing_code (0.02s) --- PASS: TestOIDC_ClaimsFromAuthCode/failed_code_exchange (0.02s) - --- PASS: TestOIDC_ClaimsFromAuthCode/no_id_token_returned (0.03s) + --- PASS: TestOIDC_ClaimsFromAuthCode/no_id_token_returned (0.02s) --- PASS: TestOIDC_ClaimsFromAuthCode/no_response_from_provider (0.02s) --- PASS: TestOIDC_ClaimsFromAuthCode/invalid_bound_audience (0.02s) --- PASS: TestOIDC_ClaimsFromAuthCode/failed_login_-_bad_nonce (0.02s) @@ -92933,37 +92849,37 @@ --- PASS: TestExtractStringMetadata/none (0.00s) --- PASS: TestExtractStringMetadata/nested_data (0.00s) === RUN TestExtractListMetadata -=== RUN TestExtractListMetadata/some_-_list -=== RUN TestExtractListMetadata/empty +=== RUN TestExtractListMetadata/error:_a_struct_isn't_stringifiable_(singular) +=== RUN TestExtractListMetadata/error:_a_slice_isn't_stringifiable_(singular) === RUN TestExtractListMetadata/all_-_list -=== RUN TestExtractListMetadata/nested_data_-_list -=== RUN TestExtractListMetadata/all_-_singular +=== RUN TestExtractListMetadata/some_-_list +=== RUN TestExtractListMetadata/JSONPointer_not_found === RUN TestExtractListMetadata/none_-_singular +=== RUN TestExtractListMetadata/nested_data_-_list +=== RUN TestExtractListMetadata/some_-_singular === RUN TestExtractListMetadata/nested_data_-_singular -=== RUN TestExtractListMetadata/error:_a_struct_isn't_stringifiable_(singular) === RUN TestExtractListMetadata/non-string-slice_data_(string) === RUN TestExtractListMetadata/none_-_list +=== RUN TestExtractListMetadata/all_-_singular === RUN TestExtractListMetadata/list_omits_empty_strings === RUN TestExtractListMetadata/JSONPointer -=== RUN TestExtractListMetadata/some_-_singular -=== RUN TestExtractListMetadata/JSONPointer_not_found -=== RUN TestExtractListMetadata/error:_a_slice_isn't_stringifiable_(singular) +=== RUN TestExtractListMetadata/empty --- PASS: TestExtractListMetadata (0.00s) - --- PASS: TestExtractListMetadata/some_-_list (0.00s) - --- PASS: TestExtractListMetadata/empty (0.00s) + --- PASS: TestExtractListMetadata/error:_a_struct_isn't_stringifiable_(singular) (0.00s) + --- PASS: TestExtractListMetadata/error:_a_slice_isn't_stringifiable_(singular) (0.00s) --- PASS: TestExtractListMetadata/all_-_list (0.00s) - --- PASS: TestExtractListMetadata/nested_data_-_list (0.00s) - --- PASS: TestExtractListMetadata/all_-_singular (0.00s) + --- PASS: TestExtractListMetadata/some_-_list (0.00s) + --- PASS: TestExtractListMetadata/JSONPointer_not_found (0.00s) --- PASS: TestExtractListMetadata/none_-_singular (0.00s) + --- PASS: TestExtractListMetadata/nested_data_-_list (0.00s) + --- PASS: TestExtractListMetadata/some_-_singular (0.00s) --- PASS: TestExtractListMetadata/nested_data_-_singular (0.00s) - --- PASS: TestExtractListMetadata/error:_a_struct_isn't_stringifiable_(singular) (0.00s) --- PASS: TestExtractListMetadata/non-string-slice_data_(string) (0.00s) --- PASS: TestExtractListMetadata/none_-_list (0.00s) + --- PASS: TestExtractListMetadata/all_-_singular (0.00s) --- PASS: TestExtractListMetadata/list_omits_empty_strings (0.00s) --- PASS: TestExtractListMetadata/JSONPointer (0.00s) - --- PASS: TestExtractListMetadata/some_-_singular (0.00s) - --- PASS: TestExtractListMetadata/JSONPointer_not_found (0.00s) - --- PASS: TestExtractListMetadata/error:_a_slice_isn't_stringifiable_(singular) (0.00s) + --- PASS: TestExtractListMetadata/empty (0.00s) === RUN TestGetClaim === RUN TestGetClaim/a === RUN TestGetClaim//a @@ -93013,51 +92929,51 @@ --- PASS: TestNormalizeList/struct_{_A_int_}{A:5} (0.00s) --- PASS: TestNormalizeList/ (0.00s) === RUN TestStringifyMetadataValue -=== RUN TestStringifyMetadataValue/int -=== RUN TestStringifyMetadataValue/uint8 -=== RUN TestStringifyMetadataValue/nil -=== RUN TestStringifyMetadataValue/true -=== RUN TestStringifyMetadataValue/float64 -=== RUN TestStringifyMetadataValue/int8 +=== RUN TestStringifyMetadataValue/string === RUN TestStringifyMetadataValue/int16 -=== RUN TestStringifyMetadataValue/int32 === RUN TestStringifyMetadataValue/uint32 -=== RUN TestStringifyMetadataValue/struct -=== RUN TestStringifyMetadataValue/string -=== RUN TestStringifyMetadataValue/json_number -=== RUN TestStringifyMetadataValue/string_slice === RUN TestStringifyMetadataValue/int_slice -=== RUN TestStringifyMetadataValue/empty_string === RUN TestStringifyMetadataValue/false +=== RUN TestStringifyMetadataValue/int8 +=== RUN TestStringifyMetadataValue/uint16 +=== RUN TestStringifyMetadataValue/uint +=== RUN TestStringifyMetadataValue/true +=== RUN TestStringifyMetadataValue/json_number +=== RUN TestStringifyMetadataValue/int32 +=== RUN TestStringifyMetadataValue/nil +=== RUN TestStringifyMetadataValue/empty_string +=== RUN TestStringifyMetadataValue/float64 === RUN TestStringifyMetadataValue/float32 === RUN TestStringifyMetadataValue/int64 +=== RUN TestStringifyMetadataValue/int +=== RUN TestStringifyMetadataValue/uint8 === RUN TestStringifyMetadataValue/uint64 -=== RUN TestStringifyMetadataValue/uint +=== RUN TestStringifyMetadataValue/string_slice === RUN TestStringifyMetadataValue/map -=== RUN TestStringifyMetadataValue/uint16 +=== RUN TestStringifyMetadataValue/struct --- PASS: TestStringifyMetadataValue (0.00s) - --- PASS: TestStringifyMetadataValue/int (0.00s) - --- PASS: TestStringifyMetadataValue/uint8 (0.00s) - --- PASS: TestStringifyMetadataValue/nil (0.00s) - --- PASS: TestStringifyMetadataValue/true (0.00s) - --- PASS: TestStringifyMetadataValue/float64 (0.00s) - --- PASS: TestStringifyMetadataValue/int8 (0.00s) + --- PASS: TestStringifyMetadataValue/string (0.00s) --- PASS: TestStringifyMetadataValue/int16 (0.00s) - --- PASS: TestStringifyMetadataValue/int32 (0.00s) --- PASS: TestStringifyMetadataValue/uint32 (0.00s) - --- PASS: TestStringifyMetadataValue/struct (0.00s) - --- PASS: TestStringifyMetadataValue/string (0.00s) - --- PASS: TestStringifyMetadataValue/json_number (0.00s) - --- PASS: TestStringifyMetadataValue/string_slice (0.00s) --- PASS: TestStringifyMetadataValue/int_slice (0.00s) - --- PASS: TestStringifyMetadataValue/empty_string (0.00s) --- PASS: TestStringifyMetadataValue/false (0.00s) + --- PASS: TestStringifyMetadataValue/int8 (0.00s) + --- PASS: TestStringifyMetadataValue/uint16 (0.00s) + --- PASS: TestStringifyMetadataValue/uint (0.00s) + --- PASS: TestStringifyMetadataValue/true (0.00s) + --- PASS: TestStringifyMetadataValue/json_number (0.00s) + --- PASS: TestStringifyMetadataValue/int32 (0.00s) + --- PASS: TestStringifyMetadataValue/nil (0.00s) + --- PASS: TestStringifyMetadataValue/empty_string (0.00s) + --- PASS: TestStringifyMetadataValue/float64 (0.00s) --- PASS: TestStringifyMetadataValue/float32 (0.00s) --- PASS: TestStringifyMetadataValue/int64 (0.00s) + --- PASS: TestStringifyMetadataValue/int (0.00s) + --- PASS: TestStringifyMetadataValue/uint8 (0.00s) --- PASS: TestStringifyMetadataValue/uint64 (0.00s) - --- PASS: TestStringifyMetadataValue/uint (0.00s) + --- PASS: TestStringifyMetadataValue/string_slice (0.00s) --- PASS: TestStringifyMetadataValue/map (0.00s) - --- PASS: TestStringifyMetadataValue/uint16 (0.00s) + --- PASS: TestStringifyMetadataValue/struct (0.00s) === RUN TestValidateAudience === RUN TestValidateAudience/boundAudiences=[]string{"a"}_audience=[]string{"a"}_strict=false === RUN TestValidateAudience/boundAudiences=[]string{"a"}_audience=[]string{"a"}_strict=true @@ -93127,11 +93043,11 @@ --- PASS: TestValidRedirect/uri="https://[::1]:5000/x/y?r=42"_allowed=[]string{"a",_"b",_"https://[::1]:5000/x/y?r=43"} (0.00s) --- PASS: TestValidRedirect/uri="%%%%%%%%%%%"_allowed=[]string{} (0.00s) PASS -ok github.com/hashicorp/consul/internal/go-sso/oidcauth 1.942s +ok github.com/hashicorp/consul/internal/go-sso/oidcauth 1.690s === RUN TestStrListContains --- PASS: TestStrListContains (0.00s) PASS -ok github.com/hashicorp/consul/internal/go-sso/oidcauth/internal/strutil 0.010s +ok github.com/hashicorp/consul/internal/go-sso/oidcauth/internal/strutil 0.011s ? github.com/hashicorp/consul/internal/go-sso/oidcauth/oidcauthtest [no test files] === RUN TestIsPrivateIP === RUN TestIsPrivateIP/10.0.0.1 @@ -93204,7 +93120,7 @@ --- PASS: TestIsIPv6/localhost (0.00s) --- PASS: TestIsIPv6/1.257.0.1 (0.00s) PASS -ok github.com/hashicorp/consul/ipaddr 0.011s +ok github.com/hashicorp/consul/ipaddr 0.007s === RUN TestDurationMinusBuffer --- PASS: TestDurationMinusBuffer (0.00s) === RUN TestDurationMinusBufferDomain @@ -93297,19 +93213,12 @@ === CONT TestJitterRandomStagger === RUN TestJitterRandomStagger/0_percent === PAUSE TestJitterRandomStagger/0_percent -=== CONT TestRetryWaiter_WaitChans === RUN TestJitterRandomStagger/10_percent === PAUSE TestJitterRandomStagger/10_percent -=== CONT TestRetryWaiter_calculateWait -=== RUN TestRetryWaiter_calculateWait/Defaults -=== PAUSE TestRetryWaiter_calculateWait/Defaults -=== RUN TestRetryWaiter_calculateWait/Minimum_Wait -=== PAUSE TestRetryWaiter_calculateWait/Minimum_Wait -=== RUN TestRetryWaiter_calculateWait/Minimum_Failures -=== PAUSE TestRetryWaiter_calculateWait/Minimum_Failures -=== RUN TestRetryWaiter_calculateWait/Maximum_Wait -=== PAUSE TestRetryWaiter_calculateWait/Maximum_Wait -=== CONT TestRetryWaiter_calculateWait/Defaults +=== RUN TestJitterRandomStagger/100_percent +=== PAUSE TestJitterRandomStagger/100_percent +=== CONT TestJitterRandomStagger/0_percent +=== CONT TestRetryWaiter_WaitChans === RUN TestRetryWaiter_WaitChans/Minimum_Wait_-_Success === PAUSE TestRetryWaiter_WaitChans/Minimum_Wait_-_Success === RUN TestRetryWaiter_WaitChans/Minimum_Wait_-_WaitIf @@ -93323,28 +93232,35 @@ === RUN TestRetryWaiter_WaitChans/Maximum_Wait_-_WaitIfErr === PAUSE TestRetryWaiter_WaitChans/Maximum_Wait_-_WaitIfErr === CONT TestRetryWaiter_WaitChans/Minimum_Wait_-_Success -=== CONT TestRetryWaiter_calculateWait/Maximum_Wait +=== CONT TestRetryWaiter_calculateWait +=== CONT TestRetryWaiter_WaitChans/Maximum_Wait_-_WaitIf +=== RUN TestRetryWaiter_calculateWait/Defaults +=== CONT TestJitterRandomStagger/100_percent +=== PAUSE TestRetryWaiter_calculateWait/Defaults +=== RUN TestRetryWaiter_calculateWait/Minimum_Wait +=== PAUSE TestRetryWaiter_calculateWait/Minimum_Wait +=== CONT TestJitterRandomStagger/10_percent +=== CONT TestRetryWaiter_WaitChans/Minimum_Wait_-_WaitIf === CONT TestRetryWaiter_WaitChans/Maximum_Wait_-_WaitIfErr +--- PASS: TestJitterRandomStagger (0.00s) + --- PASS: TestJitterRandomStagger/0_percent (0.00s) + --- PASS: TestJitterRandomStagger/100_percent (0.00s) + --- PASS: TestJitterRandomStagger/10_percent (0.00s) +=== RUN TestRetryWaiter_calculateWait/Minimum_Failures +=== PAUSE TestRetryWaiter_calculateWait/Minimum_Failures +=== RUN TestRetryWaiter_calculateWait/Maximum_Wait +=== PAUSE TestRetryWaiter_calculateWait/Maximum_Wait +=== CONT TestRetryWaiter_calculateWait/Defaults +=== CONT TestRetryWaiter_calculateWait/Maximum_Wait === CONT TestRetryWaiter_calculateWait/Minimum_Failures -=== CONT TestRetryWaiter_WaitChans/Maximum_Wait_-_WaitIf +=== CONT TestRetryWaiter_calculateWait/Minimum_Wait === CONT TestRetryWaiter_WaitChans/Maximum_Wait_-_Failed === CONT TestRetryWaiter_WaitChans/Minimum_Wait_-_WaitIfErr -=== CONT TestRetryWaiter_WaitChans/Minimum_Wait_-_WaitIf -=== CONT TestRetryWaiter_calculateWait/Minimum_Wait --- PASS: TestRetryWaiter_calculateWait (0.00s) --- PASS: TestRetryWaiter_calculateWait/Defaults (0.00s) - --- PASS: TestRetryWaiter_calculateWait/Maximum_Wait (0.00s) --- PASS: TestRetryWaiter_calculateWait/Minimum_Failures (0.00s) + --- PASS: TestRetryWaiter_calculateWait/Maximum_Wait (0.00s) --- PASS: TestRetryWaiter_calculateWait/Minimum_Wait (0.00s) -=== RUN TestJitterRandomStagger/100_percent -=== PAUSE TestJitterRandomStagger/100_percent -=== CONT TestJitterRandomStagger/0_percent -=== CONT TestJitterRandomStagger/100_percent -=== CONT TestJitterRandomStagger/10_percent ---- PASS: TestJitterRandomStagger (0.00s) - --- PASS: TestJitterRandomStagger/0_percent (0.00s) - --- PASS: TestJitterRandomStagger/100_percent (0.00s) - --- PASS: TestJitterRandomStagger/10_percent (0.00s) --- PASS: TestRetryWaiter_WaitChans (0.00s) --- PASS: TestRetryWaiter_WaitChans/Minimum_Wait_-_Success (0.20s) --- PASS: TestRetryWaiter_WaitChans/Minimum_Wait_-_WaitIf (0.20s) @@ -93353,7 +93269,7 @@ --- PASS: TestRetryWaiter_WaitChans/Maximum_Wait_-_WaitIfErr (0.25s) --- PASS: TestRetryWaiter_WaitChans/Maximum_Wait_-_Failed (0.25s) PASS -ok github.com/hashicorp/consul/lib 0.270s +ok github.com/hashicorp/consul/lib 0.268s === RUN TestHookTranslateKeys === RUN TestHookTranslateKeys/target_of_type_struct,_with_struct_receiver === RUN TestHookTranslateKeys/target_of_type_ptr,_with_struct_receiver @@ -93385,11 +93301,11 @@ === RUN TestHookWeakDecodeFromSlice_NestedOpaqueConfig --- PASS: TestHookWeakDecodeFromSlice_NestedOpaqueConfig (0.00s) PASS -ok github.com/hashicorp/consul/lib/decode 0.013s +ok github.com/hashicorp/consul/lib/decode 0.009s === RUN TestWriteAtomic --- PASS: TestWriteAtomic (0.00s) PASS -ok github.com/hashicorp/consul/lib/file 0.017s +ok github.com/hashicorp/consul/lib/file 0.007s === RUN TestDynamic === PAUSE TestDynamic === RUN TestDynamicPanic @@ -93400,10 +93316,10 @@ === CONT TestDynamicAcquire === CONT TestDynamicPanic --- PASS: TestDynamicPanic (0.00s) ---- PASS: TestDynamicAcquire (0.06s) ---- PASS: TestDynamic (0.98s) +--- PASS: TestDynamicAcquire (0.05s) +--- PASS: TestDynamic (0.91s) PASS -ok github.com/hashicorp/consul/lib/semaphore 0.994s +ok github.com/hashicorp/consul/lib/semaphore 0.936s === RUN TestContains --- PASS: TestContains (0.00s) === RUN TestEqual @@ -93425,97 +93341,97 @@ --- PASS: TestEqual/[]string{"a",_"b"}_=?=_[]string{"a",_"b"} (0.00s) --- PASS: TestEqual/[]string{"a",_"b"}_=?=_[]string{"b",_"a"} (0.00s) === RUN TestMergeSorted -=== RUN TestMergeSorted/one_and_one_dupe -=== RUN TestMergeSorted/one_and_one -=== RUN TestMergeSorted/two_and_one -=== RUN TestMergeSorted/two_and_two === RUN TestMergeSorted/two_and_two_dupe === RUN TestMergeSorted/nil === RUN TestMergeSorted/empty === RUN TestMergeSorted/one_and_none +=== RUN TestMergeSorted/one_and_one_dupe +=== RUN TestMergeSorted/one_and_one +=== RUN TestMergeSorted/two_and_one +=== RUN TestMergeSorted/two_and_two --- PASS: TestMergeSorted (0.00s) - --- PASS: TestMergeSorted/one_and_one_dupe (0.00s) - --- PASS: TestMergeSorted/one_and_one (0.00s) - --- PASS: TestMergeSorted/two_and_one (0.00s) - --- PASS: TestMergeSorted/two_and_two (0.00s) --- PASS: TestMergeSorted/two_and_two_dupe (0.00s) --- PASS: TestMergeSorted/nil (0.00s) --- PASS: TestMergeSorted/empty (0.00s) --- PASS: TestMergeSorted/one_and_none (0.00s) + --- PASS: TestMergeSorted/one_and_one_dupe (0.00s) + --- PASS: TestMergeSorted/one_and_one (0.00s) + --- PASS: TestMergeSorted/two_and_one (0.00s) + --- PASS: TestMergeSorted/two_and_two (0.00s) PASS -ok github.com/hashicorp/consul/lib/stringslice 0.017s +ok github.com/hashicorp/consul/lib/stringslice 0.008s === RUN TestInterpolateHIL -=== RUN TestInterpolateHIL/no_vars_lower=false -=== RUN TestInterpolateHIL/no_vars_lower=true -=== RUN TestInterpolateHIL/just_uppercase_var_lower=false -=== RUN TestInterpolateHIL/just_uppercase_var_lower=true -=== RUN TestInterpolateHIL/missing_map_val_lower=false -=== RUN TestInterpolateHIL/missing_map_val_lower=true -=== RUN TestInterpolateHIL/just_start_lower=false -=== RUN TestInterpolateHIL/just_start_lower=true +=== RUN TestInterpolateHIL/two_vars_lower=false +=== RUN TestInterpolateHIL/two_vars_lower=true +=== RUN TestInterpolateHIL/just_end_lower=false +=== RUN TestInterpolateHIL/just_end_lower=true +=== RUN TestInterpolateHIL/var_without_start_lower=false +=== RUN TestInterpolateHIL/var_without_start_lower=true === RUN TestInterpolateHIL/missing_map_key_lower=false === RUN TestInterpolateHIL/missing_map_key_lower=true +=== RUN TestInterpolateHIL/two_vars_missing_first_end_lower=false +=== RUN TestInterpolateHIL/two_vars_missing_first_end_lower=true === RUN TestInterpolateHIL/empty_lower=false === RUN TestInterpolateHIL/empty_lower=true -=== RUN TestInterpolateHIL/uppercase_var_in_middle_lower=false -=== RUN TestInterpolateHIL/uppercase_var_in_middle_lower=true -=== RUN TestInterpolateHIL/var_without_start_lower=false -=== RUN TestInterpolateHIL/var_without_start_lower=true === RUN TestInterpolateHIL/just_lowercase_var_lower=false === RUN TestInterpolateHIL/just_lowercase_var_lower=true -=== RUN TestInterpolateHIL/just_end_lower=false -=== RUN TestInterpolateHIL/just_end_lower=true -=== RUN TestInterpolateHIL/two_vars_missing_first_end_lower=false -=== RUN TestInterpolateHIL/two_vars_missing_first_end_lower=true +=== RUN TestInterpolateHIL/just_uppercase_var_lower=false +=== RUN TestInterpolateHIL/just_uppercase_var_lower=true +=== RUN TestInterpolateHIL/no_varname_lower=false +=== RUN TestInterpolateHIL/no_varname_lower=true +=== RUN TestInterpolateHIL/var_without_end_lower=false +=== RUN TestInterpolateHIL/var_without_end_lower=true +=== RUN TestInterpolateHIL/no_vars_lower=false +=== RUN TestInterpolateHIL/no_vars_lower=true === RUN TestInterpolateHIL/lowercase_var_in_middle_lower=false === RUN TestInterpolateHIL/lowercase_var_in_middle_lower=true -=== RUN TestInterpolateHIL/two_vars_lower=false -=== RUN TestInterpolateHIL/two_vars_lower=true +=== RUN TestInterpolateHIL/uppercase_var_in_middle_lower=false +=== RUN TestInterpolateHIL/uppercase_var_in_middle_lower=true +=== RUN TestInterpolateHIL/missing_map_val_lower=false +=== RUN TestInterpolateHIL/missing_map_val_lower=true === RUN TestInterpolateHIL/two_vars_missing_second_start_lower=false === RUN TestInterpolateHIL/two_vars_missing_second_start_lower=true +=== RUN TestInterpolateHIL/just_start_lower=false +=== RUN TestInterpolateHIL/just_start_lower=true === RUN TestInterpolateHIL/backwards_lower=false === RUN TestInterpolateHIL/backwards_lower=true -=== RUN TestInterpolateHIL/no_varname_lower=false -=== RUN TestInterpolateHIL/no_varname_lower=true -=== RUN TestInterpolateHIL/var_without_end_lower=false -=== RUN TestInterpolateHIL/var_without_end_lower=true --- PASS: TestInterpolateHIL (0.00s) - --- PASS: TestInterpolateHIL/no_vars_lower=false (0.00s) - --- PASS: TestInterpolateHIL/no_vars_lower=true (0.00s) - --- PASS: TestInterpolateHIL/just_uppercase_var_lower=false (0.00s) - --- PASS: TestInterpolateHIL/just_uppercase_var_lower=true (0.00s) - --- PASS: TestInterpolateHIL/missing_map_val_lower=false (0.00s) - --- PASS: TestInterpolateHIL/missing_map_val_lower=true (0.00s) - --- PASS: TestInterpolateHIL/just_start_lower=false (0.00s) - --- PASS: TestInterpolateHIL/just_start_lower=true (0.00s) + --- PASS: TestInterpolateHIL/two_vars_lower=false (0.00s) + --- PASS: TestInterpolateHIL/two_vars_lower=true (0.00s) + --- PASS: TestInterpolateHIL/just_end_lower=false (0.00s) + --- PASS: TestInterpolateHIL/just_end_lower=true (0.00s) + --- PASS: TestInterpolateHIL/var_without_start_lower=false (0.00s) + --- PASS: TestInterpolateHIL/var_without_start_lower=true (0.00s) --- PASS: TestInterpolateHIL/missing_map_key_lower=false (0.00s) --- PASS: TestInterpolateHIL/missing_map_key_lower=true (0.00s) + --- PASS: TestInterpolateHIL/two_vars_missing_first_end_lower=false (0.00s) + --- PASS: TestInterpolateHIL/two_vars_missing_first_end_lower=true (0.00s) --- PASS: TestInterpolateHIL/empty_lower=false (0.00s) --- PASS: TestInterpolateHIL/empty_lower=true (0.00s) - --- PASS: TestInterpolateHIL/uppercase_var_in_middle_lower=false (0.00s) - --- PASS: TestInterpolateHIL/uppercase_var_in_middle_lower=true (0.00s) - --- PASS: TestInterpolateHIL/var_without_start_lower=false (0.00s) - --- PASS: TestInterpolateHIL/var_without_start_lower=true (0.00s) --- PASS: TestInterpolateHIL/just_lowercase_var_lower=false (0.00s) --- PASS: TestInterpolateHIL/just_lowercase_var_lower=true (0.00s) - --- PASS: TestInterpolateHIL/just_end_lower=false (0.00s) - --- PASS: TestInterpolateHIL/just_end_lower=true (0.00s) - --- PASS: TestInterpolateHIL/two_vars_missing_first_end_lower=false (0.00s) - --- PASS: TestInterpolateHIL/two_vars_missing_first_end_lower=true (0.00s) + --- PASS: TestInterpolateHIL/just_uppercase_var_lower=false (0.00s) + --- PASS: TestInterpolateHIL/just_uppercase_var_lower=true (0.00s) + --- PASS: TestInterpolateHIL/no_varname_lower=false (0.00s) + --- PASS: TestInterpolateHIL/no_varname_lower=true (0.00s) + --- PASS: TestInterpolateHIL/var_without_end_lower=false (0.00s) + --- PASS: TestInterpolateHIL/var_without_end_lower=true (0.00s) + --- PASS: TestInterpolateHIL/no_vars_lower=false (0.00s) + --- PASS: TestInterpolateHIL/no_vars_lower=true (0.00s) --- PASS: TestInterpolateHIL/lowercase_var_in_middle_lower=false (0.00s) --- PASS: TestInterpolateHIL/lowercase_var_in_middle_lower=true (0.00s) - --- PASS: TestInterpolateHIL/two_vars_lower=false (0.00s) - --- PASS: TestInterpolateHIL/two_vars_lower=true (0.00s) + --- PASS: TestInterpolateHIL/uppercase_var_in_middle_lower=false (0.00s) + --- PASS: TestInterpolateHIL/uppercase_var_in_middle_lower=true (0.00s) + --- PASS: TestInterpolateHIL/missing_map_val_lower=false (0.00s) + --- PASS: TestInterpolateHIL/missing_map_val_lower=true (0.00s) --- PASS: TestInterpolateHIL/two_vars_missing_second_start_lower=false (0.00s) --- PASS: TestInterpolateHIL/two_vars_missing_second_start_lower=true (0.00s) + --- PASS: TestInterpolateHIL/just_start_lower=false (0.00s) + --- PASS: TestInterpolateHIL/just_start_lower=true (0.00s) --- PASS: TestInterpolateHIL/backwards_lower=false (0.00s) --- PASS: TestInterpolateHIL/backwards_lower=true (0.00s) - --- PASS: TestInterpolateHIL/no_varname_lower=false (0.00s) - --- PASS: TestInterpolateHIL/no_varname_lower=true (0.00s) - --- PASS: TestInterpolateHIL/var_without_end_lower=false (0.00s) - --- PASS: TestInterpolateHIL/var_without_end_lower=true (0.00s) PASS -ok github.com/hashicorp/consul/lib/template 0.012s +ok github.com/hashicorp/consul/lib/template 0.009s === RUN TestGatedWriter_impl --- PASS: TestGatedWriter_impl (0.00s) === RUN TestGatedWriter @@ -93632,14 +93548,14 @@ === CONT TestLogFile_deleteArchivesDisabled --- PASS: TestLogFile_deleteArchivesDisabled (0.00s) === CONT TestLogFile_deleteArchives -=== CONT TestLogFile_openNew ---- PASS: TestLogFile_openNew (0.00s) -=== CONT TestLogFile_byteRotation --- PASS: TestLogFile_deleteArchives (0.00s) +=== CONT TestLogFile_byteRotation --- PASS: TestLogFile_byteRotation (0.00s) +=== CONT TestLogFile_openNew +--- PASS: TestLogFile_openNew (0.00s) --- PASS: TestLogFile_timeRotation (2.00s) PASS -ok github.com/hashicorp/consul/logging 2.026s +ok github.com/hashicorp/consul/logging 2.013s === RUN TestMonitor_Start === PAUSE TestMonitor_Start === RUN TestMonitor_Stop @@ -93651,17 +93567,17 @@ === RUN TestMonitor_WriteStopped === PAUSE TestMonitor_WriteStopped === CONT TestMonitor_Start ---- PASS: TestMonitor_Start (0.00s) -=== CONT TestMonitor_WriteStopped ---- PASS: TestMonitor_WriteStopped (0.00s) === CONT TestMonitor_ZeroBufSizeDefault === CONT TestMonitor_DroppedMessages === CONT TestMonitor_Stop ---- PASS: TestMonitor_Stop (0.10s) +=== CONT TestMonitor_WriteStopped +--- PASS: TestMonitor_Start (0.00s) +--- PASS: TestMonitor_WriteStopped (0.00s) --- PASS: TestMonitor_ZeroBufSizeDefault (0.10s) --- PASS: TestMonitor_DroppedMessages (0.10s) +--- PASS: TestMonitor_Stop (0.10s) PASS -ok github.com/hashicorp/consul/logging/monitor 0.115s +ok github.com/hashicorp/consul/logging/monitor 0.112s === RUN TestHookPBTimestampToTime --- PASS: TestHookPBTimestampToTime (0.00s) === RUN TestHookTimeToPBTimestamp @@ -93669,7 +93585,7 @@ === RUN TestHookTimeToPBTimestamp_ZeroTime --- PASS: TestHookTimeToPBTimestamp_ZeroTime (0.00s) PASS -ok github.com/hashicorp/consul/proto 0.010s +ok github.com/hashicorp/consul/proto 0.007s ? github.com/hashicorp/consul/proto/pbacl [no test files] ? github.com/hashicorp/consul/proto/pbautoconf [no test files] ? github.com/hashicorp/consul/proto/pbcommon [no test files] @@ -93677,7 +93593,7 @@ ? github.com/hashicorp/consul/proto/pbconnect [no test files] === RUN TestGetEphemeralPortRange ephemeral_linux_test.go:17: min=32768, max=60999 ---- PASS: TestGetEphemeralPortRange (0.01s) +--- PASS: TestGetEphemeralPortRange (0.00s) === RUN TestTakeReturn freeport_test.go:13: DM-skipped --- SKIP: TestTakeReturn (0.00s) @@ -93696,7 +93612,7 @@ --- PASS: TestIntervalOverlap/1:4_vs_3:6 (0.00s) --- PASS: TestIntervalOverlap/1:6_vs_3:4 (0.00s) PASS -ok github.com/hashicorp/consul/sdk/freeport 0.041s +ok github.com/hashicorp/consul/sdk/freeport 0.019s ? github.com/hashicorp/consul/sdk/testutil [no test files] === RUN TestRetryer retry_test.go:12: DM-skipped @@ -93714,54 +93630,54 @@ === RUN TestArchive_hashList --- PASS: TestArchive_hashList (0.00s) === RUN TestSnapshot -2022-09-15T21:47:00.342Z [INFO] raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b2d1835d-4075-e808-961c-9a9bb97576eb Address:b2d1835d-4075-e808-961c-9a9bb97576eb}]" -2022-09-15T21:47:00.342Z [INFO] raft: entering follower state: follower="Node at b2d1835d-4075-e808-961c-9a9bb97576eb [Follower]" leader= -2022-09-15T21:47:02.241Z [WARN] raft: heartbeat timeout reached, starting election: last-leader= -2022-09-15T21:47:02.241Z [INFO] raft: entering candidate state: node="Node at b2d1835d-4075-e808-961c-9a9bb97576eb [Candidate]" term=2 -2022-09-15T21:47:02.241Z [DEBUG] raft: votes: needed=1 -2022-09-15T21:47:02.241Z [DEBUG] raft: vote granted: from=server-b2d1835d-4075-e808-961c-9a9bb97576eb term=2 tally=1 -2022-09-15T21:47:02.241Z [INFO] raft: election won: tally=1 -2022-09-15T21:47:02.241Z [INFO] raft: entering leader state: leader="Node at b2d1835d-4075-e808-961c-9a9bb97576eb [Leader]" -2022-09-15T21:47:05.015Z [INFO] raft: starting snapshot up to: index=65538 -2022-09-15T21:47:05.015Z [INFO] snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshot-snapshot718130774/before/snapshots/2-65538-1663278425015.tmp -2022-09-15T21:47:05.131Z [INFO] raft: compacting logs: from=1 to=55298 -2022-09-15T21:47:05.144Z [INFO] raft: snapshot complete up to: index=65538 -2022-09-15T21:47:07.986Z [INFO] raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-994f44f4-a63f-0d5f-3933-241f5e43ac13 Address:994f44f4-a63f-0d5f-3933-241f5e43ac13}]" -2022-09-15T21:47:07.986Z [INFO] raft: entering follower state: follower="Node at 994f44f4-a63f-0d5f-3933-241f5e43ac13 [Follower]" leader= -2022-09-15T21:47:09.580Z [WARN] raft: heartbeat timeout reached, starting election: last-leader= -2022-09-15T21:47:09.580Z [INFO] raft: entering candidate state: node="Node at 994f44f4-a63f-0d5f-3933-241f5e43ac13 [Candidate]" term=2 -2022-09-15T21:47:09.580Z [DEBUG] raft: votes: needed=1 -2022-09-15T21:47:09.580Z [DEBUG] raft: vote granted: from=server-994f44f4-a63f-0d5f-3933-241f5e43ac13 term=2 tally=1 -2022-09-15T21:47:09.580Z [INFO] raft: election won: tally=1 -2022-09-15T21:47:09.580Z [INFO] raft: entering leader state: leader="Node at 994f44f4-a63f-0d5f-3933-241f5e43ac13 [Leader]" -2022-09-15T21:47:09.975Z [INFO] snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshot-snapshot718130774/after/snapshots/2-65539-1663278429975.tmp -2022-09-15T21:47:10.027Z [INFO] raft: copied to local snapshot: bytes=16973829 -2022-09-15T21:47:10.138Z [INFO] raft: restored user snapshot: index=1 ---- PASS: TestSnapshot (9.86s) +2021-08-13T15:39:47.220Z [INFO] raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-163163f7-0124-0f0b-97bb-f8d8e0d49789 Address:163163f7-0124-0f0b-97bb-f8d8e0d49789}]" +2021-08-13T15:39:47.220Z [INFO] raft: entering follower state: follower="Node at 163163f7-0124-0f0b-97bb-f8d8e0d49789 [Follower]" leader= +2021-08-13T15:39:48.767Z [WARN] raft: heartbeat timeout reached, starting election: last-leader= +2021-08-13T15:39:48.767Z [INFO] raft: entering candidate state: node="Node at 163163f7-0124-0f0b-97bb-f8d8e0d49789 [Candidate]" term=2 +2021-08-13T15:39:48.767Z [DEBUG] raft: votes: needed=1 +2021-08-13T15:39:48.767Z [DEBUG] raft: vote granted: from=server-163163f7-0124-0f0b-97bb-f8d8e0d49789 term=2 tally=1 +2021-08-13T15:39:48.767Z [INFO] raft: election won: tally=1 +2021-08-13T15:39:48.767Z [INFO] raft: entering leader state: leader="Node at 163163f7-0124-0f0b-97bb-f8d8e0d49789 [Leader]" +2021-08-13T15:39:51.428Z [INFO] raft: starting snapshot up to: index=65538 +2021-08-13T15:39:51.429Z [INFO] snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshot-snapshot290705825/before/snapshots/2-65538-1628869191429.tmp +2021-08-13T15:39:51.527Z [INFO] raft: compacting logs: from=1 to=55298 +2021-08-13T15:39:51.536Z [INFO] raft: snapshot complete up to: index=65538 +2021-08-13T15:39:54.059Z [INFO] raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-58700a28-6739-ac08-3c81-d6a124856a5e Address:58700a28-6739-ac08-3c81-d6a124856a5e}]" +2021-08-13T15:39:54.059Z [INFO] raft: entering follower state: follower="Node at 58700a28-6739-ac08-3c81-d6a124856a5e [Follower]" leader= +2021-08-13T15:39:56.038Z [WARN] raft: heartbeat timeout reached, starting election: last-leader= +2021-08-13T15:39:56.038Z [INFO] raft: entering candidate state: node="Node at 58700a28-6739-ac08-3c81-d6a124856a5e [Candidate]" term=2 +2021-08-13T15:39:56.038Z [DEBUG] raft: votes: needed=1 +2021-08-13T15:39:56.038Z [DEBUG] raft: vote granted: from=server-58700a28-6739-ac08-3c81-d6a124856a5e term=2 tally=1 +2021-08-13T15:39:56.038Z [INFO] raft: election won: tally=1 +2021-08-13T15:39:56.038Z [INFO] raft: entering leader state: leader="Node at 58700a28-6739-ac08-3c81-d6a124856a5e [Leader]" +2021-08-13T15:39:56.422Z [INFO] snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshot-snapshot290705825/after/snapshots/2-65539-1628869196422.tmp +2021-08-13T15:39:56.468Z [INFO] raft: copied to local snapshot: bytes=16973829 +2021-08-13T15:39:56.574Z [INFO] raft: restored user snapshot: index=1 +--- PASS: TestSnapshot (9.41s) === RUN TestSnapshot_Nil --- PASS: TestSnapshot_Nil (0.00s) === RUN TestSnapshot_BadVerify --- PASS: TestSnapshot_BadVerify (0.00s) === RUN TestSnapshot_TruncatedVerify -2022-09-15T21:47:10.201Z [INFO] raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-343b831b-8f6a-b157-5df1-2441ee144d0f Address:343b831b-8f6a-b157-5df1-2441ee144d0f}]" -2022-09-15T21:47:10.201Z [INFO] raft: entering follower state: follower="Node at 343b831b-8f6a-b157-5df1-2441ee144d0f [Follower]" leader= -2022-09-15T21:47:11.365Z [WARN] raft: heartbeat timeout reached, starting election: last-leader= -2022-09-15T21:47:11.365Z [INFO] raft: entering candidate state: node="Node at 343b831b-8f6a-b157-5df1-2441ee144d0f [Candidate]" term=2 -2022-09-15T21:47:11.365Z [DEBUG] raft: votes: needed=1 -2022-09-15T21:47:11.365Z [DEBUG] raft: vote granted: from=server-343b831b-8f6a-b157-5df1-2441ee144d0f term=2 tally=1 -2022-09-15T21:47:11.365Z [INFO] raft: election won: tally=1 -2022-09-15T21:47:11.365Z [INFO] raft: entering leader state: leader="Node at 343b831b-8f6a-b157-5df1-2441ee144d0f [Leader]" -2022-09-15T21:47:14.540Z [INFO] raft: starting snapshot up to: index=65538 -2022-09-15T21:47:14.540Z [INFO] snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshot_TruncatedVerify-snapshot797136631/before/snapshots/2-65538-1663278434540.tmp -2022-09-15T21:47:14.649Z [INFO] raft: compacting logs: from=1 to=55298 -2022-09-15T21:47:14.658Z [INFO] raft: snapshot complete up to: index=65538 +2021-08-13T15:39:56.629Z [INFO] raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-3cd244ca-4736-4632-a4ed-77671255027b Address:3cd244ca-4736-4632-a4ed-77671255027b}]" +2021-08-13T15:39:56.629Z [INFO] raft: entering follower state: follower="Node at 3cd244ca-4736-4632-a4ed-77671255027b [Follower]" leader= +2021-08-13T15:39:58.203Z [WARN] raft: heartbeat timeout reached, starting election: last-leader= +2021-08-13T15:39:58.203Z [INFO] raft: entering candidate state: node="Node at 3cd244ca-4736-4632-a4ed-77671255027b [Candidate]" term=2 +2021-08-13T15:39:58.203Z [DEBUG] raft: votes: needed=1 +2021-08-13T15:39:58.203Z [DEBUG] raft: vote granted: from=server-3cd244ca-4736-4632-a4ed-77671255027b term=2 tally=1 +2021-08-13T15:39:58.203Z [INFO] raft: election won: tally=1 +2021-08-13T15:39:58.203Z [INFO] raft: entering leader state: leader="Node at 3cd244ca-4736-4632-a4ed-77671255027b [Leader]" +2021-08-13T15:40:01.354Z [INFO] raft: starting snapshot up to: index=65538 +2021-08-13T15:40:01.354Z [INFO] snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshot_TruncatedVerify-snapshot521145246/before/snapshots/2-65538-1628869201354.tmp +2021-08-13T15:40:01.456Z [INFO] raft: compacting logs: from=1 to=55298 +2021-08-13T15:40:01.465Z [INFO] raft: snapshot complete up to: index=65538 === RUN TestSnapshot_TruncatedVerify/truncate_200_bytes_from_end === RUN TestSnapshot_TruncatedVerify/truncate_16_bytes_from_end === RUN TestSnapshot_TruncatedVerify/truncate_8_bytes_from_end === RUN TestSnapshot_TruncatedVerify/truncate_4_bytes_from_end === RUN TestSnapshot_TruncatedVerify/truncate_2_bytes_from_end === RUN TestSnapshot_TruncatedVerify/truncate_1_bytes_from_end ---- PASS: TestSnapshot_TruncatedVerify (9.19s) +--- PASS: TestSnapshot_TruncatedVerify (9.35s) --- PASS: TestSnapshot_TruncatedVerify/truncate_200_bytes_from_end (0.36s) --- PASS: TestSnapshot_TruncatedVerify/truncate_16_bytes_from_end (0.36s) --- PASS: TestSnapshot_TruncatedVerify/truncate_8_bytes_from_end (0.36s) @@ -93769,30 +93685,30 @@ --- PASS: TestSnapshot_TruncatedVerify/truncate_2_bytes_from_end (0.36s) --- PASS: TestSnapshot_TruncatedVerify/truncate_1_bytes_from_end (0.36s) === RUN TestSnapshot_BadRestore -2022-09-15T21:47:19.388Z [INFO] raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-264fa024-fd73-0917-3eef-1c299dd249ca Address:264fa024-fd73-0917-3eef-1c299dd249ca}]" -2022-09-15T21:47:19.388Z [INFO] raft: entering follower state: follower="Node at 264fa024-fd73-0917-3eef-1c299dd249ca [Follower]" leader= -2022-09-15T21:47:20.981Z [WARN] raft: heartbeat timeout reached, starting election: last-leader= -2022-09-15T21:47:20.981Z [INFO] raft: entering candidate state: node="Node at 264fa024-fd73-0917-3eef-1c299dd249ca [Candidate]" term=2 -2022-09-15T21:47:20.981Z [DEBUG] raft: votes: needed=1 -2022-09-15T21:47:20.981Z [DEBUG] raft: vote granted: from=server-264fa024-fd73-0917-3eef-1c299dd249ca term=2 tally=1 -2022-09-15T21:47:20.981Z [INFO] raft: election won: tally=1 -2022-09-15T21:47:20.981Z [INFO] raft: entering leader state: leader="Node at 264fa024-fd73-0917-3eef-1c299dd249ca [Leader]" -2022-09-15T21:47:21.594Z [INFO] raft: starting snapshot up to: index=16386 -2022-09-15T21:47:21.594Z [INFO] snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshot_BadRestore-snapshot957657153/before/snapshots/2-16386-1663278441594.tmp -2022-09-15T21:47:21.628Z [INFO] raft: compacting logs: from=1 to=6146 -2022-09-15T21:47:21.629Z [INFO] raft: snapshot complete up to: index=16386 -2022-09-15T21:47:22.226Z [INFO] raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-4d0a7e83-d597-8021-1d72-8754c4f848ce Address:4d0a7e83-d597-8021-1d72-8754c4f848ce}]" -2022-09-15T21:47:22.226Z [INFO] raft: entering follower state: follower="Node at 4d0a7e83-d597-8021-1d72-8754c4f848ce [Follower]" leader= -2022-09-15T21:47:23.396Z [WARN] raft: heartbeat timeout reached, starting election: last-leader= -2022-09-15T21:47:23.396Z [INFO] raft: entering candidate state: node="Node at 4d0a7e83-d597-8021-1d72-8754c4f848ce [Candidate]" term=2 -2022-09-15T21:47:23.396Z [DEBUG] raft: votes: needed=1 -2022-09-15T21:47:23.396Z [DEBUG] raft: vote granted: from=server-4d0a7e83-d597-8021-1d72-8754c4f848ce term=2 tally=1 -2022-09-15T21:47:23.396Z [INFO] raft: election won: tally=1 -2022-09-15T21:47:23.396Z [INFO] raft: entering leader state: leader="Node at 4d0a7e83-d597-8021-1d72-8754c4f848ce [Leader]" -2022-09-15T21:47:23.398Z [ERROR] TestSnapshot_BadRestore: Failed to close snapshot decompressor: error="unexpected EOF" ---- PASS: TestSnapshot_BadRestore (4.01s) +2021-08-13T15:40:05.979Z [INFO] raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-726aad49-ee7d-466a-34ee-89cd8fb24253 Address:726aad49-ee7d-466a-34ee-89cd8fb24253}]" +2021-08-13T15:40:05.979Z [INFO] raft: entering follower state: follower="Node at 726aad49-ee7d-466a-34ee-89cd8fb24253 [Follower]" leader= +2021-08-13T15:40:07.715Z [WARN] raft: heartbeat timeout reached, starting election: last-leader= +2021-08-13T15:40:07.715Z [INFO] raft: entering candidate state: node="Node at 726aad49-ee7d-466a-34ee-89cd8fb24253 [Candidate]" term=2 +2021-08-13T15:40:07.715Z [DEBUG] raft: votes: needed=1 +2021-08-13T15:40:07.715Z [DEBUG] raft: vote granted: from=server-726aad49-ee7d-466a-34ee-89cd8fb24253 term=2 tally=1 +2021-08-13T15:40:07.715Z [INFO] raft: election won: tally=1 +2021-08-13T15:40:07.715Z [INFO] raft: entering leader state: leader="Node at 726aad49-ee7d-466a-34ee-89cd8fb24253 [Leader]" +2021-08-13T15:40:08.357Z [INFO] raft: starting snapshot up to: index=16386 +2021-08-13T15:40:08.357Z [INFO] snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshot_BadRestore-snapshot936455296/before/snapshots/2-16386-1628869208357.tmp +2021-08-13T15:40:08.383Z [INFO] raft: compacting logs: from=1 to=6146 +2021-08-13T15:40:08.384Z [INFO] raft: snapshot complete up to: index=16386 +2021-08-13T15:40:08.928Z [INFO] raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-e33a09e3-9f31-02eb-f99d-f3cf92f3978f Address:e33a09e3-9f31-02eb-f99d-f3cf92f3978f}]" +2021-08-13T15:40:08.928Z [INFO] raft: entering follower state: follower="Node at e33a09e3-9f31-02eb-f99d-f3cf92f3978f [Follower]" leader= +2021-08-13T15:40:10.700Z [WARN] raft: heartbeat timeout reached, starting election: last-leader= +2021-08-13T15:40:10.700Z [INFO] raft: entering candidate state: node="Node at e33a09e3-9f31-02eb-f99d-f3cf92f3978f [Candidate]" term=2 +2021-08-13T15:40:10.700Z [DEBUG] raft: votes: needed=1 +2021-08-13T15:40:10.700Z [DEBUG] raft: vote granted: from=server-e33a09e3-9f31-02eb-f99d-f3cf92f3978f term=2 tally=1 +2021-08-13T15:40:10.701Z [INFO] raft: election won: tally=1 +2021-08-13T15:40:10.701Z [INFO] raft: entering leader state: leader="Node at e33a09e3-9f31-02eb-f99d-f3cf92f3978f [Leader]" +2021-08-13T15:40:10.702Z [ERROR] TestSnapshot_BadRestore: Failed to close snapshot decompressor: error="unexpected EOF" +--- PASS: TestSnapshot_BadRestore (4.73s) PASS -ok github.com/hashicorp/consul/snapshot 23.106s +ok github.com/hashicorp/consul/snapshot 23.522s ? github.com/hashicorp/consul/testrpc [no test files] === RUN TestConfigurator_outgoingWrapper_OK --- PASS: TestConfigurator_outgoingWrapper_OK (0.01s) @@ -93821,7 +93737,7 @@ === RUN TestConfig_SpecifyDC --- PASS: TestConfig_SpecifyDC (0.00s) === RUN TestConfigurator_NewConfigurator -2022-09-15T21:47:00.847Z [TRACE] TestConfigurator_NewConfigurator.tlsutil: Update: version=1 +2021-08-13T15:39:48.063Z [TRACE] TestConfigurator_NewConfigurator.tlsutil: Update: version=1 --- PASS: TestConfigurator_NewConfigurator (0.00s) === RUN TestConfigurator_ErrorPropagation --- PASS: TestConfigurator_ErrorPropagation (0.01s) @@ -93836,7 +93752,7 @@ === RUN TestConfigurator_CommonTLSConfigCipherSuites --- PASS: TestConfigurator_CommonTLSConfigCipherSuites (0.00s) === RUN TestConfigurator_CommonTLSConfigGetClientCertificate ---- PASS: TestConfigurator_CommonTLSConfigGetClientCertificate (0.03s) +--- PASS: TestConfigurator_CommonTLSConfigGetClientCertificate (0.02s) === RUN TestConfigurator_CommonTLSConfigGetCertificate --- PASS: TestConfigurator_CommonTLSConfigGetCertificate (0.00s) === RUN TestConfigurator_CommonTLSConfigCAs @@ -93911,7 +93827,7 @@ --- PASS: TestGeneratePrivateKey (0.00s) --- PASS: TestGenerateCA (0.00s) PASS -ok github.com/hashicorp/consul/tlsutil 0.201s +ok github.com/hashicorp/consul/tlsutil 0.187s ? github.com/hashicorp/consul/types [no test files] ? github.com/hashicorp/consul/version [no test files] make[1]: Leaving directory '/build/consul-1.8.7+dfsg1' @@ -93960,8 +93876,8 @@ dh_gencontrol -O--buildsystem=golang -O--builddirectory=_build dh_md5sums -O--buildsystem=golang -O--builddirectory=_build dh_builddeb -O--buildsystem=golang -O--builddirectory=_build -dpkg-deb: building package 'consul' in '../consul_1.8.7+dfsg1-2_arm64.deb'. dpkg-deb: building package 'golang-github-hashicorp-consul-dev' in '../golang-github-hashicorp-consul-dev_1.8.7+dfsg1-2_all.deb'. +dpkg-deb: building package 'consul' in '../consul_1.8.7+dfsg1-2_arm64.deb'. dpkg-genbuildinfo --build=binary dpkg-genchanges --build=binary >../consul_1.8.7+dfsg1-2_arm64.changes dpkg-genchanges: info: binary-only upload (no source code included) @@ -93969,12 +93885,14 @@ dpkg-buildpackage: info: binary-only upload (no source included) dpkg-genchanges: info: not including original source code in upload I: copying local configuration +I: user script /srv/workspace/pbuilder/20322/tmp/hooks/B01_cleanup starting +I: user script /srv/workspace/pbuilder/20322/tmp/hooks/B01_cleanup finished I: unmounting dev/ptmx filesystem I: unmounting dev/pts filesystem I: unmounting dev/shm filesystem I: unmounting proc filesystem I: unmounting sys filesystem I: cleaning the build env -I: removing directory /srv/workspace/pbuilder/17411 and its subdirectories -I: Current time: Thu Sep 15 09:49:11 -12 2022 -I: pbuilder-time-stamp: 1663278551 +I: removing directory /srv/workspace/pbuilder/20322 and its subdirectories +I: Current time: Sat Aug 14 05:41:47 +14 2021 +I: pbuilder-time-stamp: 1628869307